Continuously attempt to unseal if sealed keys are supported (#6039)

* Add helper for checking if an error is a fatal error

The double-double negative was really confusing, and this pattern is used a few places in Vault. This negates the double negative, making the devx a bit easier to follow.

* Check return value of UnsealWithStoredKeys in sys/init

* Return proper error types when attempting unseal with stored key

Prior to this commit, "nil" could have meant unsupported auto-unseal, a transient error, or success. This updates the function to return the correct error type, signaling to the caller whether they should retry or fail.

* Continuously attempt to unseal if sealed keys are supported

This fixes a bug that occurs on bootstrapping an initial cluster. Given a collection of Vault nodes and an initialized storage backend, they will all go into standby waiting for initialization. After one node is initialized, the other nodes had no mechanism by which they "re-check" to see if unseal keys are present. This adds a goroutine to the server command which continually waits for unseal keys to exist. It exits in the following conditions:

- the node is unsealed
- the node does not support stored keys
- a fatal error occurs (as defined by Vault)
- the server is shutting down

In all other situations, the routine wakes up at the specified interval and attempts to unseal with the stored keys.

command/server.go

@@ -741,7 +741,7 @@ CLUSTER_SYNTHESIS_COMPLETE:
 	// Initialize the core
 	core, newCoreError := vault.NewCore(coreConfig)
 	if newCoreError != nil {
-		if !errwrap.ContainsType(newCoreError, new(vault.NonFatalError)) {
+		if vault.IsFatalError(newCoreError) {
 			c.UI.Error(fmt.Sprintf("Error initializing core: %s", newCoreError))
 			return 1
 		}
@@ -937,12 +937,31 @@ CLUSTER_SYNTHESIS_COMPLETE:
 		return 1
 	}
 
-	if err := core.UnsealWithStoredKeys(context.Background()); err != nil {
-		if !errwrap.ContainsType(err, new(vault.NonFatalError)) {
-			c.UI.Error(fmt.Sprintf("Error initializing core: %s", err))
-			return 1
+	// Attempt unsealing in a background goroutine. This is needed for when a
+	// Vault cluster with multiple servers is configured with auto-unseal but is
+	// uninitialized. Once one server initializes the storage backend, this
+	// goroutine will pick up the unseal keys and unseal this instance.
+	go func() {
+		for {
+			err := core.UnsealWithStoredKeys(context.Background())
+			if err == nil {
+				return
+			}
+
+			if vault.IsFatalError(err) {
+				c.logger.Error(fmt.Sprintf("Error unsealing core", "error", err))
+				return
+			} else {
+				c.logger.Warn(fmt.Sprintf("Failed to unseal core", "error" err))
+			}
+
+			select {
+			case <-c.ShutdownCh:
+				return
+			case <-time.After(5 * time.Second):
+			}
 		}
-	}
+	}()
 
 	// Perform service discovery registrations and initialization of
 	// HTTP server after the verifyOnly check.

http/sys_init.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/vault/vault"
 )
 
@@ -104,7 +103,7 @@ func handleSysInitPut(core *vault.Core, w http.ResponseWriter, r *http.Request)
 
 	result, initErr := core.Initialize(ctx, initParams)
 	if initErr != nil {
-		if !errwrap.ContainsType(initErr, new(vault.NonFatalError)) {
+		if vault.IsFatalError(initErr) {
 			respondError(w, http.StatusBadRequest, initErr)
 			return
 		} else {
@@ -136,7 +135,10 @@ func handleSysInitPut(core *vault.Core, w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	core.UnsealWithStoredKeys(ctx)
+	if err := core.UnsealWithStoredKeys(ctx); err != nil {
+		respondError(w, http.StatusInternalServerError, err)
+		return
+	}
 
 	respondOk(w, resp)
 }

vault/core.go

@@ -107,6 +107,16 @@ func (e *NonFatalError) Error() string {
 	return e.Err.Error()
 }
 
+// NewNonFatalError returns a new non-fatal error.
+func NewNonFatalError(err error) *NonFatalError {
+	return &NonFatalError{Err: err}
+}
+
+// IsFatalError returns true if the given error is a non-fatal error.
+func IsFatalError(err error) bool {
+	return !errwrap.ContainsType(err, new(NonFatalError))
+}
+
 // ErrInvalidKey is returned if there is a user-based error with a provided
 // unseal key. This will be shown to the user, so should not contain
 // information that is sensitive.
@@ -386,6 +396,10 @@ type Core struct {
 	// Stores the sealunwrapper for downgrade needs
 	sealUnwrapper physical.Backend
 
+	// unsealwithStoredKeysLock is a mutex that prevents multiple processes from
+	// unsealing with stored keys are the same time.
+	unsealWithStoredKeysLock sync.Mutex
+
 	// Stores any funcs that should be run on successful postUnseal
 	postUnsealFuncs []func()
 

vault/init.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 
 	"github.com/hashicorp/errwrap"
@@ -270,53 +271,62 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes
 	return results, nil
 }
 
-// UnsealWithStoredKeys performs auto-unseal using stored keys.
+// UnsealWithStoredKeys performs auto-unseal using stored keys. An error
+// return value of "nil" implies the Vault instance is unsealed.
+//
+// Callers should attempt to retry any NOnFatalErrors. Callers should
+// not re-attempt fatal errors.
 func (c *Core) UnsealWithStoredKeys(ctx context.Context) error {
+	c.unsealWithStoredKeysLock.Lock()
+	defer c.unsealWithStoredKeysLock.Unlock()
+
 	if !c.seal.StoredKeysSupported() {
 		return nil
 	}
 
 	// Disallow auto-unsealing when migrating
 	if c.IsInSealMigration() {
-		return nil
+		return NewNonFatalError(errors.New("cannot auto-unseal during seal migration"))
 	}
 
 	sealed := c.Sealed()
 	if !sealed {
+		c.Logger().Warn("attempted unseal with stored keys, but vault is already unsealed")
 		return nil
 	}
 
-	c.logger.Info("stored unseal keys supported, attempting fetch")
+	c.Logger().Info("stored unseal keys supported, attempting fetch")
 	keys, err := c.seal.GetStoredKeys(ctx)
 	if err != nil {
-		c.logger.Error("fetching stored unseal keys failed", "error", err)
-		return &NonFatalError{Err: errwrap.Wrapf("fetching stored unseal keys failed: {{err}}", err)}
+		return NewNonFatalError(errwrap.Wrapf("fetching stored unseal keys failed: {{err}}", err))
 	}
+
+	// This usually happens when auto-unseal is configured, but the servers have
+	// not been initialized yet.
 	if len(keys) == 0 {
-		c.logger.Warn("stored unseal key(s) supported but none found")
+		return NewNonFatalError(errors.New("stored unseal keys are supported, but none were found"))
+	}
+
+	unsealed := false
+	keysUsed := 0
+	for _, key := range keys {
+		unsealed, err = c.Unseal(key)
+		if err != nil {
+			return NewNonFatalError(errwrap.Wrapf("unseal with stored key failed: {{err}}", err))
+		}
+		keysUsed++
+		if unsealed {
+			break
+		}
+	}
+
+	if !unsealed {
+		// This most likely means that the user configured Vault to only store a
+		// subset of the required threshold of keys. We still consider this a
+		// "success", since trying again would yield the same result.
+		c.Logger().Warn("vault still sealed after using stored unseal keys", "stored_keys_used", keysUsed)
 	} else {
-		unsealed := false
-		keysUsed := 0
-		for _, key := range keys {
-			unsealed, err = c.Unseal(key)
-			if err != nil {
-				c.logger.Error("unseal with stored unseal key failed", "error", err)
-				return &NonFatalError{Err: errwrap.Wrapf("unseal with stored key failed: {{err}}", err)}
-			}
-			keysUsed += 1
-			if unsealed {
-				break
-			}
-		}
-		if !unsealed {
-			if c.logger.IsWarn() {
-				c.logger.Warn("stored unseal key(s) used but Vault not unsealed yet", "stored_keys_used", keysUsed)
-			}
-		} else {
-			if c.logger.IsInfo() {
-				c.logger.Info("successfully unsealed with stored key(s)", "stored_keys_used", keysUsed)
-			}
-		}
+		c.Logger().Info("unsealed with stored keys", "stored_keys_used", keysUsed)
 	}
 
 	return nil