backport of commit 91ec1a788b46c0bf12a3351e5e3339474400eee9 (#24363)

Co-authored-by: Scott Miller <smiller@hashicorp.com>

changelog/24336.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core: Fix a timeout initializing Vault by only using a short timeout persisting barrier keyring encryption counts.
+```

vault/barrier_aes_gcm.go

@@ -37,6 +37,8 @@ const (
 
 	autoRotateCheckInterval = 5 * time.Minute
 	legacyRotateReason      = "legacy rotation"
+	// The keyring is persisted before the root key.
+	keyringTimeout = 1 * time.Second
 )
 
 // Versions of the AESGCM storage methodology
@@ -211,11 +213,18 @@ func (b *AESGCMBarrier) Initialize(ctx context.Context, key, sealKey []byte, rea
 // persistKeyring is used to write out the keyring using the
 // root key to encrypt it.
 func (b *AESGCMBarrier) persistKeyring(ctx context.Context, keyring *Keyring) error {
-	const (
+	return b.persistKeyringInternal(ctx, keyring, false)
-		// The keyring is persisted before the root key.
+}
-		keyringTimeout = 1 * time.Second
-	)
 
+// persistKeyringBestEffort is like persistKeyring but 'best effort', ie times out early
+// for non critical keyring writes (encryption/rotation tracking)
+func (b *AESGCMBarrier) persistKeyringBestEffort(ctx context.Context, keyring *Keyring) error {
+	return b.persistKeyringInternal(ctx, keyring, true)
+}
+
+// persistKeyring is used to write out the keyring using the
+// root key to encrypt it.
+func (b *AESGCMBarrier) persistKeyringInternal(ctx context.Context, keyring *Keyring, bestEffort bool) error {
 	// Create the keyring entry
 	keyringBuf, err := keyring.Serialize()
 	defer memzero(keyringBuf)
@@ -241,10 +250,16 @@ func (b *AESGCMBarrier) persistKeyring(ctx context.Context, keyring *Keyring) er
 		Value: value,
 	}
 
-	// We reduce the timeout on the initial 'put' but if this succeeds we will
+	ctxKeyring := ctx
-	// allow longer later on when we try to persist the root key .
+
-	ctxKeyring, cancelKeyring := context.WithTimeout(ctx, keyringTimeout)
+	if bestEffort {
-	defer cancelKeyring()
+		// We reduce the timeout on the initial 'put' but if this succeeds we will
+		// allow longer later on when we try to persist the root key .
+		var cancelKeyring func()
+		ctxKeyring, cancelKeyring = context.WithTimeout(ctx, keyringTimeout)
+		defer cancelKeyring()
+	}
+
 	if err := b.backend.Put(ctxKeyring, pe); err != nil {
 		return fmt.Errorf("failed to persist keyring: %w", err)
 	}
@@ -1231,7 +1246,7 @@ func (b *AESGCMBarrier) persistEncryptions(ctx context.Context) error {
 			newEncs := upe + 1
 			activeKey.Encryptions += uint64(newEncs)
 			newKeyring := b.keyring.Clone()
-			err := b.persistKeyring(ctx, newKeyring)
+			err := b.persistKeyringBestEffort(ctx, newKeyring)
 			if err != nil {
 				return err
 			}