Fix race with test that mutates KeyRotateGracePeriod: make the global be a Core field instead. (#10512)

vault/core.go

@@ -534,6 +534,10 @@ type Core struct {
 	clusterHeartbeatInterval time.Duration
 
 	activityLogConfig ActivityLogCoreConfig
+
+	// KeyRotateGracePeriod is how long we allow an upgrade path
+	// for standby instances before we delete the upgrade keys
+	keyRotateGracePeriod *int64
 }
 
 // CoreConfig is used to parameterize a core
@@ -776,6 +780,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 		raftJoinDoneCh:           make(chan struct{}),
 		clusterHeartbeatInterval: clusterHeartbeatInterval,
 		activityLogConfig:        conf.ActivityLogConfig,
+		keyRotateGracePeriod:     new(int64),
 	}
 	c.standbyStopCh.Store(make(chan struct{}))
 	atomic.StoreUint32(c.sealed, 1)
@@ -796,6 +801,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 	c.clusterLeaderParams.Store((*ClusterLeaderParams)(nil))
 	c.clusterAddr.Store(conf.ClusterAddr)
 	c.activeContextCancelFunc.Store((context.CancelFunc)(nil))
+	atomic.StoreInt64(c.keyRotateGracePeriod, int64(2*time.Minute))
 
 	switch conf.ClusterCipherSuites {
 	case "tls13", "tls12":
@@ -2664,3 +2670,11 @@ func (c *Core) RateLimitResponseHeadersEnabled() bool {
 
 	return false
 }
+
+func (c *Core) KeyRotateGracePeriod() time.Duration {
+	return time.Duration(atomic.LoadInt64(c.keyRotateGracePeriod))
+}
+
+func (c *Core) SetKeyRotateGracePeriod(t time.Duration) {
+	atomic.StoreInt64(c.keyRotateGracePeriod, int64(t))
+}

vault/external_tests/raft/raft_test.go

@@ -638,7 +638,9 @@ func TestRaft_SnapshotAPI_RekeyRotate_Forward(t *testing.T) {
 				// Set the key clean up to 0 so it's cleaned immediately. This
 				// will simulate that there are no ways to upgrade to the latest
 				// term.
-				vault.KeyRotateGracePeriod = 0
+				for _, c := range cluster.Cores {
+					c.Core.SetKeyRotateGracePeriod(0)
+				}
 
 				// Rotate
 				err = leaderClient.Sys().Rotate()

vault/ha.go

@@ -44,10 +44,6 @@ const (
 )
 
 var (
-	// KeyRotateGracePeriod is how long we allow an upgrade path
-	// for standby instances before we delete the upgrade keys
-	KeyRotateGracePeriod = 2 * time.Minute
-
 	addEnterpriseHaActors func(*Core, *run.Group) chan func()            = addEnterpriseHaActorsNoop
 	interruptPerfStandby  func(chan func(), chan struct{}) chan struct{} = interruptPerfStandbyNoop
 )
@@ -882,7 +878,7 @@ func (c *Core) scheduleUpgradeCleanup(ctx context.Context) error {
 	}
 
 	// Schedule cleanup for all of them
-	time.AfterFunc(KeyRotateGracePeriod, func() {
+	time.AfterFunc(c.KeyRotateGracePeriod(), func() {
 		sealed, err := c.barrier.Sealed()
 		if err != nil {
 			c.logger.Warn("failed to check barrier status at upgrade cleanup time")

vault/logical_system.go

@@ -2540,8 +2540,8 @@ func (b *SystemBackend) handleRotate(ctx context.Context, req *logical.Request,
 		}
 
 		// Schedule the destroy of the upgrade path
-		time.AfterFunc(KeyRotateGracePeriod, func() {
-			b.Backend.Logger().Debug("cleaning up upgrade keys", "waited", KeyRotateGracePeriod)
+		time.AfterFunc(b.Core.KeyRotateGracePeriod(), func() {
+			b.Backend.Logger().Debug("cleaning up upgrade keys", "waited", b.Core.KeyRotateGracePeriod())
 			if err := b.Core.barrier.DestroyUpgrade(b.Core.activeContext, newTerm); err != nil {
 				b.Backend.Logger().Error("failed to destroy upgrade", "term", newTerm, "error", err)
 			}