Use a separate var for active node replication state (#3819)

http/sys_health.go

@@ -115,7 +115,13 @@ func getSysHealth(core *vault.Core, r *http.Request) (int, *HealthResponse, erro
 	// Check system status
 	sealed, _ := core.Sealed()
 	standby, _ := core.Standby()
-	replicationState := core.ReplicationState()
+	var replicationState consts.ReplicationState
+	if standby {
+		replicationState = core.ActiveNodeReplicationState()
+	} else {
+		replicationState = core.ReplicationState()
+	}
+
 	init, err := core.Initialized(ctx)
 	if err != nil {
 		return http.StatusInternalServerError, nil, err

vault/core.go

@@ -355,8 +355,9 @@ type Core struct {
 
 	atomicPrimaryFailoverAddrs *atomic.Value
 	// replicationState keeps the current replication state cached for quick
-	// lookup
+	// lookup; activeNodeReplicationState stores the active value on standbys
 	replicationState           *uint32
+	activeNodeReplicationState *uint32
 
 	// uiEnabled indicates whether Vault Web UI is enabled or not
 	uiEnabled bool
@@ -489,6 +490,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 		replicationState:                 new(uint32),
 		atomicPrimaryClusterAddrs:        new(atomic.Value),
 		atomicPrimaryFailoverAddrs:       new(atomic.Value),
+		activeNodeReplicationState:       new(uint32),
 	}
 
 	if conf.ClusterCipherSuites != "" {
@@ -2125,6 +2127,10 @@ func (c *Core) ReplicationState() consts.ReplicationState {
 	return consts.ReplicationState(atomic.LoadUint32(c.replicationState))
 }
 
+func (c *Core) ActiveNodeReplicationState() consts.ReplicationState {
+	return consts.ReplicationState(atomic.LoadUint32(c.activeNodeReplicationState))
+}
+
 func (c *Core) SealAccess() *SealAccess {
 	return NewSealAccess(c.seal)
 }

vault/request_forwarding.go

@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/forwarding"
 	"golang.org/x/net/http2"
 	"google.golang.org/grpc"
@@ -26,7 +27,7 @@ const (
 
 var (
 	// Making this a package var allows tests to modify
-	HeartbeatInterval = 30 * time.Second
+	HeartbeatInterval = 5 * time.Second
 )
 
 // Starts the listeners and servers necessary to handle forwarded requests
@@ -468,8 +469,8 @@ func (c *forwardingClient) startHeartbeat() {
 			}
 			// Store the active node's replication state to display in
 			// sys/health calls
-			atomic.StoreUint32(c.core.replicationState, resp.ReplicationState)
+			atomic.StoreUint32(c.core.activeNodeReplicationState, resp.ReplicationState)
-			c.core.logger.Trace("forwarding: successful heartbeat")
+			//c.core.logger.Trace("forwarding: successful heartbeat")
 		}
 
 		tick()
@@ -479,6 +480,7 @@ func (c *forwardingClient) startHeartbeat() {
 			case <-c.echoContext.Done():
 				c.echoTicker.Stop()
 				c.core.logger.Trace("forwarding: stopping heartbeating")
+				atomic.StoreUint32(c.core.activeNodeReplicationState, uint32(consts.ReplicationDisabled))
 				return
 			case <-c.echoTicker.C:
 				tick()