core: Record the time a node became active (#10489)

* core: Record the time a node became active

* Update vault/core.go

Co-authored-by: Nick Cabatoff <ncabatoff@hashicorp.com>

* Add omitempty field

* Update vendor

* Added CL entry and fixed test

* Fix test

* Fix command package tests

Co-authored-by: Nick Cabatoff <ncabatoff@hashicorp.com>

api/sys_leader.go

@@ -1,6 +1,9 @@
 package api
 
-import "context"
+import (
+	"context"
+	"time"
+)
 
 func (c *Sys) Leader() (*LeaderResponse, error) {
 	r := c.c.NewRequest("GET", "/v1/sys/leader")
@@ -21,6 +24,7 @@ func (c *Sys) Leader() (*LeaderResponse, error) {
 type LeaderResponse struct {
 	HAEnabled                bool      `json:"ha_enabled"`
 	IsSelf                   bool      `json:"is_self"`
+	ActiveTime               time.Time `json:"active_time"`
 	LeaderAddress            string    `json:"leader_address"`
 	LeaderClusterAddress     string    `json:"leader_cluster_address"`
 	PerfStandby              bool      `json:"performance_standby"`

changelog/10489.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: Added active since timestamp to the status output of active nodes.
+```

command/format.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/hashicorp/vault/api"
@@ -191,6 +192,9 @@ func (t TableFormatter) OutputSealStatusStruct(ui cli.Ui, secret *api.Secret, da
 			}
 			out = append(out, fmt.Sprintf("HA Mode | %s", mode))
 
+			if status.IsSelf && !status.ActiveTime.IsZero() {
+				out = append(out, fmt.Sprintf("Active Since | %s", status.ActiveTime.Format(time.RFC3339Nano)))
+			}
 			// This is down here just to keep ordering consistent
 			if showLeaderAddr {
 				out = append(out, fmt.Sprintf("Active Node Address | %s", status.LeaderAddress))
@@ -405,6 +409,7 @@ func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusRespo
 	// copy leaderStatus fields into sealStatusOutput for display later
 	sealStatusOutput.HAEnabled = leaderStatus.HAEnabled
 	sealStatusOutput.IsSelf = leaderStatus.IsSelf
+	sealStatusOutput.ActiveTime = leaderStatus.ActiveTime
 	sealStatusOutput.LeaderAddress = leaderStatus.LeaderAddress
 	sealStatusOutput.LeaderClusterAddress = leaderStatus.LeaderClusterAddress
 	sealStatusOutput.PerfStandby = leaderStatus.PerfStandby
@@ -433,6 +438,7 @@ type SealStatusOutput struct {
 	api.SealStatusResponse
 	HAEnabled                bool      `json:"ha_enabled"`
 	IsSelf                   bool      `json:"is_self,omitempty""`
+	ActiveTime               time.Time `json:"active_time,omitempty""`
 	LeaderAddress            string    `json:"leader_address,omitempty"`
 	LeaderClusterAddress     string    `json:"leader_cluster_address,omitempty"`
 	PerfStandby              bool      `json:"performance_standby,omitempty"`

command/format_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/hashicorp/vault/api"
@@ -178,6 +179,7 @@ func getMockStatusData(emptyFields bool) SealStatusOutput {
 			sealStatusResponseMock,
 			true,                     // HAEnabled
 			true,                     // IsSelf
+			time.Time{}.UTC(),        // ActiveTime
 			"leader address",         // LeaderAddress
 			"leader cluster address", // LeaderClusterAddress
 			true,                     // PerfStandby
@@ -208,6 +210,7 @@ func getMockStatusData(emptyFields bool) SealStatusOutput {
 			sealStatusResponseMock,
 			false,             // HAEnabled
 			false,             // IsSelf
+			time.Time{}.UTC(), // ActiveTime
 			"",                // LeaderAddress
 			"",                // LeaderClusterAddress
 			false,             // PerfStandby

http/sys_leader.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/vault/vault"
@@ -36,6 +37,9 @@ func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request
 		LeaderClusterAddress: clusterAddr,
 		PerfStandby:          core.PerfStandby(),
 	}
+	if isLeader {
+		resp.ActiveTime = core.ActiveTime()
+	}
 	if resp.PerfStandby {
 		resp.PerfStandbyLastRemoteWAL = vault.LastRemoteWAL(core)
 	} else if isLeader || !haEnabled {
@@ -50,6 +54,7 @@ func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request
 type LeaderResponse struct {
 	HAEnabled                bool      `json:"ha_enabled"`
 	IsSelf                   bool      `json:"is_self"`
+	ActiveTime               time.Time `json:"active_time,omitempty"`
 	LeaderAddress            string    `json:"leader_address"`
 	LeaderClusterAddress     string    `json:"leader_cluster_address"`
 	PerfStandby              bool      `json:"performance_standby"`

http/sys_leader_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/vault/vault"
 )
@@ -27,10 +28,11 @@ func TestSysLeader_get(t *testing.T) {
 		"leader_cluster_address":              "",
 		"performance_standby":                 false,
 		"performance_standby_last_remote_wal": json.Number("0"),
+		"active_time":                         time.Time{}.UTC().Format(time.RFC3339),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
 	if !reflect.DeepEqual(actual, expected) {
-		t.Fatalf("bad: %#v", actual)
+		t.Fatalf("bad: %#v \n%#v", actual, expected)
 	}
 }

vault/core.go

@@ -535,6 +535,10 @@ type Core struct {
 
 	activityLogConfig ActivityLogCoreConfig
 
+	// activeTime is set on active nodes indicating the time at which this node
+	// became active.
+	activeTime time.Time
+
 	// KeyRotateGracePeriod is how long we allow an upgrade path
 	// for standby instances before we delete the upgrade keys
 	keyRotateGracePeriod *int64
@@ -1856,6 +1860,10 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
 	c.clearForwardingClients()
 	c.requestForwardingConnectionLock.Unlock()
 
+	// Mark the active time. We do this first so it can be correlated to the logs
+	// for the active startup.
+	c.activeTime = time.Now().UTC()
+
 	if err := postUnsealPhysical(c); err != nil {
 		return err
 	}
@@ -2047,6 +2055,7 @@ func (c *Core) preSeal() error {
 
 	// Clear any pending funcs
 	c.postUnsealFuncs = nil
+	c.activeTime = time.Time{}
 
 	// Clear any rekey progress
 	c.barrierRekeyConfig = nil

vault/ha.go

@@ -70,6 +70,13 @@ func (c *Core) PerfStandby() bool {
 	return perfStandby
 }
 
+func (c *Core) ActiveTime() time.Time {
+	c.stateLock.RLock()
+	activeTime := c.activeTime
+	c.stateLock.RUnlock()
+	return activeTime
+}
+
 // StandbyStates is meant as a way to avoid some extra locking on the very
 // common sys/health check.
 func (c *Core) StandbyStates() (standby, perfStandby bool) {

vendor/github.com/hashicorp/vault/api/sys_leader.go

@@ -1,6 +1,9 @@
 package api
 
-import "context"
+import (
+	"context"
+	"time"
+)
 
 func (c *Sys) Leader() (*LeaderResponse, error) {
 	r := c.c.NewRequest("GET", "/v1/sys/leader")
@@ -21,6 +24,7 @@ func (c *Sys) Leader() (*LeaderResponse, error) {
 type LeaderResponse struct {
 	HAEnabled                bool      `json:"ha_enabled"`
 	IsSelf                   bool      `json:"is_self"`
+	ActiveTime               time.Time `json:"active_time"`
 	LeaderAddress            string    `json:"leader_address"`
 	LeaderClusterAddress     string    `json:"leader_cluster_address"`
 	PerfStandby              bool      `json:"performance_standby"`