From 275ca323e82380050e419ab6b5adedc656ec63c9 Mon Sep 17 00:00:00 2001 From: Brian Kassouf Date: Fri, 11 Dec 2020 16:50:19 -0800 Subject: [PATCH] core: Record the time a node became active (#10489) * core: Record the time a node became active * Update vault/core.go Co-authored-by: Nick Cabatoff * Add omitempty field * Update vendor * Added CL entry and fixed test * Fix test * Fix command package tests Co-authored-by: Nick Cabatoff --- api/sys_leader.go | 24 +++++++++++-------- changelog/10489.txt | 3 +++ command/format.go | 24 ++++++++++++------- command/format_test.go | 21 +++++++++------- http/sys_leader.go | 19 +++++++++------ http/sys_leader_test.go | 4 +++- vault/core.go | 9 +++++++ vault/ha.go | 7 ++++++ .../hashicorp/vault/api/sys_leader.go | 24 +++++++++++-------- 9 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 changelog/10489.txt diff --git a/api/sys_leader.go b/api/sys_leader.go index e8104b86a..1c6be8d88 100644 --- a/api/sys_leader.go +++ b/api/sys_leader.go @@ -1,6 +1,9 @@ package api -import "context" +import ( + "context" + "time" +) func (c *Sys) Leader() (*LeaderResponse, error) { r := c.c.NewRequest("GET", "/v1/sys/leader") @@ -19,13 +22,14 @@ func (c *Sys) Leader() (*LeaderResponse, error) { } type LeaderResponse struct { - HAEnabled bool `json:"ha_enabled"` - IsSelf bool `json:"is_self"` - LeaderAddress string `json:"leader_address"` - LeaderClusterAddress string `json:"leader_cluster_address"` - PerfStandby bool `json:"performance_standby"` - PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` - LastWAL uint64 `json:"last_wal"` - RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` - RaftAppliedIndex uint64 `json:"raft_applied_index,omitempty"` + HAEnabled bool `json:"ha_enabled"` + IsSelf bool `json:"is_self"` + ActiveTime time.Time `json:"active_time"` + LeaderAddress string `json:"leader_address"` + LeaderClusterAddress string `json:"leader_cluster_address"` + PerfStandby bool `json:"performance_standby"` + PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` + LastWAL uint64 `json:"last_wal"` + RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` + RaftAppliedIndex uint64 `json:"raft_applied_index,omitempty"` } diff --git a/changelog/10489.txt b/changelog/10489.txt new file mode 100644 index 000000000..c86cb8a74 --- /dev/null +++ b/changelog/10489.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core: Added active since timestamp to the status output of active nodes. +``` diff --git a/command/format.go b/command/format.go index 6e43a2c0b..d33bac4d4 100644 --- a/command/format.go +++ b/command/format.go @@ -7,6 +7,7 @@ import ( "os" "sort" "strings" + "time" "github.com/ghodss/yaml" "github.com/hashicorp/vault/api" @@ -191,6 +192,9 @@ func (t TableFormatter) OutputSealStatusStruct(ui cli.Ui, secret *api.Secret, da } out = append(out, fmt.Sprintf("HA Mode | %s", mode)) + if status.IsSelf && !status.ActiveTime.IsZero() { + out = append(out, fmt.Sprintf("Active Since | %s", status.ActiveTime.Format(time.RFC3339Nano))) + } // This is down here just to keep ordering consistent if showLeaderAddr { out = append(out, fmt.Sprintf("Active Node Address | %s", status.LeaderAddress)) @@ -405,6 +409,7 @@ func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusRespo // copy leaderStatus fields into sealStatusOutput for display later sealStatusOutput.HAEnabled = leaderStatus.HAEnabled sealStatusOutput.IsSelf = leaderStatus.IsSelf + sealStatusOutput.ActiveTime = leaderStatus.ActiveTime sealStatusOutput.LeaderAddress = leaderStatus.LeaderAddress sealStatusOutput.LeaderClusterAddress = leaderStatus.LeaderClusterAddress sealStatusOutput.PerfStandby = leaderStatus.PerfStandby @@ -431,13 +436,14 @@ func looksLikeDuration(k string) bool { // Currently we are adding the fields from api.LeaderResponse type SealStatusOutput struct { api.SealStatusResponse - HAEnabled bool `json:"ha_enabled"` - IsSelf bool `json:"is_self,omitempty""` - LeaderAddress string `json:"leader_address,omitempty"` - LeaderClusterAddress string `json:"leader_cluster_address,omitempty"` - PerfStandby bool `json:"performance_standby,omitempty"` - PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal,omitempty"` - LastWAL uint64 `json:"last_wal,omitempty"` - RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` - RaftAppliedIndex uint64 `json:"raft_applied_index,omitempty"` + HAEnabled bool `json:"ha_enabled"` + IsSelf bool `json:"is_self,omitempty""` + ActiveTime time.Time `json:"active_time,omitempty""` + LeaderAddress string `json:"leader_address,omitempty"` + LeaderClusterAddress string `json:"leader_cluster_address,omitempty"` + PerfStandby bool `json:"performance_standby,omitempty"` + PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal,omitempty"` + LastWAL uint64 `json:"last_wal,omitempty"` + RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` + RaftAppliedIndex uint64 `json:"raft_applied_index,omitempty"` } diff --git a/command/format_test.go b/command/format_test.go index 9fa222e4c..fafb178e1 100644 --- a/command/format_test.go +++ b/command/format_test.go @@ -6,6 +6,7 @@ import ( "os" "strings" "testing" + "time" "github.com/ghodss/yaml" "github.com/hashicorp/vault/api" @@ -178,6 +179,7 @@ func getMockStatusData(emptyFields bool) SealStatusOutput { sealStatusResponseMock, true, // HAEnabled true, // IsSelf + time.Time{}.UTC(), // ActiveTime "leader address", // LeaderAddress "leader cluster address", // LeaderClusterAddress true, // PerfStandby @@ -206,15 +208,16 @@ func getMockStatusData(emptyFields bool) SealStatusOutput { // must initialize this struct without explicit field names due to embedding status = SealStatusOutput{ sealStatusResponseMock, - false, // HAEnabled - false, // IsSelf - "", // LeaderAddress - "", // LeaderClusterAddress - false, // PerfStandby - 0, // PerfStandbyLastRemoteWAL - 0, // LastWAL - 0, // RaftCommittedIndex - 0, // RaftAppliedIndex + false, // HAEnabled + false, // IsSelf + time.Time{}.UTC(), // ActiveTime + "", // LeaderAddress + "", // LeaderClusterAddress + false, // PerfStandby + 0, // PerfStandbyLastRemoteWAL + 0, // LastWAL + 0, // RaftCommittedIndex + 0, // RaftAppliedIndex } } return status diff --git a/http/sys_leader.go b/http/sys_leader.go index 2772d98f0..8c8b69a50 100644 --- a/http/sys_leader.go +++ b/http/sys_leader.go @@ -2,6 +2,7 @@ package http import ( "net/http" + "time" "github.com/hashicorp/errwrap" "github.com/hashicorp/vault/vault" @@ -36,6 +37,9 @@ func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request LeaderClusterAddress: clusterAddr, PerfStandby: core.PerfStandby(), } + if isLeader { + resp.ActiveTime = core.ActiveTime() + } if resp.PerfStandby { resp.PerfStandbyLastRemoteWAL = vault.LastRemoteWAL(core) } else if isLeader || !haEnabled { @@ -48,13 +52,14 @@ func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request } type LeaderResponse struct { - HAEnabled bool `json:"ha_enabled"` - IsSelf bool `json:"is_self"` - LeaderAddress string `json:"leader_address"` - LeaderClusterAddress string `json:"leader_cluster_address"` - PerfStandby bool `json:"performance_standby"` - PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` - LastWAL uint64 `json:"last_wal,omitempty"` + HAEnabled bool `json:"ha_enabled"` + IsSelf bool `json:"is_self"` + ActiveTime time.Time `json:"active_time,omitempty"` + LeaderAddress string `json:"leader_address"` + LeaderClusterAddress string `json:"leader_cluster_address"` + PerfStandby bool `json:"performance_standby"` + PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` + LastWAL uint64 `json:"last_wal,omitempty"` // Raft Indexes for this node RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` diff --git a/http/sys_leader_test.go b/http/sys_leader_test.go index 6bf0e4a46..974b3a7b7 100644 --- a/http/sys_leader_test.go +++ b/http/sys_leader_test.go @@ -5,6 +5,7 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/hashicorp/vault/vault" ) @@ -27,10 +28,11 @@ func TestSysLeader_get(t *testing.T) { "leader_cluster_address": "", "performance_standby": false, "performance_standby_last_remote_wal": json.Number("0"), + "active_time": time.Time{}.UTC().Format(time.RFC3339), } testResponseStatus(t, resp, 200) testResponseBody(t, resp, &actual) if !reflect.DeepEqual(actual, expected) { - t.Fatalf("bad: %#v", actual) + t.Fatalf("bad: %#v \n%#v", actual, expected) } } diff --git a/vault/core.go b/vault/core.go index ddca75026..21c85b789 100644 --- a/vault/core.go +++ b/vault/core.go @@ -535,6 +535,10 @@ type Core struct { activityLogConfig ActivityLogCoreConfig + // activeTime is set on active nodes indicating the time at which this node + // became active. + activeTime time.Time + // KeyRotateGracePeriod is how long we allow an upgrade path // for standby instances before we delete the upgrade keys keyRotateGracePeriod *int64 @@ -1856,6 +1860,10 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c c.clearForwardingClients() c.requestForwardingConnectionLock.Unlock() + // Mark the active time. We do this first so it can be correlated to the logs + // for the active startup. + c.activeTime = time.Now().UTC() + if err := postUnsealPhysical(c); err != nil { return err } @@ -2047,6 +2055,7 @@ func (c *Core) preSeal() error { // Clear any pending funcs c.postUnsealFuncs = nil + c.activeTime = time.Time{} // Clear any rekey progress c.barrierRekeyConfig = nil diff --git a/vault/ha.go b/vault/ha.go index 88d300b15..375112ee3 100644 --- a/vault/ha.go +++ b/vault/ha.go @@ -70,6 +70,13 @@ func (c *Core) PerfStandby() bool { return perfStandby } +func (c *Core) ActiveTime() time.Time { + c.stateLock.RLock() + activeTime := c.activeTime + c.stateLock.RUnlock() + return activeTime +} + // StandbyStates is meant as a way to avoid some extra locking on the very // common sys/health check. func (c *Core) StandbyStates() (standby, perfStandby bool) { diff --git a/vendor/github.com/hashicorp/vault/api/sys_leader.go b/vendor/github.com/hashicorp/vault/api/sys_leader.go index e8104b86a..1c6be8d88 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_leader.go +++ b/vendor/github.com/hashicorp/vault/api/sys_leader.go @@ -1,6 +1,9 @@ package api -import "context" +import ( + "context" + "time" +) func (c *Sys) Leader() (*LeaderResponse, error) { r := c.c.NewRequest("GET", "/v1/sys/leader") @@ -19,13 +22,14 @@ func (c *Sys) Leader() (*LeaderResponse, error) { } type LeaderResponse struct { - HAEnabled bool `json:"ha_enabled"` - IsSelf bool `json:"is_self"` - LeaderAddress string `json:"leader_address"` - LeaderClusterAddress string `json:"leader_cluster_address"` - PerfStandby bool `json:"performance_standby"` - PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` - LastWAL uint64 `json:"last_wal"` - RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` - RaftAppliedIndex uint64 `json:"raft_applied_index,omitempty"` + HAEnabled bool `json:"ha_enabled"` + IsSelf bool `json:"is_self"` + ActiveTime time.Time `json:"active_time"` + LeaderAddress string `json:"leader_address"` + LeaderClusterAddress string `json:"leader_cluster_address"` + PerfStandby bool `json:"performance_standby"` + PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` + LastWAL uint64 `json:"last_wal"` + RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"` + RaftAppliedIndex uint64 `json:"raft_applied_index,omitempty"` }