diff --git a/api/sys_hastatus.go b/api/sys_hastatus.go new file mode 100644 index 000000000..408da0509 --- /dev/null +++ b/api/sys_hastatus.go @@ -0,0 +1,34 @@ +package api + +import ( + "context" + "time" +) + +func (c *Sys) HAStatus() (*HAStatusResponse, error) { + r := c.c.NewRequest("GET", "/v1/sys/ha-status") + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + resp, err := c.c.RawRequestWithContext(ctx, r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result HAStatusResponse + err = resp.DecodeJSON(&result) + return &result, err +} + +type HAStatusResponse struct { + Nodes []HANode +} + +type HANode struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + ActiveNode bool `json:"active_node"` + LastEcho *time.Time `json:"last_echo"` +} diff --git a/changelog/13292.txt b/changelog/13292.txt new file mode 100644 index 000000000..e3943389b --- /dev/null +++ b/changelog/13292.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core/ha: Add new mechanism for keeping track of peers talking to active node, and new 'operator members' command to view them. +``` diff --git a/command/commands.go b/command/commands.go index dc08c4a74..bf5daa747 100644 --- a/command/commands.go +++ b/command/commands.go @@ -461,6 +461,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { BaseCommand: getBaseCommand(), }, nil }, + "operator members": func() (cli.Command, error) { + return &OperatorMembersCommand{ + BaseCommand: getBaseCommand(), + }, nil + }, "path-help": func() (cli.Command, error) { return &PathHelpCommand{ BaseCommand: getBaseCommand(), diff --git a/command/operator_members.go b/command/operator_members.go new file mode 100644 index 000000000..6b163d669 --- /dev/null +++ b/command/operator_members.go @@ -0,0 +1,82 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +var ( + _ cli.Command = (*OperatorMembersCommand)(nil) + _ cli.CommandAutocomplete = (*OperatorMembersCommand)(nil) +) + +type OperatorMembersCommand struct { + *BaseCommand +} + +func (c *OperatorMembersCommand) Synopsis() string { + return "Returns the list of nodes in the cluster" +} + +func (c *OperatorMembersCommand) Help() string { + helpText := ` +Usage: vault operator members + + Provides the details of all the nodes in the cluster. + + $ vault operator members + +` + c.Flags().Help() + + return strings.TrimSpace(helpText) +} + +func (c *OperatorMembersCommand) Flags() *FlagSets { + set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat) + + return set +} + +func (c *OperatorMembersCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictAnything +} + +func (c *OperatorMembersCommand) AutocompleteFlags() complete.Flags { + return c.Flags().Completions() +} + +func (c *OperatorMembersCommand) Run(args []string) int { + f := c.Flags() + + if err := f.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + client, err := c.Client() + if err != nil { + c.UI.Error(err.Error()) + return 2 + } + + resp, err := client.Sys().HAStatus() + if err != nil { + c.UI.Error(err.Error()) + return 2 + } + + switch Format(c.UI) { + case "table": + out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"} + for _, node := range resp.Nodes { + out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho)) + } + c.UI.Output(tableOutput(out, nil)) + return 0 + default: + return OutputData(c.UI, resp) + } +} diff --git a/http/handler.go b/http/handler.go index 01085884b..02a153eb4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -93,6 +93,7 @@ var ( "/v1/sys/capabilities", "/v1/sys/capabilities-accessor", "/v1/sys/capabilities-self", + "/v1/sys/ha-status", "/v1/sys/key-status", "/v1/sys/mounts", "/v1/sys/mounts/", @@ -213,7 +214,6 @@ func Handler(props *vault.HandlerProperties) http.Handler { return printablePathCheckHandler } - type copyResponseWriter struct { wrapped http.ResponseWriter statusCode int @@ -304,7 +304,6 @@ func wrapGenericHandler(core *vault.Core, h http.Handler, props *vault.HandlerPr hostname, _ := os.Hostname() return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // This block needs to be here so that upon sending SIGHUP, custom response // headers are also reloaded into the handlers. var customHeaders map[string][]*logical.CustomHeader diff --git a/vault/core.go b/vault/core.go index d7a5e91cd..0f88233cf 100644 --- a/vault/core.go +++ b/vault/core.go @@ -2946,3 +2946,25 @@ type LicenseState struct { ExpiryTime time.Time Terminated bool } + +type PeerNode struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + LastEcho time.Time `json:"last_echo"` +} + +// GetHAPeerNodesCached returns the nodes that've sent us Echo requests recently. +func (c *Core) GetHAPeerNodesCached() []PeerNode { + var nodes []PeerNode + for itemClusterAddr, item := range c.clusterPeerClusterAddrsCache.Items() { + info := item.Object.(nodeHAConnectionInfo) + nodes = append(nodes, PeerNode{ + Hostname: info.nodeInfo.Hostname, + APIAddress: info.nodeInfo.ApiAddr, + ClusterAddress: itemClusterAddr, + LastEcho: info.lastHeartbeat, + }) + } + return nodes +} diff --git a/vault/logical_system.go b/vault/logical_system.go index 417ad7240..a801a6b5e 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -40,6 +40,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/sdk/version" "github.com/mitchellh/mapstructure" + "github.com/shirou/gopsutil/host" ) const ( @@ -4119,6 +4120,43 @@ func (b *SystemBackend) rotateBarrierKey(ctx context.Context) error { return nil } +func (b *SystemBackend) handleHAStatus(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + // We're always the leader if we're handling this request. + h, _ := host.Info() + nodes := []HAStatusNode{ + { + Hostname: h.Hostname, + APIAddress: b.Core.redirectAddr, + ClusterAddress: b.Core.ClusterAddr(), + ActiveNode: true, + }, + } + + for _, peerNode := range b.Core.GetHAPeerNodesCached() { + lastEcho := peerNode.LastEcho + nodes = append(nodes, HAStatusNode{ + Hostname: peerNode.Hostname, + APIAddress: peerNode.APIAddress, + ClusterAddress: peerNode.ClusterAddress, + LastEcho: &lastEcho, + }) + } + + return &logical.Response{ + Data: map[string]interface{}{ + "nodes": nodes, + }, + }, nil +} + +type HAStatusNode struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + ActiveNode bool `json:"active_node"` + LastEcho *time.Time `json:"last_echo"` +} + func sanitizePath(path string) string { if !strings.HasSuffix(path, "/") { path += "/" @@ -4609,6 +4647,13 @@ Enable a new audit backend or disable an existing backend. `, }, + "ha-status": { + "Provides information about the nodes in an HA cluster.", + ` + Provides the list of hosts known to the active node and when they were last heard from. + `, + }, + "key-status": { "Provides information about the backend encryption key.", ` diff --git a/vault/logical_system_integ_test.go b/vault/logical_system_integ_test.go index 4e1b1aecf..b76b8755b 100644 --- a/vault/logical_system_integ_test.go +++ b/vault/logical_system_integ_test.go @@ -9,13 +9,17 @@ import ( "time" "github.com/go-test/deep" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/builtin/plugin" "github.com/hashicorp/vault/helper/namespace" vaulthttp "github.com/hashicorp/vault/http" "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/hashicorp/vault/sdk/helper/logging" "github.com/hashicorp/vault/sdk/helper/pluginutil" "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/sdk/physical" + "github.com/hashicorp/vault/sdk/physical/inmem" lplugin "github.com/hashicorp/vault/sdk/plugin" "github.com/hashicorp/vault/sdk/plugin/mock" "github.com/hashicorp/vault/vault" @@ -301,7 +305,6 @@ func testPlugin_continueOnError(t *testing.T, btype logical.BackendType, mismatc if err != nil { t.Fatalf("err:%v", err) } - } // Trigger a sha256 mismatch or missing plugin error @@ -856,3 +859,40 @@ func TestSystemBackend_InternalUIResultantACL(t *testing.T) { t.Fatal(diff) } } + +func TestSystemBackend_HAStatus(t *testing.T) { + logger := logging.NewVaultLogger(hclog.Trace) + inm, err := inmem.NewTransactionalInmem(nil, logger) + if err != nil { + t.Fatal(err) + } + inmha, err := inmem.NewInmemHA(nil, logger) + if err != nil { + t.Fatal(err) + } + + conf := &vault.CoreConfig{ + Physical: inm, + HAPhysical: inmha.(physical.HABackend), + } + opts := &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + } + cluster := vault.NewTestCluster(t, conf, opts) + cluster.Start() + defer cluster.Cleanup() + + vault.RetryUntil(t, 15*time.Second, func() error { + // Use standby deliberately to make sure it forwards + client := cluster.Cores[1].Client + resp, err := client.Sys().HAStatus() + if err != nil { + t.Fatal(err) + } + + if len(resp.Nodes) != len(cluster.Cores) { + return fmt.Errorf("expected %d nodes, got %d", len(cluster.Cores), len(resp.Nodes)) + } + return nil + }) +} diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index 06fae5226..78ef45a8f 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -476,6 +476,19 @@ func (b *SystemBackend) statusPaths() []*framework.Path { HelpSynopsis: strings.TrimSpace(sysHelp["seal-status"][0]), HelpDescription: strings.TrimSpace(sysHelp["seal-status"][1]), }, + { + Pattern: "ha-status$", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handleHAStatus, + Summary: "Check the HA status of a Vault cluster", + }, + }, + + HelpSynopsis: strings.TrimSpace(sysHelp["ha-status"][0]), + HelpDescription: strings.TrimSpace(sysHelp["ha-status"][1]), + }, } } @@ -927,8 +940,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/ui/namespaces", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: pathInternalUINamespacesRead(b), - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: pathInternalUINamespacesRead(b), + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-ui-namespaces"][0]), @@ -938,8 +951,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/ui/resultant-acl", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalUIResultantACL, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalUIResultantACL, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-ui-resultant-acl"][0]), @@ -949,8 +962,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/counters/requests", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalCountersRequests, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalCountersRequests, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-counters-requests"][0]), @@ -960,8 +973,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/counters/tokens", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalCountersTokens, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalCountersTokens, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-counters-tokens"][0]), @@ -971,8 +984,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/counters/entities", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalCountersEntities, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalCountersEntities, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-counters-entities"][0]), diff --git a/vault/request_forwarding_rpc.go b/vault/request_forwarding_rpc.go index 0388e4c67..2cb04cb91 100644 --- a/vault/request_forwarding_rpc.go +++ b/vault/request_forwarding_rpc.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/vault/helper/forwarding" "github.com/hashicorp/vault/physical/raft" "github.com/hashicorp/vault/vault/replication" + "github.com/shirou/gopsutil/host" ) type forwardedRequestRPCServer struct { @@ -71,9 +72,18 @@ func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *fo return resp, nil } +type nodeHAConnectionInfo struct { + nodeInfo *NodeInformation + lastHeartbeat time.Time +} + func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) { + incomingNodeConnectionInfo := nodeHAConnectionInfo{ + nodeInfo: in.NodeInfo, + lastHeartbeat: time.Now(), + } if in.ClusterAddr != "" { - s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, nil, 0) + s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, incomingNodeConnectionInfo, 0) } if in.RaftAppliedIndex > 0 && len(in.RaftNodeID) > 0 && s.raftFollowerStates != nil { @@ -106,12 +116,18 @@ type forwardingClient struct { // with these requests it's useful to keep this as well func (c *forwardingClient) startHeartbeat() { go func() { + clusterAddr := c.core.ClusterAddr() + h, _ := host.Info() + ni := NodeInformation{ + ApiAddr: c.core.redirectAddr, + Hostname: h.Hostname, + Mode: "standby", + } tick := func() { - clusterAddr := c.core.ClusterAddr() - req := &EchoRequest{ Message: "ping", ClusterAddr: clusterAddr, + NodeInfo: &ni, } if raftBackend := c.core.getRaftBackend(); raftBackend != nil { diff --git a/vault/request_forwarding_service.pb.go b/vault/request_forwarding_service.pb.go index 9a8880e8d..62962be0c 100644 --- a/vault/request_forwarding_service.pb.go +++ b/vault/request_forwarding_service.pb.go @@ -225,6 +225,7 @@ type NodeInformation struct { Mode string `protobuf:"bytes,3,opt,name=mode,proto3" json:"mode,omitempty"` NodeID string `protobuf:"bytes,4,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` ReplicationState uint32 `protobuf:"varint,5,opt,name=replication_state,json=replicationState,proto3" json:"replication_state,omitempty"` + Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"` } func (x *NodeInformation) Reset() { @@ -294,6 +295,13 @@ func (x *NodeInformation) GetReplicationState() uint32 { return 0 } +func (x *NodeInformation) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + type ClientKey struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -534,7 +542,7 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{ 0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f, 0x64, - 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xa9, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x19, 0x0a, 0x08, @@ -545,46 +553,47 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{ 0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x22, 0x49, 0x0a, 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78, - 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c, - 0x0a, 0x01, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18, - 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72, - 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, - 0x72, 0x79, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, - 0x63, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65, - 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, - 0x65, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x4b, 0x65, 0x79, 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f, - 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66, - 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x14, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68, - 0x6f, 0x12, 0x12, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, - 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72, - 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, - 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, - 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, - 0x22, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, - 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, - 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, + 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0c, + 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a, 0x01, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c, 0x0a, 0x01, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x50, 0x65, 0x72, 0x66, + 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, + 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1f, + 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12, + 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, + 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x46, 0x6f, 0x72, 0x77, + 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, + 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, + 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x12, 0x2e, + 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x76, 0x61, 0x75, + 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x61, + 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c, + 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/vault/request_forwarding_service.proto b/vault/request_forwarding_service.proto index 1e54549e4..5617242b9 100644 --- a/vault/request_forwarding_service.proto +++ b/vault/request_forwarding_service.proto @@ -37,6 +37,7 @@ message NodeInformation { string mode = 3; string node_id = 4; uint32 replication_state = 5; + string hostname = 6; } message ClientKey { diff --git a/vault/testing.go b/vault/testing.go index c50eda911..966fddf9e 100644 --- a/vault/testing.go +++ b/vault/testing.go @@ -2271,3 +2271,18 @@ func (n *NoopAudit) Invalidate(ctx context.Context) { defer n.saltMutex.Unlock() n.salt = nil } + +// RetryUntil runs f until it returns a nil result or the timeout is reached. +// If a nil result hasn't been obtained by timeout, calls t.Fatal. +func RetryUntil(t testing.T, timeout time.Duration, f func() error) { + t.Helper() + deadline := time.Now().Add(timeout) + var err error + for time.Now().Before(deadline) { + if err = f(); err == nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("did not complete before deadline, err: %v", err) +} diff --git a/website/content/api-docs/system/ha-status.mdx b/website/content/api-docs/system/ha-status.mdx new file mode 100644 index 000000000..fe16a76f7 --- /dev/null +++ b/website/content/api-docs/system/ha-status.mdx @@ -0,0 +1,57 @@ +--- +layout: api +page_title: /sys/ha-status - HTTP API +description: The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster. +--- + +# `/sys/ha-status` + +The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster. +It lists the active node and the peers that it's heard from since it became active. + +## HA Status + +This endpoint returns the HA status of the Vault cluster. + +| Method | Path | +| :----- | :----------------- | +| `GET` | `/sys/ha-status` | + +### Sample Request + +```shell-session +$ curl \ + --header "X-Vault-Token: ..." \ + http://127.0.0.1:8200/v1/sys/ha-status +``` + +### Sample Response + +```json +{ + "Nodes": [ + { + "hostname": "node1", + "api_address": "http://10.0.0.2:8200", + "cluster_address": "https://10.0.0.2:8201", + "active_node": true, + "last_echo": null + }, + { + "hostname": "node2", + "api_address": "http://10.0.0.3:8200", + "cluster_address": "https://10.0.0.3:8201", + "active_node": false, + "last_echo": "2021-11-29T10:29:09.202235-05:00" + }, + { + "hostname": "node3", + "api_address": "http://10.0.0.4:8200", + "cluster_address": "https://10.0.0.4:8201", + "active_node": false, + "last_echo": "2021-11-29T10:29:07.402548-05:00" + } + ] +} + +``` diff --git a/website/content/docs/commands/operator/members.mdx b/website/content/docs/commands/operator/members.mdx new file mode 100644 index 000000000..2ee055293 --- /dev/null +++ b/website/content/docs/commands/operator/members.mdx @@ -0,0 +1,36 @@ +--- +layout: docs +page_title: operator members - Command +description: |- + The "operator members" command provides information about the nodes making up an HA cluster. +--- + +# operator members + +The `operator members` lists the active node and the peers that it's heard from +since it became active. + +## Examples + +Get the key status: + +```shell-session +$ vault operator members +Host Name API Address Cluster Address ActiveNode Last Echo +--------- ----------- --------------- ---------- --------- +node1 http://10.0.0.2:8200 https://10.0.0.2:8201 true +node2 http://10.0.0.3:8200 https://10.0.0.3:8201 false 2021-11-29 10:19:39.236409 -0500 EST +node3 http://10.0.0.4:8200 https://10.0.0.4:8201 false 2021-11-29 10:19:37.436283 -0500 EST + +``` + +## Usage + +The following flags are available in addition to the [standard set of +flags](/docs/commands) included on all commands. + +### Output Options + +- `-format` `(string: "table")` - Print the output in the given format. Valid + formats are "table", "json", or "yaml". This can also be specified via the + `VAULT_FORMAT` environment variable. diff --git a/website/data/api-docs-nav-data.json b/website/data/api-docs-nav-data.json index 761cdb7ef..14a1c9a74 100644 --- a/website/data/api-docs-nav-data.json +++ b/website/data/api-docs-nav-data.json @@ -429,6 +429,10 @@ "title": "/sys/key-status", "path": "system/key-status" }, + { + "title": "/sys/ha-status", + "path": "system/ha-status" + }, { "title": "/sys/leader", "path": "system/leader" diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index c3cc225eb..adf19fc58 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -574,6 +574,10 @@ "title": "key-status", "path": "commands/operator/key-status" }, + { + "title": "members", + "path": "commands/operator/members" + }, { "title": "migrate", "path": "commands/operator/migrate"