diff --git a/api/operator.go b/api/operator.go new file mode 100644 index 000000000..dcf65b332 --- /dev/null +++ b/api/operator.go @@ -0,0 +1,81 @@ +package api + +// Operator can be used to perform low-level operator tasks for Nomad. +type Operator struct { + c *Client +} + +// Operator returns a handle to the operator endpoints. +func (c *Client) Operator() *Operator { + return &Operator{c} +} + +// RaftServer has information about a server in the Raft configuration. +type RaftServer struct { + // ID is the unique ID for the server. These are currently the same + // as the address, but they will be changed to a real GUID in a future + // release of Nomad. + ID string + + // Node is the node name of the server, as known by Nomad, or this + // will be set to "(unknown)" otherwise. + Node string + + // Address is the IP:port of the server, used for Raft communications. + Address string + + // Leader is true if this server is the current cluster leader. + Leader bool + + // Voter is true if this server has a vote in the cluster. This might + // be false if the server is staging and still coming online, or if + // it's a non-voting server, which will be added in a future release of + // Nomad. + Voter bool +} + +// RaftConfigration is returned when querying for the current Raft configuration. +type RaftConfiguration struct { + // Servers has the list of servers in the Raft configuration. + Servers []*RaftServer + + // Index has the Raft index of this configuration. + Index uint64 +} + +// RaftGetConfiguration is used to query the current Raft peer set. +func (op *Operator) RaftGetConfiguration(q *QueryOptions) (*RaftConfiguration, error) { + r := op.c.newRequest("GET", "/v1/operator/raft/configuration") + r.setQueryOptions(q) + _, resp, err := requireOK(op.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var out RaftConfiguration + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return &out, nil +} + +// RaftRemovePeerByAddress is used to kick a stale peer (one that it in the Raft +// quorum but no longer known to Serf or the catalog) by address in the form of +// "IP:port". +func (op *Operator) RaftRemovePeerByAddress(address string, q *WriteOptions) error { + r := op.c.newRequest("DELETE", "/v1/operator/raft/peer") + r.setWriteOptions(q) + + // TODO (alexdadgar) Currently we made address a query parameter. Once + // IDs are in place this will be DELETE /v1/operator/raft/peer/. + r.params.Set("address", string(address)) + + _, resp, err := requireOK(op.c.doRequest(r)) + if err != nil { + return err + } + + resp.Body.Close() + return nil +} diff --git a/api/operator_test.go b/api/operator_test.go new file mode 100644 index 000000000..4db2c4c45 --- /dev/null +++ b/api/operator_test.go @@ -0,0 +1,36 @@ +package api + +import ( + "strings" + "testing" +) + +func TestOperator_RaftGetConfiguration(t *testing.T) { + c, s := makeClient(t, nil, nil) + defer s.Stop() + + operator := c.Operator() + out, err := operator.RaftGetConfiguration(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter { + t.Fatalf("bad: %v", out) + } +} + +func TestOperator_RaftRemovePeerByAddress(t *testing.T) { + c, s := makeClient(t, nil, nil) + defer s.Stop() + + // If we get this error, it proves we sent the address all the way + // through. + operator := c.Operator() + err := operator.RaftRemovePeerByAddress("nope", nil) + if err == nil || !strings.Contains(err.Error(), + "address \"nope\" was not found in the Raft configuration") { + t.Fatalf("err: %v", err) + } +} diff --git a/command/agent/http.go b/command/agent/http.go index 5a31f7d4e..e111baea8 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -170,6 +170,8 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/status/leader", s.wrap(s.StatusLeaderRequest)) s.mux.HandleFunc("/v1/status/peers", s.wrap(s.StatusPeersRequest)) + s.mux.HandleFunc("/v1/operator/", s.wrap(s.OperatorRequest)) + s.mux.HandleFunc("/v1/system/gc", s.wrap(s.GarbageCollectRequest)) s.mux.HandleFunc("/v1/system/reconcile/summaries", s.wrap(s.ReconcileJobSummaries)) diff --git a/command/agent/operator_endpoint.go b/command/agent/operator_endpoint.go new file mode 100644 index 000000000..0a898de4b --- /dev/null +++ b/command/agent/operator_endpoint.go @@ -0,0 +1,69 @@ +package agent + +import ( + "net/http" + "strings" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/raft" +) + +func (s *HTTPServer) OperatorRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + path := strings.TrimPrefix(req.URL.Path, "/v1/operator/raft/") + switch { + case strings.HasPrefix(path, "configuration"): + return s.OperatorRaftConfiguration(resp, req) + case strings.HasPrefix(path, "peer"): + return s.OperatorRaftPeer(resp, req) + default: + return nil, CodedError(404, ErrInvalidMethod) + } +} + +// OperatorRaftConfiguration is used to inspect the current Raft configuration. +// This supports the stale query mode in case the cluster doesn't have a leader. +func (s *HTTPServer) OperatorRaftConfiguration(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != "GET" { + resp.WriteHeader(http.StatusMethodNotAllowed) + return nil, nil + } + + var args structs.GenericRequest + if done := s.parse(resp, req, &args.Region, &args.QueryOptions); done { + return nil, nil + } + + var reply structs.RaftConfigurationResponse + if err := s.agent.RPC("Operator.RaftGetConfiguration", &args, &reply); err != nil { + return nil, err + } + + return reply, nil +} + +// OperatorRaftPeer supports actions on Raft peers. Currently we only support +// removing peers by address. +func (s *HTTPServer) OperatorRaftPeer(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != "DELETE" { + resp.WriteHeader(http.StatusMethodNotAllowed) + return nil, nil + } + + var args structs.RaftPeerByAddressRequest + s.parseRegion(req, &args.Region) + + params := req.URL.Query() + if _, ok := params["address"]; ok { + args.Address = raft.ServerAddress(params.Get("address")) + } else { + resp.WriteHeader(http.StatusBadRequest) + resp.Write([]byte("Must specify ?address with IP:port of peer to remove")) + return nil, nil + } + + var reply struct{} + if err := s.agent.RPC("Operator.RaftRemovePeerByAddress", &args, &reply); err != nil { + return nil, err + } + return nil, nil +} diff --git a/command/agent/operator_endpoint_test.go b/command/agent/operator_endpoint_test.go new file mode 100644 index 000000000..5b5de7d82 --- /dev/null +++ b/command/agent/operator_endpoint_test.go @@ -0,0 +1,58 @@ +package agent + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestHTTP_OperatorRaftConfiguration(t *testing.T) { + httpTest(t, nil, func(s *TestServer) { + body := bytes.NewBuffer(nil) + req, err := http.NewRequest("GET", "/v1/operator/raft/configuration", body) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp := httptest.NewRecorder() + obj, err := s.Server.OperatorRaftConfiguration(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp.Code != 200 { + t.Fatalf("bad code: %d", resp.Code) + } + out, ok := obj.(structs.RaftConfigurationResponse) + if !ok { + t.Fatalf("unexpected: %T", obj) + } + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter { + t.Fatalf("bad: %v", out) + } + }) +} + +func TestHTTP_OperatorRaftPeer(t *testing.T) { + httpTest(t, nil, func(s *TestServer) { + body := bytes.NewBuffer(nil) + req, err := http.NewRequest("DELETE", "/v1/operator/raft/peer?address=nope", body) + if err != nil { + t.Fatalf("err: %v", err) + } + + // If we get this error, it proves we sent the address all the + // way through. + resp := httptest.NewRecorder() + _, err = s.Server.OperatorRaftPeer(resp, req) + if err == nil || !strings.Contains(err.Error(), + "address \"nope\" was not found in the Raft configuration") { + t.Fatalf("err: %v", err) + } + }) +} diff --git a/command/job_dispatch.go b/command/job_dispatch.go index 702fa4a97..b53567509 100644 --- a/command/job_dispatch.go +++ b/command/job_dispatch.go @@ -33,11 +33,11 @@ General Options: Dispatch Options: -meta = - Meta takes a key/value pair seperated by "=". The metadata key will be - merged into the job's metadata. The job may define a default value for the - key which is overriden when dispatching. The flag can be provided more than - once to inject multiple metadata key/value pairs. Arbitrary keys are not - allowed. The parameterized job must allow the key to be merged. + Meta takes a key/value pair seperated by "=". The metadata key will be + merged into the job's metadata. The job may define a default value for the + key which is overriden when dispatching. The flag can be provided more than + once to inject multiple metadata key/value pairs. Arbitrary keys are not + allowed. The parameterized job must allow the key to be merged. -detach Return immediately instead of entering monitor mode. After job dispatch, diff --git a/command/operator.go b/command/operator.go new file mode 100644 index 000000000..db5ed3574 --- /dev/null +++ b/command/operator.go @@ -0,0 +1,32 @@ +package command + +import ( + "strings" + + "github.com/mitchellh/cli" +) + +type OperatorCommand struct { + Meta +} + +func (f *OperatorCommand) Help() string { + helpText := ` +Usage: nomad operator [options] + + Provides cluster-level tools for Nomad operators, such as interacting with + the Raft subsystem. NOTE: Use this command with extreme caution, as improper + use could lead to a Nomad outage and even loss of data. + + Run nomad operator with no arguments for help on that subcommand. +` + return strings.TrimSpace(helpText) +} + +func (f *OperatorCommand) Synopsis() string { + return "Provides cluster-level tools for Nomad operators" +} + +func (f *OperatorCommand) Run(args []string) int { + return cli.RunResultHelp +} diff --git a/command/operator_raft.go b/command/operator_raft.go new file mode 100644 index 000000000..450988e0f --- /dev/null +++ b/command/operator_raft.go @@ -0,0 +1,30 @@ +package command + +import ( + "strings" + + "github.com/mitchellh/cli" +) + +type OperatorRaftCommand struct { + Meta +} + +func (c *OperatorRaftCommand) Help() string { + helpText := ` +Usage: nomad operator raft [options] + +The Raft operator command is used to interact with Nomad's Raft subsystem. The +command can be used to verify Raft peers or in rare cases to recover quorum by +removing invalid peers. +` + return strings.TrimSpace(helpText) +} + +func (c *OperatorRaftCommand) Synopsis() string { + return "Provides access to the Raft subsystem" +} + +func (c *OperatorRaftCommand) Run(args []string) int { + return cli.RunResultHelp +} diff --git a/command/operator_raft_list.go b/command/operator_raft_list.go new file mode 100644 index 000000000..9c7bac8b6 --- /dev/null +++ b/command/operator_raft_list.go @@ -0,0 +1,82 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/ryanuber/columnize" +) + +type OperatorRaftListCommand struct { + Meta +} + +func (c *OperatorRaftListCommand) Help() string { + helpText := ` +Usage: nomad operator raft list-peers [options] + +Displays the current Raft peer configuration. + +General Options: + + ` + generalOptionsUsage() + ` + +List Peers Options: + + -stale=[true|false] + The -stale argument defaults to "false" which means the leader provides the + result. If the cluster is in an outage state without a leader, you may need + to set -stale to "true" to get the configuration from a non-leader server. +` + return strings.TrimSpace(helpText) +} + +func (c *OperatorRaftListCommand) Synopsis() string { + return "Display the current Raft peer configuration" +} + +func (c *OperatorRaftListCommand) Run(args []string) int { + var stale bool + + flags := c.Meta.FlagSet("raft", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + flags.BoolVar(&stale, "stale", false, "") + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err)) + return 1 + } + + // Set up a client. + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + operator := client.Operator() + + // Fetch the current configuration. + q := &api.QueryOptions{ + AllowStale: stale, + } + reply, err := operator.RaftGetConfiguration(q) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to retrieve raft configuration: %v", err)) + return 1 + } + + // Format it as a nice table. + result := []string{"Node|ID|Address|State|Voter"} + for _, s := range reply.Servers { + state := "follower" + if s.Leader { + state = "leader" + } + result = append(result, fmt.Sprintf("%s|%s|%s|%s|%v", + s.Node, s.ID, s.Address, state, s.Voter)) + } + c.Ui.Output(columnize.SimpleFormat(result)) + + return 0 +} diff --git a/command/operator_raft_list_test.go b/command/operator_raft_list_test.go new file mode 100644 index 000000000..b96b66f50 --- /dev/null +++ b/command/operator_raft_list_test.go @@ -0,0 +1,30 @@ +package command + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Raft_ListPeers_Implements(t *testing.T) { + var _ cli.Command = &OperatorRaftListCommand{} +} + +func TestOperator_Raft_ListPeers(t *testing.T) { + s, _, addr := testServer(t, nil) + defer s.Stop() + + ui := new(cli.MockUi) + c := &OperatorRaftListCommand{Meta: Meta{Ui: ui}} + args := []string{"-address=" + addr} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + output := strings.TrimSpace(ui.OutputWriter.String()) + if !strings.Contains(output, "leader") { + t.Fatalf("bad: %s", output) + } +} diff --git a/command/operator_raft_remove.go b/command/operator_raft_remove.go new file mode 100644 index 000000000..bb5789cfd --- /dev/null +++ b/command/operator_raft_remove.go @@ -0,0 +1,79 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" +) + +type OperatorRaftRemoveCommand struct { + Meta +} + +func (c *OperatorRaftRemoveCommand) Help() string { + helpText := ` +Usage: nomad operator raft remove-peer [options] + +Remove the Nomad server with given -peer-address from the Raft configuration. + +There are rare cases where a peer may be left behind in the Raft quorum even +though the server is no longer present and known to the cluster. This command +can be used to remove the failed server so that it is no longer affects the Raft +quorum. If the server still shows in the output of the "nomad server-members" +command, it is preferable to clean up by simply running "nomad +server-force-leave" instead of this command. + +General Options: + + ` + generalOptionsUsage() + ` + +Remove Peer Options: + + -peer-address="IP:port" + Remove a Nomad server with given address from the Raft configuration. +` + return strings.TrimSpace(helpText) +} + +func (c *OperatorRaftRemoveCommand) Synopsis() string { + return "Remove a Nomad server from the Raft configuration" +} + +func (c *OperatorRaftRemoveCommand) Run(args []string) int { + var peerAddress string + + flags := c.Meta.FlagSet("raft", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + flags.StringVar(&peerAddress, "peer-address", "", "") + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err)) + return 1 + } + + // Set up a client. + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + operator := client.Operator() + + // TODO (alexdadgar) Once we expose IDs, add support for removing + // by ID, add support for that. + if len(peerAddress) == 0 { + c.Ui.Error(fmt.Sprintf("an address is required for the peer to remove")) + return 1 + } + + // Try to kick the peer. + w := &api.WriteOptions{} + if err := operator.RaftRemovePeerByAddress(peerAddress, w); err != nil { + c.Ui.Error(fmt.Sprintf("Failed to remove raft peer: %v", err)) + return 1 + } + c.Ui.Output(fmt.Sprintf("Removed peer with address %q", peerAddress)) + + return 0 +} diff --git a/command/operator_raft_remove_test.go b/command/operator_raft_remove_test.go new file mode 100644 index 000000000..a5954d03f --- /dev/null +++ b/command/operator_raft_remove_test.go @@ -0,0 +1,32 @@ +package command + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Raft_RemovePeers_Implements(t *testing.T) { + var _ cli.Command = &OperatorRaftRemoveCommand{} +} + +func TestOperator_Raft_RemovePeer(t *testing.T) { + s, _, addr := testServer(t, nil) + defer s.Stop() + + ui := new(cli.MockUi) + c := &OperatorRaftRemoveCommand{Meta: Meta{Ui: ui}} + args := []string{"-address=" + addr, "-peer-address=nope"} + + code := c.Run(args) + if code != 1 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + // If we get this error, it proves we sent the address all they through. + output := strings.TrimSpace(ui.ErrorWriter.String()) + if !strings.Contains(output, "address \"nope\" was not found in the Raft configuration") { + t.Fatalf("bad: %s", output) + } +} diff --git a/command/operator_raft_test.go b/command/operator_raft_test.go new file mode 100644 index 000000000..a4ce6269b --- /dev/null +++ b/command/operator_raft_test.go @@ -0,0 +1,11 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Raft_Implements(t *testing.T) { + var _ cli.Command = &OperatorRaftCommand{} +} diff --git a/command/operator_test.go b/command/operator_test.go new file mode 100644 index 000000000..485f97544 --- /dev/null +++ b/command/operator_test.go @@ -0,0 +1,11 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Implements(t *testing.T) { + var _ cli.Command = &OperatorCommand{} +} diff --git a/commands.go b/commands.go index 3def861c4..9bc568dea 100644 --- a/commands.go +++ b/commands.go @@ -115,6 +115,30 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { }, nil }, + "operator": func() (cli.Command, error) { + return &command.OperatorCommand{ + Meta: meta, + }, nil + }, + + "operator raft": func() (cli.Command, error) { + return &command.OperatorRaftCommand{ + Meta: meta, + }, nil + }, + + "operator raft list-peers": func() (cli.Command, error) { + return &command.OperatorRaftListCommand{ + Meta: meta, + }, nil + }, + + "operator raft remove-peer": func() (cli.Command, error) { + return &command.OperatorRaftRemoveCommand{ + Meta: meta, + }, nil + }, + "plan": func() (cli.Command, error) { return &command.PlanCommand{ Meta: meta, diff --git a/main.go b/main.go index 35e26efd3..cc42fd279 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,8 @@ func RunCustom(args []string, commands map[string]cli.CommandFactory) int { switch k { case "executor": case "syslog": + case "operator raft", "operator raft list-peers", "operator raft remove-peer": + case "job dispatch": case "fs ls", "fs cat", "fs stat": case "check": default: diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go new file mode 100644 index 000000000..576962aa3 --- /dev/null +++ b/nomad/operator_endpoint.go @@ -0,0 +1,107 @@ +package nomad + +import ( + "fmt" + "net" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/raft" + "github.com/hashicorp/serf/serf" +) + +// Operator endpoint is used to perform low-level operator tasks for Nomad. +type Operator struct { + srv *Server +} + +// RaftGetConfiguration is used to retrieve the current Raft configuration. +func (op *Operator) RaftGetConfiguration(args *structs.GenericRequest, reply *structs.RaftConfigurationResponse) error { + if done, err := op.srv.forward("Operator.RaftGetConfiguration", args, args, reply); done { + return err + } + + // We can't fetch the leader and the configuration atomically with + // the current Raft API. + future := op.srv.raft.GetConfiguration() + if err := future.Error(); err != nil { + return err + } + + // Index the Nomad information about the servers. + serverMap := make(map[raft.ServerAddress]serf.Member) + for _, member := range op.srv.serf.Members() { + valid, parts := isNomadServer(member) + if !valid { + continue + } + + addr := (&net.TCPAddr{IP: member.Addr, Port: parts.Port}).String() + serverMap[raft.ServerAddress(addr)] = member + } + + // Fill out the reply. + leader := op.srv.raft.Leader() + reply.Index = future.Index() + for _, server := range future.Configuration().Servers { + node := "(unknown)" + if member, ok := serverMap[server.Address]; ok { + node = member.Name + } + + entry := &structs.RaftServer{ + ID: server.ID, + Node: node, + Address: server.Address, + Leader: server.Address == leader, + Voter: server.Suffrage == raft.Voter, + } + reply.Servers = append(reply.Servers, entry) + } + return nil +} + +// RaftRemovePeerByAddress is used to kick a stale peer (one that it in the Raft +// quorum but no longer known to Serf or the catalog) by address in the form of +// "IP:port". The reply argument is not used, but it required to fulfill the RPC +// interface. +func (op *Operator) RaftRemovePeerByAddress(args *structs.RaftPeerByAddressRequest, reply *struct{}) error { + if done, err := op.srv.forward("Operator.RaftRemovePeerByAddress", args, args, reply); done { + return err + } + + // Since this is an operation designed for humans to use, we will return + // an error if the supplied address isn't among the peers since it's + // likely they screwed up. + { + future := op.srv.raft.GetConfiguration() + if err := future.Error(); err != nil { + return err + } + for _, s := range future.Configuration().Servers { + if s.Address == args.Address { + goto REMOVE + } + } + return fmt.Errorf("address %q was not found in the Raft configuration", + args.Address) + } + +REMOVE: + // The Raft library itself will prevent various forms of foot-shooting, + // like making a configuration with no voters. Some consideration was + // given here to adding more checks, but it was decided to make this as + // low-level and direct as possible. We've got ACL coverage to lock this + // down, and if you are an operator, it's assumed you know what you are + // doing if you are calling this. If you remove a peer that's known to + // Serf, for example, it will come back when the leader does a reconcile + // pass. + future := op.srv.raft.RemovePeer(args.Address) + if err := future.Error(); err != nil { + op.srv.logger.Printf("[WARN] nomad.operator: Failed to remove Raft peer %q: %v", + args.Address, err) + return err + } + + op.srv.logger.Printf("[WARN] nomad.operator: Removed Raft peer %q", args.Address) + return nil +} diff --git a/nomad/operator_endpoint_test.go b/nomad/operator_endpoint_test.go new file mode 100644 index 000000000..c5ab5a611 --- /dev/null +++ b/nomad/operator_endpoint_test.go @@ -0,0 +1,109 @@ +package nomad + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/hashicorp/raft" +) + +func TestOperator_RaftGetConfiguration(t *testing.T) { + s1 := testServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + arg := structs.GenericRequest{ + QueryOptions: structs.QueryOptions{ + Region: s1.config.Region, + }, + } + var reply structs.RaftConfigurationResponse + if err := msgpackrpc.CallWithCodec(codec, "Operator.RaftGetConfiguration", &arg, &reply); err != nil { + t.Fatalf("err: %v", err) + } + + future := s1.raft.GetConfiguration() + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + if len(future.Configuration().Servers) != 1 { + t.Fatalf("bad: %v", future.Configuration().Servers) + } + me := future.Configuration().Servers[0] + expected := structs.RaftConfigurationResponse{ + Servers: []*structs.RaftServer{ + &structs.RaftServer{ + ID: me.ID, + Node: fmt.Sprintf("%v.%v", s1.config.NodeName, s1.config.Region), + Address: me.Address, + Leader: true, + Voter: true, + }, + }, + Index: future.Index(), + } + if !reflect.DeepEqual(reply, expected) { + t.Fatalf("bad: got %+v; want %+v", reply, expected) + } +} + +func TestOperator_RaftRemovePeerByAddress(t *testing.T) { + s1 := testServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Try to remove a peer that's not there. + arg := structs.RaftPeerByAddressRequest{ + Address: raft.ServerAddress(fmt.Sprintf("127.0.0.1:%d", getPort())), + } + arg.Region = s1.config.Region + var reply struct{} + err := msgpackrpc.CallWithCodec(codec, "Operator.RaftRemovePeerByAddress", &arg, &reply) + if err == nil || !strings.Contains(err.Error(), "not found in the Raft configuration") { + t.Fatalf("err: %v", err) + } + + // Add it manually to Raft. + { + future := s1.raft.AddPeer(arg.Address) + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + } + + // Make sure it's there. + { + future := s1.raft.GetConfiguration() + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + configuration := future.Configuration() + if len(configuration.Servers) != 2 { + t.Fatalf("bad: %v", configuration) + } + } + + // Remove it, now it should go through. + if err := msgpackrpc.CallWithCodec(codec, "Operator.RaftRemovePeerByAddress", &arg, &reply); err != nil { + t.Fatalf("err: %v", err) + } + + // Make sure it's not there. + { + future := s1.raft.GetConfiguration() + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + configuration := future.Configuration() + if len(configuration.Servers) != 1 { + t.Fatalf("bad: %v", configuration) + } + } +} diff --git a/nomad/server.go b/nomad/server.go index e3cae7278..35e928160 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -162,6 +162,7 @@ type endpoints struct { Region *Region Periodic *Periodic System *System + Operator *Operator } // NewServer is used to construct a new Nomad server from the @@ -639,25 +640,27 @@ func (s *Server) setupVaultClient() error { // setupRPC is used to setup the RPC listener func (s *Server) setupRPC(tlsWrap tlsutil.RegionWrapper) error { // Create endpoints - s.endpoints.Status = &Status{s} - s.endpoints.Node = &Node{srv: s} - s.endpoints.Job = &Job{s} - s.endpoints.Eval = &Eval{s} - s.endpoints.Plan = &Plan{s} s.endpoints.Alloc = &Alloc{s} - s.endpoints.Region = &Region{s} + s.endpoints.Eval = &Eval{s} + s.endpoints.Job = &Job{s} + s.endpoints.Node = &Node{srv: s} + s.endpoints.Operator = &Operator{s} s.endpoints.Periodic = &Periodic{s} + s.endpoints.Plan = &Plan{s} + s.endpoints.Region = &Region{s} + s.endpoints.Status = &Status{s} s.endpoints.System = &System{s} // Register the handlers - s.rpcServer.Register(s.endpoints.Status) - s.rpcServer.Register(s.endpoints.Node) - s.rpcServer.Register(s.endpoints.Job) - s.rpcServer.Register(s.endpoints.Eval) - s.rpcServer.Register(s.endpoints.Plan) s.rpcServer.Register(s.endpoints.Alloc) - s.rpcServer.Register(s.endpoints.Region) + s.rpcServer.Register(s.endpoints.Eval) + s.rpcServer.Register(s.endpoints.Job) + s.rpcServer.Register(s.endpoints.Node) + s.rpcServer.Register(s.endpoints.Operator) s.rpcServer.Register(s.endpoints.Periodic) + s.rpcServer.Register(s.endpoints.Plan) + s.rpcServer.Register(s.endpoints.Region) + s.rpcServer.Register(s.endpoints.Status) s.rpcServer.Register(s.endpoints.System) list, err := net.ListenTCP("tcp", s.config.RPCAddr) diff --git a/nomad/structs/operator.go b/nomad/structs/operator.go new file mode 100644 index 000000000..93b99f6fb --- /dev/null +++ b/nomad/structs/operator.go @@ -0,0 +1,49 @@ +package structs + +import ( + "github.com/hashicorp/raft" +) + +// RaftServer has information about a server in the Raft configuration. +type RaftServer struct { + // ID is the unique ID for the server. These are currently the same + // as the address, but they will be changed to a real GUID in a future + // release of Nomad. + ID raft.ServerID + + // Node is the node name of the server, as known by Nomad, or this + // will be set to "(unknown)" otherwise. + Node string + + // Address is the IP:port of the server, used for Raft communications. + Address raft.ServerAddress + + // Leader is true if this server is the current cluster leader. + Leader bool + + // Voter is true if this server has a vote in the cluster. This might + // be false if the server is staging and still coming online, or if + // it's a non-voting server, which will be added in a future release of + // Nomad. + Voter bool +} + +// RaftConfigrationResponse is returned when querying for the current Raft +// configuration. +type RaftConfigurationResponse struct { + // Servers has the list of servers in the Raft configuration. + Servers []*RaftServer + + // Index has the Raft index of this configuration. + Index uint64 +} + +// RaftPeerByAddressRequest is used by the Operator endpoint to apply a Raft +// operation on a specific Raft peer by address in the form of "IP:port". +type RaftPeerByAddressRequest struct { + // Address is the peer to remove, in the form "IP:port". + Address raft.ServerAddress + + // WriteRequest holds the Region for this request. + WriteRequest +} diff --git a/website/source/docs/commands/operator-index.html.md.erb b/website/source/docs/commands/operator-index.html.md.erb new file mode 100644 index 000000000..ba9aa66c8 --- /dev/null +++ b/website/source/docs/commands/operator-index.html.md.erb @@ -0,0 +1,31 @@ +--- +layout: "docs" +page_title: "Commands: operator" +sidebar_current: "docs-commands-operator" +description: > + The operator command provides cluster-level tools for Nomad operators. +--- + +# Nomad Operator + +Command: `nomad operator` + +The `operator` command provides cluster-level tools for Nomad operators, such +as interacting with the Raft subsystem. This was added in Nomad 0.5.5. + +~> Use this command with extreme caution, as improper use could lead to a Nomad +outage and even loss of data. + +See the [Outage Recovery](TODO alexdadgar) guide for some examples of how +this command is used. For an API to perform these operations programatically, +please see the documentation for the [Operator](/docs/agent/http/operator.html) +endpoint. + +## Usage + +Usage: `nomad operator [options]` + +Run `nomad operator ` with no arguments for help on that subcommand. +The following subcommands are available: + +* `raft` - View and modify Nomad's Raft configuration. diff --git a/website/source/docs/commands/operator-raft-list-peers.html.md.erb b/website/source/docs/commands/operator-raft-list-peers.html.md.erb new file mode 100644 index 000000000..f83ccc984 --- /dev/null +++ b/website/source/docs/commands/operator-raft-list-peers.html.md.erb @@ -0,0 +1,62 @@ +--- +layout: "docs" +page_title: "Commands: operator raft list-peers" +sidebar_current: "docs-commands-operator-raft-list-peers" +description: > + Display the current Raft peer configuration. +--- + +# Command: `operator raft list-peers` + +The Raft list-peers command is used to display the current Raft peer +configuration. + +See the [Outage Recovery](TODO alexdadgar) guide for some examples of how +this command is used. For an API to perform these operations programatically, +please see the documentation for the [Operator](/docs/http/operator.html) +endpoint. + +## Usage + +``` +nomad operator raft list-peers [options] +``` + +## General Options + +<%= partial "docs/commands/_general_options" %> + +## List Peers Options + +* `-stale`: The stale argument defaults to "false" which means the leader +provides the result. If the cluster is in an outage state without a leader, you +may need to set `-stale` to "true" to get the configuration from a non-leader +server. + +## Examples + +An example output with three servers is as follows: + +``` +$ nomad operator raft list-peers +Node ID Address State Voter +nomad-server01.global 10.10.11.5:4647 10.10.11.5:4647 follower true +nomad-server02.global 10.10.11.6:4647 10.10.11.6:4647 leader true +nomad-server03.global 10.10.11.7:4647 10.10.11.7:4647 follower true + +``` + +* `Node` is the node name of the server, as known to Nomad, or "(unknown)" if +the node is stale and not known. + +* `ID` is the ID of the server. This is the same as the `Address` but may be +upgraded to a GUID in a future version of Nomad. + +* `Address` is the IP:port for the server. + +* `State` is either "follower" or "leader" depending on the server's role in the +Raft configuration. + +* `Voter` is "true" or "false", indicating if the server has a vote in the Raft +configuration. Future versions of Nomad may add support for non-voting servers. + diff --git a/website/source/docs/commands/operator-raft-remove-peer.html.md.erb b/website/source/docs/commands/operator-raft-remove-peer.html.md.erb new file mode 100644 index 000000000..f1b30e719 --- /dev/null +++ b/website/source/docs/commands/operator-raft-remove-peer.html.md.erb @@ -0,0 +1,41 @@ +--- +layout: "docs" +page_title: "Commands: operator raft remove-peer" +sidebar_current: "docs-commands-operator-raft-remove-peer" +description: > + Remove a Nomad server from the Raft configuration. +--- + +# Command: `operator raft remove-peer` + +Remove the Nomad server with given address from the Raft configuration. + +There are rare cases where a peer may be left behind in the Raft quorum even +though the server is no longer present and known to the cluster. This command +can be used to remove the failed server so that it is no longer affects the Raft +quorum. If the server still shows in the output of the [`nomad +server-members`](/docs/commands/server-members.html) command, it is preferable +to clean up by simply running [`nomad +server-force-leave`](/docs/commands/server-force-leave.html) instead of this +command. + +See the [Outage Recovery](TODO alexdadgar) guide for some examples of how +this command is used. For an API to perform these operations programatically, +please see the documentation for the [Operator](/docs/http/operator.html) +endpoint. + +## Usage + +``` +nomad operator raft remove-peer [options] +``` + +## General Options + +<%= partial "docs/commands/_general_options" %> + +## Remove Peer Options + +* `-peer-address`: Remove a Nomad server with given address from the Raft +configuration. The format is "IP:port" + diff --git a/website/source/docs/http/operator.html.md b/website/source/docs/http/operator.html.md new file mode 100644 index 000000000..5b27e7b15 --- /dev/null +++ b/website/source/docs/http/operator.html.md @@ -0,0 +1,166 @@ +--- +layout: "http" +page_title: "HTTP API: /v1/operator/" +sidebar_current: "docs-http-operator" +description: > + The '/v1/operator/' endpoints provides cluster-level tools for Nomad + operators. +--- + +# /v1/operator + +The Operator endpoint provides cluster-level tools for Nomad operators, such +as interacting with the Raft subsystem. This was added in Nomad 0.5.5 + +~> Use this interface with extreme caution, as improper use could lead to a + Nomad outage and even loss of data. + +See the [Outage Recovery](/docs/guides/outage.html) guide for some examples of how +these capabilities are used. For a CLI to perform these operations manually, please +see the documentation for the [`nomad operator`](/docs/commands/operator-index.html) +command. + +By default, the agent's local region is used; another region can be specified +using the `?region=` query parameter. + +## GET + +
+
Description
+
+ Query the status of a client node registered with Nomad. +
+ +
Method
+
GET
+ +
URL
+
`/v1/operator/raft/configuration`
+ +
Parameters
+
+
    +
  • + stale + optional + If the cluster doesn't currently have a leader an error will be + returned. You can use the `?stale` query parameter to read the Raft + configuration from any of the Nomad servers. +
    • +
+
+ +
Returns
+
+ + ```javascript +{ + "Servers": [ + { + "ID": "127.0.0.1:4647", + "Node": "alice", + "Address": "127.0.0.1:4647", + "Leader": true, + "Voter": true + }, + { + "ID": "127.0.0.2:4647", + "Node": "bob", + "Address": "127.0.0.2:4647", + "Leader": false, + "Voter": true + }, + { + "ID": "127.0.0.3:4647", + "Node": "carol", + "Address": "127.0.0.3:4647", + "Leader": false, + "Voter": true + } + ], + "Index": 22 +} + ``` + +
+ +
Field Reference
+
+ +
    +
  • + Servers + The returned `Servers` array has information about the servers in the Raft + peer configuration. See the `Server` block for a description of its fields: +
    • +
  • + Index + The `Index` value is the Raft corresponding to this configuration. The + latest configuration may not yet be committed if changes are in flight. +
    • +
+ + `Server` Fields: +
    +
  • + ID + `ID` is the ID of the server. This is the same as the `Address` but may + be upgraded to a GUID in a future version of Nomad. +
    • +
  • + Node + `Node` is the node name of the server, as known to Nomad, or "(unknown)" if + the node is stale and not known. +
    • +
  • + Address + `Address` is the IP:port for the server. +
    • +
  • + Leader + `Leader` is either "true" or "false" depending on the server's role in the + Raft configuration. +
    • +
  • + Voter + `Voter` is "true" or "false", indicating if the server has a vote in the Raft + configuration. Future versions of Nomad may add support for non-voting servers. +
    • +
+ +
+
+ + +## DELETE + +
+
Description
+
+ Remove the Nomad server with given address from the Raft configuration. The + return code signifies success or failure. +
+ +
Method
+
DELETE
+ +
URL
+
`/v1/operator/raft/peer`
+ +
Parameters
+
+
    +
  • + address + required + The address specifies the server to remove and is given as an `IP:port`. + The port number is usually 4647, unless configured otherwise. Nothing is + required in the body of the request. +
    • +
+
+ +
Returns
+
None
+ +
diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 48fb8ebdd..9c62e22b3 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -228,6 +228,17 @@ > node-status + > + operator + + > plan diff --git a/website/source/layouts/http.erb b/website/source/layouts/http.erb index 5fb7d7dc8..6e6105e7f 100644 --- a/website/source/layouts/http.erb +++ b/website/source/layouts/http.erb @@ -117,6 +117,10 @@ Status + > + Operator + + > System