package nomad import ( "context" "fmt" "strconv" metrics "github.com/armon/go-metrics" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/raft" autopilot "github.com/hashicorp/raft-autopilot" "github.com/hashicorp/serf/serf" ) const ( // AutopilotRZTag is the Serf tag to use for the redundancy zone value // when passing the server metadata to Autopilot. AutopilotRZTag = "ap_zone" // AutopilotRZTag is the Serf tag to use for the custom version value // when passing the server metadata to Autopilot. AutopilotVersionTag = "ap_version" ) // AutopilotDelegate is a Nomad delegate for autopilot operations. It implements // the autopilot.ApplicationIntegration interface, and the methods required for // that interface have been documented as such below. type AutopilotDelegate struct { server *Server } // AutopilotConfig is used to retrieve the latest configuration from the Nomad // delegate. This method is required to implement the ApplicationIntegration // interface. func (d *AutopilotDelegate) AutopilotConfig() *autopilot.Config { c := d.server.getOrCreateAutopilotConfig() if c == nil { return nil } conf := &autopilot.Config{ CleanupDeadServers: c.CleanupDeadServers, LastContactThreshold: c.LastContactThreshold, MaxTrailingLogs: c.MaxTrailingLogs, MinQuorum: c.MinQuorum, ServerStabilizationTime: c.ServerStabilizationTime, Ext: autopilotConfigExt(c), } return conf } // FetchServerStats will be called by autopilot to request Nomad fetch the // server stats out of band. This method is required to implement the // ApplicationIntegration interface func (d *AutopilotDelegate) FetchServerStats(ctx context.Context, servers map[raft.ServerID]*autopilot.Server) map[raft.ServerID]*autopilot.ServerStats { return d.server.statsFetcher.Fetch(ctx, servers) } // KnownServers will be called by autopilot to request the list of servers known // to Nomad. This method is required to implement the ApplicationIntegration // interface func (d *AutopilotDelegate) KnownServers() map[raft.ServerID]*autopilot.Server { return d.server.autopilotServers() } // NotifyState will be called when the autopilot state is updated. The Nomad // leader heartbeats a metric for monitoring based on this information. This // method is required to implement the ApplicationIntegration interface func (d *AutopilotDelegate) NotifyState(state *autopilot.State) { if d.server.raft.State() == raft.Leader { metrics.SetGauge([]string{"nomad", "autopilot", "failure_tolerance"}, float32(state.FailureTolerance)) if state.Healthy { metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 1) } else { metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 0) } } } // RemoveFailedServer will be called by autopilot to notify Nomad to remove the // server in a failed state. This method is required to implement the // ApplicationIntegration interface. (Note this is expected to return // immediately so we'll spawn a goroutine for it.) func (d *AutopilotDelegate) RemoveFailedServer(failedSrv *autopilot.Server) { go func() { err := d.server.RemoveFailedNode(failedSrv.Name) if err != nil { d.server.logger.Error("could not remove failed server", "server", string(failedSrv.ID), "error", err, ) } }() } // MinRaftProtocol returns the lowest supported Raft protocol among alive // servers func (s *Server) MinRaftProtocol() (int, error) { return minRaftProtocol(s.serf.Members(), isNomadServer) } // GetClusterHealth is used to get the current health of the servers, as known // by the leader. func (s *Server) GetClusterHealth() *structs.OperatorHealthReply { state := s.autopilot.GetState() if state == nil { // this behavior seems odd but its functionally equivalent to 1.8.5 where if // autopilot didn't have a health reply yet it would just return no error return nil } health := &structs.OperatorHealthReply{ Healthy: state.Healthy, FailureTolerance: state.FailureTolerance, } for _, srv := range state.Servers { srvHealth := structs.ServerHealth{ ID: string(srv.Server.ID), Name: srv.Server.Name, Address: string(srv.Server.Address), Version: srv.Server.Version, Leader: srv.State == autopilot.RaftLeader, Voter: srv.State == autopilot.RaftLeader || srv.State == autopilot.RaftVoter, LastContact: srv.Stats.LastContact, LastTerm: srv.Stats.LastTerm, LastIndex: srv.Stats.LastIndex, Healthy: srv.Health.Healthy, StableSince: srv.Health.StableSince, } switch srv.Server.NodeStatus { case autopilot.NodeAlive: srvHealth.SerfStatus = serf.StatusAlive case autopilot.NodeLeft: srvHealth.SerfStatus = serf.StatusLeft case autopilot.NodeFailed: srvHealth.SerfStatus = serf.StatusFailed default: srvHealth.SerfStatus = serf.StatusNone } health.Servers = append(health.Servers, srvHealth) } return health } // ------------------- // helper functions func minRaftProtocol(members []serf.Member, serverFunc func(serf.Member) (bool, *serverParts)) (int, error) { minVersion := -1 for _, m := range members { if m.Status != serf.StatusAlive { continue } ok, server := serverFunc(m) if !ok { return -1, fmt.Errorf("not a Nomad server") } if server == nil { continue } vsn, ok := m.Tags["raft_vsn"] if !ok { vsn = "1" } raftVsn, err := strconv.Atoi(vsn) if err != nil { return -1, err } if minVersion == -1 || raftVsn < minVersion { minVersion = raftVsn } } if minVersion == -1 { return minVersion, fmt.Errorf("No servers found") } return minVersion, nil } func (s *Server) autopilotServers() map[raft.ServerID]*autopilot.Server { servers := make(map[raft.ServerID]*autopilot.Server) for _, member := range s.serf.Members() { srv, err := s.autopilotServer(member) if err != nil { s.logger.Warn("Error parsing server info", "name", member.Name, "error", err) continue } else if srv == nil { // this member was a client or in another region continue } servers[srv.ID] = srv } return servers } func (s *Server) autopilotServer(m serf.Member) (*autopilot.Server, error) { ok, srv := isNomadServer(m) if !ok { return nil, nil } if srv.Region != s.Region() { return nil, nil } return s.autopilotServerFromMetadata(srv) } func (s *Server) autopilotServerFromMetadata(srv *serverParts) (*autopilot.Server, error) { server := &autopilot.Server{ Name: srv.Name, ID: raft.ServerID(srv.ID), Address: raft.ServerAddress(srv.Addr.String()), Version: srv.Build.String(), RaftVersion: srv.RaftVersion, Ext: s.autopilotServerExt(srv), } switch srv.Status { case serf.StatusLeft: server.NodeStatus = autopilot.NodeLeft case serf.StatusAlive, serf.StatusLeaving: // we want to treat leaving as alive to prevent autopilot from // prematurely removing the node. server.NodeStatus = autopilot.NodeAlive case serf.StatusFailed: server.NodeStatus = autopilot.NodeFailed default: server.NodeStatus = autopilot.NodeUnknown } members := s.serf.Members() for _, member := range members { if member.Name == srv.Name { server.Meta = member.Tags break } } return server, nil }