package raft import ( "bytes" "errors" "fmt" "io" "log" "os" "strconv" "sync" "time" "github.com/armon/go-metrics" ) const ( minCheckInterval = 10 * time.Millisecond ) var ( keyCurrentTerm = []byte("CurrentTerm") keyLastVoteTerm = []byte("LastVoteTerm") keyLastVoteCand = []byte("LastVoteCand") // ErrLeader is returned when an operation can't be completed on a // leader node. ErrLeader = errors.New("node is the leader") // ErrNotLeader is returned when an operation can't be completed on a // follower or candidate node. ErrNotLeader = errors.New("node is not the leader") // ErrLeadershipLost is returned when a leader fails to commit a log entry // because it's been deposed in the process. ErrLeadershipLost = errors.New("leadership lost while committing log") // ErrRaftShutdown is returned when operations are requested against an // inactive Raft. ErrRaftShutdown = errors.New("raft is already shutdown") // ErrEnqueueTimeout is returned when a command fails due to a timeout. ErrEnqueueTimeout = errors.New("timed out enqueuing operation") // ErrKnownPeer is returned when trying to add a peer to the configuration // that already exists. ErrKnownPeer = errors.New("peer already known") // ErrUnknownPeer is returned when trying to remove a peer from the // configuration that doesn't exist. ErrUnknownPeer = errors.New("peer is unknown") // ErrNothingNewToSnapshot is returned when trying to create a snapshot // but there's nothing new commited to the FSM since we started. ErrNothingNewToSnapshot = errors.New("Nothing new to snapshot") ) // commitTuple is used to send an index that was committed, // with an optional associated future that should be invoked. type commitTuple struct { log *Log future *logFuture } // leaderState is state that is used while we are a leader. type leaderState struct { commitCh chan struct{} inflight *inflight replState map[string]*followerReplication notify map[*verifyFuture]struct{} stepDown chan struct{} } // Raft implements a Raft node. type Raft struct { raftState // applyCh is used to async send logs to the main thread to // be committed and applied to the FSM. applyCh chan *logFuture // Configuration provided at Raft initialization conf *Config // FSM is the client state machine to apply commands to fsm FSM // fsmCommitCh is used to trigger async application of logs to the fsm fsmCommitCh chan commitTuple // fsmRestoreCh is used to trigger a restore from snapshot fsmRestoreCh chan *restoreFuture // fsmSnapshotCh is used to trigger a new snapshot being taken fsmSnapshotCh chan *reqSnapshotFuture // lastContact is the last time we had contact from the // leader node. This can be used to gauge staleness. lastContact time.Time lastContactLock sync.RWMutex // Leader is the current cluster leader leader string leaderLock sync.RWMutex // leaderCh is used to notify of leadership changes leaderCh chan bool // leaderState used only while state is leader leaderState leaderState // Stores our local addr localAddr string // Used for our logging logger *log.Logger // LogStore provides durable storage for logs logs LogStore // Track our known peers peerCh chan *peerFuture peers []string peerStore PeerStore // RPC chan comes from the transport layer rpcCh <-chan RPC // Shutdown channel to exit, protected to prevent concurrent exits shutdown bool shutdownCh chan struct{} shutdownLock sync.Mutex // snapshots is used to store and retrieve snapshots snapshots SnapshotStore // snapshotCh is used for user triggered snapshots snapshotCh chan *snapshotFuture // stable is a StableStore implementation for durable state // It provides stable storage for many fields in raftState stable StableStore // The transport layer we use trans Transport // verifyCh is used to async send verify futures to the main thread // to verify we are still the leader verifyCh chan *verifyFuture } // NewRaft is used to construct a new Raft node. It takes a configuration, as well // as implementations of various interfaces that are required. If we have any old state, // such as snapshots, logs, peers, etc, all those will be restored when creating the // Raft node. func NewRaft(conf *Config, fsm FSM, logs LogStore, stable StableStore, snaps SnapshotStore, peerStore PeerStore, trans Transport) (*Raft, error) { // Validate the configuration if err := ValidateConfig(conf); err != nil { return nil, err } // Ensure we have a LogOutput var logger *log.Logger if conf.Logger != nil { logger = conf.Logger } else { if conf.LogOutput == nil { conf.LogOutput = os.Stderr } logger = log.New(conf.LogOutput, "", log.LstdFlags) } // Try to restore the current term currentTerm, err := stable.GetUint64(keyCurrentTerm) if err != nil && err.Error() != "not found" { return nil, fmt.Errorf("failed to load current term: %v", err) } // Read the last log value lastIdx, err := logs.LastIndex() if err != nil { return nil, fmt.Errorf("failed to find last log: %v", err) } // Get the log var lastLog Log if lastIdx > 0 { if err := logs.GetLog(lastIdx, &lastLog); err != nil { return nil, fmt.Errorf("failed to get last log: %v", err) } } // Construct the list of peers that excludes us localAddr := trans.LocalAddr() peers, err := peerStore.Peers() if err != nil { return nil, fmt.Errorf("failed to get list of peers: %v", err) } peers = ExcludePeer(peers, localAddr) // Create Raft struct r := &Raft{ applyCh: make(chan *logFuture), conf: conf, fsm: fsm, fsmCommitCh: make(chan commitTuple, 128), fsmRestoreCh: make(chan *restoreFuture), fsmSnapshotCh: make(chan *reqSnapshotFuture), leaderCh: make(chan bool), localAddr: localAddr, logger: logger, logs: logs, peerCh: make(chan *peerFuture), peers: peers, peerStore: peerStore, rpcCh: trans.Consumer(), snapshots: snaps, snapshotCh: make(chan *snapshotFuture), shutdownCh: make(chan struct{}), stable: stable, trans: trans, verifyCh: make(chan *verifyFuture, 64), } // Initialize as a follower r.setState(Follower) // Start as leader if specified. This should only be used // for testing purposes. if conf.StartAsLeader { r.setState(Leader) r.setLeader(r.localAddr) } // Restore the current term and the last log r.setCurrentTerm(currentTerm) r.setLastLogIndex(lastLog.Index) r.setLastLogTerm(lastLog.Term) // Attempt to restore a snapshot if there are any if err := r.restoreSnapshot(); err != nil { return nil, err } // Setup a heartbeat fast-path to avoid head-of-line // blocking where possible. It MUST be safe for this // to be called concurrently with a blocking RPC. trans.SetHeartbeatHandler(r.processHeartbeat) // Start the background work r.goFunc(r.run) r.goFunc(r.runFSM) r.goFunc(r.runSnapshots) return r, nil } // Leader is used to return the current leader of the cluster. // It may return empty string if there is no current leader // or the leader is unknown. func (r *Raft) Leader() string { r.leaderLock.RLock() leader := r.leader r.leaderLock.RUnlock() return leader } // setLeader is used to modify the current leader of the cluster func (r *Raft) setLeader(leader string) { r.leaderLock.Lock() r.leader = leader r.leaderLock.Unlock() } // Apply is used to apply a command to the FSM in a highly consistent // manner. This returns a future that can be used to wait on the application. // An optional timeout can be provided to limit the amount of time we wait // for the command to be started. This must be run on the leader or it // will fail. func (r *Raft) Apply(cmd []byte, timeout time.Duration) ApplyFuture { metrics.IncrCounter([]string{"raft", "apply"}, 1) var timer <-chan time.Time if timeout > 0 { timer = time.After(timeout) } // Create a log future, no index or term yet logFuture := &logFuture{ log: Log{ Type: LogCommand, Data: cmd, }, } logFuture.init() select { case <-timer: return errorFuture{ErrEnqueueTimeout} case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} case r.applyCh <- logFuture: return logFuture } } // Barrier is used to issue a command that blocks until all preceeding // operations have been applied to the FSM. It can be used to ensure the // FSM reflects all queued writes. An optional timeout can be provided to // limit the amount of time we wait for the command to be started. This // must be run on the leader or it will fail. func (r *Raft) Barrier(timeout time.Duration) Future { metrics.IncrCounter([]string{"raft", "barrier"}, 1) var timer <-chan time.Time if timeout > 0 { timer = time.After(timeout) } // Create a log future, no index or term yet logFuture := &logFuture{ log: Log{ Type: LogBarrier, }, } logFuture.init() select { case <-timer: return errorFuture{ErrEnqueueTimeout} case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} case r.applyCh <- logFuture: return logFuture } } // VerifyLeader is used to ensure the current node is still // the leader. This can be done to prevent stale reads when a // new leader has potentially been elected. func (r *Raft) VerifyLeader() Future { metrics.IncrCounter([]string{"raft", "verify_leader"}, 1) verifyFuture := &verifyFuture{} verifyFuture.init() select { case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} case r.verifyCh <- verifyFuture: return verifyFuture } } // AddPeer is used to add a new peer into the cluster. This must be // run on the leader or it will fail. func (r *Raft) AddPeer(peer string) Future { logFuture := &logFuture{ log: Log{ Type: LogAddPeer, peer: peer, }, } logFuture.init() select { case r.applyCh <- logFuture: return logFuture case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} } } // RemovePeer is used to remove a peer from the cluster. If the // current leader is being removed, it will cause a new election // to occur. This must be run on the leader or it will fail. func (r *Raft) RemovePeer(peer string) Future { logFuture := &logFuture{ log: Log{ Type: LogRemovePeer, peer: peer, }, } logFuture.init() select { case r.applyCh <- logFuture: return logFuture case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} } } // SetPeers is used to forcibly replace the set of internal peers and // the peerstore with the ones specified. This can be considered unsafe. func (r *Raft) SetPeers(p []string) Future { peerFuture := &peerFuture{ peers: p, } peerFuture.init() select { case r.peerCh <- peerFuture: return peerFuture case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} } } // Shutdown is used to stop the Raft background routines. // This is not a graceful operation. Provides a future that // can be used to block until all background routines have exited. func (r *Raft) Shutdown() Future { r.shutdownLock.Lock() defer r.shutdownLock.Unlock() if !r.shutdown { close(r.shutdownCh) r.shutdown = true r.setState(Shutdown) } return &shutdownFuture{r} } // Snapshot is used to manually force Raft to take a snapshot. // Returns a future that can be used to block until complete. func (r *Raft) Snapshot() Future { snapFuture := &snapshotFuture{} snapFuture.init() select { case r.snapshotCh <- snapFuture: return snapFuture case <-r.shutdownCh: return errorFuture{ErrRaftShutdown} } } // State is used to return the current raft state. func (r *Raft) State() RaftState { return r.getState() } // LeaderCh is used to get a channel which delivers signals on // acquiring or losing leadership. It sends true if we become // the leader, and false if we lose it. The channel is not buffered, // and does not block on writes. func (r *Raft) LeaderCh() <-chan bool { return r.leaderCh } func (r *Raft) String() string { return fmt.Sprintf("Node at %s [%v]", r.localAddr, r.getState()) } // LastContact returns the time of last contact by a leader. // This only makes sense if we are currently a follower. func (r *Raft) LastContact() time.Time { r.lastContactLock.RLock() last := r.lastContact r.lastContactLock.RUnlock() return last } // Stats is used to return a map of various internal stats. This should only // be used for informative purposes or debugging. func (r *Raft) Stats() map[string]string { toString := func(v uint64) string { return strconv.FormatUint(v, 10) } s := map[string]string{ "state": r.getState().String(), "term": toString(r.getCurrentTerm()), "last_log_index": toString(r.getLastLogIndex()), "last_log_term": toString(r.getLastLogTerm()), "commit_index": toString(r.getCommitIndex()), "applied_index": toString(r.getLastApplied()), "fsm_pending": toString(uint64(len(r.fsmCommitCh))), "last_snapshot_index": toString(r.getLastSnapshotIndex()), "last_snapshot_term": toString(r.getLastSnapshotTerm()), "num_peers": toString(uint64(len(r.peers))), } last := r.LastContact() if last.IsZero() { s["last_contact"] = "never" } else if r.getState() == Leader { s["last_contact"] = "0" } else { s["last_contact"] = fmt.Sprintf("%v", time.Now().Sub(last)) } return s } // LastIndex returns the last index in stable storage, // either from the last log or from the last snapshot. func (r *Raft) LastIndex() uint64 { return r.getLastIndex() } // AppliedIndex returns the last index applied to the FSM. // This is generally lagging behind the last index, especially // for indexes that are persisted but have not yet been considered // committed by the leader. func (r *Raft) AppliedIndex() uint64 { return r.getLastApplied() } // runFSM is a long running goroutine responsible for applying logs // to the FSM. This is done async of other logs since we don't want // the FSM to block our internal operations. func (r *Raft) runFSM() { var lastIndex, lastTerm uint64 for { select { case req := <-r.fsmRestoreCh: // Open the snapshot meta, source, err := r.snapshots.Open(req.ID) if err != nil { req.respond(fmt.Errorf("failed to open snapshot %v: %v", req.ID, err)) continue } // Attempt to restore start := time.Now() if err := r.fsm.Restore(source); err != nil { req.respond(fmt.Errorf("failed to restore snapshot %v: %v", req.ID, err)) source.Close() continue } source.Close() metrics.MeasureSince([]string{"raft", "fsm", "restore"}, start) // Update the last index and term lastIndex = meta.Index lastTerm = meta.Term req.respond(nil) case req := <-r.fsmSnapshotCh: // Is there something to snapshot? if lastIndex == 0 { req.respond(ErrNothingNewToSnapshot) continue } // Get our peers peers, err := r.peerStore.Peers() if err != nil { req.respond(err) continue } // Start a snapshot start := time.Now() snap, err := r.fsm.Snapshot() metrics.MeasureSince([]string{"raft", "fsm", "snapshot"}, start) // Respond to the request req.index = lastIndex req.term = lastTerm req.peers = peers req.snapshot = snap req.respond(err) case commitTuple := <-r.fsmCommitCh: // Apply the log if a command var resp interface{} if commitTuple.log.Type == LogCommand { start := time.Now() resp = r.fsm.Apply(commitTuple.log) metrics.MeasureSince([]string{"raft", "fsm", "apply"}, start) } // Update the indexes lastIndex = commitTuple.log.Index lastTerm = commitTuple.log.Term // Invoke the future if given if commitTuple.future != nil { commitTuple.future.response = resp commitTuple.future.respond(nil) } case <-r.shutdownCh: return } } } // run is a long running goroutine that runs the Raft FSM. func (r *Raft) run() { for { // Check if we are doing a shutdown select { case <-r.shutdownCh: // Clear the leader to prevent forwarding r.setLeader("") return default: } // Enter into a sub-FSM switch r.getState() { case Follower: r.runFollower() case Candidate: r.runCandidate() case Leader: r.runLeader() } } } // runFollower runs the FSM for a follower. func (r *Raft) runFollower() { didWarn := false r.logger.Printf("[INFO] raft: %v entering Follower state", r) metrics.IncrCounter([]string{"raft", "state", "follower"}, 1) heartbeatTimer := randomTimeout(r.conf.HeartbeatTimeout) for { select { case rpc := <-r.rpcCh: r.processRPC(rpc) case a := <-r.applyCh: // Reject any operations since we are not the leader a.respond(ErrNotLeader) case v := <-r.verifyCh: // Reject any operations since we are not the leader v.respond(ErrNotLeader) case p := <-r.peerCh: // Set the peers r.peers = ExcludePeer(p.peers, r.localAddr) p.respond(r.peerStore.SetPeers(p.peers)) case <-heartbeatTimer: // Restart the heartbeat timer heartbeatTimer = randomTimeout(r.conf.HeartbeatTimeout) // Check if we have had a successful contact lastContact := r.LastContact() if time.Now().Sub(lastContact) < r.conf.HeartbeatTimeout { continue } // Heartbeat failed! Transition to the candidate state r.setLeader("") if len(r.peers) == 0 && !r.conf.EnableSingleNode { if !didWarn { r.logger.Printf("[WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.") didWarn = true } } else { r.logger.Printf("[WARN] raft: Heartbeat timeout reached, starting election") metrics.IncrCounter([]string{"raft", "transition", "heartbeat_timeout"}, 1) r.setState(Candidate) return } case <-r.shutdownCh: return } } } // runCandidate runs the FSM for a candidate. func (r *Raft) runCandidate() { r.logger.Printf("[INFO] raft: %v entering Candidate state", r) metrics.IncrCounter([]string{"raft", "state", "candidate"}, 1) // Start vote for us, and set a timeout voteCh := r.electSelf() electionTimer := randomTimeout(r.conf.ElectionTimeout) // Tally the votes, need a simple majority grantedVotes := 0 votesNeeded := r.quorumSize() r.logger.Printf("[DEBUG] raft: Votes needed: %d", votesNeeded) for r.getState() == Candidate { select { case rpc := <-r.rpcCh: r.processRPC(rpc) case vote := <-voteCh: // Check if the term is greater than ours, bail if vote.Term > r.getCurrentTerm() { r.logger.Printf("[DEBUG] raft: Newer term discovered, fallback to follower") r.setState(Follower) r.setCurrentTerm(vote.Term) return } // Check if the vote is granted if vote.Granted { grantedVotes++ r.logger.Printf("[DEBUG] raft: Vote granted from %s. Tally: %d", vote.voter, grantedVotes) } // Check if we've become the leader if grantedVotes >= votesNeeded { r.logger.Printf("[INFO] raft: Election won. Tally: %d", grantedVotes) r.setState(Leader) r.setLeader(r.localAddr) return } case a := <-r.applyCh: // Reject any operations since we are not the leader a.respond(ErrNotLeader) case v := <-r.verifyCh: // Reject any operations since we are not the leader v.respond(ErrNotLeader) case p := <-r.peerCh: // Set the peers r.peers = ExcludePeer(p.peers, r.localAddr) p.respond(r.peerStore.SetPeers(p.peers)) // Become a follower again r.setState(Follower) return case <-electionTimer: // Election failed! Restart the election. We simply return, // which will kick us back into runCandidate r.logger.Printf("[WARN] raft: Election timeout reached, restarting election") return case <-r.shutdownCh: return } } } // runLeader runs the FSM for a leader. Do the setup here and drop into // the leaderLoop for the hot loop. func (r *Raft) runLeader() { r.logger.Printf("[INFO] raft: %v entering Leader state", r) metrics.IncrCounter([]string{"raft", "state", "leader"}, 1) // Notify that we are the leader asyncNotifyBool(r.leaderCh, true) // Push to the notify channel if given if notify := r.conf.NotifyCh; notify != nil { select { case notify <- true: case <-r.shutdownCh: } } // Setup leader state r.leaderState.commitCh = make(chan struct{}, 1) r.leaderState.inflight = newInflight(r.leaderState.commitCh) r.leaderState.replState = make(map[string]*followerReplication) r.leaderState.notify = make(map[*verifyFuture]struct{}) r.leaderState.stepDown = make(chan struct{}, 1) // Cleanup state on step down defer func() { // Since we were the leader previously, we update our // last contact time when we step down, so that we are not // reporting a last contact time from before we were the // leader. Otherwise, to a client it would seem our data // is extremely stale. r.setLastContact() // Stop replication for _, p := range r.leaderState.replState { close(p.stopCh) } // Cancel inflight requests r.leaderState.inflight.Cancel(ErrLeadershipLost) // Respond to any pending verify requests for future := range r.leaderState.notify { future.respond(ErrLeadershipLost) } // Clear all the state r.leaderState.commitCh = nil r.leaderState.inflight = nil r.leaderState.replState = nil r.leaderState.notify = nil r.leaderState.stepDown = nil // If we are stepping down for some reason, no known leader. // We may have stepped down due to an RPC call, which would // provide the leader, so we cannot always blank this out. r.leaderLock.Lock() if r.leader == r.localAddr { r.leader = "" } r.leaderLock.Unlock() // Notify that we are not the leader asyncNotifyBool(r.leaderCh, false) // Push to the notify channel if given if notify := r.conf.NotifyCh; notify != nil { select { case notify <- false: case <-r.shutdownCh: // On shutdown, make a best effort but do not block select { case notify <- false: default: } } } }() // Start a replication routine for each peer for _, peer := range r.peers { r.startReplication(peer) } // Dispatch a no-op log first. Instead of LogNoop, // we use a LogAddPeer with our peerset. This acts like // a no-op as well, but when doing an initial bootstrap, ensures // that all nodes share a common peerset. peerSet := append([]string{r.localAddr}, r.peers...) noop := &logFuture{ log: Log{ Type: LogAddPeer, Data: encodePeers(peerSet, r.trans), }, } r.dispatchLogs([]*logFuture{noop}) // Disable EnableSingleNode after we've been elected leader. // This is to prevent a split brain in the future, if we are removed // from the cluster and then elect ourself as leader. if r.conf.DisableBootstrapAfterElect && r.conf.EnableSingleNode { r.logger.Printf("[INFO] raft: Disabling EnableSingleNode (bootstrap)") r.conf.EnableSingleNode = false } // Sit in the leader loop until we step down r.leaderLoop() } // startReplication is a helper to setup state and start async replication to a peer. func (r *Raft) startReplication(peer string) { lastIdx := r.getLastIndex() s := &followerReplication{ peer: peer, inflight: r.leaderState.inflight, stopCh: make(chan uint64, 1), triggerCh: make(chan struct{}, 1), currentTerm: r.getCurrentTerm(), matchIndex: 0, nextIndex: lastIdx + 1, lastContact: time.Now(), notifyCh: make(chan struct{}, 1), stepDown: r.leaderState.stepDown, } r.leaderState.replState[peer] = s r.goFunc(func() { r.replicate(s) }) asyncNotifyCh(s.triggerCh) } // leaderLoop is the hot loop for a leader. It is invoked // after all the various leader setup is done. func (r *Raft) leaderLoop() { // stepDown is used to track if there is an inflight log that // would cause us to lose leadership (specifically a RemovePeer of // ourselves). If this is the case, we must not allow any logs to // be processed in parallel, otherwise we are basing commit on // only a single peer (ourself) and replicating to an undefined set // of peers. stepDown := false lease := time.After(r.conf.LeaderLeaseTimeout) for r.getState() == Leader { select { case rpc := <-r.rpcCh: r.processRPC(rpc) case <-r.leaderState.stepDown: r.setState(Follower) case <-r.leaderState.commitCh: // Get the committed messages committed := r.leaderState.inflight.Committed() for e := committed.Front(); e != nil; e = e.Next() { // Measure the commit time commitLog := e.Value.(*logFuture) metrics.MeasureSince([]string{"raft", "commitTime"}, commitLog.dispatch) // Increment the commit index idx := commitLog.log.Index r.setCommitIndex(idx) r.processLogs(idx, commitLog) } case v := <-r.verifyCh: if v.quorumSize == 0 { // Just dispatched, start the verification r.verifyLeader(v) } else if v.votes < v.quorumSize { // Early return, means there must be a new leader r.logger.Printf("[WARN] raft: New leader elected, stepping down") r.setState(Follower) delete(r.leaderState.notify, v) v.respond(ErrNotLeader) } else { // Quorum of members agree, we are still leader delete(r.leaderState.notify, v) v.respond(nil) } case p := <-r.peerCh: p.respond(ErrLeader) case newLog := <-r.applyCh: // Group commit, gather all the ready commits ready := []*logFuture{newLog} for i := 0; i < r.conf.MaxAppendEntries; i++ { select { case newLog := <-r.applyCh: ready = append(ready, newLog) default: break } } // Handle any peer set changes n := len(ready) for i := 0; i < n; i++ { // Fail all future transactions once stepDown is on if stepDown { ready[i].respond(ErrNotLeader) ready[i], ready[n-1] = ready[n-1], nil n-- i-- continue } // Special case AddPeer and RemovePeer log := ready[i] if log.log.Type != LogAddPeer && log.log.Type != LogRemovePeer { continue } // Check if this log should be ignored. The logs can be // reordered here since we have not yet assigned an index // and are not violating any promises. if !r.preparePeerChange(log) { ready[i], ready[n-1] = ready[n-1], nil n-- i-- continue } // Apply peer set changes early and check if we will step // down after the commit of this log. If so, we must not // allow any future entries to make progress to avoid undefined // behavior. if ok := r.processLog(&log.log, nil, true); ok { stepDown = true } } // Nothing to do if all logs are invalid if n == 0 { continue } // Dispatch the logs ready = ready[:n] r.dispatchLogs(ready) case <-lease: // Check if we've exceeded the lease, potentially stepping down maxDiff := r.checkLeaderLease() // Next check interval should adjust for the last node we've // contacted, without going negative checkInterval := r.conf.LeaderLeaseTimeout - maxDiff if checkInterval < minCheckInterval { checkInterval = minCheckInterval } // Renew the lease timer lease = time.After(checkInterval) case <-r.shutdownCh: return } } } // verifyLeader must be called from the main thread for safety. // Causes the followers to attempt an immediate heartbeat. func (r *Raft) verifyLeader(v *verifyFuture) { // Current leader always votes for self v.votes = 1 // Set the quorum size, hot-path for single node v.quorumSize = r.quorumSize() if v.quorumSize == 1 { v.respond(nil) return } // Track this request v.notifyCh = r.verifyCh r.leaderState.notify[v] = struct{}{} // Trigger immediate heartbeats for _, repl := range r.leaderState.replState { repl.notifyLock.Lock() repl.notify = append(repl.notify, v) repl.notifyLock.Unlock() asyncNotifyCh(repl.notifyCh) } } // checkLeaderLease is used to check if we can contact a quorum of nodes // within the last leader lease interval. If not, we need to step down, // as we may have lost connectivity. Returns the maximum duration without // contact. func (r *Raft) checkLeaderLease() time.Duration { // Track contacted nodes, we can always contact ourself contacted := 1 // Check each follower var maxDiff time.Duration now := time.Now() for peer, f := range r.leaderState.replState { diff := now.Sub(f.LastContact()) if diff <= r.conf.LeaderLeaseTimeout { contacted++ if diff > maxDiff { maxDiff = diff } } else { // Log at least once at high value, then debug. Otherwise it gets very verbose. if diff <= 3*r.conf.LeaderLeaseTimeout { r.logger.Printf("[WARN] raft: Failed to contact %v in %v", peer, diff) } else { r.logger.Printf("[DEBUG] raft: Failed to contact %v in %v", peer, diff) } } metrics.AddSample([]string{"raft", "leader", "lastContact"}, float32(diff/time.Millisecond)) } // Verify we can contact a quorum quorum := r.quorumSize() if contacted < quorum { r.logger.Printf("[WARN] raft: Failed to contact quorum of nodes, stepping down") r.setState(Follower) metrics.IncrCounter([]string{"raft", "transition", "leader_lease_timeout"}, 1) } return maxDiff } // quorumSize is used to return the quorum size func (r *Raft) quorumSize() int { return ((len(r.peers) + 1) / 2) + 1 } // preparePeerChange checks if a LogAddPeer or LogRemovePeer should be performed, // and properly formats the data field on the log before dispatching it. func (r *Raft) preparePeerChange(l *logFuture) bool { // Check if this is a known peer p := l.log.peer knownPeer := PeerContained(r.peers, p) || r.localAddr == p // Ignore known peers on add if l.log.Type == LogAddPeer && knownPeer { l.respond(ErrKnownPeer) return false } // Ignore unknown peers on remove if l.log.Type == LogRemovePeer && !knownPeer { l.respond(ErrUnknownPeer) return false } // Construct the peer set var peerSet []string if l.log.Type == LogAddPeer { peerSet = append([]string{p, r.localAddr}, r.peers...) } else { peerSet = ExcludePeer(append([]string{r.localAddr}, r.peers...), p) } // Setup the log l.log.Data = encodePeers(peerSet, r.trans) return true } // dispatchLog is called to push a log to disk, mark it // as inflight and begin replication of it. func (r *Raft) dispatchLogs(applyLogs []*logFuture) { now := time.Now() defer metrics.MeasureSince([]string{"raft", "leader", "dispatchLog"}, now) term := r.getCurrentTerm() lastIndex := r.getLastIndex() logs := make([]*Log, len(applyLogs)) for idx, applyLog := range applyLogs { applyLog.dispatch = now applyLog.log.Index = lastIndex + uint64(idx) + 1 applyLog.log.Term = term applyLog.policy = newMajorityQuorum(len(r.peers) + 1) logs[idx] = &applyLog.log } // Write the log entry locally if err := r.logs.StoreLogs(logs); err != nil { r.logger.Printf("[ERR] raft: Failed to commit logs: %v", err) for _, applyLog := range applyLogs { applyLog.respond(err) } r.setState(Follower) return } // Add this to the inflight logs, commit r.leaderState.inflight.StartAll(applyLogs) // Update the last log since it's on disk now r.setLastLogIndex(lastIndex + uint64(len(applyLogs))) r.setLastLogTerm(term) // Notify the replicators of the new log for _, f := range r.leaderState.replState { asyncNotifyCh(f.triggerCh) } } // processLogs is used to process all the logs from the lastApplied // up to the given index. func (r *Raft) processLogs(index uint64, future *logFuture) { // Reject logs we've applied already lastApplied := r.getLastApplied() if index <= lastApplied { r.logger.Printf("[WARN] raft: Skipping application of old log: %d", index) return } // Apply all the preceding logs for idx := r.getLastApplied() + 1; idx <= index; idx++ { // Get the log, either from the future or from our log store if future != nil && future.log.Index == idx { r.processLog(&future.log, future, false) } else { l := new(Log) if err := r.logs.GetLog(idx, l); err != nil { r.logger.Printf("[ERR] raft: Failed to get log at %d: %v", idx, err) panic(err) } r.processLog(l, nil, false) } // Update the lastApplied index and term r.setLastApplied(idx) } } // processLog is invoked to process the application of a single committed log. // Returns if this log entry would cause us to stepDown after it commits. func (r *Raft) processLog(l *Log, future *logFuture, precommit bool) (stepDown bool) { switch l.Type { case LogBarrier: // Barrier is handled by the FSM fallthrough case LogCommand: // Forward to the fsm handler select { case r.fsmCommitCh <- commitTuple{l, future}: case <-r.shutdownCh: if future != nil { future.respond(ErrRaftShutdown) } } // Return so that the future is only responded to // by the FSM handler when the application is done return case LogAddPeer: fallthrough case LogRemovePeer: peers := decodePeers(l.Data, r.trans) r.logger.Printf("[DEBUG] raft: Node %v updated peer set (%v): %v", r.localAddr, l.Type, peers) // If the peer set does not include us, remove all other peers removeSelf := !PeerContained(peers, r.localAddr) && l.Type == LogRemovePeer if removeSelf { // Mark that this operation will cause us to step down as // leader. This prevents the future logs from being Applied // from this leader. stepDown = true // We only modify the peers after the commit, otherwise we // would be using a quorum size of 1 for the RemovePeer operation. // This is used with the stepDown guard to prevent any other logs. if !precommit { r.peers = nil r.peerStore.SetPeers([]string{r.localAddr}) } } else { r.peers = ExcludePeer(peers, r.localAddr) r.peerStore.SetPeers(peers) } // Handle replication if we are the leader if r.getState() == Leader { for _, p := range r.peers { if _, ok := r.leaderState.replState[p]; !ok { r.logger.Printf("[INFO] raft: Added peer %v, starting replication", p) r.startReplication(p) } } } // Stop replication for old nodes if r.getState() == Leader && !precommit { var toDelete []string for _, repl := range r.leaderState.replState { if !PeerContained(r.peers, repl.peer) { r.logger.Printf("[INFO] raft: Removed peer %v, stopping replication (Index: %d)", repl.peer, l.Index) // Replicate up to this index and stop repl.stopCh <- l.Index close(repl.stopCh) toDelete = append(toDelete, repl.peer) } } for _, name := range toDelete { delete(r.leaderState.replState, name) } } // Handle removing ourself if removeSelf && !precommit { if r.conf.ShutdownOnRemove { r.logger.Printf("[INFO] raft: Removed ourself, shutting down") r.Shutdown() } else { r.logger.Printf("[INFO] raft: Removed ourself, transitioning to follower") r.setState(Follower) } } case LogNoop: // Ignore the no-op default: r.logger.Printf("[ERR] raft: Got unrecognized log type: %#v", l) } // Invoke the future if given if future != nil && !precommit { future.respond(nil) } return } // processRPC is called to handle an incoming RPC request. func (r *Raft) processRPC(rpc RPC) { switch cmd := rpc.Command.(type) { case *AppendEntriesRequest: r.appendEntries(rpc, cmd) case *RequestVoteRequest: r.requestVote(rpc, cmd) case *InstallSnapshotRequest: r.installSnapshot(rpc, cmd) default: r.logger.Printf("[ERR] raft: Got unexpected command: %#v", rpc.Command) rpc.Respond(nil, fmt.Errorf("unexpected command")) } } // processHeartbeat is a special handler used just for heartbeat requests // so that they can be fast-pathed if a transport supports it. func (r *Raft) processHeartbeat(rpc RPC) { defer metrics.MeasureSince([]string{"raft", "rpc", "processHeartbeat"}, time.Now()) // Check if we are shutdown, just ignore the RPC select { case <-r.shutdownCh: return default: } // Ensure we are only handling a heartbeat switch cmd := rpc.Command.(type) { case *AppendEntriesRequest: r.appendEntries(rpc, cmd) default: r.logger.Printf("[ERR] raft: Expected heartbeat, got command: %#v", rpc.Command) rpc.Respond(nil, fmt.Errorf("unexpected command")) } } // appendEntries is invoked when we get an append entries RPC call. func (r *Raft) appendEntries(rpc RPC, a *AppendEntriesRequest) { defer metrics.MeasureSince([]string{"raft", "rpc", "appendEntries"}, time.Now()) // Setup a response resp := &AppendEntriesResponse{ Term: r.getCurrentTerm(), LastLog: r.getLastIndex(), Success: false, NoRetryBackoff: false, } var rpcErr error defer func() { rpc.Respond(resp, rpcErr) }() // Ignore an older term if a.Term < r.getCurrentTerm() { return } // Increase the term if we see a newer one, also transition to follower // if we ever get an appendEntries call if a.Term > r.getCurrentTerm() || r.getState() != Follower { // Ensure transition to follower r.setState(Follower) r.setCurrentTerm(a.Term) resp.Term = a.Term } // Save the current leader r.setLeader(r.trans.DecodePeer(a.Leader)) // Verify the last log entry if a.PrevLogEntry > 0 { lastIdx, lastTerm := r.getLastEntry() var prevLogTerm uint64 if a.PrevLogEntry == lastIdx { prevLogTerm = lastTerm } else { var prevLog Log if err := r.logs.GetLog(a.PrevLogEntry, &prevLog); err != nil { r.logger.Printf("[WARN] raft: Failed to get previous log: %d %v (last: %d)", a.PrevLogEntry, err, lastIdx) resp.NoRetryBackoff = true return } prevLogTerm = prevLog.Term } if a.PrevLogTerm != prevLogTerm { r.logger.Printf("[WARN] raft: Previous log term mis-match: ours: %d remote: %d", prevLogTerm, a.PrevLogTerm) resp.NoRetryBackoff = true return } } // Process any new entries if n := len(a.Entries); n > 0 { start := time.Now() first := a.Entries[0] last := a.Entries[n-1] // Delete any conflicting entries lastLogIdx := r.getLastLogIndex() if first.Index <= lastLogIdx { r.logger.Printf("[WARN] raft: Clearing log suffix from %d to %d", first.Index, lastLogIdx) if err := r.logs.DeleteRange(first.Index, lastLogIdx); err != nil { r.logger.Printf("[ERR] raft: Failed to clear log suffix: %v", err) return } } // Append the entry if err := r.logs.StoreLogs(a.Entries); err != nil { r.logger.Printf("[ERR] raft: Failed to append to logs: %v", err) return } // Update the lastLog r.setLastLogIndex(last.Index) r.setLastLogTerm(last.Term) metrics.MeasureSince([]string{"raft", "rpc", "appendEntries", "storeLogs"}, start) } // Update the commit index if a.LeaderCommitIndex > 0 && a.LeaderCommitIndex > r.getCommitIndex() { start := time.Now() idx := min(a.LeaderCommitIndex, r.getLastIndex()) r.setCommitIndex(idx) r.processLogs(idx, nil) metrics.MeasureSince([]string{"raft", "rpc", "appendEntries", "processLogs"}, start) } // Everything went well, set success resp.Success = true r.setLastContact() return } // requestVote is invoked when we get an request vote RPC call. func (r *Raft) requestVote(rpc RPC, req *RequestVoteRequest) { defer metrics.MeasureSince([]string{"raft", "rpc", "requestVote"}, time.Now()) // Setup a response resp := &RequestVoteResponse{ Term: r.getCurrentTerm(), Peers: encodePeers(r.peers, r.trans), Granted: false, } var rpcErr error defer func() { rpc.Respond(resp, rpcErr) }() // Check if we have an existing leader [who's not the candidate] candidate := r.trans.DecodePeer(req.Candidate) if leader := r.Leader(); leader != "" && leader != candidate { r.logger.Printf("[WARN] raft: Rejecting vote request from %v since we have a leader: %v", candidate, leader) return } // Ignore an older term if req.Term < r.getCurrentTerm() { return } // Increase the term if we see a newer one if req.Term > r.getCurrentTerm() { // Ensure transition to follower r.setState(Follower) r.setCurrentTerm(req.Term) resp.Term = req.Term } // Check if we have voted yet lastVoteTerm, err := r.stable.GetUint64(keyLastVoteTerm) if err != nil && err.Error() != "not found" { r.logger.Printf("[ERR] raft: Failed to get last vote term: %v", err) return } lastVoteCandBytes, err := r.stable.Get(keyLastVoteCand) if err != nil && err.Error() != "not found" { r.logger.Printf("[ERR] raft: Failed to get last vote candidate: %v", err) return } // Check if we've voted in this election before if lastVoteTerm == req.Term && lastVoteCandBytes != nil { r.logger.Printf("[INFO] raft: Duplicate RequestVote for same term: %d", req.Term) if bytes.Compare(lastVoteCandBytes, req.Candidate) == 0 { r.logger.Printf("[WARN] raft: Duplicate RequestVote from candidate: %s", req.Candidate) resp.Granted = true } return } // Reject if their term is older lastIdx, lastTerm := r.getLastEntry() if lastTerm > req.LastLogTerm { r.logger.Printf("[WARN] raft: Rejecting vote request from %v since our last term is greater (%d, %d)", candidate, lastTerm, req.LastLogTerm) return } if lastIdx > req.LastLogIndex { r.logger.Printf("[WARN] raft: Rejecting vote request from %v since our last index is greater (%d, %d)", candidate, lastIdx, req.LastLogIndex) return } // Persist a vote for safety if err := r.persistVote(req.Term, req.Candidate); err != nil { r.logger.Printf("[ERR] raft: Failed to persist vote: %v", err) return } resp.Granted = true return } // installSnapshot is invoked when we get a InstallSnapshot RPC call. // We must be in the follower state for this, since it means we are // too far behind a leader for log replay. func (r *Raft) installSnapshot(rpc RPC, req *InstallSnapshotRequest) { defer metrics.MeasureSince([]string{"raft", "rpc", "installSnapshot"}, time.Now()) // Setup a response resp := &InstallSnapshotResponse{ Term: r.getCurrentTerm(), Success: false, } var rpcErr error defer func() { rpc.Respond(resp, rpcErr) }() // Ignore an older term if req.Term < r.getCurrentTerm() { return } // Increase the term if we see a newer one if req.Term > r.getCurrentTerm() { // Ensure transition to follower r.setState(Follower) r.setCurrentTerm(req.Term) resp.Term = req.Term } // Save the current leader r.setLeader(r.trans.DecodePeer(req.Leader)) // Create a new snapshot sink, err := r.snapshots.Create(req.LastLogIndex, req.LastLogTerm, req.Peers) if err != nil { r.logger.Printf("[ERR] raft: Failed to create snapshot to install: %v", err) rpcErr = fmt.Errorf("failed to create snapshot: %v", err) return } // Spill the remote snapshot to disk n, err := io.Copy(sink, rpc.Reader) if err != nil { sink.Cancel() r.logger.Printf("[ERR] raft: Failed to copy snapshot: %v", err) rpcErr = err return } // Check that we received it all if n != req.Size { sink.Cancel() r.logger.Printf("[ERR] raft: Failed to receive whole snapshot: %d / %d", n, req.Size) rpcErr = fmt.Errorf("short read") return } // Finalize the snapshot if err := sink.Close(); err != nil { r.logger.Printf("[ERR] raft: Failed to finalize snapshot: %v", err) rpcErr = err return } r.logger.Printf("[INFO] raft: Copied %d bytes to local snapshot", n) // Restore snapshot future := &restoreFuture{ID: sink.ID()} future.init() select { case r.fsmRestoreCh <- future: case <-r.shutdownCh: future.respond(ErrRaftShutdown) return } // Wait for the restore to happen if err := future.Error(); err != nil { r.logger.Printf("[ERR] raft: Failed to restore snapshot: %v", err) rpcErr = err return } // Update the lastApplied so we don't replay old logs r.setLastApplied(req.LastLogIndex) // Update the last stable snapshot info r.setLastSnapshotIndex(req.LastLogIndex) r.setLastSnapshotTerm(req.LastLogTerm) // Restore the peer set peers := decodePeers(req.Peers, r.trans) r.peers = ExcludePeer(peers, r.localAddr) r.peerStore.SetPeers(peers) // Compact logs, continue even if this fails if err := r.compactLogs(req.LastLogIndex); err != nil { r.logger.Printf("[ERR] raft: Failed to compact logs: %v", err) } r.logger.Printf("[INFO] raft: Installed remote snapshot") resp.Success = true r.setLastContact() return } // setLastContact is used to set the last contact time to now func (r *Raft) setLastContact() { r.lastContactLock.Lock() r.lastContact = time.Now() r.lastContactLock.Unlock() } type voteResult struct { RequestVoteResponse voter string } // electSelf is used to send a RequestVote RPC to all peers, // and vote for ourself. This has the side affecting of incrementing // the current term. The response channel returned is used to wait // for all the responses (including a vote for ourself). func (r *Raft) electSelf() <-chan *voteResult { // Create a response channel respCh := make(chan *voteResult, len(r.peers)+1) // Increment the term r.setCurrentTerm(r.getCurrentTerm() + 1) // Construct the request lastIdx, lastTerm := r.getLastEntry() req := &RequestVoteRequest{ Term: r.getCurrentTerm(), Candidate: r.trans.EncodePeer(r.localAddr), LastLogIndex: lastIdx, LastLogTerm: lastTerm, } // Construct a function to ask for a vote askPeer := func(peer string) { r.goFunc(func() { defer metrics.MeasureSince([]string{"raft", "candidate", "electSelf"}, time.Now()) resp := &voteResult{voter: peer} err := r.trans.RequestVote(peer, req, &resp.RequestVoteResponse) if err != nil { r.logger.Printf("[ERR] raft: Failed to make RequestVote RPC to %v: %v", peer, err) resp.Term = req.Term resp.Granted = false } // If we are not a peer, we could have been removed but failed // to receive the log message. OR it could mean an improperly configured // cluster. Either way, we should warn if err == nil { peerSet := decodePeers(resp.Peers, r.trans) if !PeerContained(peerSet, r.localAddr) { r.logger.Printf("[WARN] raft: Remote peer %v does not have local node %v as a peer", peer, r.localAddr) } } respCh <- resp }) } // For each peer, request a vote for _, peer := range r.peers { askPeer(peer) } // Persist a vote for ourselves if err := r.persistVote(req.Term, req.Candidate); err != nil { r.logger.Printf("[ERR] raft: Failed to persist vote : %v", err) return nil } // Include our own vote respCh <- &voteResult{ RequestVoteResponse: RequestVoteResponse{ Term: req.Term, Granted: true, }, voter: r.localAddr, } return respCh } // persistVote is used to persist our vote for safety. func (r *Raft) persistVote(term uint64, candidate []byte) error { if err := r.stable.SetUint64(keyLastVoteTerm, term); err != nil { return err } if err := r.stable.Set(keyLastVoteCand, candidate); err != nil { return err } return nil } // setCurrentTerm is used to set the current term in a durable manner. func (r *Raft) setCurrentTerm(t uint64) { // Persist to disk first if err := r.stable.SetUint64(keyCurrentTerm, t); err != nil { panic(fmt.Errorf("failed to save current term: %v", err)) } r.raftState.setCurrentTerm(t) } // setState is used to update the current state. Any state // transition causes the known leader to be cleared. This means // that leader should be set only after updating the state. func (r *Raft) setState(state RaftState) { r.setLeader("") r.raftState.setState(state) } // runSnapshots is a long running goroutine used to manage taking // new snapshots of the FSM. It runs in parallel to the FSM and // main goroutines, so that snapshots do not block normal operation. func (r *Raft) runSnapshots() { for { select { case <-randomTimeout(r.conf.SnapshotInterval): // Check if we should snapshot if !r.shouldSnapshot() { continue } // Trigger a snapshot if err := r.takeSnapshot(); err != nil { r.logger.Printf("[ERR] raft: Failed to take snapshot: %v", err) } case future := <-r.snapshotCh: // User-triggered, run immediately err := r.takeSnapshot() if err != nil { r.logger.Printf("[ERR] raft: Failed to take snapshot: %v", err) } future.respond(err) case <-r.shutdownCh: return } } } // shouldSnapshot checks if we meet the conditions to take // a new snapshot. func (r *Raft) shouldSnapshot() bool { // Check the last snapshot index lastSnap := r.getLastSnapshotIndex() // Check the last log index lastIdx, err := r.logs.LastIndex() if err != nil { r.logger.Printf("[ERR] raft: Failed to get last log index: %v", err) return false } // Compare the delta to the threshold delta := lastIdx - lastSnap return delta >= r.conf.SnapshotThreshold } // takeSnapshot is used to take a new snapshot. func (r *Raft) takeSnapshot() error { defer metrics.MeasureSince([]string{"raft", "snapshot", "takeSnapshot"}, time.Now()) // Create a snapshot request req := &reqSnapshotFuture{} req.init() // Wait for dispatch or shutdown select { case r.fsmSnapshotCh <- req: case <-r.shutdownCh: return ErrRaftShutdown } // Wait until we get a response if err := req.Error(); err != nil { if err != ErrNothingNewToSnapshot { err = fmt.Errorf("failed to start snapshot: %v", err) } return err } defer req.snapshot.Release() // Log that we are starting the snapshot r.logger.Printf("[INFO] raft: Starting snapshot up to %d", req.index) // Encode the peerset peerSet := encodePeers(req.peers, r.trans) // Create a new snapshot start := time.Now() sink, err := r.snapshots.Create(req.index, req.term, peerSet) if err != nil { return fmt.Errorf("failed to create snapshot: %v", err) } metrics.MeasureSince([]string{"raft", "snapshot", "create"}, start) // Try to persist the snapshot start = time.Now() if err := req.snapshot.Persist(sink); err != nil { sink.Cancel() return fmt.Errorf("failed to persist snapshot: %v", err) } metrics.MeasureSince([]string{"raft", "snapshot", "persist"}, start) // Close and check for error if err := sink.Close(); err != nil { return fmt.Errorf("failed to close snapshot: %v", err) } // Update the last stable snapshot info r.setLastSnapshotIndex(req.index) r.setLastSnapshotTerm(req.term) // Compact the logs if err := r.compactLogs(req.index); err != nil { return err } // Log completion r.logger.Printf("[INFO] raft: Snapshot to %d complete", req.index) return nil } // compactLogs takes the last inclusive index of a snapshot // and trims the logs that are no longer needed. func (r *Raft) compactLogs(snapIdx uint64) error { defer metrics.MeasureSince([]string{"raft", "compactLogs"}, time.Now()) // Determine log ranges to compact minLog, err := r.logs.FirstIndex() if err != nil { return fmt.Errorf("failed to get first log index: %v", err) } // Check if we have enough logs to truncate if r.getLastLogIndex() <= r.conf.TrailingLogs { return nil } // Truncate up to the end of the snapshot, or `TrailingLogs` // back from the head, which ever is further back. This ensures // at least `TrailingLogs` entries, but does not allow logs // after the snapshot to be removed. maxLog := min(snapIdx, r.getLastLogIndex()-r.conf.TrailingLogs) // Log this r.logger.Printf("[INFO] raft: Compacting logs from %d to %d", minLog, maxLog) // Compact the logs if err := r.logs.DeleteRange(minLog, maxLog); err != nil { return fmt.Errorf("log compaction failed: %v", err) } return nil } // restoreSnapshot attempts to restore the latest snapshots, and fails // if none of them can be restored. This is called at initialization time, // and is completely unsafe to call at any other time. func (r *Raft) restoreSnapshot() error { snapshots, err := r.snapshots.List() if err != nil { r.logger.Printf("[ERR] raft: Failed to list snapshots: %v", err) return err } // Try to load in order of newest to oldest for _, snapshot := range snapshots { _, source, err := r.snapshots.Open(snapshot.ID) if err != nil { r.logger.Printf("[ERR] raft: Failed to open snapshot %v: %v", snapshot.ID, err) continue } defer source.Close() if err := r.fsm.Restore(source); err != nil { r.logger.Printf("[ERR] raft: Failed to restore snapshot %v: %v", snapshot.ID, err) continue } // Log success r.logger.Printf("[INFO] raft: Restored from snapshot %v", snapshot.ID) // Update the lastApplied so we don't replay old logs r.setLastApplied(snapshot.Index) // Update the last stable snapshot info r.setLastSnapshotIndex(snapshot.Index) r.setLastSnapshotTerm(snapshot.Term) // Success! return nil } // If we had snapshots and failed to load them, its an error if len(snapshots) > 0 { return fmt.Errorf("failed to load any existing snapshots") } return nil }