From 2d1ac1f7d0916e1f844e0248499a9cebe5717e85 Mon Sep 17 00:00:00 2001
From: Dhia Ayachi <dhia@hashicorp.com>
Date: Thu, 21 Oct 2021 12:44:31 -0400
Subject: [PATCH] try to perform a leadership transfer when leaving (#11376)

* try to perform a leadership transfer when leaving

* add a changelog
---
 .changelog/11376.txt   |  3 +++
 agent/consul/server.go | 28 ++++++++++++++++++++++++++--
 2 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 .changelog/11376.txt

diff --git a/.changelog/11376.txt b/.changelog/11376.txt
new file mode 100644
index 000000000..f3924d102
--- /dev/null
+++ b/.changelog/11376.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+try to transfer leadership to another server when leaving.
+```
diff --git a/agent/consul/server.go b/agent/consul/server.go
index 00fe5e378..5996bd435 100644
--- a/agent/consul/server.go
+++ b/agent/consul/server.go
@@ -17,6 +17,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/hashicorp/go-version"
+
 	"github.com/armon/go-metrics"
 	connlimit "github.com/hashicorp/go-connlimit"
 	"github.com/hashicorp/go-hclog"
@@ -92,6 +94,8 @@ const (
 	// from Serf with the Catalog. If this is exhausted we will drop updates,
 	// and wait for a periodic reconcile.
 	reconcileChSize = 256
+
+	LeaderTransferMinVersion = "1.6.0"
 )
 
 const (
@@ -992,8 +996,28 @@ func (s *Server) Leave() error {
 	// removed for some reasonable period of time.
 	isLeader := s.IsLeader()
 	if isLeader && numPeers > 1 {
-		if err := s.autopilot.RemoveServer(raft.ServerID(s.config.NodeID)); err != nil {
-			s.logger.Error("failed to remove ourself as a Raft peer", "error", err)
+		leadershipTransferVersion := version.Must(version.NewVersion(LeaderTransferMinVersion))
+		removeServer := false
+		if ok, _ := ServersInDCMeetMinimumVersion(s, s.config.Datacenter, leadershipTransferVersion); !ok {
+			// Transfer leadership to another node then leave the cluster
+			future := s.raft.LeadershipTransfer()
+			if err := future.Error(); err != nil {
+				s.logger.Error("failed to transfer leadership, removing the server", "error", err)
+				// leadership transfer failed, fallback to removing the server from raft
+				removeServer = true
+			} else {
+				// we are not leader anymore, continue the flow to leave as follower
+				isLeader = false
+			}
+		} else {
+			// Leadership transfer is not available in the current version, fallback to removing the server from raft
+			removeServer = true
+		}
+		if removeServer {
+			future := s.raft.RemoveServer(raft.ServerID(s.config.NodeID), 0, 0)
+			if err := future.Error(); err != nil {
+				s.logger.Error("failed to remove ourself as raft peer", "error", err)
+			}
 		}
 	}