Merge pull request #3908 from hashicorp/f-improved-autopilot-shutdown

Improve autopilot shutdown to be idempotent

agent/consul/autopilot/autopilot.go

@@ -38,8 +38,10 @@ type Autopilot struct {
 	clusterHealth     OperatorHealthReply
 	clusterHealthLock sync.RWMutex
 
+	enabled      bool
 	removeDeadCh chan struct{}
 	shutdownCh   chan struct{}
+	shutdownLock sync.Mutex
 	waitGroup    sync.WaitGroup
 }
 
@@ -62,6 +64,14 @@ func NewAutopilot(logger *log.Logger, delegate Delegate, interval, healthInterva
 }
 
 func (a *Autopilot) Start() {
+	a.shutdownLock.Lock()
+	defer a.shutdownLock.Unlock()
+
+	// Nothing to do
+	if a.enabled {
+		return
+	}
+
 	a.shutdownCh = make(chan struct{})
 	a.waitGroup = sync.WaitGroup{}
 	a.clusterHealth = OperatorHealthReply{}
@@ -69,11 +79,21 @@ func (a *Autopilot) Start() {
 	a.waitGroup.Add(2)
 	go a.run()
 	go a.serverHealthLoop()
+	a.enabled = true
 }
 
 func (a *Autopilot) Stop() {
+	a.shutdownLock.Lock()
+	defer a.shutdownLock.Unlock()
+
+	// Nothing to do
+	if !a.enabled {
+		return
+	}
+
 	close(a.shutdownCh)
 	a.waitGroup.Wait()
+	a.enabled = false
 }
 
 // run periodically looks for nonvoting servers to promote and dead servers to remove.

agent/consul/autopilot_test.go

@@ -11,6 +11,20 @@ import (
 	"github.com/hashicorp/serf/serf"
 )
 
+func TestAutopilot_IdempotentShutdown(t *testing.T) {
+	dir1, s1 := testServerWithConfig(t, nil)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	retry.Run(t, func(r *retry.R) { r.Check(waitForLeader(s1)) })
+
+	s1.autopilot.Start()
+	s1.autopilot.Start()
+	s1.autopilot.Start()
+	s1.autopilot.Stop()
+	s1.autopilot.Stop()
+	s1.autopilot.Stop()
+}
+
 func TestAutopilot_CleanupDeadServer(t *testing.T) {
 	t.Parallel()
 	for i := 1; i <= 3; i++ {