Merge pull request #3231 from hashicorp/issue_3230

Fixes deadlock between barrier write and leader notify channel read .…

agent/consul/leader.go

@@ -25,6 +25,7 @@ const (
 	ConsulServiceID                     = "consul"
 	ConsulServiceName                   = "consul"
 	newLeaderEvent                      = "consul:new-leader"
+	barrierWriteTimeout                 = 2 * time.Minute
 )
 
 // monitorLeadership is used to monitor if we acquire or lose our role
@@ -35,13 +36,13 @@ func (s *Server) monitorLeadership() {
 	// leaderCh, which is only notified best-effort. Doing this ensures
 	// that we get all notifications in order, which is required for
 	// cleanup and to ensure we never run multiple leader loops.
-	leaderCh := s.leaderCh
+	raftNotifyCh := s.raftNotifyCh
 
 	var wg sync.WaitGroup
 	var stopCh chan struct{}
 	for {
 		select {
-		case isLeader := <-leaderCh:
+		case isLeader := <-raftNotifyCh:
 			if isLeader {
 				stopCh = make(chan struct{})
 				wg.Add(1)
@@ -96,10 +97,10 @@ RECONCILE:
 
 	// Apply a raft barrier to ensure our FSM is caught up
 	start := time.Now()
-	barrier := s.raft.Barrier(0)
+	barrier := s.raft.Barrier(barrierWriteTimeout)
 	if err := barrier.Error(); err != nil {
 		s.logger.Printf("[ERR] consul: failed to wait for barrier: %v", err)
-		goto WAIT
+		return
 	}
 	metrics.MeasureSince([]string{"consul", "leader", "barrier"}, start)
 

agent/consul/server.go

@@ -134,9 +134,9 @@ type Server struct {
 	raftTransport *raft.NetworkTransport
 	raftInmem     *raft.InmemStore
 
-	// leaderCh set up by setupRaft() and ensures that we get reliable leader
+	// raftNotifyCh is set up by setupRaft() and ensures that we get reliable leader
 	// transition notifications from the Raft layer.
-	leaderCh <-chan bool
+	raftNotifyCh <-chan bool
 
 	// reconcileCh is used to pass events from the serf handler
 	// into the leader manager, so that the strong state can be
@@ -601,9 +601,9 @@ func (s *Server) setupRaft() error {
 	}
 
 	// Set up a channel for reliable leader notifications.
-	leaderCh := make(chan bool, 1)
-	s.config.RaftConfig.NotifyCh = leaderCh
-	s.leaderCh = leaderCh
+	raftNotifyCh := make(chan bool, 1)
+	s.config.RaftConfig.NotifyCh = raftNotifyCh
+	s.raftNotifyCh = raftNotifyCh
 
 	// Setup the Raft store.
 	s.raft, err = raft.NewRaft(s.config.RaftConfig, s.fsm, log, stable, snap, trans)