diff --git a/command/agent/agent.go b/command/agent/agent.go index 788085a50..de6259639 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -803,6 +803,9 @@ func (a *Agent) Reload(newConfig *Config) error { // GetConfigCopy creates a replica of the agent's config, excluding locks func (a *Agent) GetConfig() *Config { + a.configLock.Lock() + defer a.configLock.Unlock() + return a.config } diff --git a/nomad/raft_rpc.go b/nomad/raft_rpc.go index 2d45ae87b..4cfcca400 100644 --- a/nomad/raft_rpc.go +++ b/nomad/raft_rpc.go @@ -113,18 +113,3 @@ func (l *RaftLayer) Dial(address raft.ServerAddress, timeout time.Duration) (net } return conn, err } - -// ReloadTLS will re-initialize the TLS wrapper on the fly -func (l *RaftLayer) ReloadTLS(tlsWrap tlsutil.Wrapper) { - l.closeLock.Lock() - defer l.closeLock.Unlock() - - if !l.closed { - l.closed = true - close(l.closeCh) - } - - l.tlsWrap = tlsWrap - l.closeCh = make(chan struct{}) - l.closed = false -} diff --git a/nomad/server.go b/nomad/server.go index 99730f475..599b441bd 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -99,21 +99,28 @@ type Server struct { leaderCh <-chan bool raft *raft.Raft raftLayer *RaftLayer - raftStore *raftboltdb.BoltStore - raftInmem *raft.InmemStore - raftTransport *raft.NetworkTransport + raftLayerLock sync.Mutex + + raftStore *raftboltdb.BoltStore + raftInmem *raft.InmemStore + + raftTransport *raft.NetworkTransport + raftTransportLock sync.Mutex // fsm is the state machine used with Raft fsm *nomadFSM // rpcListener is used to listen for incoming connections - rpcListener net.Listener + rpcListener net.Listener + rpcListenerLock sync.Mutex + rpcServer *rpc.Server rpcAdvertise net.Addr // rpcTLS is the TLS config for incoming TLS requests - rpcTLS *tls.Config - rpcCancel context.CancelFunc + rpcTLS *tls.Config + rpcCancel context.CancelFunc + rpcTLSLock sync.Mutex // peers is used to track the known Nomad servers. This is // used for region forwarding and clustering. @@ -365,9 +372,8 @@ func (s *Server) ReloadTLSConnections(newTLSConfig *config.TLSConfig) error { s.logger.Printf("[INFO] nomad: reloading server connections due to configuration changes") s.configLock.Lock() - defer s.configLock.Unlock() - s.config.TLSConfig = newTLSConfig + s.configLock.Unlock() var tlsWrap tlsutil.RegionWrapper var incomingTLS *tls.Config @@ -390,16 +396,20 @@ func (s *Server) ReloadTLSConnections(newTLSConfig *config.TLSConfig) error { s.logger.Printf("[ERR] nomad: No TLS Context to reset") return fmt.Errorf("Unable to reset tls context") } - - s.rpcTLS = incomingTLS - s.rpcCancel() + + s.rpcTLSLock.Lock() + s.rpcTLS = incomingTLS + s.rpcTLSLock.Unlock() + + s.raftTransportLock.Lock() + defer s.raftTransportLock.Unlock() s.raftTransport.Close() - s.raftLayer.Close() s.connPool.ReloadTLS(tlsWrap) // reinitialize our rpc listener + s.rpcListenerLock.Lock() s.rpcListener.Close() time.Sleep(500 * time.Millisecond) list, err := net.ListenTCP("tcp", s.config.RPCAddr) @@ -407,15 +417,21 @@ func (s *Server) ReloadTLSConnections(newTLSConfig *config.TLSConfig) error { s.logger.Printf("[ERR] nomad: No TLS listener to reload") return err } + s.rpcListener = list // reinitialize the cancel context ctx, cancel := context.WithCancel(context.Background()) s.rpcCancel = cancel + s.rpcListenerLock.Unlock() + go s.listen(ctx) + s.raftLayerLock.Lock() + s.raftLayer.Close() wrapper := tlsutil.RegionSpecificWrapper(s.config.Region, tlsWrap) - s.raftLayer.ReloadTLS(wrapper) + s.raftLayer = NewRaftLayer(s.rpcAdvertise, wrapper) + s.raftLayerLock.Unlock() // re-initialize the network transport with a re-initialized stream layer trans := raft.NewNetworkTransport(s.raftLayer, 3, s.config.RaftTimeout,