agent: enable reloading of tls config (#5419)

This PR introduces reloading tls configuration. Consul will now be able to reload the TLS configuration which previously required a restart. It is not yet possible to turn TLS ON or OFF with these changes. Only when TLS is already turned on, the configuration can be reloaded. Most importantly the certificates and CAs.

agent/agent.go

@@ -393,7 +393,11 @@ func (a *Agent) Start() error {
 	// waiting to discover a consul server
 	consulCfg.ServerUp = a.sync.SyncFull.Trigger
 
-	a.tlsConfigurator = tlsutil.NewConfigurator(c.ToTLSUtilConfig())
+	tlsConfigurator, err := tlsutil.NewConfigurator(c.ToTLSUtilConfig(), a.logger)
+	if err != nil {
+		return err
+	}
+	a.tlsConfigurator = tlsConfigurator
 
 	// Setup either the client or the server.
 	if c.ServerMode {
@@ -662,10 +666,7 @@ func (a *Agent) listenHTTP() ([]*HTTPServer, error) {
 			var tlscfg *tls.Config
 			_, isTCP := l.(*tcpKeepAliveListener)
 			if isTCP && proto == "https" {
-				tlscfg, err = a.tlsConfigurator.IncomingHTTPSConfig()
+				tlscfg = a.tlsConfigurator.IncomingHTTPSConfig()
-				if err != nil {
-					return err
-				}
 				l = tls.NewListener(l, tlscfg)
 			}
 			srv := &HTTPServer{
@@ -2232,11 +2233,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 				chkType.Interval = checks.MinInterval
 			}
 
-			a.tlsConfigurator.AddCheck(string(check.CheckID), chkType.TLSSkipVerify)
+			tlsClientConfig := a.tlsConfigurator.OutgoingTLSConfigForCheck(chkType.TLSSkipVerify)
-			tlsClientConfig, err := a.tlsConfigurator.OutgoingTLSConfigForCheck(string(check.CheckID))
-			if err != nil {
-				return fmt.Errorf("Failed to set up TLS: %v", err)
-			}
 
 			http := &checks.CheckHTTP{
 				Notify:          a.State,
@@ -2287,12 +2284,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 
 			var tlsClientConfig *tls.Config
 			if chkType.GRPCUseTLS {
-				var err error
+				tlsClientConfig = a.tlsConfigurator.OutgoingTLSConfigForCheck(chkType.TLSSkipVerify)
-				a.tlsConfigurator.AddCheck(string(check.CheckID), chkType.TLSSkipVerify)
-				tlsClientConfig, err = a.tlsConfigurator.OutgoingTLSConfigForCheck(string(check.CheckID))
-				if err != nil {
-					return fmt.Errorf("Failed to set up TLS: %v", err)
-				}
 			}
 
 			grpc := &checks.CheckGRPC{
@@ -2431,7 +2423,6 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error {
 		return fmt.Errorf("CheckID missing")
 	}
 
-	a.tlsConfigurator.RemoveCheck(string(checkID))
 	a.cancelCheckMonitors(checkID)
 	a.State.RemoveCheck(checkID)
 
@@ -3559,6 +3550,10 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error {
 	// the checks and service registrations.
 	a.loadTokens(newCfg)
 
+	if err := a.tlsConfigurator.Update(newCfg.ToTLSUtilConfig()); err != nil {
+		return fmt.Errorf("Failed reloading tls configuration: %s", err)
+	}
+
 	// Reload service/check definitions and metadata.
 	if err := a.loadServices(newCfg); err != nil {
 		return fmt.Errorf("Failed reloading services: %s", err)

agent/agent_test.go

@@ -2,6 +2,7 @@ package agent
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -3366,11 +3367,7 @@ func TestAgent_SetupProxyManager(t *testing.T) {
 		ports { http = -1 }
 		data_dir = "` + dataDir + `"
 	`
-	c := TestConfig(
+	a, err := NewUnstartedAgent(t, t.Name(), hcl)
-		// randomPortsSource(false),
-		config.Source{Name: t.Name(), Format: "hcl", Data: hcl},
-	)
-	a, err := New(c)
 	require.NoError(t, err)
 	require.Error(t, a.setupProxyManager(), "setupProxyManager should fail with invalid HTTP API config")
 
@@ -3378,11 +3375,7 @@ func TestAgent_SetupProxyManager(t *testing.T) {
 		ports { http = 8001 }
 		data_dir = "` + dataDir + `"
 	`
-	c = TestConfig(
+	a, err = NewUnstartedAgent(t, t.Name(), hcl)
-		// randomPortsSource(false),
-		config.Source{Name: t.Name(), Format: "hcl", Data: hcl},
-	)
-	a, err = New(c)
 	require.NoError(t, err)
 	require.NoError(t, a.setupProxyManager())
 }
@@ -3543,3 +3536,107 @@ func TestAgent_loadTokens(t *testing.T) {
 		require.Equal("foxtrot", a.tokens.ReplicationToken())
 	})
 }
+
+func TestAgent_ReloadConfigOutgoingRPCConfig(t *testing.T) {
+	t.Parallel()
+	dataDir := testutil.TempDir(t, "agent") // we manage the data dir
+	defer os.RemoveAll(dataDir)
+	hcl := `
+		data_dir = "` + dataDir + `"
+		verify_outgoing = true
+		ca_file = "../test/ca/root.cer"
+		cert_file = "../test/key/ourdomain.cer"
+		key_file = "../test/key/ourdomain.key"
+		verify_server_hostname = false
+	`
+	a, err := NewUnstartedAgent(t, t.Name(), hcl)
+	require.NoError(t, err)
+	tlsConf := a.tlsConfigurator.OutgoingRPCConfig()
+	require.True(t, tlsConf.InsecureSkipVerify)
+	require.Len(t, tlsConf.ClientCAs.Subjects(), 1)
+	require.Len(t, tlsConf.RootCAs.Subjects(), 1)
+
+	hcl = `
+		data_dir = "` + dataDir + `"
+		verify_outgoing = true
+		ca_path = "../test/ca_path"
+		cert_file = "../test/key/ourdomain.cer"
+		key_file = "../test/key/ourdomain.key"
+		verify_server_hostname = true
+	`
+	c := TestConfig(config.Source{Name: t.Name(), Format: "hcl", Data: hcl})
+	require.NoError(t, a.ReloadConfig(c))
+	tlsConf = a.tlsConfigurator.OutgoingRPCConfig()
+	require.False(t, tlsConf.InsecureSkipVerify)
+	require.Len(t, tlsConf.RootCAs.Subjects(), 2)
+	require.Len(t, tlsConf.ClientCAs.Subjects(), 2)
+}
+
+func TestAgent_ReloadConfigIncomingRPCConfig(t *testing.T) {
+	t.Parallel()
+	dataDir := testutil.TempDir(t, "agent") // we manage the data dir
+	defer os.RemoveAll(dataDir)
+	hcl := `
+		data_dir = "` + dataDir + `"
+		verify_outgoing = true
+		ca_file = "../test/ca/root.cer"
+		cert_file = "../test/key/ourdomain.cer"
+		key_file = "../test/key/ourdomain.key"
+		verify_server_hostname = false
+	`
+	a, err := NewUnstartedAgent(t, t.Name(), hcl)
+	require.NoError(t, err)
+	tlsConf := a.tlsConfigurator.IncomingRPCConfig()
+	require.NotNil(t, tlsConf.GetConfigForClient)
+	tlsConf, err = tlsConf.GetConfigForClient(nil)
+	require.NoError(t, err)
+	require.NotNil(t, tlsConf)
+	require.True(t, tlsConf.InsecureSkipVerify)
+	require.Len(t, tlsConf.ClientCAs.Subjects(), 1)
+	require.Len(t, tlsConf.RootCAs.Subjects(), 1)
+
+	hcl = `
+		data_dir = "` + dataDir + `"
+		verify_outgoing = true
+		ca_path = "../test/ca_path"
+		cert_file = "../test/key/ourdomain.cer"
+		key_file = "../test/key/ourdomain.key"
+		verify_server_hostname = true
+	`
+	c := TestConfig(config.Source{Name: t.Name(), Format: "hcl", Data: hcl})
+	require.NoError(t, a.ReloadConfig(c))
+	tlsConf, err = tlsConf.GetConfigForClient(nil)
+	require.NoError(t, err)
+	require.False(t, tlsConf.InsecureSkipVerify)
+	require.Len(t, tlsConf.ClientCAs.Subjects(), 2)
+	require.Len(t, tlsConf.RootCAs.Subjects(), 2)
+}
+
+func TestAgent_ReloadConfigTLSConfigFailure(t *testing.T) {
+	t.Parallel()
+	dataDir := testutil.TempDir(t, "agent") // we manage the data dir
+	defer os.RemoveAll(dataDir)
+	hcl := `
+		data_dir = "` + dataDir + `"
+		verify_outgoing = true
+		ca_file = "../test/ca/root.cer"
+		cert_file = "../test/key/ourdomain.cer"
+		key_file = "../test/key/ourdomain.key"
+		verify_server_hostname = false
+	`
+	a, err := NewUnstartedAgent(t, t.Name(), hcl)
+	require.NoError(t, err)
+	tlsConf := a.tlsConfigurator.IncomingRPCConfig()
+
+	hcl = `
+		data_dir = "` + dataDir + `"
+		verify_incoming = true
+	`
+	c := TestConfig(config.Source{Name: t.Name(), Format: "hcl", Data: hcl})
+	require.Error(t, a.ReloadConfig(c))
+	tlsConf, err = tlsConf.GetConfigForClient(nil)
+	require.NoError(t, err)
+	require.Equal(t, tls.NoClientCert, tlsConf.ClientAuth)
+	require.Len(t, tlsConf.ClientCAs.Subjects(), 1)
+	require.Len(t, tlsConf.RootCAs.Subjects(), 1)
+}

agent/config/runtime.go

@@ -1580,12 +1580,13 @@ func (c *RuntimeConfig) Sanitized() map[string]interface{} {
 	return sanitize("rt", reflect.ValueOf(c)).Interface().(map[string]interface{})
 }
 
-func (c *RuntimeConfig) ToTLSUtilConfig() *tlsutil.Config {
+func (c *RuntimeConfig) ToTLSUtilConfig() tlsutil.Config {
-	return &tlsutil.Config{
+	return tlsutil.Config{
 		VerifyIncoming:           c.VerifyIncoming,
 		VerifyIncomingRPC:        c.VerifyIncomingRPC,
 		VerifyIncomingHTTPS:      c.VerifyIncomingHTTPS,
 		VerifyOutgoing:           c.VerifyOutgoing,
+		VerifyServerHostname:     c.VerifyServerHostname,
 		CAFile:                   c.CAFile,
 		CAPath:                   c.CAPath,
 		CertFile:                 c.CertFile,

agent/config/runtime_test.go

@@ -5434,6 +5434,7 @@ func TestRuntime_ToTLSUtilConfig(t *testing.T) {
 		VerifyIncomingRPC:           true,
 		VerifyIncomingHTTPS:         true,
 		VerifyOutgoing:              true,
+		VerifyServerHostname:        true,
 		CAFile:                      "a",
 		CAPath:                      "b",
 		CertFile:                    "c",
@@ -5450,6 +5451,7 @@ func TestRuntime_ToTLSUtilConfig(t *testing.T) {
 	require.Equal(t, c.VerifyIncomingRPC, r.VerifyIncomingRPC)
 	require.Equal(t, c.VerifyIncomingHTTPS, r.VerifyIncomingHTTPS)
 	require.Equal(t, c.VerifyOutgoing, r.VerifyOutgoing)
+	require.Equal(t, c.VerifyServerHostname, r.VerifyServerHostname)
 	require.Equal(t, c.CAFile, r.CAFile)
 	require.Equal(t, c.CAPath, r.CAPath)
 	require.Equal(t, c.CertFile, r.CertFile)

agent/consul/client.go

@@ -86,10 +86,16 @@ type Client struct {
 	EnterpriseClient
 }
 
-// NewClient is used to construct a new Consul client from the
+// NewClient is used to construct a new Consul client from the configuration,
-// configuration, potentially returning an error
+// potentially returning an error.
+// NewClient only used to help setting up a client for testing. Normal code
+// exercises NewClientLogger.
 func NewClient(config *Config) (*Client, error) {
-	return NewClientLogger(config, nil, tlsutil.NewConfigurator(config.ToTLSUtilConfig()))
+	c, err := tlsutil.NewConfigurator(config.ToTLSUtilConfig(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return NewClientLogger(config, nil, c)
 }
 
 func NewClientLogger(config *Config, logger *log.Logger, tlsConfigurator *tlsutil.Configurator) (*Client, error) {
@@ -113,12 +119,6 @@ func NewClientLogger(config *Config, logger *log.Logger, tlsConfigurator *tlsuti
 		config.LogOutput = os.Stderr
 	}
 
-	// Create the tls Wrapper
-	tlsWrap, err := tlsConfigurator.OutgoingRPCWrapper()
-	if err != nil {
-		return nil, err
-	}
-
 	// Create a logger
 	if logger == nil {
 		logger = log.New(config.LogOutput, "", log.LstdFlags)
@@ -129,7 +129,7 @@ func NewClientLogger(config *Config, logger *log.Logger, tlsConfigurator *tlsuti
 		LogOutput:  config.LogOutput,
 		MaxTime:    clientRPCConnMaxIdle,
 		MaxStreams: clientMaxStreams,
-		TLSWrapper: tlsWrap,
+		TLSWrapper: tlsConfigurator.OutgoingRPCWrapper(),
 		ForceTLS:   config.VerifyOutgoing,
 	}
 
@@ -158,6 +158,7 @@ func NewClientLogger(config *Config, logger *log.Logger, tlsConfigurator *tlsuti
 		CacheConfig: clientACLCacheConfig,
 		Sentinel:    nil,
 	}
+	var err error
 	if c.acls, err = NewACLResolver(&aclConfig); err != nil {
 		c.Shutdown()
 		return nil, fmt.Errorf("Failed to create ACL resolver: %v", err)

agent/consul/config.go

@@ -379,8 +379,8 @@ type Config struct {
 	CAConfig *structs.CAConfiguration
 }
 
-func (c *Config) ToTLSUtilConfig() *tlsutil.Config {
+func (c *Config) ToTLSUtilConfig() tlsutil.Config {
-	return &tlsutil.Config{
+	return tlsutil.Config{
 		VerifyIncoming:           c.VerifyIncoming,
 		VerifyOutgoing:           c.VerifyOutgoing,
 		CAFile:                   c.CAFile,

agent/consul/server.go

@@ -252,11 +252,17 @@ type Server struct {
 	EnterpriseServer
 }
 
+// NewServer is only used to help setting up a server for testing. Normal code
+// exercises NewServerLogger.
 func NewServer(config *Config) (*Server, error) {
-	return NewServerLogger(config, nil, new(token.Store), tlsutil.NewConfigurator(config.ToTLSUtilConfig()))
+	c, err := tlsutil.NewConfigurator(config.ToTLSUtilConfig(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return NewServerLogger(config, nil, new(token.Store), c)
 }
 
-// NewServer is used to construct a new Consul server from the
+// NewServerLogger is used to construct a new Consul server from the
 // configuration, potentially returning an error
 func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store, tlsConfigurator *tlsutil.Configurator) (*Server, error) {
 	// Check the protocol version.
@@ -296,18 +302,6 @@ func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store, tl
 		}
 	}
 
-	// Create the TLS wrapper for outgoing connections.
-	tlsWrap, err := tlsConfigurator.OutgoingRPCWrapper()
-	if err != nil {
-		return nil, err
-	}
-
-	// Get the incoming TLS config.
-	incomingTLS, err := tlsConfigurator.IncomingRPCConfig()
-	if err != nil {
-		return nil, err
-	}
-
 	// Create the tombstone GC.
 	gc, err := state.NewTombstoneGC(config.TombstoneTTL, config.TombstoneTTLGranularity)
 	if err != nil {
@@ -322,7 +316,7 @@ func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store, tl
 		LogOutput:  config.LogOutput,
 		MaxTime:    serverRPCCache,
 		MaxStreams: serverMaxStreams,
-		TLSWrapper: tlsWrap,
+		TLSWrapper: tlsConfigurator.OutgoingRPCWrapper(),
 		ForceTLS:   config.VerifyOutgoing,
 	}
 
@@ -338,7 +332,7 @@ func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store, tl
 		reconcileCh:      make(chan serf.Member, reconcileChSize),
 		router:           router.NewRouter(logger, config.Datacenter),
 		rpcServer:        rpc.NewServer(),
-		rpcTLS:           incomingTLS,
+		rpcTLS:           tlsConfigurator.IncomingRPCConfig(),
 		reassertLeaderCh: make(chan chan error),
 		segmentLAN:       make(map[string]*serf.Serf, len(config.Segments)),
 		sessionTimers:    NewSessionTimers(),
@@ -373,7 +367,7 @@ func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store, tl
 	}
 
 	// Initialize the RPC layer.
-	if err := s.setupRPC(tlsWrap); err != nil {
+	if err := s.setupRPC(tlsConfigurator.OutgoingRPCWrapper()); err != nil {
 		s.Shutdown()
 		return nil, fmt.Errorf("Failed to start RPC layer: %v", err)
 	}

agent/consul/server_test.go

@@ -179,7 +179,11 @@ func newServer(c *Config) (*Server, error) {
 		w = os.Stderr
 	}
 	logger := log.New(w, c.NodeName+" - ", log.LstdFlags|log.Lmicroseconds)
-	srv, err := NewServerLogger(c, logger, new(token.Store), tlsutil.NewConfigurator(c.ToTLSUtilConfig()))
+	tlsConf, err := tlsutil.NewConfigurator(c.ToTLSUtilConfig(), logger)
+	if err != nil {
+		return nil, err
+	}
+	srv, err := NewServerLogger(c, logger, new(token.Store), tlsConf)
 	if err != nil {
 		return nil, err
 	}

agent/testagent.go

@@ -18,9 +18,11 @@ import (
 	metrics "github.com/armon/go-metrics"
 	uuid "github.com/hashicorp/go-uuid"
 
+	"github.com/hashicorp/consul/agent/ae"
 	"github.com/hashicorp/consul/agent/config"
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/consul"
+	"github.com/hashicorp/consul/agent/local"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/lib/freeport"
@@ -103,6 +105,24 @@ func NewTestAgent(t *testing.T, name string, hcl string) *TestAgent {
 	return a
 }
 
+func NewUnstartedAgent(t *testing.T, name string, hcl string) (*Agent, error) {
+	c := TestConfig(config.Source{Name: name, Format: "hcl", Data: hcl})
+	a, err := New(c)
+	if err != nil {
+		return nil, err
+	}
+	a.State = local.NewState(LocalConfig(c), a.logger, a.tokens)
+	a.sync = ae.NewStateSyncer(a.State, c.AEInterval, a.shutdownCh, a.logger)
+	a.delegate = &consul.Client{}
+	a.State.TriggerSyncChanges = a.sync.SyncChanges.Trigger
+	tlsConfigurator, err := tlsutil.NewConfigurator(c.ToTLSUtilConfig(), nil)
+	if err != nil {
+		return nil, err
+	}
+	a.tlsConfigurator = tlsConfigurator
+	return a, nil
+}
+
 // Start starts a test agent. It fails the test if the agent could not be started.
 func (a *TestAgent) Start(t *testing.T) *TestAgent {
 	require := require.New(t)
@@ -149,7 +169,9 @@ func (a *TestAgent) Start(t *testing.T) *TestAgent {
 		agent.LogWriter = a.LogWriter
 		agent.logger = log.New(logOutput, a.Name+" - ", log.LstdFlags|log.Lmicroseconds)
 		agent.MemSink = metrics.NewInmemSink(1*time.Second, time.Minute)
-		agent.tlsConfigurator = tlsutil.NewConfigurator(a.Config.ToTLSUtilConfig())
+		tlsConfigurator, err := tlsutil.NewConfigurator(a.Config.ToTLSUtilConfig(), nil)
+		require.NoError(err)
+		agent.tlsConfigurator = tlsConfigurator
 
 		// we need the err var in the next exit condition
 		if err := agent.Start(); err == nil {

tlsutil/config.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"log"
 	"net"
 	"strings"
 	"sync"
@@ -23,6 +24,7 @@ type Wrapper func(conn net.Conn) (net.Conn, error)
 
 // TLSLookup maps the tls_min_version configuration to the internal value
 var TLSLookup = map[string]uint16{
+	"":      tls.VersionTLS10, // default in golang
 	"tls10": tls.VersionTLS10,
 	"tls11": tls.VersionTLS11,
 	"tls12": tls.VersionTLS12,
@@ -114,14 +116,7 @@ type Config struct {
 
 // KeyPair is used to open and parse a certificate and key file
 func (c *Config) KeyPair() (*tls.Certificate, error) {
-	if c.CertFile == "" || c.KeyFile == "" {
+	return loadKeyPair(c.CertFile, c.KeyFile)
-		return nil, nil
-	}
-	cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to load cert/key pair: %v", err)
-	}
-	return &cert, err
 }
 
 // SpecificDC is used to invoke a static datacenter
@@ -135,6 +130,268 @@ func SpecificDC(dc string, tlsWrap DCWrapper) Wrapper {
 	}
 }
 
+// Configurator holds a Config and is responsible for generating all the
+// *tls.Config necessary for Consul. Except the one in the api package.
+type Configurator struct {
+	sync.RWMutex
+	base    *Config
+	cert    *tls.Certificate
+	cas     *x509.CertPool
+	logger  *log.Logger
+	version int
+}
+
+// NewConfigurator creates a new Configurator and sets the provided
+// configuration.
+func NewConfigurator(config Config, logger *log.Logger) (*Configurator, error) {
+	c := &Configurator{logger: logger}
+	err := c.Update(config)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// Update updates the internal configuration which is used to generate
+// *tls.Config.
+// This function acquires a write lock because it writes the new config.
+func (c *Configurator) Update(config Config) error {
+	cert, err := loadKeyPair(config.CertFile, config.KeyFile)
+	if err != nil {
+		return err
+	}
+	cas, err := loadCAs(config.CAFile, config.CAPath)
+	if err != nil {
+		return err
+	}
+
+	if err = c.check(config, cas, cert); err != nil {
+		return err
+	}
+	c.Lock()
+	c.base = &config
+	c.cert = cert
+	c.cas = cas
+	c.version++
+	c.Unlock()
+	c.log("Update")
+	return nil
+}
+
+func (c *Configurator) check(config Config, cas *x509.CertPool, cert *tls.Certificate) error {
+	// Check if a minimum TLS version was set
+	if config.TLSMinVersion != "" {
+		if _, ok := TLSLookup[config.TLSMinVersion]; !ok {
+			return fmt.Errorf("TLSMinVersion: value %s not supported, please specify one of [tls10,tls11,tls12]", config.TLSMinVersion)
+		}
+	}
+
+	// Ensure we have a CA if VerifyOutgoing is set
+	if config.VerifyOutgoing && cas == nil {
+		return fmt.Errorf("VerifyOutgoing set, and no CA certificate provided!")
+	}
+
+	// Ensure we have a CA and cert if VerifyIncoming is set
+	if config.VerifyIncoming || config.VerifyIncomingRPC || config.VerifyIncomingHTTPS {
+		if cas == nil {
+			return fmt.Errorf("VerifyIncoming set, and no CA certificate provided!")
+		}
+		if cert == nil {
+			return fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!")
+		}
+	}
+	return nil
+}
+
+func loadKeyPair(certFile, keyFile string) (*tls.Certificate, error) {
+	if certFile == "" || keyFile == "" {
+		return nil, nil
+	}
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to load cert/key pair: %v", err)
+	}
+	return &cert, nil
+}
+
+func loadCAs(caFile, caPath string) (*x509.CertPool, error) {
+	if caFile != "" {
+		return rootcerts.LoadCAFile(caFile)
+	} else if caPath != "" {
+		pool, err := rootcerts.LoadCAPath(caPath)
+		if err != nil {
+			return nil, err
+		}
+		// make sure to not return an empty pool because this is not
+		// the users intention when providing a path for CAs.
+		if len(pool.Subjects()) == 0 {
+			return nil, fmt.Errorf("Error loading CA: path %q has no CAs", caPath)
+		}
+		return pool, nil
+	}
+	return nil, nil
+}
+
+// commonTLSConfig generates a *tls.Config from the base configuration the
+// Configurator has. It accepts an additional flag in case a config is needed
+// for incoming TLS connections.
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) commonTLSConfig(additionalVerifyIncomingFlag bool) *tls.Config {
+	c.RLock()
+	defer c.RUnlock()
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: !c.base.VerifyServerHostname,
+	}
+
+	// Set the cipher suites
+	if len(c.base.CipherSuites) != 0 {
+		tlsConfig.CipherSuites = c.base.CipherSuites
+	}
+
+	tlsConfig.PreferServerCipherSuites = c.base.PreferServerCipherSuites
+
+	tlsConfig.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+		return c.cert, nil
+	}
+	tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+		return c.cert, nil
+	}
+
+	tlsConfig.ClientCAs = c.cas
+	tlsConfig.RootCAs = c.cas
+
+	// This is possible because TLSLookup also contains "" with golang's
+	// default (tls10). And because the initial check makes sure the
+	// version correctly matches.
+	tlsConfig.MinVersion = TLSLookup[c.base.TLSMinVersion]
+
+	// Set ClientAuth if necessary
+	if c.base.VerifyIncoming || additionalVerifyIncomingFlag {
+		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	return tlsConfig
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) outgoingRPCTLSDisabled() bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.cas == nil && !c.base.VerifyOutgoing
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) someValuesFromConfig() (bool, bool, string) {
+	c.RLock()
+	defer c.RUnlock()
+	return c.base.VerifyServerHostname, c.base.VerifyOutgoing, c.base.Domain
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) verifyIncomingRPC() bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.base.VerifyIncomingRPC
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) verifyIncomingHTTPS() bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.base.VerifyIncomingHTTPS
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) enableAgentTLSForChecks() bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.base.EnableAgentTLSForChecks
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) serverNameOrNodeName() string {
+	c.RLock()
+	defer c.RUnlock()
+	if c.base.ServerName != "" {
+		return c.base.ServerName
+	}
+	return c.base.NodeName
+}
+
+// IncomingRPCConfig generates a *tls.Config for incoming RPC connections.
+func (c *Configurator) IncomingRPCConfig() *tls.Config {
+	c.log("IncomingRPCConfig")
+	config := c.commonTLSConfig(c.verifyIncomingRPC())
+	config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) {
+		return c.IncomingRPCConfig(), nil
+	}
+	return config
+}
+
+// IncomingHTTPSConfig generates a *tls.Config for incoming HTTPS connections.
+func (c *Configurator) IncomingHTTPSConfig() *tls.Config {
+	c.log("IncomingHTTPSConfig")
+	config := c.commonTLSConfig(c.verifyIncomingHTTPS())
+	config.NextProtos = []string{"h2", "http/1.1"}
+	config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) {
+		return c.IncomingHTTPSConfig(), nil
+	}
+	return config
+}
+
+// IncomingTLSConfig generates a *tls.Config for outgoing TLS connections for
+// checks. This function is separated because there is an extra flag to
+// consider for checks. EnableAgentTLSForChecks and InsecureSkipVerify has to
+// be checked for checks.
+func (c *Configurator) OutgoingTLSConfigForCheck(skipVerify bool) *tls.Config {
+	c.log("OutgoingTLSConfigForCheck")
+	if !c.enableAgentTLSForChecks() {
+		return &tls.Config{
+			InsecureSkipVerify: skipVerify,
+		}
+	}
+
+	config := c.commonTLSConfig(false)
+	config.InsecureSkipVerify = skipVerify
+	config.ServerName = c.serverNameOrNodeName()
+
+	return config
+}
+
+// OutgoingRPCConfig generates a *tls.Config for outgoing RPC connections. If
+// there is a CA or VerifyOutgoing is set, a *tls.Config will be provided,
+// otherwise we assume that no TLS should be used.
+func (c *Configurator) OutgoingRPCConfig() *tls.Config {
+	c.log("OutgoingRPCConfig")
+	if c.outgoingRPCTLSDisabled() {
+		return nil
+	}
+	return c.commonTLSConfig(false)
+}
+
+// OutgoingRPCWrapper wraps the result of OutgoingRPCConfig in a DCWrapper. It
+// decides if verify server hostname should be used.
+func (c *Configurator) OutgoingRPCWrapper() DCWrapper {
+	c.log("OutgoingRPCWrapper")
+	if c.outgoingRPCTLSDisabled() {
+		return nil
+	}
+
+	// Generate the wrapper based on dc
+	return func(dc string, conn net.Conn) (net.Conn, error) {
+		return c.wrapTLSClient(dc, conn)
+	}
+}
+
+// This function acquires a read lock because it reads from the config.
+func (c *Configurator) log(name string) {
+	if c.logger != nil {
+		c.RLock()
+		defer c.RUnlock()
+		c.logger.Printf("[DEBUG] tlsutil: %s with version %d", name, c.version)
+	}
+}
+
 // Wrap a net.Conn into a client tls connection, performing any
 // additional verification as needed.
 //
@@ -145,20 +402,28 @@ func SpecificDC(dc string, tlsWrap DCWrapper) Wrapper {
 // node names, we don't verify the certificate DNS names. Since go 1.3
 // no longer supports this mode of operation, we have to do it
 // manually.
-func (c *Config) wrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) {
+func (c *Configurator) wrapTLSClient(dc string, conn net.Conn) (net.Conn, error) {
 	var err error
 	var tlsConn *tls.Conn
 
-	tlsConn = tls.Client(conn, tlsConfig)
+	config := c.OutgoingRPCConfig()
+	verifyServerHostname, verifyOutgoing, domain := c.someValuesFromConfig()
+
+	if verifyServerHostname {
+		// Strip the trailing '.' from the domain if any
+		domain = strings.TrimSuffix(domain, ".")
+		config.ServerName = "server." + dc + "." + domain
+	}
+	tlsConn = tls.Client(conn, config)
 
 	// If crypto/tls is doing verification, there's no need to do
 	// our own.
-	if tlsConfig.InsecureSkipVerify == false {
+	if !config.InsecureSkipVerify {
 		return tlsConn, nil
 	}
 
 	// If verification is not turned on, don't do it.
-	if !c.VerifyOutgoing {
+	if !verifyOutgoing {
 		return tlsConn, nil
 	}
 
@@ -170,7 +435,7 @@ func (c *Config) wrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn,
 	// The following is lightly-modified from the doFullHandshake
 	// method in crypto/tls's handshake_client.go.
 	opts := x509.VerifyOptions{
-		Roots:         tlsConfig.RootCAs,
+		Roots:         config.RootCAs,
 		CurrentTime:   time.Now(),
 		DNSName:       "",
 		Intermediates: x509.NewCertPool(),
@@ -193,198 +458,6 @@ func (c *Config) wrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn,
 	return tlsConn, err
 }
 
-// Configurator holds a Config and is responsible for generating all the
-// *tls.Config necessary for Consul. Except the one in the api package.
-type Configurator struct {
-	sync.Mutex
-	base   *Config
-	checks map[string]bool
-}
-
-// NewConfigurator creates a new Configurator and sets the provided
-// configuration.
-// Todo (Hans): should config be a value instead a pointer to avoid side
-// effects?
-func NewConfigurator(config *Config) *Configurator {
-	return &Configurator{base: config, checks: map[string]bool{}}
-}
-
-// Update updates the internal configuration which is used to generate
-// *tls.Config.
-func (c *Configurator) Update(config *Config) {
-	c.Lock()
-	defer c.Unlock()
-	c.base = config
-}
-
-// commonTLSConfig generates a *tls.Config from the base configuration the
-// Configurator has. It accepts an additional flag in case a config is needed
-// for incoming TLS connections.
-func (c *Configurator) commonTLSConfig(additionalVerifyIncomingFlag bool) (*tls.Config, error) {
-	if c.base == nil {
-		return nil, fmt.Errorf("No base config")
-	}
-
-	tlsConfig := &tls.Config{
-		InsecureSkipVerify: !c.base.VerifyServerHostname,
-	}
-
-	// Set the cipher suites
-	if len(c.base.CipherSuites) != 0 {
-		tlsConfig.CipherSuites = c.base.CipherSuites
-	}
-	if c.base.PreferServerCipherSuites {
-		tlsConfig.PreferServerCipherSuites = true
-	}
-
-	// Add cert/key
-	cert, err := c.base.KeyPair()
-	if err != nil {
-		return nil, err
-	} else if cert != nil {
-		tlsConfig.Certificates = []tls.Certificate{*cert}
-	}
-
-	// Check if a minimum TLS version was set
-	if c.base.TLSMinVersion != "" {
-		tlsvers, ok := TLSLookup[c.base.TLSMinVersion]
-		if !ok {
-			return nil, fmt.Errorf("TLSMinVersion: value %s not supported, please specify one of [tls10,tls11,tls12]", c.base.TLSMinVersion)
-		}
-		tlsConfig.MinVersion = tlsvers
-	}
-
-	// Ensure we have a CA if VerifyOutgoing is set
-	if c.base.VerifyOutgoing && c.base.CAFile == "" && c.base.CAPath == "" {
-		return nil, fmt.Errorf("VerifyOutgoing set, and no CA certificate provided!")
-	}
-
-	// Parse the CA certs if any
-	if c.base.CAFile != "" {
-		pool, err := rootcerts.LoadCAFile(c.base.CAFile)
-		if err != nil {
-			return nil, err
-		}
-		tlsConfig.ClientCAs = pool
-		tlsConfig.RootCAs = pool
-	} else if c.base.CAPath != "" {
-		pool, err := rootcerts.LoadCAPath(c.base.CAPath)
-		if err != nil {
-			return nil, err
-		}
-		tlsConfig.ClientCAs = pool
-		tlsConfig.RootCAs = pool
-	}
-
-	// Set ClientAuth if necessary
-	if c.base.VerifyIncoming || additionalVerifyIncomingFlag {
-		if c.base.CAFile == "" && c.base.CAPath == "" {
-			return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!")
-		}
-		if len(tlsConfig.Certificates) == 0 {
-			return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!")
-		}
-
-		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
-	}
-
-	return tlsConfig, nil
-}
-
-// IncomingRPCConfig generates a *tls.Config for incoming RPC connections.
-func (c *Configurator) IncomingRPCConfig() (*tls.Config, error) {
-	return c.commonTLSConfig(c.base.VerifyIncomingRPC)
-}
-
-// IncomingHTTPSConfig generates a *tls.Config for incoming HTTPS connections.
-func (c *Configurator) IncomingHTTPSConfig() (*tls.Config, error) {
-	return c.commonTLSConfig(c.base.VerifyIncomingHTTPS)
-}
-
-// IncomingTLSConfig generates a *tls.Config for outgoing TLS connections for
-// checks. This function is separated because there is an extra flag to
-// consider for checks. EnableAgentTLSForChecks and InsecureSkipVerify has to
-// be checked for checks.
-func (c *Configurator) OutgoingTLSConfigForCheck(id string) (*tls.Config, error) {
-	if !c.base.EnableAgentTLSForChecks {
-		return &tls.Config{
-			InsecureSkipVerify: c.getSkipVerifyForCheck(id),
-		}, nil
-	}
-
-	tlsConfig, err := c.commonTLSConfig(false)
-	if err != nil {
-		return nil, err
-	}
-	tlsConfig.InsecureSkipVerify = c.getSkipVerifyForCheck(id)
-	tlsConfig.ServerName = c.base.ServerName
-	if tlsConfig.ServerName == "" {
-		tlsConfig.ServerName = c.base.NodeName
-	}
-
-	return tlsConfig, nil
-}
-
-// OutgoingRPCConfig generates a *tls.Config for outgoing RPC connections. If
-// there is a CA or VerifyOutgoing is set, a *tls.Config will be provided,
-// otherwise we assume that no TLS should be used.
-func (c *Configurator) OutgoingRPCConfig() (*tls.Config, error) {
-	useTLS := c.base.CAFile != "" || c.base.CAPath != "" || c.base.VerifyOutgoing
-	if !useTLS {
-		return nil, nil
-	}
-	return c.commonTLSConfig(false)
-}
-
-// OutgoingRPCWrapper wraps the result of OutgoingRPCConfig in a DCWrapper. It
-// decides if verify server hostname should be used.
-func (c *Configurator) OutgoingRPCWrapper() (DCWrapper, error) {
-	// Get the TLS config
-	tlsConfig, err := c.OutgoingRPCConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	// Check if TLS is not enabled
-	if tlsConfig == nil {
-		return nil, nil
-	}
-
-	// Generate the wrapper based on hostname verification
-	wrapper := func(dc string, conn net.Conn) (net.Conn, error) {
-		if c.base.VerifyServerHostname {
-			// Strip the trailing '.' from the domain if any
-			domain := strings.TrimSuffix(c.base.Domain, ".")
-			tlsConfig = tlsConfig.Clone()
-			tlsConfig.ServerName = "server." + dc + "." + domain
-		}
-		return c.base.wrapTLSClient(conn, tlsConfig)
-	}
-
-	return wrapper, nil
-}
-
-// AddCheck adds a check to the internal check map together with the skipVerify
-// value, which is used when generating a *tls.Config for this check.
-func (c *Configurator) AddCheck(id string, skipVerify bool) {
-	c.Lock()
-	defer c.Unlock()
-	c.checks[id] = skipVerify
-}
-
-// RemoveCheck removes a check from the internal check map.
-func (c *Configurator) RemoveCheck(id string) {
-	c.Lock()
-	defer c.Unlock()
-	delete(c.checks, id)
-}
-
-func (c *Configurator) getSkipVerifyForCheck(id string) bool {
-	c.Lock()
-	defer c.Unlock()
-	return c.checks[id]
-}
-
 // ParseCiphers parse ciphersuites from the comma-separated string into
 // recognized slice
 func ParseCiphers(cipherStr string) ([]uint16, error) {

website/source/docs/agent/options.html.md

@@ -1731,6 +1731,8 @@ items which are reloaded include:
 * Services
 * Watches
 * HTTP Client Address
+* TLS Configuration
+  * Please be aware that this is currently limited to reload a configuration that is already TLS enabled. You cannot enable or disable TLS only with reloading.
 * <a href="#node_meta">Node Metadata</a>
 * <a href="#telemetry-prefix_filter">Metric Prefix Filter</a>
 * <a href="#discard_check_output">Discard Check Output</a>