VAULT-9883: Agent Reloadable Config (#18638)

* Update command/agent.go
* Attempt to only reload log level and certs
* Mimicked 'server' test for cert reload in 'agent'

Co-authored-by: Nick Cabatoff <ncabatoff@hashicorp.com>

Left out the `c.config` tweak that meant changes to lots of lines of code within the `Run` function of Agent command. :)

changelog/18638.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: allows some parts of config to be reloaded without requiring a restart.
+```

command/agent.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	ctconfig "github.com/hashicorp/consul-template/config"
+	"github.com/hashicorp/go-multierror"
 
 	"github.com/hashicorp/vault/command/agent/sink/inmem"
 
@@ -24,6 +25,7 @@ import (
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/gatedwriter"
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
+	"github.com/hashicorp/go-secure-stdlib/reloadutil"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/command/agent/auth"
 	"github.com/hashicorp/vault/command/agent/auth/alicloud"
@@ -75,9 +77,14 @@ type AgentCommand struct {
 	*BaseCommand
 	logFlags logFlags
 
+	config *agentConfig.Config
+
 	ShutdownCh chan struct{}
 	SighupCh   chan struct{}
 
+	tlsReloadFuncsLock sync.RWMutex
+	tlsReloadFuncs     []reloadutil.ReloadFunc
+
 	logWriter io.Writer
 	logGate   *gatedwriter.Writer
 	logger    log.Logger
@@ -87,7 +94,8 @@ type AgentCommand struct {
 
 	cleanupGuard sync.Once
 
-	startedCh chan (struct{}) // for tests
+	startedCh  chan struct{} // for tests
+	reloadedCh chan struct{} // for tests
 
 	flagConfigs        []string
 	flagExitAfterAuth  bool
@@ -102,7 +110,7 @@ func (c *AgentCommand) Help() string {
 	helpText := `
 Usage: vault agent [options]
 
-  This command starts a Vault agent that can perform automatic authentication
+  This command starts a Vault Agent that can perform automatic authentication
   in certain environments.
 
   Start an agent with a configuration file:
@@ -193,76 +201,24 @@ func (c *AgentCommand) Run(args []string) int {
 		return 1
 	}
 
-	config := agentConfig.NewConfig()
+	config, err := c.loadConfig(c.flagConfigs)
-
-	for _, configPath := range c.flagConfigs {
-		configFromPath, err := agentConfig.LoadConfig(configPath)
-		if err != nil {
-			c.UI.Error(fmt.Sprintf("Error loading configuration from %s: %s", configPath, err))
-			return 1
-		}
-		config = config.Merge(configFromPath)
-	}
-
-	err := config.ValidateConfig()
 	if err != nil {
-		c.UI.Error(fmt.Sprintf("Error loading configuration: %s", err))
+		c.outputErrors(err)
 		return 1
 	}
 
 	if config.AutoAuth == nil {
-		c.UI.Info("No auto_auth block found in config, not starting automatic authentication feature")
+		c.UI.Info("No auto_auth block found in config, the automatic authentication feature will not be started")
 	}
 
-	c.updateConfig(f, config)
+	c.updateConfig(f, config) // This only needs to happen on start-up to aggregate config from flags and env vars
+	c.config = config
 
-	// Parse all the log related config
+	l, err := c.newLogger()
-	logLevel, err := logging.ParseLogLevel(config.LogLevel)
 	if err != nil {
-		c.UI.Error(err.Error())
+		c.outputErrors(err)
 		return 1
 	}
-
-	logFormat, err := logging.ParseLogFormat(config.LogFormat)
-	if err != nil {
-		c.UI.Error(err.Error())
-		return 1
-	}
-
-	logRotateDuration, err := parseutil.ParseDurationSecond(config.LogRotateDuration)
-	if err != nil {
-		c.UI.Error(err.Error())
-		return 1
-	}
-
-	logRotateBytes, err := parseutil.ParseInt(config.LogRotateBytes)
-	if err != nil {
-		c.UI.Error(err.Error())
-		return 1
-	}
-
-	logRotateMaxFiles, err := parseutil.ParseInt(config.LogRotateMaxFiles)
-	if err != nil {
-		c.UI.Error(err.Error())
-		return 1
-	}
-
-	logCfg := &logging.LogConfig{
-		Name:              "vault-agent",
-		LogLevel:          logLevel,
-		LogFormat:         logFormat,
-		LogFilePath:       config.LogFile,
-		LogRotateDuration: logRotateDuration,
-		LogRotateBytes:    int(logRotateBytes),
-		LogRotateMaxFiles: int(logRotateMaxFiles),
-	}
-
-	l, err := logging.Setup(logCfg, c.logWriter)
-	if err != nil {
-		c.UI.Error(err.Error())
-		return 1
-	}
-
 	c.logger = l
 
 	infoKeys := make([]string, 0, 10)
@@ -289,7 +245,7 @@ func (c *AgentCommand) Run(args []string) int {
 		if os.Getenv("VAULT_TEST_VERIFY_ONLY_DUMP_CONFIG") != "" {
 			c.UI.Output(fmt.Sprintf(
 				"\nConfiguration:\n%s\n",
-				pretty.Sprint(*config)))
+				pretty.Sprint(*c.config)))
 		}
 		return 0
 	}
@@ -364,7 +320,7 @@ func (c *AgentCommand) Run(args []string) int {
 				}
 				s, err := file.NewFileSink(config)
 				if err != nil {
-					c.UI.Error(fmt.Errorf("Error creating file sink: %w", err).Error())
+					c.UI.Error(fmt.Errorf("error creating file sink: %w", err).Error())
 					return 1
 				}
 				config.Sink = s
@@ -504,7 +460,7 @@ func (c *AgentCommand) Run(args []string) int {
 
 	// Output the header that the agent has started
 	if !c.logFlags.flagCombineLogs {
-		c.UI.Output("==> Vault agent started! Log data will stream in below:\n")
+		c.UI.Output("==> Vault Agent started! Log data will stream in below:\n")
 	}
 
 	var leaseCache *cache.LeaseCache
@@ -708,9 +664,13 @@ func (c *AgentCommand) Run(args []string) int {
 	if len(config.Templates) > 0 {
 		config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType})
 	}
+
+	// Ensure we've added all the reload funcs for TLS before anyone triggers a reload.
+	c.tlsReloadFuncsLock.Lock()
+
 	for i, lnConfig := range config.Listeners {
 		var ln net.Listener
-		var tlsConf *tls.Config
+		var tlsCfg *tls.Config
 
 		if lnConfig.Type == listenerutil.BufConnType {
 			inProcListener := bufconn.Listen(1024 * 1024)
@@ -719,11 +679,17 @@ func (c *AgentCommand) Run(args []string) int {
 			}
 			ln = inProcListener
 		} else {
-			ln, tlsConf, err = cache.StartListener(lnConfig)
+			lnBundle, err := cache.StartListener(lnConfig)
 			if err != nil {
 				c.UI.Error(fmt.Sprintf("Error starting listener: %v", err))
 				return 1
 			}
+
+			tlsCfg = lnBundle.TLSConfig
+			ln = lnBundle.Listener
+
+			// Track the reload func, so we can reload later if needed.
+			c.tlsReloadFuncs = append(c.tlsReloadFuncs, lnBundle.TLSReloadFunc)
 		}
 
 		listeners = append(listeners, ln)
@@ -768,7 +734,7 @@ func (c *AgentCommand) Run(args []string) int {
 		}
 
 		scheme := "https://"
-		if tlsConf == nil {
+		if tlsCfg == nil {
 			scheme = "http://"
 		}
 		if ln.Addr().Network() == "unix" {
@@ -781,7 +747,7 @@ func (c *AgentCommand) Run(args []string) int {
 
 		server := &http.Server{
 			Addr:              ln.Addr().String(),
-			TLSConfig:         tlsConf,
+			TLSConfig:         tlsCfg,
 			Handler:           mux,
 			ReadHeaderTimeout: 10 * time.Second,
 			ReadTimeout:       30 * time.Second,
@@ -792,6 +758,8 @@ func (c *AgentCommand) Run(args []string) int {
 		go server.Serve(ln)
 	}
 
+	c.tlsReloadFuncsLock.Unlock()
+
 	// Ensure that listeners are closed at all the exits
 	listenerCloseFunc := func() {
 		for _, ln := range listeners {
@@ -805,28 +773,43 @@ func (c *AgentCommand) Run(args []string) int {
 		close(c.startedCh)
 	}
 
-	// Listen for signals
-	// TODO: implement support for SIGHUP reloading of configuration
-	// signal.Notify(c.signalCh)
-
 	var g run.Group
 
+	g.Add(func() error {
+		for {
+			select {
+			case <-c.SighupCh:
+				c.UI.Output("==> Vault Agent config reload triggered")
+				err := c.reloadConfig(c.flagConfigs)
+				if err != nil {
+					c.outputErrors(err)
+				}
+				// Send the 'reloaded' message on the relevant channel
+				select {
+				case c.reloadedCh <- struct{}{}:
+				default:
+				}
+			case <-ctx.Done():
+				return nil
+			}
+		}
+	}, func(error) {
+		cancelFunc()
+	})
+
 	// This run group watches for signal termination
 	g.Add(func() error {
 		for {
 			select {
 			case <-c.ShutdownCh:
-				c.UI.Output("==> Vault agent shutdown triggered")
+				c.UI.Output("==> Vault Agent shutdown triggered")
 				// Notify systemd that the server is shutting down
-				c.notifySystemd(systemd.SdNotifyStopping)
+				// Let the lease cache know this is a shutdown; no need to evict everything
-				// Let the lease cache know this is a shutdown; no need to evict
-				// everything
 				if leaseCache != nil {
 					leaseCache.SetShuttingDown(true)
 				}
 				return nil
 			case <-ctx.Done():
-				c.notifySystemd(systemd.SdNotifyStopping)
 				return nil
 			case <-winsvc.ShutdownChannel():
 				return nil
@@ -874,9 +857,9 @@ func (c *AgentCommand) Run(args []string) int {
 
 		ts := template.NewServer(&template.ServerConfig{
 			Logger:        c.logger.Named("template.server"),
-			LogLevel:      logLevel,
+			LogLevel:      c.logger.GetLevel(),
 			LogWriter:     c.logWriter,
-			AgentConfig:   config,
+			AgentConfig:   c.config,
 			Namespace:     templateNamespace,
 			ExitAfterAuth: config.ExitAfterAuth,
 		})
@@ -940,7 +923,7 @@ func (c *AgentCommand) Run(args []string) int {
 	// Server configuration output
 	padding := 24
 	sort.Strings(infoKeys)
-	c.UI.Output("==> Vault agent configuration:\n")
+	c.UI.Output("==> Vault Agent configuration:\n")
 	for _, k := range infoKeys {
 		c.UI.Output(fmt.Sprintf(
 			"%s%s: %s",
@@ -968,13 +951,14 @@ func (c *AgentCommand) Run(args []string) int {
 		}
 	}()
 
+	var exitCode int
 	if err := g.Run(); err != nil {
 		c.logger.Error("runtime error encountered", "error", err)
 		c.UI.Error("Error encountered during run, refer to logs for more details.")
-		return 1
+		exitCode = 1
 	}
-
+	c.notifySystemd(systemd.SdNotifyStopping)
-	return 0
+	return exitCode
 }
 
 // updateConfig ensures that the config object accurately reflects the desired
@@ -1219,3 +1203,170 @@ func (c *AgentCommand) handleQuit(enabled bool) http.Handler {
 		close(c.ShutdownCh)
 	})
 }
+
+// newLogger creates a logger based on parsed config field on the Agent Command struct.
+func (c *AgentCommand) newLogger() (log.InterceptLogger, error) {
+	if c.config == nil {
+		return nil, fmt.Errorf("cannot create logger, no config")
+	}
+
+	var errors error
+
+	// Parse all the log related config
+	logLevel, err := logging.ParseLogLevel(c.config.LogLevel)
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	logFormat, err := logging.ParseLogFormat(c.config.LogFormat)
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	logRotateDuration, err := parseutil.ParseDurationSecond(c.config.LogRotateDuration)
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	logRotateBytes, err := parseutil.ParseInt(c.config.LogRotateBytes)
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	logRotateMaxFiles, err := parseutil.ParseInt(c.config.LogRotateMaxFiles)
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	if errors != nil {
+		return nil, errors
+	}
+
+	logCfg := &logging.LogConfig{
+		Name:              "vault-agent",
+		LogLevel:          logLevel,
+		LogFormat:         logFormat,
+		LogFilePath:       c.config.LogFile,
+		LogRotateDuration: logRotateDuration,
+		LogRotateBytes:    int(logRotateBytes),
+		LogRotateMaxFiles: int(logRotateMaxFiles),
+	}
+
+	l, err := logging.Setup(logCfg, c.logWriter)
+	if err != nil {
+		return nil, err
+	}
+
+	return l, nil
+}
+
+// loadConfig attempts to generate an Agent config from the file(s) specified.
+func (c *AgentCommand) loadConfig(paths []string) (*agentConfig.Config, error) {
+	var errors error
+	cfg := agentConfig.NewConfig()
+
+	for _, configPath := range paths {
+		configFromPath, err := agentConfig.LoadConfig(configPath)
+		if err != nil {
+			errors = multierror.Append(errors, fmt.Errorf("error loading configuration from %s: %w", configPath, err))
+		} else {
+			cfg = cfg.Merge(configFromPath)
+		}
+	}
+
+	if errors != nil {
+		return nil, errors
+	}
+
+	if err := cfg.ValidateConfig(); err != nil {
+		return nil, fmt.Errorf("error validating configuration: %w", err)
+	}
+
+	return cfg, nil
+}
+
+// reloadConfig will attempt to reload the config from file(s) and adjust certain
+// config values without requiring a restart of the Vault Agent.
+// If config is retrieved without error it is stored in the config field of the AgentCommand.
+// This operation is not atomic and could result in updated config but partially applied config settings.
+// The error returned from this func may be a multierror.
+// This function will most likely be called due to Vault Agent receiving a SIGHUP signal.
+// Currently only reloading the following are supported:
+// * log level
+// * TLS certs for listeners
+func (c *AgentCommand) reloadConfig(paths []string) error {
+	// Notify systemd that the server is reloading
+	c.notifySystemd(systemd.SdNotifyReloading)
+	defer c.notifySystemd(systemd.SdNotifyReady)
+
+	var errors error
+
+	// Reload the config
+	cfg, err := c.loadConfig(paths)
+	if err != nil {
+		// Returning single error as we won't continue with bad config and won't 'commit' it.
+		return err
+	}
+	c.config = cfg
+
+	// Update the log level
+	err = c.reloadLogLevel()
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	// Update certs
+	err = c.reloadCerts()
+	if err != nil {
+		errors = multierror.Append(errors, err)
+	}
+
+	return errors
+}
+
+// reloadLogLevel will attempt to update the log level for the logger attached
+// to the AgentComment struct using the value currently set in config.
+func (c *AgentCommand) reloadLogLevel() error {
+	logLevel, err := logging.ParseLogLevel(c.config.LogLevel)
+	if err != nil {
+		return err
+	}
+
+	c.logger.SetLevel(logLevel)
+
+	return nil
+}
+
+// reloadCerts will attempt to reload certificates using a reload func which
+// was provided when the listeners were configured, only funcs that were appended
+// to the AgentCommand slice will be invoked.
+// This function returns a multierror type so that every func can report an error
+// if it encounters one.
+func (c *AgentCommand) reloadCerts() error {
+	var errors error
+
+	c.tlsReloadFuncsLock.RLock()
+	defer c.tlsReloadFuncsLock.RUnlock()
+
+	for _, reloadFunc := range c.tlsReloadFuncs {
+		err := reloadFunc()
+		if err != nil {
+			errors = multierror.Append(errors, err)
+		}
+	}
+
+	return errors
+}
+
+// outputErrors will take an error or multierror and handle outputting each to the UI
+func (c *AgentCommand) outputErrors(err error) {
+	if err != nil {
+		if me, ok := err.(*multierror.Error); ok {
+			for _, err := range me.Errors {
+				c.UI.Error(err.Error())
+			}
+		} else {
+			c.UI.Error(err.Error())
+		}
+	}
+}

command/agent/cache/listener.go

@@ -6,12 +6,19 @@ import (
 	"net"
 	"strings"
 
+	"github.com/hashicorp/go-secure-stdlib/reloadutil"
 	"github.com/hashicorp/vault/command/server"
 	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/internalshared/listenerutil"
 )
 
-func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, error) {
+type ListenerBundle struct {
+	Listener      net.Listener
+	TLSConfig     *tls.Config
+	TLSReloadFunc reloadutil.ReloadFunc
+}
+
+func StartListener(lnConfig *configutil.Listener) (*ListenerBundle, error) {
 	addr := lnConfig.Address
 
 	var ln net.Listener
@@ -31,7 +38,7 @@ func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, er
 
 		ln, err = net.Listen(bindProto, addr)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 		ln = &server.TCPKeepAliveListener{ln.(*net.TCPListener)}
 
@@ -48,21 +55,27 @@ func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, er
 		}
 		ln, err = listenerutil.UnixSocketListener(addr, uConfig)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 
 	default:
-		return nil, nil, fmt.Errorf("invalid listener type: %q", lnConfig.Type)
+		return nil, fmt.Errorf("invalid listener type: %q", lnConfig.Type)
 	}
 
 	props := map[string]string{"addr": ln.Addr().String()}
-	tlsConf, _, err := listenerutil.TLSConfig(lnConfig, props, nil)
+	tlsConf, reloadFunc, err := listenerutil.TLSConfig(lnConfig, props, nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	if tlsConf != nil {
 		ln = tls.NewListener(ln, tlsConf)
 	}
 
-	return ln, tlsConf, nil
+	cfg := &ListenerBundle{
+		Listener:      ln,
+		TLSConfig:     tlsConf,
+		TLSReloadFunc: reloadFunc,
+	}
+
+	return cfg, nil
 }

command/agent/test-fixtures/reload/reload_bar.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAwF7sRAyUiLcd6es6VeaTRUBOusFFGkmKJ5lU351waCJqXFju
+Z6i/SQYNAAnnRgotXSTE1fIPjE2kZNH1hvqE5IpTGgAwy50xpjJrrBBI6e9lyKqj
+7T8gLVNBvtC0cpQi+pGrszEI0ckDQCSZHqi/PAzcpmLUgh2KMrgagT+YlN35KHtl
+/bQ/Fsn+kqykVqNw69n/CDKNKdDHn1qPwiX9q/fTMj3EG6g+3ntKrUOh8V/gHKPz
+q8QGP/wIud2K+tTSorVXr/4zx7xgzlbJkCakzcQQiP6K+paPnDRlE8fK+1gRRyR7
+XCzyp0irUl8G1NjYAR/tVWxiUhlk/jZutb8PpwIDAQABAoIBAEOzJELuindyujxQ
+ZD9G3h1I/GwNCFyv9Mbq10u7BIwhUH0fbwdcA7WXQ4v38ERd4IkfH4aLoZ0m1ewF
+V/sgvxQO+h/0YTfHImny5KGxOXfaoF92bipYROKuojydBmQsbgLwsRRm9UufCl3Q
+g3KewG5JuH112oPQEYq379v8nZ4FxC3Ano1OFBTm9UhHIAX1Dn22kcHOIIw8jCsQ
+zp7TZOW+nwtkS41cBwhvV4VIeL6yse2UgbOfRVRwI7B0OtswS5VgW3wysO2mTDKt
+V/WCmeht1il/6ZogEHgi/mvDCKpj20wQ1EzGnPdFLdiFJFylf0oufQD/7N/uezbC
+is0qJEECgYEA3AE7SeLpe3SZApj2RmE2lcD9/Saj1Y30PznxB7M7hK0sZ1yXEbtS
+Qf894iDDD/Cn3ufA4xk/K52CXgAcqvH/h2geG4pWLYsT1mdWhGftprtOMCIvJvzU
+8uWJzKdOGVMG7R59wNgEpPDZDpBISjexwQsFo3aw1L/H1/Sa8cdY3a0CgYEA39hB
+1oLmGRyE32Q4GF/srG4FqKL1EsbISGDUEYTnaYg2XiM43gu3tC/ikfclk27Jwc2L
+m7cA5FxxaEyfoOgfAizfU/uWTAbx9GoXgWsO0hWSN9+YNq61gc5WKoHyrJ/rfrti
+y5d7k0OCeBxckLqGDuJqICQ0myiz0El6FU8h5SMCgYEAuhigmiNC9JbwRu40g9v/
+XDVfox9oPmBRVpogdC78DYKeqN/9OZaGQiUxp3GnDni2xyqqUm8srCwT9oeJuF/z
+kgpUTV96/hNCuH25BU8UC5Es1jJUSFpdlwjqwx5SRcGhfjnojZMseojwUg1h2MW7
+qls0bc0cTxnaZaYW2qWRWhECgYBrT0cwyQv6GdvxJCBoPwQ9HXmFAKowWC+H0zOX
+Onmd8/jsZEJM4J0uuo4Jn8vZxBDg4eL9wVuiHlcXwzP7dYv4BP8DSechh2rS21Ft
+b59pQ4IXWw+jl1nYYsyYEDgAXaIN3VNder95N7ICVsZhc6n01MI/qlu1zmt1fOQT
+9x2utQKBgHI9SbsfWfbGiu6oLS3+9V1t4dORhj8D8b7z3trvECrD6tPhxoZqtfrH
+4apKr3OKRSXk3K+1K6pkMHJHunspucnA1ChXLhzfNF08BSRJkQDGYuaRLS6VGgab
+JZTl54bGvO1GkszEBE/9QFcqNVtWGMWXnUPwNNv8t//yJT5rvQil
+-----END RSA PRIVATE KEY-----

command/agent/test-fixtures/reload/reload_bar.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDQzCCAiugAwIBAgIULLCz3mZKmg2xy3rWCud0f1zcmBwwDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYwMzEwMDIzNjQ0WhcNMzYw
+MzA1MDEzNzE0WjAaMRgwFgYDVQQDEw9iYXIuZXhhbXBsZS5jb20wggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAXuxEDJSItx3p6zpV5pNFQE66wUUaSYon
+mVTfnXBoImpcWO5nqL9JBg0ACedGCi1dJMTV8g+MTaRk0fWG+oTkilMaADDLnTGm
+MmusEEjp72XIqqPtPyAtU0G+0LRylCL6kauzMQjRyQNAJJkeqL88DNymYtSCHYoy
+uBqBP5iU3fkoe2X9tD8Wyf6SrKRWo3Dr2f8IMo0p0MefWo/CJf2r99MyPcQbqD7e
+e0qtQ6HxX+Aco/OrxAY//Ai53Yr61NKitVev/jPHvGDOVsmQJqTNxBCI/or6lo+c
+NGUTx8r7WBFHJHtcLPKnSKtSXwbU2NgBH+1VbGJSGWT+Nm61vw+nAgMBAAGjgYQw
+gYEwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBSVoF8F
+7qbzSryIFrldurAG78LvSjAfBgNVHSMEGDAWgBRzDNvqF/Tq21OgWs13B5YydZjl
+vzAgBgNVHREEGTAXgg9iYXIuZXhhbXBsZS5jb22HBH8AAAEwDQYJKoZIhvcNAQEL
+BQADggEBAGmz2N282iT2IaEZvOmzIE4znHGkvoxZmrr/2byq5PskBg9ysyCHfUvw
+SFA8U7jWjezKTnGRUu5blB+yZdjrMtB4AePWyEqtkJwVsZ2SPeP+9V2gNYK4iktP
+UF3aIgBbAbw8rNuGIIB0T4D+6Zyo9Y3MCygs6/N4bRPZgLhewWn1ilklfnl3eqaC
+a+JY1NBuTgCMa28NuC+Hy3mCveqhI8tFNiOthlLdgAEbuQaOuNutAG73utZ2aq6Q
+W4pajFm3lEf5zt7Lo6ZCFtY/Q8jjURJ9e4O7VjXcqIhBM5bSMI6+fgQyOH0SLboj
+RNanJ2bcyF1iPVyPBGzV3dF0ngYzxEY=
+-----END CERTIFICATE-----

command/agent/test-fixtures/reload/reload_ca.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNTCCAh2gAwIBAgIUBeVo+Ce2BrdRT1cogKvJLtdOky8wDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYwMzEwMDIzNTM4WhcNMzYw
+MzA1MDIzNjA4WjAWMRQwEgYDVQQDEwtleGFtcGxlLmNvbTCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAPTQGWPRIOECGeJB6tR/ftvvtioC9f84fY2QdJ5k
+JBupXjPAGYKgS4MGzyT5bz9yY400tCtmh6h7p9tZwHl/TElTugtLQ/8ilMbJTiOM
+SiyaMDPHiMJJYKTjm9bu6bKeU1qPZ0Cryes4rygbqs7w2XPgA2RxNmDh7JdX7/h+
+VB5onBmv8g4WFSayowGyDcJWWCbu5yv6ZdH1bqQjgRzQ5xp17WXNmvlzdp2vate/
+9UqPdA8sdJzW/91Gvmros0o/FnG7c2pULhk22wFqO8t2HRjKb3nuxALEJvqoPvad
+KjpDTaq1L1ZzxcB7wvWyhy/lNLZL7jiNWy0mN1YB0UpSWdECAwEAAaN7MHkwDgYD
+VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFHMM2+oX9Orb
+U6BazXcHljJ1mOW/MB8GA1UdIwQYMBaAFHMM2+oX9OrbU6BazXcHljJ1mOW/MBYG
+A1UdEQQPMA2CC2V4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQAp17XsOaT9
+hculRqrFptn3+zkH3HrIckHm+28R5xYT8ASFXFcLFugGizJAXVL5lvsRVRIwCoOX
+Nhi8XSNEFP640VbHcEl81I84bbRIIDS+Yheu6JDZGemTaDYLv1J3D5SHwgoM+nyf
+oTRgotUCIXcwJHmTpWEUkZFKuqBxsoTGzk0jO8wOP6xoJkzxVVG5PvNxs924rxY8
+Y8iaLdDfMeT7Pi0XIliBa/aSp/iqSW8XKyJl5R5vXg9+DOgZUrVzIxObaF5RBl/a
+mJOeklJBdNVzQm5+iMpO42lu0TA9eWtpP+YiUEXU17XDvFeQWOocFbQ1Peo0W895
+XRz2GCwCNyvW
+-----END CERTIFICATE-----

command/agent/test-fixtures/reload/reload_foo.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpgIBAAKCAQEAzNyVieSti9XBb5/celB5u8YKRJv3mQS9A4/X0mqY1ePznt1i
+ilG7OmG0yM2VAk0ceIAQac3Bsn74jxn2cDlrrVniPXcNgYtMtW0kRqNEo4doo4EX
+xZguS9vNBu29useHhif1TGX/pA3dgvaVycUCjzTEVk6qI8UEehMK6gEGZb7nOr0A
+A9nipSqoeHpDLe3a4KVqj1vtlJKUvD2i1MuBuQ130cB1K9rufLCShGu7mEgzEosc
+gr+K3Bf03IejbeVRyIfLtgj1zuvV1katec75UqRA/bsvt5G9JfJqiZ9mwFN0vp3g
+Cr7pdQBSBQ2q4yf9s8CuY5c5w9fl3F8f5QFQoQIDAQABAoIBAQCbCb1qNFRa5ZSV
+I8i6ELlwMDqJHfhOJ9XcIjpVljLAfNlcu3Ld92jYkCU/asaAjVckotbJG9yhd5Io
+yp9E40/oS4P6vGTOS1vsWgMAKoPBtrKsOwCAm+E9q8UIn1fdSS/5ibgM74x+3bds
+a62Em8KKGocUQkhk9a+jq1GxMsFisbHRxEHvClLmDMgGnW3FyGmWwT6yZLPSC0ey
+szmmjt3ouP8cLAOmSjzcQBMmEZpQMCgR6Qckg6nrLQAGzZyTdCd875wbGA57DpWX
+Lssn95+A5EFvr/6b7DkXeIFCrYBFFa+UQN3PWGEQ6Zjmiw4VgV2vO8yX2kCLlUhU
+02bL393ZAoGBAPXPD/0yWINbKUPcRlx/WfWQxfz0bu50ytwIXzVK+pRoAMuNqehK
+BJ6kNzTTBq40u+IZ4f5jbLDulymR+4zSkirLE7CyWFJOLNI/8K4Pf5DJUgNdrZjJ
+LCtP9XRdxiPatQF0NGfdgHlSJh+/CiRJP4AgB17AnB/4z9/M0ZlJGVrzAoGBANVa
+69P3Rp/WPBQv0wx6f0tWppJolWekAHKcDIdQ5HdOZE5CPAYSlTrTUW3uJuqMwU2L
+M0Er2gIPKWIR5X+9r7Fvu9hQW6l2v3xLlcrGPiapp3STJvuMxzhRAmXmu3bZfVn1
+Vn7Vf1jPULHtTFSlNFEvYG5UJmygK9BeyyVO5KMbAoGBAMCyAibLQPg4jrDUDZSV
+gUAwrgUO2ae1hxHWvkxY6vdMUNNByuB+pgB3W4/dnm8Sh/dHsxJpftt1Lqs39ar/
+p/ZEHLt4FCTxg9GOrm7FV4t5RwG8fko36phJpnIC0UFqQltRbYO+8OgqrhhU+u5X
+PaCDe0OcWsf1lYAsYGN6GpZhAoGBAMJ5Ksa9+YEODRs1cIFKUyd/5ztC2xRqOAI/
+3WemQ2nAacuvsfizDZVeMzYpww0+maAuBt0btI719PmwaGmkpDXvK+EDdlmkpOwO
+FY6MXvBs6fdnfjwCWUErDi2GQFAX9Jt/9oSL5JU1+08DhvUM1QA/V/2Y9KFE6kr3
+bOIn5F4LAoGBAKQzH/AThDGhT3hwr4ktmReF3qKxBgxzjVa8veXtkY5VWwyN09iT
+jnTTt6N1CchZoK5WCETjdzNYP7cuBTcV4d3bPNRiJmxXaNVvx3Tlrk98OiffT8Qa
+5DO/Wfb43rNHYXBjU6l0n2zWcQ4PUSSbu0P0bM2JTQPRCqSthXvSHw2P
+-----END RSA PRIVATE KEY-----

command/agent/test-fixtures/reload/reload_foo.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDQzCCAiugAwIBAgIUFVW6i/M+yJUsDrXWgRKO/Dnb+L4wDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYwMzEwMDIzNjA1WhcNMzYw
+MzA1MDEzNjM1WjAaMRgwFgYDVQQDEw9mb28uZXhhbXBsZS5jb20wggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDM3JWJ5K2L1cFvn9x6UHm7xgpEm/eZBL0D
+j9fSapjV4/Oe3WKKUbs6YbTIzZUCTRx4gBBpzcGyfviPGfZwOWutWeI9dw2Bi0y1
+bSRGo0Sjh2ijgRfFmC5L280G7b26x4eGJ/VMZf+kDd2C9pXJxQKPNMRWTqojxQR6
+EwrqAQZlvuc6vQAD2eKlKqh4ekMt7drgpWqPW+2UkpS8PaLUy4G5DXfRwHUr2u58
+sJKEa7uYSDMSixyCv4rcF/Tch6Nt5VHIh8u2CPXO69XWRq15zvlSpED9uy+3kb0l
+8mqJn2bAU3S+neAKvul1AFIFDarjJ/2zwK5jlznD1+XcXx/lAVChAgMBAAGjgYQw
+gYEwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBRNJoOJ
+dnazDiuqLhV6truQ4cRe9jAfBgNVHSMEGDAWgBRzDNvqF/Tq21OgWs13B5YydZjl
+vzAgBgNVHREEGTAXgg9mb28uZXhhbXBsZS5jb22HBH8AAAEwDQYJKoZIhvcNAQEL
+BQADggEBAHzv67mtbxMWcuMsxCFBN1PJNAyUDZVCB+1gWhk59EySbVg81hWJDCBy
+fl3TKjz3i7wBGAv+C2iTxmwsSJbda22v8JQbuscXIfLFbNALsPzF+J0vxAgJs5Gc
+sDbfJ7EQOIIOVKQhHLYnQoLnigSSPc1kd0JjYyHEBjgIaSuXgRRTBAeqLiBMx0yh
+RKL1lQ+WoBU/9SXUZZkwokqWt5G7khi5qZkNxVXZCm8VGPg0iywf6gGyhI1SU5S2
+oR219S6kA4JY/stw1qne85/EmHmoImHGt08xex3GoU72jKAjsIpqRWopcD/+uene
+Tc9nn3fTQW/Z9fsoJ5iF5OdJnDEswqE=
+-----END CERTIFICATE-----

command/agent_test.go

@@ -1,6 +1,8 @@
 package command
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -14,7 +16,7 @@ import (
 	"testing"
 	"time"
 
-	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-hclog"
 	vaultjwt "github.com/hashicorp/vault-plugin-auth-jwt"
 	logicalKv "github.com/hashicorp/vault-plugin-secrets-kv"
 	"github.com/hashicorp/vault/api"
@@ -34,7 +36,8 @@ import (
 
 const (
 	BasicHclConfig = `
-log_file = "/foo/bar/juan.log"
+log_file = "TMPDIR/juan.log"
+log_level="warn"
 vault {
 	address = "http://127.0.0.1:8200"
 	retry {
@@ -44,7 +47,25 @@ vault {
 
 listener "tcp" {
 	address = "127.0.0.1:8100"
-	tls_disable = true
+	tls_disable = false
+	tls_cert_file = "TMPDIR/reload_cert.pem"
+  	tls_key_file  = "TMPDIR/reload_key.pem"
+}`
+	BasicHclConfig2 = `
+log_file = "TMPDIR/juan.log"
+log_level="debug"
+vault {
+	address = "http://127.0.0.1:8200"
+	retry {
+		num_retries = 5
+	}
+}
+
+listener "tcp" {
+	address = "127.0.0.1:8100"
+	tls_disable = false
+	tls_cert_file = "TMPDIR/reload_cert.pem"
+  	tls_key_file  = "TMPDIR/reload_key.pem"
 }`
 )
 
@@ -57,7 +78,10 @@ func testAgentCommand(tb testing.TB, logger hclog.Logger) (*cli.MockUi, *AgentCo
 			UI: ui,
 		},
 		ShutdownCh: MakeShutdownCh(),
+		SighupCh:   MakeSighupCh(),
 		logger:     logger,
+		startedCh:  make(chan struct{}, 5),
+		reloadedCh: make(chan struct{}, 5),
 	}
 }
 
@@ -1512,7 +1536,6 @@ vault {
 %s
 %s
 `, serverClient.Address(), retryConf, cacheConfig, listenConfig)
-
 			configPath := makeTempFile(t, "config.hcl", config)
 			defer os.Remove(configPath)
 
@@ -2075,7 +2098,7 @@ func TestAgent_LogFile_CliOverridesConfig(t *testing.T) {
 	}
 
 	// Sanity check that the config value is the current value
-	assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile)
+	assert.Equal(t, "TMPDIR/juan.log", cfg.LogFile)
 
 	// Initialize the command and parse any flags
 	cmd := &AgentCommand{BaseCommand: &BaseCommand{}}
@@ -2089,7 +2112,7 @@ func TestAgent_LogFile_CliOverridesConfig(t *testing.T) {
 	// Update the config based on the inputs.
 	cmd.updateConfig(f, cfg)
 
-	assert.NotEqual(t, "/foo/bar/juan.log", cfg.LogFile)
+	assert.NotEqual(t, "TMPDIR/juan.log", cfg.LogFile)
 	assert.NotEqual(t, "/squiggle/logs.txt", cfg.LogFile)
 	assert.Equal(t, "/foo/bar/test.log", cfg.LogFile)
 }
@@ -2103,7 +2126,7 @@ func TestAgent_LogFile_Config(t *testing.T) {
 	}
 
 	// Sanity check that the config value is the current value
-	assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile, "sanity check on log config failed")
+	assert.Equal(t, "TMPDIR/juan.log", cfg.LogFile, "sanity check on log config failed")
 
 	// Parse the cli flags (but we pass in an empty slice)
 	cmd := &AgentCommand{BaseCommand: &BaseCommand{}}
@@ -2115,7 +2138,180 @@ func TestAgent_LogFile_Config(t *testing.T) {
 
 	cmd.updateConfig(f, cfg)
 
-	assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile, "actual config check")
+	assert.Equal(t, "TMPDIR/juan.log", cfg.LogFile, "actual config check")
+}
+
+func TestAgent_Config_NewLogger_Default(t *testing.T) {
+	cmd := &AgentCommand{BaseCommand: &BaseCommand{}}
+	cmd.config = agentConfig.NewConfig()
+	logger, err := cmd.newLogger()
+
+	assert.NoError(t, err)
+	assert.NotNil(t, logger)
+	assert.Equal(t, hclog.Info.String(), logger.GetLevel().String())
+}
+
+func TestAgent_Config_ReloadLogLevel(t *testing.T) {
+	cmd := &AgentCommand{BaseCommand: &BaseCommand{}}
+	var err error
+	tempDir := t.TempDir()
+
+	// Load an initial config
+	hcl := strings.ReplaceAll(BasicHclConfig, "TMPDIR", tempDir)
+	configFile := populateTempFile(t, "agent-config.hcl", hcl)
+	cmd.config, err = agentConfig.LoadConfigFile(configFile.Name())
+	if err != nil {
+		t.Fatal("Cannot load config to test update/merge", err)
+	}
+
+	// Tweak the loaded config to make sure we can put log files into a temp dir
+	// and systemd log attempts work fine, this would usually happen during Run.
+	cmd.logWriter = os.Stdout
+	cmd.logger, err = cmd.newLogger()
+	if err != nil {
+		t.Fatal("logger required for systemd log messages", err)
+	}
+
+	// Sanity check
+	assert.Equal(t, "warn", cmd.config.LogLevel)
+
+	// Load a new config
+	hcl = strings.ReplaceAll(BasicHclConfig2, "TMPDIR", tempDir)
+	configFile = populateTempFile(t, "agent-config.hcl", hcl)
+	err = cmd.reloadConfig([]string{configFile.Name()})
+	assert.NoError(t, err)
+	assert.Equal(t, "debug", cmd.config.LogLevel)
+}
+
+func TestAgent_Config_ReloadTls(t *testing.T) {
+	var wg sync.WaitGroup
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal("unable to get current working directory")
+	}
+	workingDir := filepath.Join(wd, "/agent/test-fixtures/reload")
+	fooCert := "reload_foo.pem"
+	fooKey := "reload_foo.key"
+
+	barCert := "reload_bar.pem"
+	barKey := "reload_bar.key"
+
+	reloadCert := "reload_cert.pem"
+	reloadKey := "reload_key.pem"
+	caPem := "reload_ca.pem"
+
+	tempDir := t.TempDir()
+
+	// Set up initial 'foo' certs
+	inBytes, err := os.ReadFile(filepath.Join(workingDir, fooCert))
+	if err != nil {
+		t.Fatal("unable to read cert required for test", fooCert, err)
+	}
+	err = os.WriteFile(filepath.Join(tempDir, reloadCert), inBytes, 0o777)
+	if err != nil {
+		t.Fatal("unable to write temp cert required for test", reloadCert, err)
+	}
+
+	inBytes, err = os.ReadFile(filepath.Join(workingDir, fooKey))
+	if err != nil {
+		t.Fatal("unable to read cert key required for test", fooKey, err)
+	}
+	err = os.WriteFile(filepath.Join(tempDir, reloadKey), inBytes, 0o777)
+	if err != nil {
+		t.Fatal("unable to write temp cert key required for test", reloadKey, err)
+	}
+
+	inBytes, err = os.ReadFile(filepath.Join(workingDir, caPem))
+	if err != nil {
+		t.Fatal("unable to read CA pem required for test", caPem, err)
+	}
+	certPool := x509.NewCertPool()
+	ok := certPool.AppendCertsFromPEM(inBytes)
+	if !ok {
+		t.Fatal("not ok when appending CA cert")
+	}
+
+	replacedHcl := strings.ReplaceAll(BasicHclConfig, "TMPDIR", tempDir)
+	configFile := populateTempFile(t, "agent-config.hcl", replacedHcl)
+
+	// Set up Agent/cmd
+	logger := logging.NewVaultLogger(hclog.Trace)
+	ui, cmd := testAgentCommand(t, logger)
+
+	wg.Add(1)
+	args := []string{"-config", configFile.Name()}
+	go func() {
+		if code := cmd.Run(args); code != 0 {
+			output := ui.ErrorWriter.String() + ui.OutputWriter.String()
+			t.Errorf("got a non-zero exit status: %s", output)
+		}
+		wg.Done()
+	}()
+
+	testCertificateName := func(cn string) error {
+		conn, err := tls.Dial("tcp", "127.0.0.1:8100", &tls.Config{
+			RootCAs: certPool,
+		})
+		if err != nil {
+			return err
+		}
+		defer conn.Close()
+		if err = conn.Handshake(); err != nil {
+			return err
+		}
+		servName := conn.ConnectionState().PeerCertificates[0].Subject.CommonName
+		if servName != cn {
+			return fmt.Errorf("expected %s, got %s", cn, servName)
+		}
+		return nil
+	}
+
+	// Start
+	select {
+	case <-cmd.startedCh:
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timeout")
+	}
+
+	if err := testCertificateName("foo.example.com"); err != nil {
+		t.Fatalf("certificate name didn't check out: %s", err)
+	}
+
+	// Swap out certs
+	inBytes, err = os.ReadFile(filepath.Join(workingDir, barCert))
+	if err != nil {
+		t.Fatal("unable to read cert required for test", barCert, err)
+	}
+	err = os.WriteFile(filepath.Join(tempDir, reloadCert), inBytes, 0o777)
+	if err != nil {
+		t.Fatal("unable to write temp cert required for test", reloadCert, err)
+	}
+
+	inBytes, err = os.ReadFile(filepath.Join(workingDir, barKey))
+	if err != nil {
+		t.Fatal("unable to read cert key required for test", barKey, err)
+	}
+	err = os.WriteFile(filepath.Join(tempDir, reloadKey), inBytes, 0o777)
+	if err != nil {
+		t.Fatal("unable to write temp cert key required for test", reloadKey, err)
+	}
+
+	// Reload
+	cmd.SighupCh <- struct{}{}
+	select {
+	case <-cmd.reloadedCh:
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timeout")
+	}
+
+	if err := testCertificateName("bar.example.com"); err != nil {
+		t.Fatalf("certificate name didn't check out: %s", err)
+	}
+
+	// Shut down
+	cmd.ShutdownCh <- struct{}{}
+
+	wg.Wait()
 }
 
 // Get a randomly assigned port and then free it again before returning it.

command/commands.go

@@ -259,6 +259,7 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 					UI: serverCmdUi,
 				},
 				ShutdownCh: MakeShutdownCh(),
+				SighupCh:   MakeSighupCh(),
 			}, nil
 		},
 		"audit": func() (cli.Command, error) {

website/content/docs/agent/index.mdx

@@ -186,6 +186,9 @@ These are the currently-available general configuration options:
 
 - `listener` <code>([listener][listener]: <optional\>)</code> - Specifies the addresses and ports on which the Agent will respond to requests.
 
+  ~> **Note:** On `SIGHUP` (`kill -SIGHUP $(pidof vault)`), Vault Agent will attempt to reload listener TLS configuration.
+  This method can be used to refresh certificates used by Vault Agent without having to restart its process.
+
 - `pid_file` `(string: "")` - Path to the file in which the agent's Process ID
   (PID) should be stored
 
@@ -213,6 +216,9 @@ These are the currently-available general configuration options:
 
 - `log_level` - Equivalent to the [`-log-level` command-line flag](#_log_level).
 
+  ~> **Note:** On `SIGHUP` (`kill -SIGHUP $(pidof vault)`), Vault Agent will update the log level to the value
+  specified by configuration file (including overriding values set using CLI or environment variable parameters).
+
 - `log_format` - Equivalent to the [`-log-format` command-line flag](#_log_format).
 
 - `log_file` - Equivalent to the [`-log-file` command-line flag](#_log_file).