From e4685c10ef66e73ea2adbd7bcf4958db395490df Mon Sep 17 00:00:00 2001 From: Peter Wilson Date: Tue, 10 Jan 2023 17:45:34 +0000 Subject: [PATCH] VAULT-9883: Agent Reloadable Config (#18638) * Update command/agent.go * Attempt to only reload log level and certs * Mimicked 'server' test for cert reload in 'agent' Co-authored-by: Nick Cabatoff Left out the `c.config` tweak that meant changes to lots of lines of code within the `Run` function of Agent command. :) --- changelog/18638.txt | 3 + command/agent.go | 317 +++++++++++++----- command/agent/cache/listener.go | 27 +- .../agent/test-fixtures/reload/reload_bar.key | 27 ++ .../agent/test-fixtures/reload/reload_bar.pem | 20 ++ .../agent/test-fixtures/reload/reload_ca.pem | 20 ++ .../agent/test-fixtures/reload/reload_foo.key | 27 ++ .../agent/test-fixtures/reload/reload_foo.pem | 20 ++ command/agent_test.go | 212 +++++++++++- command/commands.go | 1 + website/content/docs/agent/index.mdx | 6 + 11 files changed, 582 insertions(+), 98 deletions(-) create mode 100644 changelog/18638.txt create mode 100644 command/agent/test-fixtures/reload/reload_bar.key create mode 100644 command/agent/test-fixtures/reload/reload_bar.pem create mode 100644 command/agent/test-fixtures/reload/reload_ca.pem create mode 100644 command/agent/test-fixtures/reload/reload_foo.key create mode 100644 command/agent/test-fixtures/reload/reload_foo.pem diff --git a/changelog/18638.txt b/changelog/18638.txt new file mode 100644 index 000000000..727c85a66 --- /dev/null +++ b/changelog/18638.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: allows some parts of config to be reloaded without requiring a restart. +``` \ No newline at end of file diff --git a/command/agent.go b/command/agent.go index 9a7aa1192..2d6dcd208 100644 --- a/command/agent.go +++ b/command/agent.go @@ -17,6 +17,7 @@ import ( "time" ctconfig "github.com/hashicorp/consul-template/config" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/vault/command/agent/sink/inmem" @@ -24,6 +25,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-secure-stdlib/gatedwriter" "github.com/hashicorp/go-secure-stdlib/parseutil" + "github.com/hashicorp/go-secure-stdlib/reloadutil" "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/command/agent/auth" "github.com/hashicorp/vault/command/agent/auth/alicloud" @@ -75,9 +77,14 @@ type AgentCommand struct { *BaseCommand logFlags logFlags + config *agentConfig.Config + ShutdownCh chan struct{} SighupCh chan struct{} + tlsReloadFuncsLock sync.RWMutex + tlsReloadFuncs []reloadutil.ReloadFunc + logWriter io.Writer logGate *gatedwriter.Writer logger log.Logger @@ -87,7 +94,8 @@ type AgentCommand struct { cleanupGuard sync.Once - startedCh chan (struct{}) // for tests + startedCh chan struct{} // for tests + reloadedCh chan struct{} // for tests flagConfigs []string flagExitAfterAuth bool @@ -102,7 +110,7 @@ func (c *AgentCommand) Help() string { helpText := ` Usage: vault agent [options] - This command starts a Vault agent that can perform automatic authentication + This command starts a Vault Agent that can perform automatic authentication in certain environments. Start an agent with a configuration file: @@ -193,76 +201,24 @@ func (c *AgentCommand) Run(args []string) int { return 1 } - config := agentConfig.NewConfig() - - for _, configPath := range c.flagConfigs { - configFromPath, err := agentConfig.LoadConfig(configPath) - if err != nil { - c.UI.Error(fmt.Sprintf("Error loading configuration from %s: %s", configPath, err)) - return 1 - } - config = config.Merge(configFromPath) - } - - err := config.ValidateConfig() + config, err := c.loadConfig(c.flagConfigs) if err != nil { - c.UI.Error(fmt.Sprintf("Error loading configuration: %s", err)) + c.outputErrors(err) return 1 } if config.AutoAuth == nil { - c.UI.Info("No auto_auth block found in config, not starting automatic authentication feature") + c.UI.Info("No auto_auth block found in config, the automatic authentication feature will not be started") } - c.updateConfig(f, config) + c.updateConfig(f, config) // This only needs to happen on start-up to aggregate config from flags and env vars + c.config = config - // Parse all the log related config - logLevel, err := logging.ParseLogLevel(config.LogLevel) + l, err := c.newLogger() if err != nil { - c.UI.Error(err.Error()) + c.outputErrors(err) return 1 } - - logFormat, err := logging.ParseLogFormat(config.LogFormat) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - - logRotateDuration, err := parseutil.ParseDurationSecond(config.LogRotateDuration) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - - logRotateBytes, err := parseutil.ParseInt(config.LogRotateBytes) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - - logRotateMaxFiles, err := parseutil.ParseInt(config.LogRotateMaxFiles) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - - logCfg := &logging.LogConfig{ - Name: "vault-agent", - LogLevel: logLevel, - LogFormat: logFormat, - LogFilePath: config.LogFile, - LogRotateDuration: logRotateDuration, - LogRotateBytes: int(logRotateBytes), - LogRotateMaxFiles: int(logRotateMaxFiles), - } - - l, err := logging.Setup(logCfg, c.logWriter) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - c.logger = l infoKeys := make([]string, 0, 10) @@ -289,7 +245,7 @@ func (c *AgentCommand) Run(args []string) int { if os.Getenv("VAULT_TEST_VERIFY_ONLY_DUMP_CONFIG") != "" { c.UI.Output(fmt.Sprintf( "\nConfiguration:\n%s\n", - pretty.Sprint(*config))) + pretty.Sprint(*c.config))) } return 0 } @@ -364,7 +320,7 @@ func (c *AgentCommand) Run(args []string) int { } s, err := file.NewFileSink(config) if err != nil { - c.UI.Error(fmt.Errorf("Error creating file sink: %w", err).Error()) + c.UI.Error(fmt.Errorf("error creating file sink: %w", err).Error()) return 1 } config.Sink = s @@ -504,7 +460,7 @@ func (c *AgentCommand) Run(args []string) int { // Output the header that the agent has started if !c.logFlags.flagCombineLogs { - c.UI.Output("==> Vault agent started! Log data will stream in below:\n") + c.UI.Output("==> Vault Agent started! Log data will stream in below:\n") } var leaseCache *cache.LeaseCache @@ -708,9 +664,13 @@ func (c *AgentCommand) Run(args []string) int { if len(config.Templates) > 0 { config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType}) } + + // Ensure we've added all the reload funcs for TLS before anyone triggers a reload. + c.tlsReloadFuncsLock.Lock() + for i, lnConfig := range config.Listeners { var ln net.Listener - var tlsConf *tls.Config + var tlsCfg *tls.Config if lnConfig.Type == listenerutil.BufConnType { inProcListener := bufconn.Listen(1024 * 1024) @@ -719,11 +679,17 @@ func (c *AgentCommand) Run(args []string) int { } ln = inProcListener } else { - ln, tlsConf, err = cache.StartListener(lnConfig) + lnBundle, err := cache.StartListener(lnConfig) if err != nil { c.UI.Error(fmt.Sprintf("Error starting listener: %v", err)) return 1 } + + tlsCfg = lnBundle.TLSConfig + ln = lnBundle.Listener + + // Track the reload func, so we can reload later if needed. + c.tlsReloadFuncs = append(c.tlsReloadFuncs, lnBundle.TLSReloadFunc) } listeners = append(listeners, ln) @@ -768,7 +734,7 @@ func (c *AgentCommand) Run(args []string) int { } scheme := "https://" - if tlsConf == nil { + if tlsCfg == nil { scheme = "http://" } if ln.Addr().Network() == "unix" { @@ -781,7 +747,7 @@ func (c *AgentCommand) Run(args []string) int { server := &http.Server{ Addr: ln.Addr().String(), - TLSConfig: tlsConf, + TLSConfig: tlsCfg, Handler: mux, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, @@ -792,6 +758,8 @@ func (c *AgentCommand) Run(args []string) int { go server.Serve(ln) } + c.tlsReloadFuncsLock.Unlock() + // Ensure that listeners are closed at all the exits listenerCloseFunc := func() { for _, ln := range listeners { @@ -805,28 +773,43 @@ func (c *AgentCommand) Run(args []string) int { close(c.startedCh) } - // Listen for signals - // TODO: implement support for SIGHUP reloading of configuration - // signal.Notify(c.signalCh) - var g run.Group + g.Add(func() error { + for { + select { + case <-c.SighupCh: + c.UI.Output("==> Vault Agent config reload triggered") + err := c.reloadConfig(c.flagConfigs) + if err != nil { + c.outputErrors(err) + } + // Send the 'reloaded' message on the relevant channel + select { + case c.reloadedCh <- struct{}{}: + default: + } + case <-ctx.Done(): + return nil + } + } + }, func(error) { + cancelFunc() + }) + // This run group watches for signal termination g.Add(func() error { for { select { case <-c.ShutdownCh: - c.UI.Output("==> Vault agent shutdown triggered") + c.UI.Output("==> Vault Agent shutdown triggered") // Notify systemd that the server is shutting down - c.notifySystemd(systemd.SdNotifyStopping) - // Let the lease cache know this is a shutdown; no need to evict - // everything + // Let the lease cache know this is a shutdown; no need to evict everything if leaseCache != nil { leaseCache.SetShuttingDown(true) } return nil case <-ctx.Done(): - c.notifySystemd(systemd.SdNotifyStopping) return nil case <-winsvc.ShutdownChannel(): return nil @@ -874,9 +857,9 @@ func (c *AgentCommand) Run(args []string) int { ts := template.NewServer(&template.ServerConfig{ Logger: c.logger.Named("template.server"), - LogLevel: logLevel, + LogLevel: c.logger.GetLevel(), LogWriter: c.logWriter, - AgentConfig: config, + AgentConfig: c.config, Namespace: templateNamespace, ExitAfterAuth: config.ExitAfterAuth, }) @@ -940,7 +923,7 @@ func (c *AgentCommand) Run(args []string) int { // Server configuration output padding := 24 sort.Strings(infoKeys) - c.UI.Output("==> Vault agent configuration:\n") + c.UI.Output("==> Vault Agent configuration:\n") for _, k := range infoKeys { c.UI.Output(fmt.Sprintf( "%s%s: %s", @@ -968,13 +951,14 @@ func (c *AgentCommand) Run(args []string) int { } }() + var exitCode int if err := g.Run(); err != nil { c.logger.Error("runtime error encountered", "error", err) c.UI.Error("Error encountered during run, refer to logs for more details.") - return 1 + exitCode = 1 } - - return 0 + c.notifySystemd(systemd.SdNotifyStopping) + return exitCode } // updateConfig ensures that the config object accurately reflects the desired @@ -1219,3 +1203,170 @@ func (c *AgentCommand) handleQuit(enabled bool) http.Handler { close(c.ShutdownCh) }) } + +// newLogger creates a logger based on parsed config field on the Agent Command struct. +func (c *AgentCommand) newLogger() (log.InterceptLogger, error) { + if c.config == nil { + return nil, fmt.Errorf("cannot create logger, no config") + } + + var errors error + + // Parse all the log related config + logLevel, err := logging.ParseLogLevel(c.config.LogLevel) + if err != nil { + errors = multierror.Append(errors, err) + } + + logFormat, err := logging.ParseLogFormat(c.config.LogFormat) + if err != nil { + errors = multierror.Append(errors, err) + } + + logRotateDuration, err := parseutil.ParseDurationSecond(c.config.LogRotateDuration) + if err != nil { + errors = multierror.Append(errors, err) + } + + logRotateBytes, err := parseutil.ParseInt(c.config.LogRotateBytes) + if err != nil { + errors = multierror.Append(errors, err) + } + + logRotateMaxFiles, err := parseutil.ParseInt(c.config.LogRotateMaxFiles) + if err != nil { + errors = multierror.Append(errors, err) + } + + if errors != nil { + return nil, errors + } + + logCfg := &logging.LogConfig{ + Name: "vault-agent", + LogLevel: logLevel, + LogFormat: logFormat, + LogFilePath: c.config.LogFile, + LogRotateDuration: logRotateDuration, + LogRotateBytes: int(logRotateBytes), + LogRotateMaxFiles: int(logRotateMaxFiles), + } + + l, err := logging.Setup(logCfg, c.logWriter) + if err != nil { + return nil, err + } + + return l, nil +} + +// loadConfig attempts to generate an Agent config from the file(s) specified. +func (c *AgentCommand) loadConfig(paths []string) (*agentConfig.Config, error) { + var errors error + cfg := agentConfig.NewConfig() + + for _, configPath := range paths { + configFromPath, err := agentConfig.LoadConfig(configPath) + if err != nil { + errors = multierror.Append(errors, fmt.Errorf("error loading configuration from %s: %w", configPath, err)) + } else { + cfg = cfg.Merge(configFromPath) + } + } + + if errors != nil { + return nil, errors + } + + if err := cfg.ValidateConfig(); err != nil { + return nil, fmt.Errorf("error validating configuration: %w", err) + } + + return cfg, nil +} + +// reloadConfig will attempt to reload the config from file(s) and adjust certain +// config values without requiring a restart of the Vault Agent. +// If config is retrieved without error it is stored in the config field of the AgentCommand. +// This operation is not atomic and could result in updated config but partially applied config settings. +// The error returned from this func may be a multierror. +// This function will most likely be called due to Vault Agent receiving a SIGHUP signal. +// Currently only reloading the following are supported: +// * log level +// * TLS certs for listeners +func (c *AgentCommand) reloadConfig(paths []string) error { + // Notify systemd that the server is reloading + c.notifySystemd(systemd.SdNotifyReloading) + defer c.notifySystemd(systemd.SdNotifyReady) + + var errors error + + // Reload the config + cfg, err := c.loadConfig(paths) + if err != nil { + // Returning single error as we won't continue with bad config and won't 'commit' it. + return err + } + c.config = cfg + + // Update the log level + err = c.reloadLogLevel() + if err != nil { + errors = multierror.Append(errors, err) + } + + // Update certs + err = c.reloadCerts() + if err != nil { + errors = multierror.Append(errors, err) + } + + return errors +} + +// reloadLogLevel will attempt to update the log level for the logger attached +// to the AgentComment struct using the value currently set in config. +func (c *AgentCommand) reloadLogLevel() error { + logLevel, err := logging.ParseLogLevel(c.config.LogLevel) + if err != nil { + return err + } + + c.logger.SetLevel(logLevel) + + return nil +} + +// reloadCerts will attempt to reload certificates using a reload func which +// was provided when the listeners were configured, only funcs that were appended +// to the AgentCommand slice will be invoked. +// This function returns a multierror type so that every func can report an error +// if it encounters one. +func (c *AgentCommand) reloadCerts() error { + var errors error + + c.tlsReloadFuncsLock.RLock() + defer c.tlsReloadFuncsLock.RUnlock() + + for _, reloadFunc := range c.tlsReloadFuncs { + err := reloadFunc() + if err != nil { + errors = multierror.Append(errors, err) + } + } + + return errors +} + +// outputErrors will take an error or multierror and handle outputting each to the UI +func (c *AgentCommand) outputErrors(err error) { + if err != nil { + if me, ok := err.(*multierror.Error); ok { + for _, err := range me.Errors { + c.UI.Error(err.Error()) + } + } else { + c.UI.Error(err.Error()) + } + } +} diff --git a/command/agent/cache/listener.go b/command/agent/cache/listener.go index c11867ac1..ec1ddf2c9 100644 --- a/command/agent/cache/listener.go +++ b/command/agent/cache/listener.go @@ -6,12 +6,19 @@ import ( "net" "strings" + "github.com/hashicorp/go-secure-stdlib/reloadutil" "github.com/hashicorp/vault/command/server" "github.com/hashicorp/vault/internalshared/configutil" "github.com/hashicorp/vault/internalshared/listenerutil" ) -func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, error) { +type ListenerBundle struct { + Listener net.Listener + TLSConfig *tls.Config + TLSReloadFunc reloadutil.ReloadFunc +} + +func StartListener(lnConfig *configutil.Listener) (*ListenerBundle, error) { addr := lnConfig.Address var ln net.Listener @@ -31,7 +38,7 @@ func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, er ln, err = net.Listen(bindProto, addr) if err != nil { - return nil, nil, err + return nil, err } ln = &server.TCPKeepAliveListener{ln.(*net.TCPListener)} @@ -48,21 +55,27 @@ func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, er } ln, err = listenerutil.UnixSocketListener(addr, uConfig) if err != nil { - return nil, nil, err + return nil, err } default: - return nil, nil, fmt.Errorf("invalid listener type: %q", lnConfig.Type) + return nil, fmt.Errorf("invalid listener type: %q", lnConfig.Type) } props := map[string]string{"addr": ln.Addr().String()} - tlsConf, _, err := listenerutil.TLSConfig(lnConfig, props, nil) + tlsConf, reloadFunc, err := listenerutil.TLSConfig(lnConfig, props, nil) if err != nil { - return nil, nil, err + return nil, err } if tlsConf != nil { ln = tls.NewListener(ln, tlsConf) } - return ln, tlsConf, nil + cfg := &ListenerBundle{ + Listener: ln, + TLSConfig: tlsConf, + TLSReloadFunc: reloadFunc, + } + + return cfg, nil } diff --git a/command/agent/test-fixtures/reload/reload_bar.key b/command/agent/test-fixtures/reload/reload_bar.key new file mode 100644 index 000000000..10849fbe1 --- /dev/null +++ b/command/agent/test-fixtures/reload/reload_bar.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwF7sRAyUiLcd6es6VeaTRUBOusFFGkmKJ5lU351waCJqXFju +Z6i/SQYNAAnnRgotXSTE1fIPjE2kZNH1hvqE5IpTGgAwy50xpjJrrBBI6e9lyKqj +7T8gLVNBvtC0cpQi+pGrszEI0ckDQCSZHqi/PAzcpmLUgh2KMrgagT+YlN35KHtl +/bQ/Fsn+kqykVqNw69n/CDKNKdDHn1qPwiX9q/fTMj3EG6g+3ntKrUOh8V/gHKPz +q8QGP/wIud2K+tTSorVXr/4zx7xgzlbJkCakzcQQiP6K+paPnDRlE8fK+1gRRyR7 +XCzyp0irUl8G1NjYAR/tVWxiUhlk/jZutb8PpwIDAQABAoIBAEOzJELuindyujxQ +ZD9G3h1I/GwNCFyv9Mbq10u7BIwhUH0fbwdcA7WXQ4v38ERd4IkfH4aLoZ0m1ewF +V/sgvxQO+h/0YTfHImny5KGxOXfaoF92bipYROKuojydBmQsbgLwsRRm9UufCl3Q +g3KewG5JuH112oPQEYq379v8nZ4FxC3Ano1OFBTm9UhHIAX1Dn22kcHOIIw8jCsQ +zp7TZOW+nwtkS41cBwhvV4VIeL6yse2UgbOfRVRwI7B0OtswS5VgW3wysO2mTDKt +V/WCmeht1il/6ZogEHgi/mvDCKpj20wQ1EzGnPdFLdiFJFylf0oufQD/7N/uezbC +is0qJEECgYEA3AE7SeLpe3SZApj2RmE2lcD9/Saj1Y30PznxB7M7hK0sZ1yXEbtS +Qf894iDDD/Cn3ufA4xk/K52CXgAcqvH/h2geG4pWLYsT1mdWhGftprtOMCIvJvzU +8uWJzKdOGVMG7R59wNgEpPDZDpBISjexwQsFo3aw1L/H1/Sa8cdY3a0CgYEA39hB +1oLmGRyE32Q4GF/srG4FqKL1EsbISGDUEYTnaYg2XiM43gu3tC/ikfclk27Jwc2L +m7cA5FxxaEyfoOgfAizfU/uWTAbx9GoXgWsO0hWSN9+YNq61gc5WKoHyrJ/rfrti +y5d7k0OCeBxckLqGDuJqICQ0myiz0El6FU8h5SMCgYEAuhigmiNC9JbwRu40g9v/ +XDVfox9oPmBRVpogdC78DYKeqN/9OZaGQiUxp3GnDni2xyqqUm8srCwT9oeJuF/z +kgpUTV96/hNCuH25BU8UC5Es1jJUSFpdlwjqwx5SRcGhfjnojZMseojwUg1h2MW7 +qls0bc0cTxnaZaYW2qWRWhECgYBrT0cwyQv6GdvxJCBoPwQ9HXmFAKowWC+H0zOX +Onmd8/jsZEJM4J0uuo4Jn8vZxBDg4eL9wVuiHlcXwzP7dYv4BP8DSechh2rS21Ft +b59pQ4IXWw+jl1nYYsyYEDgAXaIN3VNder95N7ICVsZhc6n01MI/qlu1zmt1fOQT +9x2utQKBgHI9SbsfWfbGiu6oLS3+9V1t4dORhj8D8b7z3trvECrD6tPhxoZqtfrH +4apKr3OKRSXk3K+1K6pkMHJHunspucnA1ChXLhzfNF08BSRJkQDGYuaRLS6VGgab +JZTl54bGvO1GkszEBE/9QFcqNVtWGMWXnUPwNNv8t//yJT5rvQil +-----END RSA PRIVATE KEY----- diff --git a/command/agent/test-fixtures/reload/reload_bar.pem b/command/agent/test-fixtures/reload/reload_bar.pem new file mode 100644 index 000000000..a8217be5c --- /dev/null +++ b/command/agent/test-fixtures/reload/reload_bar.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQzCCAiugAwIBAgIULLCz3mZKmg2xy3rWCud0f1zcmBwwDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYwMzEwMDIzNjQ0WhcNMzYw +MzA1MDEzNzE0WjAaMRgwFgYDVQQDEw9iYXIuZXhhbXBsZS5jb20wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAXuxEDJSItx3p6zpV5pNFQE66wUUaSYon +mVTfnXBoImpcWO5nqL9JBg0ACedGCi1dJMTV8g+MTaRk0fWG+oTkilMaADDLnTGm +MmusEEjp72XIqqPtPyAtU0G+0LRylCL6kauzMQjRyQNAJJkeqL88DNymYtSCHYoy +uBqBP5iU3fkoe2X9tD8Wyf6SrKRWo3Dr2f8IMo0p0MefWo/CJf2r99MyPcQbqD7e +e0qtQ6HxX+Aco/OrxAY//Ai53Yr61NKitVev/jPHvGDOVsmQJqTNxBCI/or6lo+c +NGUTx8r7WBFHJHtcLPKnSKtSXwbU2NgBH+1VbGJSGWT+Nm61vw+nAgMBAAGjgYQw +gYEwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBSVoF8F +7qbzSryIFrldurAG78LvSjAfBgNVHSMEGDAWgBRzDNvqF/Tq21OgWs13B5YydZjl +vzAgBgNVHREEGTAXgg9iYXIuZXhhbXBsZS5jb22HBH8AAAEwDQYJKoZIhvcNAQEL +BQADggEBAGmz2N282iT2IaEZvOmzIE4znHGkvoxZmrr/2byq5PskBg9ysyCHfUvw +SFA8U7jWjezKTnGRUu5blB+yZdjrMtB4AePWyEqtkJwVsZ2SPeP+9V2gNYK4iktP +UF3aIgBbAbw8rNuGIIB0T4D+6Zyo9Y3MCygs6/N4bRPZgLhewWn1ilklfnl3eqaC +a+JY1NBuTgCMa28NuC+Hy3mCveqhI8tFNiOthlLdgAEbuQaOuNutAG73utZ2aq6Q +W4pajFm3lEf5zt7Lo6ZCFtY/Q8jjURJ9e4O7VjXcqIhBM5bSMI6+fgQyOH0SLboj +RNanJ2bcyF1iPVyPBGzV3dF0ngYzxEY= +-----END CERTIFICATE----- diff --git a/command/agent/test-fixtures/reload/reload_ca.pem b/command/agent/test-fixtures/reload/reload_ca.pem new file mode 100644 index 000000000..72a74440c --- /dev/null +++ b/command/agent/test-fixtures/reload/reload_ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNTCCAh2gAwIBAgIUBeVo+Ce2BrdRT1cogKvJLtdOky8wDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYwMzEwMDIzNTM4WhcNMzYw +MzA1MDIzNjA4WjAWMRQwEgYDVQQDEwtleGFtcGxlLmNvbTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAPTQGWPRIOECGeJB6tR/ftvvtioC9f84fY2QdJ5k +JBupXjPAGYKgS4MGzyT5bz9yY400tCtmh6h7p9tZwHl/TElTugtLQ/8ilMbJTiOM +SiyaMDPHiMJJYKTjm9bu6bKeU1qPZ0Cryes4rygbqs7w2XPgA2RxNmDh7JdX7/h+ +VB5onBmv8g4WFSayowGyDcJWWCbu5yv6ZdH1bqQjgRzQ5xp17WXNmvlzdp2vate/ +9UqPdA8sdJzW/91Gvmros0o/FnG7c2pULhk22wFqO8t2HRjKb3nuxALEJvqoPvad +KjpDTaq1L1ZzxcB7wvWyhy/lNLZL7jiNWy0mN1YB0UpSWdECAwEAAaN7MHkwDgYD +VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFHMM2+oX9Orb +U6BazXcHljJ1mOW/MB8GA1UdIwQYMBaAFHMM2+oX9OrbU6BazXcHljJ1mOW/MBYG +A1UdEQQPMA2CC2V4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQAp17XsOaT9 +hculRqrFptn3+zkH3HrIckHm+28R5xYT8ASFXFcLFugGizJAXVL5lvsRVRIwCoOX +Nhi8XSNEFP640VbHcEl81I84bbRIIDS+Yheu6JDZGemTaDYLv1J3D5SHwgoM+nyf +oTRgotUCIXcwJHmTpWEUkZFKuqBxsoTGzk0jO8wOP6xoJkzxVVG5PvNxs924rxY8 +Y8iaLdDfMeT7Pi0XIliBa/aSp/iqSW8XKyJl5R5vXg9+DOgZUrVzIxObaF5RBl/a +mJOeklJBdNVzQm5+iMpO42lu0TA9eWtpP+YiUEXU17XDvFeQWOocFbQ1Peo0W895 +XRz2GCwCNyvW +-----END CERTIFICATE----- diff --git a/command/agent/test-fixtures/reload/reload_foo.key b/command/agent/test-fixtures/reload/reload_foo.key new file mode 100644 index 000000000..86e6cce63 --- /dev/null +++ b/command/agent/test-fixtures/reload/reload_foo.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEAzNyVieSti9XBb5/celB5u8YKRJv3mQS9A4/X0mqY1ePznt1i +ilG7OmG0yM2VAk0ceIAQac3Bsn74jxn2cDlrrVniPXcNgYtMtW0kRqNEo4doo4EX +xZguS9vNBu29useHhif1TGX/pA3dgvaVycUCjzTEVk6qI8UEehMK6gEGZb7nOr0A +A9nipSqoeHpDLe3a4KVqj1vtlJKUvD2i1MuBuQ130cB1K9rufLCShGu7mEgzEosc +gr+K3Bf03IejbeVRyIfLtgj1zuvV1katec75UqRA/bsvt5G9JfJqiZ9mwFN0vp3g +Cr7pdQBSBQ2q4yf9s8CuY5c5w9fl3F8f5QFQoQIDAQABAoIBAQCbCb1qNFRa5ZSV +I8i6ELlwMDqJHfhOJ9XcIjpVljLAfNlcu3Ld92jYkCU/asaAjVckotbJG9yhd5Io +yp9E40/oS4P6vGTOS1vsWgMAKoPBtrKsOwCAm+E9q8UIn1fdSS/5ibgM74x+3bds +a62Em8KKGocUQkhk9a+jq1GxMsFisbHRxEHvClLmDMgGnW3FyGmWwT6yZLPSC0ey +szmmjt3ouP8cLAOmSjzcQBMmEZpQMCgR6Qckg6nrLQAGzZyTdCd875wbGA57DpWX +Lssn95+A5EFvr/6b7DkXeIFCrYBFFa+UQN3PWGEQ6Zjmiw4VgV2vO8yX2kCLlUhU +02bL393ZAoGBAPXPD/0yWINbKUPcRlx/WfWQxfz0bu50ytwIXzVK+pRoAMuNqehK +BJ6kNzTTBq40u+IZ4f5jbLDulymR+4zSkirLE7CyWFJOLNI/8K4Pf5DJUgNdrZjJ +LCtP9XRdxiPatQF0NGfdgHlSJh+/CiRJP4AgB17AnB/4z9/M0ZlJGVrzAoGBANVa +69P3Rp/WPBQv0wx6f0tWppJolWekAHKcDIdQ5HdOZE5CPAYSlTrTUW3uJuqMwU2L +M0Er2gIPKWIR5X+9r7Fvu9hQW6l2v3xLlcrGPiapp3STJvuMxzhRAmXmu3bZfVn1 +Vn7Vf1jPULHtTFSlNFEvYG5UJmygK9BeyyVO5KMbAoGBAMCyAibLQPg4jrDUDZSV +gUAwrgUO2ae1hxHWvkxY6vdMUNNByuB+pgB3W4/dnm8Sh/dHsxJpftt1Lqs39ar/ +p/ZEHLt4FCTxg9GOrm7FV4t5RwG8fko36phJpnIC0UFqQltRbYO+8OgqrhhU+u5X +PaCDe0OcWsf1lYAsYGN6GpZhAoGBAMJ5Ksa9+YEODRs1cIFKUyd/5ztC2xRqOAI/ +3WemQ2nAacuvsfizDZVeMzYpww0+maAuBt0btI719PmwaGmkpDXvK+EDdlmkpOwO +FY6MXvBs6fdnfjwCWUErDi2GQFAX9Jt/9oSL5JU1+08DhvUM1QA/V/2Y9KFE6kr3 +bOIn5F4LAoGBAKQzH/AThDGhT3hwr4ktmReF3qKxBgxzjVa8veXtkY5VWwyN09iT +jnTTt6N1CchZoK5WCETjdzNYP7cuBTcV4d3bPNRiJmxXaNVvx3Tlrk98OiffT8Qa +5DO/Wfb43rNHYXBjU6l0n2zWcQ4PUSSbu0P0bM2JTQPRCqSthXvSHw2P +-----END RSA PRIVATE KEY----- diff --git a/command/agent/test-fixtures/reload/reload_foo.pem b/command/agent/test-fixtures/reload/reload_foo.pem new file mode 100644 index 000000000..c8b868bcd --- /dev/null +++ b/command/agent/test-fixtures/reload/reload_foo.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQzCCAiugAwIBAgIUFVW6i/M+yJUsDrXWgRKO/Dnb+L4wDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYwMzEwMDIzNjA1WhcNMzYw +MzA1MDEzNjM1WjAaMRgwFgYDVQQDEw9mb28uZXhhbXBsZS5jb20wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDM3JWJ5K2L1cFvn9x6UHm7xgpEm/eZBL0D +j9fSapjV4/Oe3WKKUbs6YbTIzZUCTRx4gBBpzcGyfviPGfZwOWutWeI9dw2Bi0y1 +bSRGo0Sjh2ijgRfFmC5L280G7b26x4eGJ/VMZf+kDd2C9pXJxQKPNMRWTqojxQR6 +EwrqAQZlvuc6vQAD2eKlKqh4ekMt7drgpWqPW+2UkpS8PaLUy4G5DXfRwHUr2u58 +sJKEa7uYSDMSixyCv4rcF/Tch6Nt5VHIh8u2CPXO69XWRq15zvlSpED9uy+3kb0l +8mqJn2bAU3S+neAKvul1AFIFDarjJ/2zwK5jlznD1+XcXx/lAVChAgMBAAGjgYQw +gYEwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBRNJoOJ +dnazDiuqLhV6truQ4cRe9jAfBgNVHSMEGDAWgBRzDNvqF/Tq21OgWs13B5YydZjl +vzAgBgNVHREEGTAXgg9mb28uZXhhbXBsZS5jb22HBH8AAAEwDQYJKoZIhvcNAQEL +BQADggEBAHzv67mtbxMWcuMsxCFBN1PJNAyUDZVCB+1gWhk59EySbVg81hWJDCBy +fl3TKjz3i7wBGAv+C2iTxmwsSJbda22v8JQbuscXIfLFbNALsPzF+J0vxAgJs5Gc +sDbfJ7EQOIIOVKQhHLYnQoLnigSSPc1kd0JjYyHEBjgIaSuXgRRTBAeqLiBMx0yh +RKL1lQ+WoBU/9SXUZZkwokqWt5G7khi5qZkNxVXZCm8VGPg0iywf6gGyhI1SU5S2 +oR219S6kA4JY/stw1qne85/EmHmoImHGt08xex3GoU72jKAjsIpqRWopcD/+uene +Tc9nn3fTQW/Z9fsoJ5iF5OdJnDEswqE= +-----END CERTIFICATE----- diff --git a/command/agent_test.go b/command/agent_test.go index 13dc17cd5..8b918a6e6 100644 --- a/command/agent_test.go +++ b/command/agent_test.go @@ -1,6 +1,8 @@ package command import ( + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io/ioutil" @@ -14,7 +16,7 @@ import ( "testing" "time" - hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-hclog" vaultjwt "github.com/hashicorp/vault-plugin-auth-jwt" logicalKv "github.com/hashicorp/vault-plugin-secrets-kv" "github.com/hashicorp/vault/api" @@ -34,7 +36,8 @@ import ( const ( BasicHclConfig = ` -log_file = "/foo/bar/juan.log" +log_file = "TMPDIR/juan.log" +log_level="warn" vault { address = "http://127.0.0.1:8200" retry { @@ -44,7 +47,25 @@ vault { listener "tcp" { address = "127.0.0.1:8100" - tls_disable = true + tls_disable = false + tls_cert_file = "TMPDIR/reload_cert.pem" + tls_key_file = "TMPDIR/reload_key.pem" +}` + BasicHclConfig2 = ` +log_file = "TMPDIR/juan.log" +log_level="debug" +vault { + address = "http://127.0.0.1:8200" + retry { + num_retries = 5 + } +} + +listener "tcp" { + address = "127.0.0.1:8100" + tls_disable = false + tls_cert_file = "TMPDIR/reload_cert.pem" + tls_key_file = "TMPDIR/reload_key.pem" }` ) @@ -57,7 +78,10 @@ func testAgentCommand(tb testing.TB, logger hclog.Logger) (*cli.MockUi, *AgentCo UI: ui, }, ShutdownCh: MakeShutdownCh(), + SighupCh: MakeSighupCh(), logger: logger, + startedCh: make(chan struct{}, 5), + reloadedCh: make(chan struct{}, 5), } } @@ -1512,7 +1536,6 @@ vault { %s %s `, serverClient.Address(), retryConf, cacheConfig, listenConfig) - configPath := makeTempFile(t, "config.hcl", config) defer os.Remove(configPath) @@ -2075,7 +2098,7 @@ func TestAgent_LogFile_CliOverridesConfig(t *testing.T) { } // Sanity check that the config value is the current value - assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile) + assert.Equal(t, "TMPDIR/juan.log", cfg.LogFile) // Initialize the command and parse any flags cmd := &AgentCommand{BaseCommand: &BaseCommand{}} @@ -2089,7 +2112,7 @@ func TestAgent_LogFile_CliOverridesConfig(t *testing.T) { // Update the config based on the inputs. cmd.updateConfig(f, cfg) - assert.NotEqual(t, "/foo/bar/juan.log", cfg.LogFile) + assert.NotEqual(t, "TMPDIR/juan.log", cfg.LogFile) assert.NotEqual(t, "/squiggle/logs.txt", cfg.LogFile) assert.Equal(t, "/foo/bar/test.log", cfg.LogFile) } @@ -2103,7 +2126,7 @@ func TestAgent_LogFile_Config(t *testing.T) { } // Sanity check that the config value is the current value - assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile, "sanity check on log config failed") + assert.Equal(t, "TMPDIR/juan.log", cfg.LogFile, "sanity check on log config failed") // Parse the cli flags (but we pass in an empty slice) cmd := &AgentCommand{BaseCommand: &BaseCommand{}} @@ -2115,7 +2138,180 @@ func TestAgent_LogFile_Config(t *testing.T) { cmd.updateConfig(f, cfg) - assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile, "actual config check") + assert.Equal(t, "TMPDIR/juan.log", cfg.LogFile, "actual config check") +} + +func TestAgent_Config_NewLogger_Default(t *testing.T) { + cmd := &AgentCommand{BaseCommand: &BaseCommand{}} + cmd.config = agentConfig.NewConfig() + logger, err := cmd.newLogger() + + assert.NoError(t, err) + assert.NotNil(t, logger) + assert.Equal(t, hclog.Info.String(), logger.GetLevel().String()) +} + +func TestAgent_Config_ReloadLogLevel(t *testing.T) { + cmd := &AgentCommand{BaseCommand: &BaseCommand{}} + var err error + tempDir := t.TempDir() + + // Load an initial config + hcl := strings.ReplaceAll(BasicHclConfig, "TMPDIR", tempDir) + configFile := populateTempFile(t, "agent-config.hcl", hcl) + cmd.config, err = agentConfig.LoadConfigFile(configFile.Name()) + if err != nil { + t.Fatal("Cannot load config to test update/merge", err) + } + + // Tweak the loaded config to make sure we can put log files into a temp dir + // and systemd log attempts work fine, this would usually happen during Run. + cmd.logWriter = os.Stdout + cmd.logger, err = cmd.newLogger() + if err != nil { + t.Fatal("logger required for systemd log messages", err) + } + + // Sanity check + assert.Equal(t, "warn", cmd.config.LogLevel) + + // Load a new config + hcl = strings.ReplaceAll(BasicHclConfig2, "TMPDIR", tempDir) + configFile = populateTempFile(t, "agent-config.hcl", hcl) + err = cmd.reloadConfig([]string{configFile.Name()}) + assert.NoError(t, err) + assert.Equal(t, "debug", cmd.config.LogLevel) +} + +func TestAgent_Config_ReloadTls(t *testing.T) { + var wg sync.WaitGroup + wd, err := os.Getwd() + if err != nil { + t.Fatal("unable to get current working directory") + } + workingDir := filepath.Join(wd, "/agent/test-fixtures/reload") + fooCert := "reload_foo.pem" + fooKey := "reload_foo.key" + + barCert := "reload_bar.pem" + barKey := "reload_bar.key" + + reloadCert := "reload_cert.pem" + reloadKey := "reload_key.pem" + caPem := "reload_ca.pem" + + tempDir := t.TempDir() + + // Set up initial 'foo' certs + inBytes, err := os.ReadFile(filepath.Join(workingDir, fooCert)) + if err != nil { + t.Fatal("unable to read cert required for test", fooCert, err) + } + err = os.WriteFile(filepath.Join(tempDir, reloadCert), inBytes, 0o777) + if err != nil { + t.Fatal("unable to write temp cert required for test", reloadCert, err) + } + + inBytes, err = os.ReadFile(filepath.Join(workingDir, fooKey)) + if err != nil { + t.Fatal("unable to read cert key required for test", fooKey, err) + } + err = os.WriteFile(filepath.Join(tempDir, reloadKey), inBytes, 0o777) + if err != nil { + t.Fatal("unable to write temp cert key required for test", reloadKey, err) + } + + inBytes, err = os.ReadFile(filepath.Join(workingDir, caPem)) + if err != nil { + t.Fatal("unable to read CA pem required for test", caPem, err) + } + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(inBytes) + if !ok { + t.Fatal("not ok when appending CA cert") + } + + replacedHcl := strings.ReplaceAll(BasicHclConfig, "TMPDIR", tempDir) + configFile := populateTempFile(t, "agent-config.hcl", replacedHcl) + + // Set up Agent/cmd + logger := logging.NewVaultLogger(hclog.Trace) + ui, cmd := testAgentCommand(t, logger) + + wg.Add(1) + args := []string{"-config", configFile.Name()} + go func() { + if code := cmd.Run(args); code != 0 { + output := ui.ErrorWriter.String() + ui.OutputWriter.String() + t.Errorf("got a non-zero exit status: %s", output) + } + wg.Done() + }() + + testCertificateName := func(cn string) error { + conn, err := tls.Dial("tcp", "127.0.0.1:8100", &tls.Config{ + RootCAs: certPool, + }) + if err != nil { + return err + } + defer conn.Close() + if err = conn.Handshake(); err != nil { + return err + } + servName := conn.ConnectionState().PeerCertificates[0].Subject.CommonName + if servName != cn { + return fmt.Errorf("expected %s, got %s", cn, servName) + } + return nil + } + + // Start + select { + case <-cmd.startedCh: + case <-time.After(5 * time.Second): + t.Fatalf("timeout") + } + + if err := testCertificateName("foo.example.com"); err != nil { + t.Fatalf("certificate name didn't check out: %s", err) + } + + // Swap out certs + inBytes, err = os.ReadFile(filepath.Join(workingDir, barCert)) + if err != nil { + t.Fatal("unable to read cert required for test", barCert, err) + } + err = os.WriteFile(filepath.Join(tempDir, reloadCert), inBytes, 0o777) + if err != nil { + t.Fatal("unable to write temp cert required for test", reloadCert, err) + } + + inBytes, err = os.ReadFile(filepath.Join(workingDir, barKey)) + if err != nil { + t.Fatal("unable to read cert key required for test", barKey, err) + } + err = os.WriteFile(filepath.Join(tempDir, reloadKey), inBytes, 0o777) + if err != nil { + t.Fatal("unable to write temp cert key required for test", reloadKey, err) + } + + // Reload + cmd.SighupCh <- struct{}{} + select { + case <-cmd.reloadedCh: + case <-time.After(5 * time.Second): + t.Fatalf("timeout") + } + + if err := testCertificateName("bar.example.com"); err != nil { + t.Fatalf("certificate name didn't check out: %s", err) + } + + // Shut down + cmd.ShutdownCh <- struct{}{} + + wg.Wait() } // Get a randomly assigned port and then free it again before returning it. diff --git a/command/commands.go b/command/commands.go index 9327ae72c..d11388e98 100644 --- a/command/commands.go +++ b/command/commands.go @@ -259,6 +259,7 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { UI: serverCmdUi, }, ShutdownCh: MakeShutdownCh(), + SighupCh: MakeSighupCh(), }, nil }, "audit": func() (cli.Command, error) { diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index e0156baba..f1b6beca1 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -186,6 +186,9 @@ These are the currently-available general configuration options: - `listener` ([listener][listener]: ) - Specifies the addresses and ports on which the Agent will respond to requests. + ~> **Note:** On `SIGHUP` (`kill -SIGHUP $(pidof vault)`), Vault Agent will attempt to reload listener TLS configuration. + This method can be used to refresh certificates used by Vault Agent without having to restart its process. + - `pid_file` `(string: "")` - Path to the file in which the agent's Process ID (PID) should be stored @@ -213,6 +216,9 @@ These are the currently-available general configuration options: - `log_level` - Equivalent to the [`-log-level` command-line flag](#_log_level). + ~> **Note:** On `SIGHUP` (`kill -SIGHUP $(pidof vault)`), Vault Agent will update the log level to the value + specified by configuration file (including overriding values set using CLI or environment variable parameters). + - `log_format` - Equivalent to the [`-log-format` command-line flag](#_log_format). - `log_file` - Equivalent to the [`-log-file` command-line flag](#_log_file).