VAULT-9900: Log rotation for 'agent' and 'server' commands (#18031)

* Work to unify log-file for agent/server and add rotation
* Updates to rotation code, tried to centralise the log config setup
* logging + tests
* Move LogFile to ShareConfig in test
* Docs
This commit is contained in:
Peter Wilson 2022-11-29 14:07:04 +00:00 committed by GitHub
parent 08e89a7e9e
commit 33e6a3a87c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 916 additions and 400 deletions

3
changelog/18031.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:feature
logging: Vault agent and server commands support log file and log rotation.
```

View File

@ -19,6 +19,7 @@ import (
systemd "github.com/coreos/go-systemd/daemon"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-secure-stdlib/gatedwriter"
"github.com/hashicorp/go-secure-stdlib/parseutil"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/command/agent/auth"
"github.com/hashicorp/vault/command/agent/auth/alicloud"
@ -69,6 +70,7 @@ const (
type AgentCommand struct {
*BaseCommand
logFlags logFlags
ShutdownCh chan struct{}
SighupCh chan struct{}
@ -84,13 +86,9 @@ type AgentCommand struct {
startedCh chan (struct{}) // for tests
flagConfigs []string
flagLogLevel string
flagLogFile string
flagExitAfterAuth bool
flagConfigs []string
flagExitAfterAuth bool
flagTestVerifyOnly bool
flagCombineLogs bool
}
func (c *AgentCommand) Synopsis() string {
@ -119,6 +117,9 @@ func (c *AgentCommand) Flags() *FlagSets {
f := set.NewFlagSet("Command Options")
// Augment with the log flags
f.addLogFlags(&c.logFlags)
f.StringSliceVar(&StringSliceVar{
Name: "config",
Target: &c.flagConfigs,
@ -130,23 +131,6 @@ func (c *AgentCommand) Flags() *FlagSets {
"contain only agent directives.",
})
f.StringVar(&StringVar{
Name: flagNameLogLevel,
Target: &c.flagLogLevel,
Default: "info",
EnvVar: EnvVaultLogLevel,
Completion: complete.PredictSet("trace", "debug", "info", "warn", "error"),
Usage: "Log verbosity level. Supported values (in order of detail) are " +
"\"trace\", \"debug\", \"info\", \"warn\", and \"error\".",
})
f.StringVar(&StringVar{
Name: flagNameLogFile,
Target: &c.flagLogFile,
EnvVar: EnvVaultLogFile,
Usage: "Path to the log file that Vault should use for logging",
})
f.BoolVar(&BoolVar{
Name: flagNameAgentExitAfterAuth,
Target: &c.flagExitAfterAuth,
@ -163,15 +147,6 @@ func (c *AgentCommand) Flags() *FlagSets {
// no warranty or backwards-compatibility promise. Do not use these flags
// in production. Do not build automation using these flags. Unless you are
// developing against Vault, you should not need any of these flags.
// TODO: should the below flags be public?
f.BoolVar(&BoolVar{
Name: "combine-logs",
Target: &c.flagCombineLogs,
Default: false,
Hidden: true,
})
f.BoolVar(&BoolVar{
Name: "test-verify-only",
Target: &c.flagTestVerifyOnly,
@ -204,7 +179,8 @@ func (c *AgentCommand) Run(args []string) int {
// start logging too early.
c.logGate = gatedwriter.NewWriter(os.Stderr)
c.logWriter = c.logGate
if c.flagCombineLogs {
if c.logFlags.flagCombineLogs {
c.logWriter = os.Stdout
}
@ -237,9 +213,9 @@ func (c *AgentCommand) Run(args []string) int {
c.UI.Info("No auto_auth block found in config file, not starting automatic authentication feature")
}
config = c.aggregateConfig(f, config)
c.updateConfig(f, config)
// Build the logger using level, format and path
// Parse all the log related config
logLevel, err := logging.ParseLogLevel(config.LogLevel)
if err != nil {
c.UI.Error(err.Error())
@ -252,7 +228,34 @@ func (c *AgentCommand) Run(args []string) int {
return 1
}
logCfg := logging.NewLogConfig("agent", logLevel, logFormat, config.LogFile)
logRotateDuration, err := parseutil.ParseDurationSecond(config.LogRotateDuration)
if err != nil {
c.UI.Error(err.Error())
return 1
}
logRotateBytes, err := parseutil.ParseInt(config.LogRotateBytes)
if err != nil {
c.UI.Error(err.Error())
return 1
}
logRotateMaxFiles, err := parseutil.ParseInt(config.LogRotateMaxFiles)
if err != nil {
c.UI.Error(err.Error())
return 1
}
logCfg := &logging.LogConfig{
Name: "vault-agent",
LogLevel: logLevel,
LogFormat: logFormat,
LogFilePath: config.LogFile,
LogRotateDuration: logRotateDuration,
LogRotateBytes: int(logRotateBytes),
LogRotateMaxFiles: int(logRotateMaxFiles),
}
l, err := logging.Setup(logCfg, c.logWriter)
if err != nil {
c.UI.Error(err.Error())
@ -263,7 +266,7 @@ func (c *AgentCommand) Run(args []string) int {
infoKeys := make([]string, 0, 10)
info := make(map[string]string)
info["log level"] = c.flagLogLevel
info["log level"] = config.LogLevel
infoKeys = append(infoKeys, "log level")
infoKeys = append(infoKeys, "version")
@ -457,7 +460,7 @@ func (c *AgentCommand) Run(args []string) int {
}
// Output the header that the agent has started
if !c.flagCombineLogs {
if !c.logFlags.flagCombineLogs {
c.UI.Output("==> Vault agent started! Log data will stream in below:\n")
}
@ -926,31 +929,19 @@ func (c *AgentCommand) Run(args []string) int {
return 0
}
// aggregateConfig ensures that the config object accurately reflects the desired
// updateConfig ensures that the config object accurately reflects the desired
// settings as configured by the user. It applies the relevant config setting based
// on the precedence (env var overrides file config, cli overrides env var).
// It mutates the config object supplied and returns the updated object.
func (c *AgentCommand) aggregateConfig(f *FlagSets, config *agentConfig.Config) *agentConfig.Config {
// It mutates the config object supplied.
func (c *AgentCommand) updateConfig(f *FlagSets, config *agentConfig.Config) {
f.updateLogConfig(config.SharedConfig)
f.Visit(func(fl *flag.Flag) {
if fl.Name == flagNameAgentExitAfterAuth {
config.ExitAfterAuth = c.flagExitAfterAuth
}
})
c.setStringFlag(f, config.LogFile, &StringVar{
Name: flagNameLogFile,
EnvVar: EnvVaultLogFile,
Target: &c.flagLogFile,
})
config.LogFile = c.flagLogFile
c.setStringFlag(f, config.LogLevel, &StringVar{
Name: flagNameLogLevel,
EnvVar: EnvVaultLogLevel,
Target: &c.flagLogLevel,
})
config.LogLevel = c.flagLogLevel
c.setStringFlag(f, config.Vault.Address, &StringVar{
Name: flagNameAddress,
Target: &c.flagAddress,
@ -1000,8 +991,6 @@ func (c *AgentCommand) aggregateConfig(f *FlagSets, config *agentConfig.Config)
EnvVar: api.EnvVaultTLSServerName,
})
config.Vault.TLSServerName = c.flagTLSServerName
return config
}
// verifyRequestHeader wraps an http.Handler inside a Handler that checks for

View File

@ -37,7 +37,6 @@ type Config struct {
DisableKeepAlivesCaching bool `hcl:"-"`
DisableKeepAlivesTemplating bool `hcl:"-"`
DisableKeepAlivesAutoAuth bool `hcl:"-"`
LogFile string `hcl:"log_file"`
}
const (

View File

@ -198,6 +198,7 @@ func TestLoadConfigFile(t *testing.T) {
expected := &Config{
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
LogFile: "/var/log/vault/vault-agent.log",
},
AutoAuth: &AutoAuth{
Method: &Method{
@ -237,7 +238,6 @@ func TestLoadConfigFile(t *testing.T) {
NumRetries: 12,
},
},
LogFile: "/var/log/vault/vault-agent.log",
}
config.Prune()

View File

@ -2250,7 +2250,7 @@ cache {}
wg.Wait()
}
func TestAgent_LogFile_EnvVarOverridesConfig(t *testing.T) {
func TestAgent_LogFile_CliOverridesConfig(t *testing.T) {
// Create basic config
configFile := populateTempFile(t, "agent-config.hcl", BasicHclConfig)
cfg, err := agentConfig.LoadConfig(configFile.Name())
@ -2261,50 +2261,6 @@ func TestAgent_LogFile_EnvVarOverridesConfig(t *testing.T) {
// Sanity check that the config value is the current value
assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile)
// Make sure the env var is configured
oldEnvVarLogFile := os.Getenv(EnvVaultLogFile)
os.Setenv(EnvVaultLogFile, "/squiggle/logs.txt")
if oldEnvVarLogFile == "" {
defer os.Unsetenv(EnvVaultLogFile)
} else {
defer os.Setenv(EnvVaultLogFile, oldEnvVarLogFile)
}
// Initialize the command and parse any flags
cmd := &AgentCommand{BaseCommand: &BaseCommand{}}
f := cmd.Flags()
err = f.Parse([]string{})
if err != nil {
t.Fatal(err)
}
// Update the config based on the inputs.
cfg = cmd.aggregateConfig(f, cfg)
assert.NotEqual(t, "/foo/bar/juan.log", cfg.LogFile)
assert.Equal(t, "/squiggle/logs.txt", cfg.LogFile)
}
func TestAgent_LogFile_CliOverridesEnvVar(t *testing.T) {
// Create basic config
configFile := populateTempFile(t, "agent-config.hcl", BasicHclConfig)
cfg, err := agentConfig.LoadConfig(configFile.Name())
if err != nil {
t.Fatal("Cannot load config to test update/merge", err)
}
// Sanity check that the config value is the current value
assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile)
// Make sure the env var is configured
oldEnvVarLogFile := os.Getenv(EnvVaultLogFile)
os.Setenv(EnvVaultLogFile, "/squiggle/logs.txt")
if oldEnvVarLogFile == "" {
defer os.Unsetenv(EnvVaultLogFile)
} else {
defer os.Setenv(EnvVaultLogFile, oldEnvVarLogFile)
}
// Initialize the command and parse any flags
cmd := &AgentCommand{BaseCommand: &BaseCommand{}}
f := cmd.Flags()
@ -2315,7 +2271,7 @@ func TestAgent_LogFile_CliOverridesEnvVar(t *testing.T) {
}
// Update the config based on the inputs.
cfg = cmd.aggregateConfig(f, cfg)
cmd.updateConfig(f, cfg)
assert.NotEqual(t, "/foo/bar/juan.log", cfg.LogFile)
assert.NotEqual(t, "/squiggle/logs.txt", cfg.LogFile)
@ -2323,9 +2279,6 @@ func TestAgent_LogFile_CliOverridesEnvVar(t *testing.T) {
}
func TestAgent_LogFile_Config(t *testing.T) {
// Sanity check, remove any env var
os.Unsetenv(EnvVaultLogFile)
configFile := populateTempFile(t, "agent-config.hcl", BasicHclConfig)
cfg, err := agentConfig.LoadConfig(configFile.Name())
@ -2344,7 +2297,7 @@ func TestAgent_LogFile_Config(t *testing.T) {
t.Fatal(err)
}
cfg = cmd.aggregateConfig(f, cfg)
cmd.updateConfig(f, cfg)
assert.Equal(t, "/foo/bar/juan.log", cfg.LogFile, "actual config check")
}

View File

@ -82,8 +82,8 @@ const (
EnvVaultLicensePath = "VAULT_LICENSE_PATH"
// EnvVaultDetailed is to output detailed information (e.g., ListResponseWithInfo).
EnvVaultDetailed = `VAULT_DETAILED`
// EnvVaultLogFile is used to specify the path to the log file that Vault should use for logging
EnvVaultLogFile = "VAULT_LOG_FILE"
// EnvVaultLogFormat is used to specify the log format. Supported values are "standard" and "json"
EnvVaultLogFormat = "VAULT_LOG_FORMAT"
// EnvVaultLogLevel is used to specify the log level applied to logging
// Supported log levels: Trace, Debug, Error, Warn, Info
EnvVaultLogLevel = "VAULT_LOG_LEVEL"
@ -141,8 +141,18 @@ const (
flagNameUserLockoutDisable = "user-lockout-disable"
// flagNameDisableRedirects is used to prevent the client from honoring a single redirect as a response to a request
flagNameDisableRedirects = "disable-redirects"
// flagNameCombineLogs is used to specify whether log output should be combined and sent to stdout
flagNameCombineLogs = "combine-logs"
// flagNameLogFile is used to specify the path to the log file that Vault should use for logging
flagNameLogFile = "log-file"
// flagNameLogRotateBytes is the flag used to specify the number of bytes a log file should be before it is rotated.
flagNameLogRotateBytes = "log-rotate-bytes"
// flagNameLogRotateDuration is the flag used to specify the duration after which a log file should be rotated.
flagNameLogRotateDuration = "log-rotate-duration"
// flagNameLogRotateMaxFiles is the flag used to specify the maximum number of older/archived log files to keep.
flagNameLogRotateMaxFiles = "log-rotate-max-files"
// flagNameLogFormat is the flag used to specify the log format. Supported values are "standard" and "json"
flagNameLogFormat = "log-format"
// flagNameLogLevel is used to specify the log level applied to logging
// Supported log levels: Trace, Debug, Error, Warn, Info
flagNameLogLevel = "log-level"

154
command/log_flags.go Normal file
View File

@ -0,0 +1,154 @@
package command
import (
"flag"
"os"
"strings"
"github.com/hashicorp/vault/internalshared/configutil"
"github.com/posener/complete"
)
// logFlags are the 'log' related flags that can be shared across commands.
type logFlags struct {
flagCombineLogs bool
flagLogLevel string
flagLogFormat string
flagLogFile string
flagLogRotateBytes string
flagLogRotateDuration string
flagLogRotateMaxFiles string
}
type provider = func(key string) (string, bool)
// valuesProvider has the intention of providing a way to supply a func with a
// way to retrieve values for flags and environment variables without having to
// directly call a specific implementation. The reasoning for its existence is
// to facilitate testing.
type valuesProvider struct {
flagProvider provider
envVarProvider provider
}
// addLogFlags will add the set of 'log' related flags to a flag set.
func (f *FlagSet) addLogFlags(l *logFlags) {
f.BoolVar(&BoolVar{
Name: flagNameCombineLogs,
Target: &l.flagCombineLogs,
Default: false,
Hidden: true,
})
f.StringVar(&StringVar{
Name: flagNameLogLevel,
Target: &l.flagLogLevel,
Default: notSetValue,
EnvVar: EnvVaultLogLevel,
Completion: complete.PredictSet("trace", "debug", "info", "warn", "error"),
Usage: "Log verbosity level. Supported values (in order of detail) are " +
"\"trace\", \"debug\", \"info\", \"warn\", and \"error\".",
})
f.StringVar(&StringVar{
Name: flagNameLogFormat,
Target: &l.flagLogFormat,
Default: notSetValue,
EnvVar: EnvVaultLogFormat,
Completion: complete.PredictSet("standard", "json"),
Usage: `Log format. Supported values are "standard" and "json".`,
})
f.StringVar(&StringVar{
Name: flagNameLogFile,
Target: &l.flagLogFile,
Usage: "Path to the log file that Vault should use for logging",
})
f.StringVar(&StringVar{
Name: flagNameLogRotateBytes,
Target: &l.flagLogRotateBytes,
Usage: "Number of bytes that should be written to a log before it needs to be rotated. " +
"Unless specified, there is no limit to the number of bytes that can be written to a log file",
})
f.StringVar(&StringVar{
Name: flagNameLogRotateDuration,
Target: &l.flagLogRotateDuration,
Usage: "The maximum duration a log should be written to before it needs to be rotated. " +
"Must be a duration value such as 30s",
})
f.StringVar(&StringVar{
Name: flagNameLogRotateMaxFiles,
Target: &l.flagLogRotateMaxFiles,
Usage: "The maximum number of older log file archives to keep",
})
}
// getValue will attempt to find the flag with the corresponding flag name (key)
// and return the value along with a bool representing whether of not the flag had been found/set.
func (f *FlagSets) getValue(flagName string) (string, bool) {
var result string
var isFlagSpecified bool
if f != nil {
f.Visit(func(fl *flag.Flag) {
if fl.Name == flagName {
result = fl.Value.String()
isFlagSpecified = true
}
})
}
return result, isFlagSpecified
}
// getAggregatedConfigValue uses the provided keys to check CLI flags and environment
// variables for values that may be used to override any specified configuration.
// If nothing can be found in flags/env vars or config, the 'fallback' (default) value will be provided.
func (p *valuesProvider) getAggregatedConfigValue(flagKey, envVarKey, current, fallback string) string {
var result string
current = strings.TrimSpace(current)
flg, flgFound := p.flagProvider(flagKey)
env, envFound := p.envVarProvider(envVarKey)
switch {
case flgFound:
result = flg
case envFound:
// Use value from env var
result = env
case current != "":
// Use value from config
result = current
default:
// Use the default value
result = fallback
}
return result
}
// updateLogConfig will accept a shared config and specifically attempt to update the 'log' related config keys.
// For each 'log' key we aggregate file config/env vars and CLI flags to select the one with the highest precedence.
// This method mutates the config object passed into it.
func (f *FlagSets) updateLogConfig(config *configutil.SharedConfig) {
p := &valuesProvider{
flagProvider: func(key string) (string, bool) { return f.getValue(key) },
envVarProvider: func(key string) (string, bool) {
if key == "" {
return "", false
}
return os.LookupEnv(key)
},
}
config.LogLevel = p.getAggregatedConfigValue(flagNameLogLevel, EnvVaultLogLevel, config.LogLevel, "info")
config.LogFormat = p.getAggregatedConfigValue(flagNameLogFormat, EnvVaultLogFormat, config.LogFormat, "")
config.LogFile = p.getAggregatedConfigValue(flagNameLogFile, "", config.LogFile, "")
config.LogRotateDuration = p.getAggregatedConfigValue(flagNameLogRotateDuration, "", config.LogRotateDuration, "")
config.LogRotateBytes = p.getAggregatedConfigValue(flagNameLogRotateBytes, "", config.LogRotateBytes, "")
config.LogRotateMaxFiles = p.getAggregatedConfigValue(flagNameLogRotateMaxFiles, "", config.LogRotateMaxFiles, "")
}

75
command/log_flags_test.go Normal file
View File

@ -0,0 +1,75 @@
package command
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestLogFlags_ValuesProvider(t *testing.T) {
cases := map[string]struct {
flagKey string
envVarKey string
current string
fallback string
want string
}{
"only-fallback": {
flagKey: "invalid",
envVarKey: "invalid",
current: "",
fallback: "foo",
want: "foo",
},
"only-config": {
flagKey: "invalid",
envVarKey: "invalid",
current: "bar",
fallback: "",
want: "bar",
},
"flag-missing": {
flagKey: "invalid",
envVarKey: "valid-env-var",
current: "my-config-value1",
fallback: "",
want: "envVarValue",
},
"envVar-missing": {
flagKey: "valid-flag",
envVarKey: "invalid",
current: "my-config-value1",
fallback: "",
want: "flagValue",
},
"all-present": {
flagKey: "valid-flag",
envVarKey: "valid-env-var",
current: "my-config-value1",
fallback: "foo",
want: "flagValue",
},
}
// Sneaky little fake provider
fakeProvider := func(key string) (string, bool) {
switch key {
case "valid-flag":
return "flagValue", true
case "valid-env-var":
return "envVarValue", true
}
return "", false
}
vp := valuesProvider{
flagProvider: fakeProvider,
envVarProvider: fakeProvider,
}
for _, tc := range cases {
got := vp.getAggregatedConfigValue(tc.flagKey, tc.envVarKey, tc.current, tc.fallback)
assert.Equal(t, tc.want, got)
}
}

View File

@ -29,12 +29,14 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-secure-stdlib/gatedwriter"
"github.com/hashicorp/go-secure-stdlib/mlock"
"github.com/hashicorp/go-secure-stdlib/parseutil"
"github.com/hashicorp/go-secure-stdlib/reloadutil"
"github.com/hashicorp/vault/audit"
config2 "github.com/hashicorp/vault/command/config"
"github.com/hashicorp/vault/command/server"
"github.com/hashicorp/vault/helper/builtinplugins"
"github.com/hashicorp/vault/helper/constants"
loghelper "github.com/hashicorp/vault/helper/logging"
"github.com/hashicorp/vault/helper/metricsutil"
"github.com/hashicorp/vault/helper/namespace"
vaulthttp "github.com/hashicorp/vault/http"
@ -42,7 +44,6 @@ import (
"github.com/hashicorp/vault/internalshared/listenerutil"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/helper/jsonutil"
"github.com/hashicorp/vault/sdk/helper/logging"
"github.com/hashicorp/vault/sdk/helper/strutil"
"github.com/hashicorp/vault/sdk/helper/useragent"
"github.com/hashicorp/vault/sdk/logical"
@ -84,6 +85,7 @@ const (
type ServerCommand struct {
*BaseCommand
logFlags logFlags
AuditBackends map[string]audit.Factory
CredentialBackends map[string]logical.Factory
@ -98,9 +100,9 @@ type ServerCommand struct {
WaitGroup *sync.WaitGroup
logOutput io.Writer
gatedWriter *gatedwriter.Writer
logger hclog.InterceptLogger
logWriter io.Writer
logGate *gatedwriter.Writer
logger hclog.InterceptLogger
cleanupGuard sync.Once
@ -112,10 +114,7 @@ type ServerCommand struct {
allLoggers []hclog.Logger
// new stuff
flagConfigs []string
flagLogLevel string
flagLogFormat string
flagRecovery bool
flagDev bool
flagDevTLS bool
@ -136,7 +135,6 @@ type ServerCommand struct {
flagDevTransactional bool
flagDevAutoSeal bool
flagTestVerifyOnly bool
flagCombineLogs bool
flagTestServerConfig bool
flagDevConsul bool
flagExitOnCoreShutdown bool
@ -175,6 +173,9 @@ func (c *ServerCommand) Flags() *FlagSets {
f := set.NewFlagSet("Command Options")
// Augment with the log flags
f.addLogFlags(&c.logFlags)
f.StringSliceVar(&StringSliceVar{
Name: "config",
Target: &c.flagConfigs,
@ -189,25 +190,6 @@ func (c *ServerCommand) Flags() *FlagSets {
".hcl or .json are loaded.",
})
f.StringVar(&StringVar{
Name: "log-level",
Target: &c.flagLogLevel,
Default: notSetValue,
EnvVar: "VAULT_LOG_LEVEL",
Completion: complete.PredictSet("trace", "debug", "info", "warn", "error"),
Usage: "Log verbosity level. Supported values (in order of detail) are " +
"\"trace\", \"debug\", \"info\", \"warn\", and \"error\".",
})
f.StringVar(&StringVar{
Name: "log-format",
Target: &c.flagLogFormat,
Default: notSetValue,
EnvVar: "VAULT_LOG_FORMAT",
Completion: complete.PredictSet("standard", "json"),
Usage: `Log format. Supported values are "standard" and "json".`,
})
f.BoolVar(&BoolVar{
Name: "exit-on-core-shutdown",
Target: &c.flagExitOnCoreShutdown,
@ -373,13 +355,6 @@ func (c *ServerCommand) Flags() *FlagSets {
})
// TODO: should the below flags be public?
f.BoolVar(&BoolVar{
Name: "combine-logs",
Target: &c.flagCombineLogs,
Default: false,
Hidden: true,
})
f.BoolVar(&BoolVar{
Name: "test-verify-only",
Target: &c.flagTestVerifyOnly,
@ -409,8 +384,8 @@ func (c *ServerCommand) AutocompleteFlags() complete.Flags {
func (c *ServerCommand) flushLog() {
c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
Output: c.logOutput,
}, c.gatedWriter)
Output: c.logWriter,
}, c.logGate)
}
func (c *ServerCommand) parseConfig() (*server.Config, []configutil.ConfigError, error) {
@ -457,20 +432,15 @@ func (c *ServerCommand) runRecoveryMode() int {
return 1
}
level, logLevelString, logLevelWasNotSet, logFormat, err := c.processLogLevelAndFormat(config)
// Update the 'log' related aspects of shared config based on config/env var/cli
c.Flags().updateLogConfig(config.SharedConfig)
l, err := c.configureLogging(config)
if err != nil {
c.UI.Error(err.Error())
return 1
}
c.logger = hclog.NewInterceptLogger(&hclog.LoggerOptions{
Output: c.gatedWriter,
Level: level,
IndependentLevels: true,
// Note that if logFormat is either unspecified or standard, then
// the resulting logger's format will be standard.
JSONFormat: logFormat == logging.JSONFormat,
})
c.logger = l
c.allLoggers = append(c.allLoggers, l)
// reporting Errors found in the config
for _, cErr := range configErrors {
@ -480,15 +450,6 @@ func (c *ServerCommand) runRecoveryMode() int {
// Ensure logging is flushed if initialization fails
defer c.flushLog()
logLevelStr, err := c.adjustLogLevel(config, logLevelWasNotSet)
if err != nil {
c.UI.Error(err.Error())
return 1
}
if logLevelStr != "" {
logLevelString = logLevelStr
}
// create GRPC logger
namedGRPCLogFaker := c.logger.Named("grpclogfaker")
grpclog.SetLogger(&grpclogFaker{
@ -533,7 +494,7 @@ func (c *ServerCommand) runRecoveryMode() int {
infoKeys := make([]string, 0, 10)
info := make(map[string]string)
info["log level"] = logLevelString
info["log level"] = config.LogLevel
infoKeys = append(infoKeys, "log level")
var barrierSeal vault.Seal
@ -598,7 +559,7 @@ func (c *ServerCommand) runRecoveryMode() int {
Physical: backend,
StorageType: config.Storage.Type,
Seal: barrierSeal,
LogLevel: logLevelString,
LogLevel: config.LogLevel,
Logger: c.logger,
DisableMlock: config.DisableMlock,
RecoveryMode: c.flagRecovery,
@ -630,7 +591,7 @@ func (c *ServerCommand) runRecoveryMode() int {
// Initialize the listeners
lns := make([]listenerutil.Listener, 0, len(config.Listeners))
for _, lnConfig := range config.Listeners {
ln, _, _, err := server.NewListener(lnConfig, c.gatedWriter, c.UI)
ln, _, _, err := server.NewListener(lnConfig, c.logGate, c.UI)
if err != nil {
c.UI.Error(fmt.Sprintf("Error initializing listener of type %s: %s", lnConfig.Type, err))
return 1
@ -726,7 +687,7 @@ func (c *ServerCommand) runRecoveryMode() int {
c.UI.Warn("")
}
if !c.flagCombineLogs {
if !c.logFlags.flagCombineLogs {
c.UI.Output("==> Vault server started! Log data will stream in below:\n")
}
@ -778,82 +739,6 @@ func logProxyEnvironmentVariables(logger hclog.Logger) {
"https_proxy", cfgMap["https_proxy"], "no_proxy", cfgMap["no_proxy"])
}
func (c *ServerCommand) adjustLogLevel(config *server.Config, logLevelWasNotSet bool) (string, error) {
var logLevelString string
if config.LogLevel != "" && logLevelWasNotSet {
configLogLevel := strings.ToLower(strings.TrimSpace(config.LogLevel))
logLevelString = configLogLevel
switch configLogLevel {
case "trace":
c.logger.SetLevel(hclog.Trace)
case "debug":
c.logger.SetLevel(hclog.Debug)
case "notice", "info", "":
c.logger.SetLevel(hclog.Info)
case "warn", "warning":
c.logger.SetLevel(hclog.Warn)
case "err", "error":
c.logger.SetLevel(hclog.Error)
default:
return "", fmt.Errorf("unknown log level: %s", config.LogLevel)
}
}
return logLevelString, nil
}
func (c *ServerCommand) processLogLevelAndFormat(config *server.Config) (hclog.Level, string, bool, logging.LogFormat, error) {
// Create a logger. We wrap it in a gated writer so that it doesn't
// start logging too early.
c.logOutput = os.Stderr
if c.flagCombineLogs {
c.logOutput = os.Stdout
}
c.gatedWriter = gatedwriter.NewWriter(c.logOutput)
var level hclog.Level
var logLevelWasNotSet bool
logFormat := logging.UnspecifiedFormat
logLevelString := c.flagLogLevel
c.flagLogLevel = strings.ToLower(strings.TrimSpace(c.flagLogLevel))
switch c.flagLogLevel {
case notSetValue, "":
logLevelWasNotSet = true
logLevelString = "info"
level = hclog.Info
case "trace":
level = hclog.Trace
case "debug":
level = hclog.Debug
case "notice", "info":
level = hclog.Info
case "warn", "warning":
level = hclog.Warn
case "err", "error":
level = hclog.Error
default:
return level, logLevelString, logLevelWasNotSet, logFormat, fmt.Errorf("unknown log level: %s", c.flagLogLevel)
}
if c.flagLogFormat != notSetValue {
var err error
logFormat, err = logging.ParseLogFormat(c.flagLogFormat)
if err != nil {
return level, logLevelString, logLevelWasNotSet, logFormat, err
}
}
if logFormat == logging.UnspecifiedFormat {
logFormat = logging.ParseEnvLogFormat()
}
if logFormat == logging.UnspecifiedFormat {
var err error
logFormat, err = logging.ParseLogFormat(config.LogFormat)
if err != nil {
return level, logLevelString, logLevelWasNotSet, logFormat, err
}
}
return level, logLevelString, logLevelWasNotSet, logFormat, nil
}
type quiescenceSink struct {
t *time.Timer
}
@ -945,7 +830,7 @@ func (c *ServerCommand) InitListeners(config *server.Config, disableClustering b
var errMsg error
for i, lnConfig := range config.Listeners {
ln, props, reloadFunc, err := server.NewListener(lnConfig, c.gatedWriter, c.UI)
ln, props, reloadFunc, err := server.NewListener(lnConfig, c.logGate, c.UI)
if err != nil {
errMsg = fmt.Errorf("Error initializing listener of type %s: %s", lnConfig.Type, err)
return 1, nil, nil, errMsg
@ -1026,6 +911,13 @@ func (c *ServerCommand) Run(args []string) int {
return 1
}
c.logGate = gatedwriter.NewWriter(os.Stderr)
c.logWriter = c.logGate
if c.logFlags.flagCombineLogs {
c.logWriter = os.Stdout
}
if c.flagRecovery {
return c.runRecoveryMode()
}
@ -1147,31 +1039,20 @@ func (c *ServerCommand) Run(args []string) int {
return 1
}
level, logLevelString, logLevelWasNotSet, logFormat, err := c.processLogLevelAndFormat(config)
f.updateLogConfig(config.SharedConfig)
// Set 'trace' log level for the following 'dev' clusters
if c.flagDevThreeNode || c.flagDevFourCluster {
config.LogLevel = "trace"
}
l, err := c.configureLogging(config)
if err != nil {
c.UI.Error(err.Error())
return 1
}
config.LogFormat = logFormat.String()
if c.flagDevThreeNode || c.flagDevFourCluster {
c.logger = hclog.NewInterceptLogger(&hclog.LoggerOptions{
Mutex: &sync.Mutex{},
Output: c.gatedWriter,
Level: hclog.Trace,
IndependentLevels: true,
})
} else {
c.logger = hclog.NewInterceptLogger(&hclog.LoggerOptions{
Output: c.gatedWriter,
Level: level,
IndependentLevels: true,
// Note that if logFormat is either unspecified or standard, then
// the resulting logger's format will be standard.
JSONFormat: logFormat == logging.JSONFormat,
})
}
c.logger = l
c.allLoggers = append(c.allLoggers, l)
// reporting Errors found in the config
for _, cErr := range configErrors {
@ -1181,17 +1062,6 @@ func (c *ServerCommand) Run(args []string) int {
// Ensure logging is flushed if initialization fails
defer c.flushLog()
c.allLoggers = []hclog.Logger{c.logger}
logLevelStr, err := c.adjustLogLevel(config, logLevelWasNotSet)
if err != nil {
c.UI.Error(err.Error())
return 1
}
if logLevelStr != "" {
logLevelString = logLevelStr
}
// create GRPC logger
namedGRPCLogFaker := c.logger.Named("grpclogfaker")
c.allLoggers = append(c.allLoggers, namedGRPCLogFaker)
@ -1296,7 +1166,7 @@ func (c *ServerCommand) Run(args []string) int {
infoKeys := make([]string, 0, 10)
info := make(map[string]string)
info["log level"] = logLevelString
info["log level"] = config.LogLevel
infoKeys = append(infoKeys, "log level")
barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(c, config, infoKeys, info)
// Check error here
@ -1586,7 +1456,7 @@ func (c *ServerCommand) Run(args []string) int {
}
// Output the header that the server has started
if !c.flagCombineLogs {
if !c.logFlags.flagCombineLogs {
c.UI.Output("==> Vault server started! Log data will stream in below:\n")
}
@ -1640,7 +1510,6 @@ func (c *ServerCommand) Run(args []string) int {
// Check for new log level
var config *server.Config
var level hclog.Level
var configErrors []configutil.ConfigError
for _, path := range c.flagConfigs {
current, err := server.LoadConfig(path)
@ -1686,20 +1555,10 @@ func (c *ServerCommand) Run(args []string) int {
c.logger.Error(err.Error())
}
// Reload log level for loggers
if config.LogLevel != "" {
configLogLevel := strings.ToLower(strings.TrimSpace(config.LogLevel))
switch configLogLevel {
case "trace":
level = hclog.Trace
case "debug":
level = hclog.Debug
case "notice", "info", "":
level = hclog.Info
case "warn", "warning":
level = hclog.Warn
case "err", "error":
level = hclog.Error
default:
level, err := loghelper.ParseLogLevel(config.LogLevel)
if err != nil {
c.logger.Error("unknown log level found on reload", "level", config.LogLevel)
goto RUNRELOADFUNCS
}
@ -1792,6 +1651,48 @@ func (c *ServerCommand) Run(args []string) int {
return retCode
}
// configureLogging takes the configuration and attempts to parse config values into 'log' friendly configuration values
// If all goes to plan, a logger is created and setup.
func (c *ServerCommand) configureLogging(config *server.Config) (hclog.InterceptLogger, error) {
// Parse all the log related config
logLevel, err := loghelper.ParseLogLevel(config.LogLevel)
if err != nil {
return nil, err
}
logFormat, err := loghelper.ParseLogFormat(config.LogFormat)
if err != nil {
return nil, err
}
logRotateDuration, err := parseutil.ParseDurationSecond(config.LogRotateDuration)
if err != nil {
return nil, err
}
logRotateBytes, err := parseutil.ParseInt(config.LogRotateBytes)
if err != nil {
return nil, err
}
logRotateMaxFiles, err := parseutil.ParseInt(config.LogRotateMaxFiles)
if err != nil {
return nil, err
}
logCfg := &loghelper.LogConfig{
Name: "vault",
LogLevel: logLevel,
LogFormat: logFormat,
LogFilePath: config.LogFile,
LogRotateDuration: logRotateDuration,
LogRotateBytes: int(logRotateBytes),
LogRotateMaxFiles: int(logRotateMaxFiles),
}
return loghelper.Setup(logCfg, c.logWriter)
}
func (c *ServerCommand) reloadHCPLink(hcpLinkVault *hcp_link.WrappedHCPLinkVault, conf *server.Config, core *vault.Core, hcpLogger hclog.Logger) (*hcp_link.WrappedHCPLinkVault, error) {
// trigger a shutdown
if hcpLinkVault != nil {

View File

@ -1,12 +1,20 @@
package logging
import (
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/hashicorp/go-multierror"
)
var now = time.Now
type LogFile struct {
// Name of the log file
fileName string
@ -14,20 +22,28 @@ type LogFile struct {
// Path to the log file
logPath string
// duration between each file rotation operation
duration time.Duration
// lastCreated represents the creation time of the latest log
lastCreated time.Time
// fileInfo is the pointer to the current file being written to
fileInfo *os.File
// maxBytes is the maximum number of desired bytes for a log file
maxBytes int
// bytesWritten is the number of bytes written in the current log file
bytesWritten int64
// Max rotated files to keep before removing them.
maxArchivedFiles int
// acquire is the mutex utilized to ensure we have no concurrency issues
acquire sync.Mutex
}
func NewLogFile(logPath string, fileName string) *LogFile {
return &LogFile{
fileName: strings.TrimSpace(fileName),
logPath: strings.TrimSpace(logPath),
}
}
// Write is used to implement io.Writer
func (l *LogFile) Write(b []byte) (n int, err error) {
l.acquire.Lock()
@ -38,19 +54,95 @@ func (l *LogFile) Write(b []byte) (n int, err error) {
return 0, err
}
}
// Check for the last contact and rotate if necessary
if err := l.rotate(); err != nil {
return 0, err
}
return l.fileInfo.Write(b)
bytesWritten, err := l.fileInfo.Write(b)
if bytesWritten > 0 {
l.bytesWritten += int64(bytesWritten)
}
return bytesWritten, err
}
func (l *LogFile) fileNamePattern() string {
// Extract the file extension
fileExt := filepath.Ext(l.fileName)
// If we have no file extension we append .log
if fileExt == "" {
fileExt = ".log"
}
// Remove the file extension from the filename
return strings.TrimSuffix(l.fileName, fileExt) + "-%s" + fileExt
}
func (l *LogFile) openNew() error {
newFilePath := filepath.Join(l.logPath, l.fileName)
fileNamePattern := l.fileNamePattern()
// Try to open an existing file or create a new one if it doesn't exist.
filePointer, err := os.OpenFile(newFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o640)
createTime := now()
newFileName := fmt.Sprintf(fileNamePattern, strconv.FormatInt(createTime.UnixNano(), 10))
newFilePath := filepath.Join(l.logPath, newFileName)
// Try creating a file. We truncate the file because we are the only authority to write the logs
filePointer, err := os.OpenFile(newFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o640)
if err != nil {
return err
}
// New file, new 'bytes' tracker, new creation time :) :)
l.fileInfo = filePointer
l.lastCreated = createTime
l.bytesWritten = 0
return nil
}
func (l *LogFile) rotate() error {
// Get the time from the last point of contact
timeElapsed := time.Since(l.lastCreated)
// Rotate if we hit the byte file limit or the time limit
if (l.bytesWritten >= int64(l.maxBytes) && (l.maxBytes > 0)) || timeElapsed >= l.duration {
if err := l.fileInfo.Close(); err != nil {
return err
}
if err := l.pruneFiles(); err != nil {
return err
}
return l.openNew()
}
return nil
}
func (l *LogFile) pruneFiles() error {
if l.maxArchivedFiles == 0 {
return nil
}
pattern := filepath.Join(l.logPath, fmt.Sprintf(l.fileNamePattern(), "*"))
matches, err := filepath.Glob(pattern)
if err != nil {
return err
}
switch {
case l.maxArchivedFiles < 0:
return removeFiles(matches)
case len(matches) < l.maxArchivedFiles:
return nil
}
sort.Strings(matches)
last := len(matches) - l.maxArchivedFiles
return removeFiles(matches[:last])
}
func removeFiles(files []string) (err error) {
for _, file := range files {
if fileError := os.Remove(file); fileError != nil {
err = multierror.Append(err, fmt.Errorf("error removing file %s: %v", file, fileError))
}
}
return err
}

View File

@ -2,13 +2,22 @@ package logging
import (
"os"
"path/filepath"
"sort"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLogFile_openNew(t *testing.T) {
logFile := NewLogFile(t.TempDir(), "vault-agent.log")
logFile := &LogFile{
fileName: "vault.log",
logPath: t.TempDir(),
duration: defaultRotateDuration,
}
err := logFile.openNew()
require.NoError(t, err)
@ -20,3 +29,126 @@ func TestLogFile_openNew(t *testing.T) {
require.NoError(t, err)
require.Contains(t, string(content), msg)
}
func TestLogFile_Rotation_MaxDuration(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
tempDir := t.TempDir()
logFile := LogFile{
fileName: "vault.log",
logPath: tempDir,
duration: 50 * time.Millisecond,
}
_, err := logFile.Write([]byte("Hello World"))
assert.NoError(t, err, "error writing rotation max duration part 1")
time.Sleep(3 * logFile.duration)
_, err = logFile.Write([]byte("Second File"))
assert.NoError(t, err, "error writing rotation max duration part 2")
require.Len(t, listDir(t, tempDir), 2)
}
func TestLogFile_Rotation_MaxBytes(t *testing.T) {
tempDir := t.TempDir()
logFile := LogFile{
fileName: "somefile.log",
logPath: tempDir,
maxBytes: 10,
duration: defaultRotateDuration,
}
_, err := logFile.Write([]byte("Hello World"))
assert.NoError(t, err, "error writing rotation max bytes part 1")
_, err = logFile.Write([]byte("Second File"))
assert.NoError(t, err, "error writing rotation max bytes part 2")
require.Len(t, listDir(t, tempDir), 2)
}
func TestLogFile_PruneFiles(t *testing.T) {
tempDir := t.TempDir()
logFile := LogFile{
fileName: "vault.log",
logPath: tempDir,
maxBytes: 10,
duration: defaultRotateDuration,
maxArchivedFiles: 1,
}
_, err := logFile.Write([]byte("[INFO] Hello World"))
assert.NoError(t, err, "error writing during prune files test part 1")
_, err = logFile.Write([]byte("[INFO] Second File"))
assert.NoError(t, err, "error writing during prune files test part 1")
_, err = logFile.Write([]byte("[INFO] Third File"))
assert.NoError(t, err, "error writing during prune files test part 1")
logFiles := listDir(t, tempDir)
sort.Strings(logFiles)
require.Len(t, logFiles, 2)
content, err := os.ReadFile(filepath.Join(tempDir, logFiles[0]))
require.NoError(t, err)
require.Contains(t, string(content), "Second File")
content, err = os.ReadFile(filepath.Join(tempDir, logFiles[1]))
require.NoError(t, err)
require.Contains(t, string(content), "Third File")
}
func TestLogFile_PruneFiles_Disabled(t *testing.T) {
tempDir := t.TempDir()
logFile := LogFile{
fileName: "somename.log",
logPath: tempDir,
maxBytes: 10,
duration: defaultRotateDuration,
maxArchivedFiles: 0,
}
_, err := logFile.Write([]byte("[INFO] Hello World"))
assert.NoError(t, err, "error writing during prune files - disabled test part 1")
_, err = logFile.Write([]byte("[INFO] Second File"))
assert.NoError(t, err, "error writing during prune files - disabled test part 2")
_, err = logFile.Write([]byte("[INFO] Third File"))
assert.NoError(t, err, "error writing during prune files - disabled test part 3")
require.Len(t, listDir(t, tempDir), 3)
}
func TestLogFile_FileRotation_Disabled(t *testing.T) {
tempDir := t.TempDir()
logFile := LogFile{
fileName: "vault.log",
logPath: tempDir,
maxBytes: 10,
maxArchivedFiles: -1,
}
_, err := logFile.Write([]byte("[INFO] Hello World"))
assert.NoError(t, err, "error writing during rotation disabled test part 1")
_, err = logFile.Write([]byte("[INFO] Second File"))
assert.NoError(t, err, "error writing during rotation disabled test part 2")
_, err = logFile.Write([]byte("[INFO] Third File"))
assert.NoError(t, err, "error writing during rotation disabled test part 3")
require.Len(t, listDir(t, tempDir), 1)
}
func listDir(t *testing.T, name string) []string {
t.Helper()
fh, err := os.Open(name)
require.NoError(t, err)
files, err := fh.Readdirnames(100)
require.NoError(t, err)
return files
}

View File

@ -6,8 +6,10 @@ import (
"io"
"path/filepath"
"strings"
"time"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
)
const (
@ -16,27 +18,41 @@ const (
JSONFormat
)
// defaultRotateDuration is the default time taken by the agent to rotate logs
const defaultRotateDuration = 24 * time.Hour
type LogFormat int
// LogConfig should be used to supply configuration when creating a new Vault logger
type LogConfig struct {
name string
logLevel log.Level
logFormat LogFormat
logFilePath string
// Name is the name the returned logger will use to prefix log lines.
Name string
// LogLevel is the minimum level to be logged.
LogLevel log.Level
// LogFormat is the log format to use, supported formats are 'standard' and 'json'.
LogFormat LogFormat
// LogFilePath is the path to write the logs to the user specified file.
LogFilePath string
// LogRotateDuration is the user specified time to rotate logs
LogRotateDuration time.Duration
// LogRotateBytes is the user specified byte limit to rotate logs
LogRotateBytes int
// LogRotateMaxFiles is the maximum number of past archived log files to keep
LogRotateMaxFiles int
}
func NewLogConfig(name string, logLevel log.Level, logFormat LogFormat, logFilePath string) LogConfig {
return LogConfig{
name: name,
logLevel: logLevel,
logFormat: logFormat,
logFilePath: strings.TrimSpace(logFilePath),
}
func (c *LogConfig) isLevelInvalid() bool {
return c.LogLevel == log.NoLevel || c.LogLevel == log.Off || c.LogLevel.String() == "unknown"
}
func (c LogConfig) IsFormatJson() bool {
return c.logFormat == JSONFormat
func (c *LogConfig) isFormatJson() bool {
return c.LogFormat == JSONFormat
}
// Stringer implementation
@ -65,11 +81,30 @@ func (w noErrorWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
// parseFullPath takes a full path intended to be the location for log files and
// breaks it down into a directory and a file name. It checks both of these for
// the common globbing character '*' and returns an error if it is present.
func parseFullPath(fullPath string) (directory, fileName string, err error) {
directory, fileName = filepath.Split(fullPath)
globChars := "*?["
if strings.ContainsAny(directory, globChars) {
err = multierror.Append(err, fmt.Errorf("directory contains glob character"))
}
if fileName == "" {
fileName = "vault.log"
} else if strings.ContainsAny(fileName, globChars) {
err = multierror.Append(err, fmt.Errorf("file name contains globbing character"))
}
return directory, fileName, err
}
// Setup creates a new logger with the specified configuration and writer
func Setup(config LogConfig, w io.Writer) (log.InterceptLogger, error) {
func Setup(config *LogConfig, w io.Writer) (log.InterceptLogger, error) {
// Validate the log level
if config.logLevel.String() == "unknown" {
return nil, fmt.Errorf("invalid log level: %v", config.logLevel)
if config.isLevelInvalid() {
return nil, fmt.Errorf("invalid log level: %v", config.LogLevel)
}
// If out is os.Stdout and Vault is being run as a Windows Service, writes will
@ -77,24 +112,39 @@ func Setup(config LogConfig, w io.Writer) (log.InterceptLogger, error) {
// noErrorWriter is used as a wrapper to suppress any errors when writing to out.
writers := []io.Writer{noErrorWriter{w: w}}
if config.logFilePath != "" {
dir, fileName := filepath.Split(config.logFilePath)
if fileName == "" {
fileName = "vault-agent.log"
// Create a file logger if the user has specified the path to the log file
if config.LogFilePath != "" {
dir, fileName, err := parseFullPath(config.LogFilePath)
if err != nil {
return nil, err
}
if config.LogRotateDuration == 0 {
config.LogRotateDuration = defaultRotateDuration
}
logFile := &LogFile{
fileName: fileName,
logPath: dir,
duration: config.LogRotateDuration,
maxBytes: config.LogRotateBytes,
maxArchivedFiles: config.LogRotateMaxFiles,
}
if err := logFile.pruneFiles(); err != nil {
return nil, fmt.Errorf("failed to prune log files: %w", err)
}
logFile := NewLogFile(dir, fileName)
if err := logFile.openNew(); err != nil {
return nil, fmt.Errorf("failed to set up file logging: %w", err)
return nil, fmt.Errorf("failed to setup logging: %w", err)
}
writers = append(writers, logFile)
}
logger := log.NewInterceptLogger(&log.LoggerOptions{
Name: config.name,
Level: config.logLevel,
Name: config.Name,
Level: config.LogLevel,
Output: io.MultiWriter(writers...),
JSONFormat: config.IsFormatJson(),
JSONFormat: config.isFormatJson(),
})
return logger, nil
}

View File

@ -13,22 +13,27 @@ import (
)
func TestLogger_SetupBasic(t *testing.T) {
cfg := NewLogConfig("test-system", log.Info, StandardFormat, t.TempDir()+"test.log")
cfg := &LogConfig{Name: "test-system", LogLevel: log.Info}
logger, err := Setup(cfg, nil)
require.NoError(t, err)
require.NotNil(t, logger)
require.Equal(t, logger.Name(), "test-system")
require.True(t, logger.IsInfo())
}
func TestLogger_SetupInvalidLogLevel(t *testing.T) {
cfg := NewLogConfig("test-system", 999, StandardFormat, t.TempDir()+"test.log")
cfg := &LogConfig{}
_, err := Setup(cfg, nil)
assert.Containsf(t, err.Error(), "invalid log level", "expected error %s", err)
}
func TestLogger_SetupLoggerErrorLevel(t *testing.T) {
cfg := NewLogConfig("test-system", log.Error, StandardFormat, t.TempDir()+"test.log")
cfg := &LogConfig{
LogLevel: log.Error,
}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
@ -40,15 +45,15 @@ func TestLogger_SetupLoggerErrorLevel(t *testing.T) {
output := buf.String()
require.Contains(t, output, "[ERROR] test-system: test error msg")
require.NotContains(t, output, "[INFO] test-system: test info msg")
require.Contains(t, output, "[ERROR] test error msg")
require.NotContains(t, output, "[INFO] test info msg")
}
func TestLogger_SetupLoggerDebugLevel(t *testing.T) {
cfg := NewLogConfig("test-system", log.Debug, StandardFormat, t.TempDir()+"test.log")
cfg := LogConfig{LogLevel: log.Debug}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
logger, err := Setup(&cfg, &buf)
require.NoError(t, err)
require.NotNil(t, logger)
@ -57,12 +62,15 @@ func TestLogger_SetupLoggerDebugLevel(t *testing.T) {
output := buf.String()
require.Contains(t, output, "[INFO] test-system: test info msg")
require.Contains(t, output, "[DEBUG] test-system: test debug msg")
require.Contains(t, output, "[INFO] test info msg")
require.Contains(t, output, "[DEBUG] test debug msg")
}
func TestLogger_SetupLoggerWithName(t *testing.T) {
cfg := NewLogConfig("test-system", log.Debug, StandardFormat, t.TempDir()+"test.log")
cfg := &LogConfig{
LogLevel: log.Debug,
Name: "test-system",
}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
@ -75,7 +83,11 @@ func TestLogger_SetupLoggerWithName(t *testing.T) {
}
func TestLogger_SetupLoggerWithJSON(t *testing.T) {
cfg := NewLogConfig("test-system", log.Debug, JSONFormat, t.TempDir()+"test.log")
cfg := &LogConfig{
LogLevel: log.Debug,
LogFormat: JSONFormat,
Name: "test-system",
}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
@ -95,7 +107,11 @@ func TestLogger_SetupLoggerWithJSON(t *testing.T) {
func TestLogger_SetupLoggerWithValidLogPath(t *testing.T) {
tmpDir := t.TempDir()
cfg := NewLogConfig("test-system", log.Info, StandardFormat, tmpDir+"/")
cfg := &LogConfig{
LogLevel: log.Info,
LogFilePath: tmpDir, //+ "/",
}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
@ -104,7 +120,10 @@ func TestLogger_SetupLoggerWithValidLogPath(t *testing.T) {
}
func TestLogger_SetupLoggerWithInValidLogPath(t *testing.T) {
cfg := NewLogConfig("test-system", log.Info, StandardFormat, "nonexistentdir/")
cfg := &LogConfig{
LogLevel: log.Info,
LogFilePath: "nonexistentdir/",
}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
@ -116,9 +135,14 @@ func TestLogger_SetupLoggerWithInValidLogPath(t *testing.T) {
func TestLogger_SetupLoggerWithInValidLogPathPermission(t *testing.T) {
tmpDir := "/tmp/" + t.Name()
os.Mkdir(tmpDir, 0o000)
err := os.Mkdir(tmpDir, 0o000)
assert.NoError(t, err, "unexpected error testing with invalid log path permission")
defer os.RemoveAll(tmpDir)
cfg := NewLogConfig("test-system", log.Info, StandardFormat, tmpDir+"/")
cfg := &LogConfig{
LogLevel: log.Info,
LogFilePath: tmpDir + "/",
}
var buf bytes.Buffer
logger, err := Setup(cfg, &buf)
@ -126,3 +150,47 @@ func TestLogger_SetupLoggerWithInValidLogPathPermission(t *testing.T) {
require.True(t, errors.Is(err, os.ErrPermission))
require.Nil(t, logger)
}
func TestLogger_SetupLoggerWithInvalidLogFilePath(t *testing.T) {
cases := map[string]struct {
path string
message string
}{
"file name *": {
path: "/this/isnt/ok/juan*.log",
message: "file name contains globbing character",
},
"file name ?": {
path: "/this/isnt/ok/juan?.log",
message: "file name contains globbing character",
},
"file name [": {
path: "/this/isnt/ok/[juan].log",
message: "file name contains globbing character",
},
"directory path *": {
path: "/this/isnt/ok/*/qwerty.log",
message: "directory contains glob character",
},
"directory path ?": {
path: "/this/isnt/ok/?/qwerty.log",
message: "directory contains glob character",
},
"directory path [": {
path: "/this/isnt/ok/[foo]/qwerty.log",
message: "directory contains glob character",
},
}
for name, tc := range cases {
name := name
tc := tc
cfg := &LogConfig{
LogLevel: log.Info,
LogFilePath: tc.path,
}
_, err := Setup(cfg, &bytes.Buffer{})
assert.Error(t, err, "%s: expected error due to *", name)
assert.Contains(t, err.Error(), tc.message, "%s: error message does not match: %s", name, err.Error())
}
}

View File

@ -38,8 +38,12 @@ type SharedConfig struct {
// LogFormat specifies the log format. Valid values are "standard" and
// "json". The values are case-insenstive. If no log format is specified,
// then standard format will be used.
LogFormat string `hcl:"log_format"`
LogLevel string `hcl:"log_level"`
LogFormat string `hcl:"log_format"`
LogLevel string `hcl:"log_level"`
LogFile string `hcl:"log_file"`
LogRotateBytes string `hcl:"log_rotate_bytes"`
LogRotateDuration string `hcl:"log_rotate_duration"`
LogRotateMaxFiles string `hcl:"log_rotate_max_files"`
PidFile string `hcl:"pid_file"`

View File

@ -63,6 +63,26 @@ func (c *SharedConfig) Merge(c2 *SharedConfig) *SharedConfig {
result.LogFormat = c2.LogFormat
}
result.LogFile = c.LogFile
if c2.LogFile != "" {
result.LogFile = c2.LogFile
}
result.LogRotateBytes = c.LogRotateBytes
if c2.LogRotateBytes != "" {
result.LogRotateBytes = c2.LogRotateBytes
}
result.LogRotateMaxFiles = c.LogRotateMaxFiles
if c2.LogRotateMaxFiles != "" {
result.LogRotateMaxFiles = c2.LogRotateMaxFiles
}
result.LogRotateDuration = c.LogRotateDuration
if c2.LogRotateDuration != "" {
result.LogRotateDuration = c2.LogRotateDuration
}
result.PidFile = c.PidFile
if c2.PidFile != "" {
result.PidFile = c2.PidFile

View File

@ -79,21 +79,19 @@ to add the Vault integration code.
Vault Agent aims to remove this initial hurdle to adopt Vault by providing a
more scalable and simpler way for applications to integrate with Vault.
## What is Vault Agent?
Vault Agent is a client daemon that provides the following features:
- [Auto-Auth][autoauth] - Automatically authenticate to Vault and manage the
token renewal process for locally-retrieved dynamic secrets.
token renewal process for locally-retrieved dynamic secrets.
- [Caching][caching] - Allows client-side caching of responses containing newly
created tokens and responses containing leased secrets generated off of these
newly created tokens. The agent also manages the renewals of the cached tokens and leases.
created tokens and responses containing leased secrets generated off of these
newly created tokens. The agent also manages the renewals of the cached tokens and leases.
- [Windows Service][winsvc] - Allows running the Vault Agent as a Windows
service.
service.
- [Templating][template] - Allows rendering of user-supplied templates by Vault
Agent, using the token generated by the Auto-Auth step.
Agent, using the token generated by the Auto-Auth step.
## Auto-Auth
@ -129,7 +127,38 @@ See the [caching](/docs/agent/caching#api) page for details on the cache API.
### Command Options
- `-log-file` `(string: "")` - If specified, should contain the full file path to use for outputting log files from Vault.
- `-log-level` ((#\_log_level)) `(string: "info")` - Log verbosity level. Supported values (in
order of descending detail) are `trace`, `debug`, `info`, `warn`, and `error`. This can
also be specified via the `VAULT_LOG_LEVEL` environment variable.
- `-log-format` ((#\_log_format)) `(string: "standard")` - Log format. Supported values
are `standard` and `json`. This can also be specified via the
`VAULT_LOG_FORMAT` environment variable.
- `-log-file` ((#\_log_file)) - writes all the Vault agent log messages
to a file. This value is used as a prefix for the log file name. The current timestamp
is appended to the file name. If the value ends in a path separator, `vault-agent`
will be appended to the value. If the file name is missing an extension, `.log`
is appended. For example, setting `log-file` to `/var/log/` would result in a log
file path of `/var/log/vault-agent-{timestamp}.log`. `log-file` can be combined with
[`-log-rotate-bytes`](#_log_rotate_bytes) and [`-log-rotate-duration`](#_log_rotate_duration)
for a fine-grained log rotation experience.
This can also be specified via the `VAULT_LOG_FILE` environment variable.
- `-log-rotate-bytes` ((#\_log_rotate_bytes)) - to specify the number of
bytes that should be written to a log before it needs to be rotated. Unless specified,
there is no limit to the number of bytes that can be written to a log file.
This can also be specified via the `VAULT_LOG_ROTATE_BYTES` environment variable.
- `-log-rotate-duration` ((#\_log_rotate_duration)) - to specify the maximum
duration a log should be written to before it needs to be rotated. Must be a duration
value such as 30s. Defaults to 24h.
This can also be specified via the `VAULT_LOG_ROTATE_DURATION` environment variable.
- `-log-rotate-max-files` ((#\_log_rotate_max_files)) - to specify the maximum
number of older log file archives to keep. Defaults to `0` (no files are ever deleted).
Set to `-1` to discard old log files when a new one is created.
This can also be specified via the `VAULT_LOG_ROTATE_MAX_FILES` environment variable.
### Configuration File Options
@ -151,11 +180,11 @@ These are the currently-available general configuration option:
token was retrieved and all sinks successfully wrote it
- `disable_idle_connections` `(string array: [])` - A list of strings that disables idle connections for various features in Vault Agent.
Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS`
Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS`
environment variable as a comma separated string. This environment variable will override any values found in a configuration file.
- `disable_keep_alives` `(string array: [])` - A list of strings that disables keep alives for various features in Vault Agent.
Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES`
Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES`
environment variable as a comma separated string. This environment variable will override any values found in a configuration file.
- `template` <code>([template][template]: <optional\>)</code> - Specifies options used for templating Vault secrets to files.
@ -166,14 +195,26 @@ These are the currently-available general configuration option:
reporting system. See the [telemetry Stanza](/docs/agent#telemetry-stanza) section below
for a list of metrics specific to Agent.
- `log_level` - Equivalent to the [`-log-level` command-line flag](#_log_level).
- `log_format` - Equivalent to the [`-log-format` command-line flag](#_log_format).
- `log_file` - Equivalent to the [`-log-file` command-line flag](#_log_file).
- `log_rotate_duration` - Equivalent to the [`-log-rotate-duration` command-line flag](#_log_rotate_duration).
- `log_rotate_bytes` - Equivalent to the [`-log-rotate-bytes` command-line flag](#_log_rotate_bytes).
- `log_rotate_max_files` - Equivalent to the [`-log-rotate-max-files` command-line flag](#_log_rotate_max_files).
### vault Stanza
There can at most be one top level `vault` block and it has the following
configuration entries:
- `address` `(string: <optional>)` - The address of the Vault server to
connect to. This should be a Fully Qualified Domain Name (FQDN) or IP
such as `https://vault-fqdn:8200` or `https://172.16.9.8:8200`.
- `address` `(string: <optional>)` - The address of the Vault server to
connect to. This should be a Fully Qualified Domain Name (FQDN) or IP
such as `https://vault-fqdn:8200` or `https://172.16.9.8:8200`.
This value can be overridden by setting the `VAULT_ADDR` environment variable.
- `ca_cert` `(string: <optional>)` - Path on the local disk to a single PEM-encoded
@ -274,17 +315,15 @@ runtime metrics about its performance, the auto-auth and the cache status:
| `vault.agent.cache.hit` | Number of cache hits | counter |
| `vault.agent.cache.miss` | Number of cache misses | counter |
## Start Vault Agent
To run Vault Agent:
1. [Download](/downloads) the Vault binary where the client application runs
(virtual machine, Kubernetes pod, etc.)
(virtual machine, Kubernetes pod, etc.)
1. Create a Vault Agent configuration file. (See the [Example
Configuration](#example-configuration) section for an example configuration.)
Configuration](#example-configuration) section for an example configuration.)
1. Start a Vault Agent with the configuration file.
@ -379,4 +418,4 @@ template {
[listener]: /docs/agent#listener-stanza
[listener_main]: /docs/configuration/listener/tcp
[winsvc]: /docs/agent/winsvc
[telemetry]: /docs/configuration/telemetry
[telemetry]: /docs/configuration/telemetry

View File

@ -331,10 +331,6 @@ precedence over [#VAULT_LICENSE_PATH](#vault_license_path) and
[Enterprise, Server only] Specify a path to a license on disk to use for this node.
This takes precedence over [license_path in config](/docs/configuration#license_path).
### `VAULT_LOG_FILE`
(Agent only) If provided, specifies the full path to a log file Vault should use to write its logs.
### `VAULT_MAX_RETRIES`
Maximum number of retries when certain error codes are encountered. The default

View File

@ -52,21 +52,42 @@ flags](/docs/commands) included on all commands.
multiple configurations. If the path is a directory, all files which end in
.hcl or .json are loaded.
- `-log-level` `(string: "info")` - Log verbosity level. Supported values (in
order of detail) are "trace", "debug", "info", "warn", and "err". This can
also be specified via the VAULT_LOG_LEVEL environment variable.
- `-log-level` ((#\_log_level)) `(string: "info")` - Log verbosity level. Supported values (in
order of descending detail) are `trace`, `debug`, `info`, `warn`, and `error`. This can
also be specified via the `VAULT_LOG_LEVEL` environment variable.
- `-log-format` `(string: "standard")` - Log format. Supported values
are "standard" and "json". This can also be specified via the
VAULT_LOG_FORMAT environment variable.
- `-log-format` ((#\_log_format)) `(string: "standard")` - Log format. Supported values
are `standard` and `json`. This can also be specified via the
`VAULT_LOG_FORMAT` environment variable.
- `-log-file` ((#\_log_file)) - writes all the Vault log messages
to a file. This value is used as a prefix for the log file name. The current timestamp
is appended to the file name. If the value ends in a path separator, `vault`
will be appended to the value. If the file name is missing an extension, `.log`
is appended. For example, setting `log-file` to `/var/log/` would result in a log
file path of `/var/log/vault-{timestamp}.log`. `log-file` can be combined with
[`-log-rotate-bytes`](#_log_rotate_bytes) and [`-log-rotate-duration`](#_log_rotate_duration)
for a fine-grained log rotation experience.
- `-log-rotate-bytes` ((#\_log_rotate_bytes)) - to specify the number of
bytes that should be written to a log before it needs to be rotated. Unless specified,
there is no limit to the number of bytes that can be written to a log file.
- `-log-rotate-duration` ((#\_log_rotate_duration)) - to specify the maximum
duration a log should be written to before it needs to be rotated. Must be a duration
value such as 30s. Defaults to 24h.
- `-log-rotate-max-files` ((#\_log_rotate_max_files)) - to specify the maximum
number of older log file archives to keep. Defaults to 0 (no files are ever deleted).
Set to -1 to discard old log files when a new one is created.
- `VAULT_ALLOW_PENDING_REMOVAL_MOUNTS` `(bool: false)` - (environment variable)
Allow Vault to be started with builtin engines which have the `Pending Removal`
deprecation state. This is a temporary stopgap in place in order to perform an
upgrade and disable these engines. Once these engines are marked `Removed` (in
the next major release of Vault), the environment variable will no longer work
and a downgrade must be performed in order to remove the offending engines. For
more information, see the [deprecation faq](/docs/deprecation/faq/#q-what-are-the-phases-of-deprecation).
Allow Vault to be started with builtin engines which have the `Pending Removal`
deprecation state. This is a temporary stopgap in place in order to perform an
upgrade and disable these engines. Once these engines are marked `Removed` (in
the next major release of Vault), the environment variable will no longer work
and a downgrade must be performed in order to remove the offending engines. For
more information, see the [deprecation faq](/docs/deprecation/faq/#q-what-are-the-phases-of-deprecation).
### Dev Options
@ -75,7 +96,7 @@ more information, see the [deprecation faq](/docs/deprecation/faq/#q-what-are-th
production.
- `-dev-tls` `(bool: false)` - Enable TLS development mode. In this mode, Vault runs
in-memory and starts unsealed with a generated TLS CA, certificate and key.
in-memory and starts unsealed with a generated TLS CA, certificate and key.
As the name implies, do not run "dev" mode in production.
- `-dev-tls-cert-dir` `(string: "")` - Directory where generated TLS files are created if `-dev-tls` is specified. If left unset, files are generated in a temporary directory.

View File

@ -123,27 +123,17 @@ to specify where the configuration is.
@include 'plugin-file-permissions-check.mdx'
- `plugin_file_uid` `(integer: 0)` Uid of the plugin directories and plugin binaries if they
are owned by an user other than the user running Vault. This only needs to be set if the
file permissions check is enabled via the environment variable `VAULT_ENABLE_FILE_PERMISSIONS_CHECK`.
are owned by an user other than the user running Vault. This only needs to be set if the
file permissions check is enabled via the environment variable `VAULT_ENABLE_FILE_PERMISSIONS_CHECK`.
- `plugin_file_permissions` `(string: "")` Octal permission string of the plugin
directories and plugin binaries if they have write or execute permissions for group or others.
This only needs to be set if the file permissions check is enabled via the environment variable
`VAULT_ENABLE_FILE_PERMISSIONS_CHECK`.
directories and plugin binaries if they have write or execute permissions for group or others.
This only needs to be set if the file permissions check is enabled via the environment variable
`VAULT_ENABLE_FILE_PERMISSIONS_CHECK`.
- `telemetry` `([Telemetry][telemetry]: <none>)` Specifies the telemetry
reporting system.
- `log_level` `(string: "")` Specifies the log level to use; overridden by
CLI and env var parameters. On SIGHUP (`sudo kill -s HUP` *pid of vault*), Vault will update the log level to the
current value specified here (including overriding the CLI/env var
parameters). Not all parts of Vault's logging can have its level be changed
dynamically this way; in particular, secrets/auth plugins are currently not
updated dynamically. Supported log levels: Trace, Debug, Error, Warn, Info.
- `log_format` `(string: "")` Specifies the log format to use; overridden by
CLI and env var parameters. Supported log formats: "standard", "json".
- `default_lease_ttl` `(string: "768h")` Specifies the default lease duration
for tokens and secrets. This is specified using a label suffix like `"30s"` or
`"1h"`. This value cannot be larger than `max_lease_ttl`.
@ -187,6 +177,26 @@ to specify where the configuration is.
participating in a Raft cluster, this header will be omitted, whether this configuration
option is enabled or not.
- `log_level` `(string: "info")` - Log verbosity level.
Supported values (in order of descending detail) are `trace`, `debug`, `info`, `warn`, and `error`.
This can also be specified via the `VAULT_LOG_LEVEL` environment variable.
~> Note: On SIGHUP (`sudo kill -s HUP` _pid of vault_), if a valid value is specified, Vault will update the existing log level,
overriding (even if specified) both the CLI flag and environment variable.
~> Note: Not all parts of Vault's logging can have its log level be changed dynamically this way; in particular,
secrets/auth plugins are currently not updated dynamically.
- `log_format` - Equivalent to the [`-log-format` command-line flag](/docs/commands/server#_log_format).
- `log_file` - Equivalent to the [`-log-file` command-line flag](/docs/commands/server#_log_file).
- `log_rotate_duration` - Equivalent to the [`-log-rotate-duration` command-line flag](/docs/commands/server#_log_rotate_duration).
- `log_rotate_bytes` - Equivalent to the [`-log-rotate-bytes` command-line flag](/docs/commands/server#_log_rotate_bytes).
- `log_rotate_max_files` - Equivalent to the [`-log-rotate-max-files` command-line flag](/docs/commands/server#_log_rotate_max_files).
### High Availability Parameters
The following parameters are used on backends that support [high availability][high-availability].
@ -227,7 +237,7 @@ The following parameters are only used with Vault Enterprise
node will disable this feature when this node is Active or Standby. It's
recommended to sync this setting across all nodes in the cluster.
- `license_path` `(string: "")` - Path to license file. This can also be
- `license_path` `(string: "")` - Path to license file. This can also be
provided via the environment variable `VAULT_LICENSE_PATH`, or the license
itself can be provided in the environment variable `VAULT_LICENSE`.