From 87350f927f0e57d08116e0264567829d280f34c8 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Thu, 15 Sep 2022 14:00:31 -0400 Subject: [PATCH] agent/auto-auth: add exit_on_err configurable (#17091) * agent/auto-auth: add exit_on_err configurable * changelog * Update backoff function to quit * Clarify doc * Fix test --- changelog/17091.txt | 3 + command/agent.go | 1 + command/agent/auth/auth.go | 132 ++++++++++++------ command/agent/auth/auth_test.go | 4 +- command/agent/config/config.go | 1 + command/agent/config/config_test.go | 64 +++++++-- .../config-method-exit-on-err.hcl | 21 +++ website/content/docs/agent/autoauth/index.mdx | 5 + 8 files changed, 176 insertions(+), 55 deletions(-) create mode 100644 changelog/17091.txt create mode 100644 command/agent/config/test-fixtures/config-method-exit-on-err.hcl diff --git a/changelog/17091.txt b/changelog/17091.txt new file mode 100644 index 000000000..2b2c04109 --- /dev/null +++ b/changelog/17091.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent/auto-auth: Add `exit_on_err` which when set to true, will cause Agent to exit if any errors are encountered during authentication. +``` diff --git a/command/agent.go b/command/agent.go index a815c9c03..b90665683 100644 --- a/command/agent.go +++ b/command/agent.go @@ -849,6 +849,7 @@ func (c *AgentCommand) Run(args []string) int { EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials, EnableTemplateTokenCh: enableTokenCh, Token: previousToken, + ExitOnError: config.AutoAuth.Method.ExitOnError, }) ss := sink.NewSinkServer(&sink.SinkServerConfig{ diff --git a/command/agent/auth/auth.go b/command/agent/auth/auth.go index 7406d7c8c..854052adc 100644 --- a/command/agent/auth/auth.go +++ b/command/agent/auth/auth.go @@ -58,6 +58,7 @@ type AuthHandler struct { minBackoff time.Duration enableReauthOnNewCredentials bool enableTemplateTokenCh bool + exitOnError bool } type AuthHandlerConfig struct { @@ -69,6 +70,7 @@ type AuthHandlerConfig struct { Token string EnableReauthOnNewCredentials bool EnableTemplateTokenCh bool + ExitOnError bool } func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { @@ -86,12 +88,17 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { maxBackoff: conf.MaxBackoff, enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials, enableTemplateTokenCh: conf.EnableTemplateTokenCh, + exitOnError: conf.ExitOnError, } return ah } -func backoffOrQuit(ctx context.Context, backoff *agentBackoff) { +func backoff(ctx context.Context, backoff *agentBackoff) bool { + if backoff.exitOnErr { + return false + } + select { case <-time.After(backoff.current): case <-ctx.Done(): @@ -100,6 +107,7 @@ func backoffOrQuit(ctx context.Context, backoff *agentBackoff) { // Increase exponential backoff for the next time if we don't // successfully auth/renew/etc. backoff.next() + return true } func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { @@ -111,9 +119,9 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { ah.minBackoff = defaultMinBackoff } - backoff := newAgentBackoff(ah.minBackoff, ah.maxBackoff) + backoffCfg := newAgentBackoff(ah.minBackoff, ah.maxBackoff, ah.exitOnError) - if backoff.min >= backoff.max { + if backoffCfg.min >= backoffCfg.max { return errors.New("auth handler: min_backoff cannot be greater than max_backoff") } @@ -167,9 +175,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { clientToUse, err = am.(AuthMethodWithClient).AuthClient(ah.client) if err != nil { ah.logger.Error("error creating client for authentication call", "error", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + + return err } default: clientToUse = ah.client @@ -189,10 +201,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { secret, err = clientToUse.Auth().Token().LookupSelfWithContext(ctx) if err != nil { - ah.logger.Error("could not look up token", "err", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("could not look up token", "err", err, "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } duration, _ := secret.Data["ttl"].(json.Number).Int64() @@ -206,20 +221,26 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { path, header, data, err = am.Authenticate(ctx, ah.client) if err != nil { - ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } } if ah.wrapTTL > 0 { wrapClient, err := clientToUse.Clone() if err != nil { - ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } wrapClient.SetWrappingLookupFunc(func(string, string) string { return ah.wrapTTL.String() @@ -238,33 +259,45 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { secret, err = clientToUse.Logical().WriteWithContext(ctx, path, data) // Check errors/sanity if err != nil { - ah.logger.Error("error authenticating", "error", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("error authenticating", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } } switch { case ah.wrapTTL > 0: if secret.WrapInfo == nil { - ah.logger.Error("authentication returned nil wrap info", "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("authentication returned nil wrap info", "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } if secret.WrapInfo.Token == "" { - ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } wrappedResp, err := jsonutil.EncodeJSON(secret.WrapInfo) if err != nil { - ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } ah.logger.Info("authentication successful, sending wrapped token to sinks and pausing") ah.OutputCh <- string(wrappedResp) @@ -273,7 +306,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset() + backoffCfg.reset() select { case <-ctx.Done(): @@ -287,16 +320,22 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { default: if secret == nil || secret.Auth == nil { - ah.logger.Error("authentication returned nil auth info", "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("authentication returned nil auth info", "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } if secret.Auth.ClientToken == "" { - ah.logger.Error("authentication returned empty client token", "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("authentication returned empty client token", "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } ah.logger.Info("authentication successful, sending token to sinks") ah.OutputCh <- secret.Auth.ClientToken @@ -305,7 +344,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset() + backoffCfg.reset() } if watcher != nil { @@ -316,10 +355,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { Secret: secret, }) if err != nil { - ah.logger.Error("error creating lifetime watcher, backing off and retrying", "error", err, "backoff", backoff) - backoffOrQuit(ctx, backoff) + ah.logger.Error("error creating lifetime watcher", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1) - continue + + if backoff(ctx, backoffCfg) { + continue + } + return err } // Start the renewal process @@ -357,12 +399,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { // agentBackoff tracks exponential backoff state. type agentBackoff struct { - min time.Duration - max time.Duration - current time.Duration + min time.Duration + max time.Duration + current time.Duration + exitOnErr bool } -func newAgentBackoff(min, max time.Duration) *agentBackoff { +func newAgentBackoff(min, max time.Duration, exitErr bool) *agentBackoff { if max <= 0 { max = defaultMaxBackoff } @@ -372,9 +415,10 @@ func newAgentBackoff(min, max time.Duration) *agentBackoff { } return &agentBackoff{ - current: min, - max: max, - min: min, + current: min, + max: max, + min: min, + exitOnErr: exitErr, } } diff --git a/command/agent/auth/auth_test.go b/command/agent/auth/auth_test.go index e0c2442d3..950134274 100644 --- a/command/agent/auth/auth_test.go +++ b/command/agent/auth/auth_test.go @@ -109,7 +109,7 @@ consumption: func TestAgentBackoff(t *testing.T) { max := 1024 * time.Second - backoff := newAgentBackoff(defaultMinBackoff, max) + backoff := newAgentBackoff(defaultMinBackoff, max, false) // Test initial value if backoff.current != defaultMinBackoff { @@ -159,7 +159,7 @@ func TestAgentMinBackoffCustom(t *testing.T) { for _, test := range tests { max := 1024 * time.Second - backoff := newAgentBackoff(test.minBackoff, max) + backoff := newAgentBackoff(test.minBackoff, max, false) // Test initial value if backoff.current != test.want { diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 0c8da4545..b32772564 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -130,6 +130,7 @@ type Method struct { MaxBackoffRaw interface{} `hcl:"max_backoff"` MaxBackoff time.Duration `hcl:"-"` Namespace string `hcl:"namespace"` + ExitOnError bool `hcl:"exit_on_err"` Config map[string]interface{} } diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 3565e196f..8a5491891 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -260,10 +260,11 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) { }, AutoAuth: &AutoAuth{ Method: &Method{ - Type: "aws", - MountPath: "auth/aws", - WrapTTL: 5 * time.Minute, - MaxBackoff: 2 * time.Minute, + Type: "aws", + MountPath: "auth/aws", + ExitOnError: false, + WrapTTL: 5 * time.Minute, + MaxBackoff: 2 * time.Minute, Config: map[string]interface{}{ "role": "foobar", }, @@ -302,11 +303,56 @@ func TestLoadConfigFile_Method_InitialBackoff(t *testing.T) { }, AutoAuth: &AutoAuth{ Method: &Method{ - Type: "aws", - MountPath: "auth/aws", - WrapTTL: 5 * time.Minute, - MinBackoff: 5 * time.Second, - MaxBackoff: 2 * time.Minute, + Type: "aws", + MountPath: "auth/aws", + ExitOnError: false, + WrapTTL: 5 * time.Minute, + MinBackoff: 5 * time.Second, + MaxBackoff: 2 * time.Minute, + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Retry: &Retry{ + NumRetries: 12, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Method_ExitOnErr(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-method-exit-on-err.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + ExitOnError: true, + WrapTTL: 5 * time.Minute, + MinBackoff: 5 * time.Second, + MaxBackoff: 2 * time.Minute, Config: map[string]interface{}{ "role": "foobar", }, diff --git a/command/agent/config/test-fixtures/config-method-exit-on-err.hcl b/command/agent/config/test-fixtures/config-method-exit-on-err.hcl new file mode 100644 index 000000000..c52140102 --- /dev/null +++ b/command/agent/config/test-fixtures/config-method-exit-on-err.hcl @@ -0,0 +1,21 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + wrap_ttl = 300 + exit_on_err = true + config = { + role = "foobar" + } + max_backoff = "2m" + min_backoff = "5s" + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + } +} diff --git a/website/content/docs/agent/autoauth/index.mdx b/website/content/docs/agent/autoauth/index.mdx index 0df2782cb..fd3f61841 100644 --- a/website/content/docs/agent/autoauth/index.mdx +++ b/website/content/docs/agent/autoauth/index.mdx @@ -152,6 +152,11 @@ These are common configuration values that live within the `method` block: duration between retries, and **not** the duration that retries will be performed before giving up. Uses [duration format strings](/docs/concepts/duration-format). +- `exit_on_err` `(bool: false)` - When set to true, Vault Agent will exit if any + errors occur during authentication. This configurable only affects login attempts + for new tokens (either intial or expired tokens) and will not exit for errors on + valid token renewals. + - `config` `(object: required)` - Configuration of the method itself. See the sidebar for information about each method.