agent/auto-auth: add exit_on_err configurable (#17091)

* agent/auto-auth: add exit_on_err configurable

* changelog

* Update backoff function to quit

* Clarify doc

* Fix test

changelog/17091.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent/auto-auth: Add `exit_on_err` which when set to true, will cause Agent to exit if any errors are encountered during authentication.
+```

command/agent.go

@@ -849,6 +849,7 @@ func (c *AgentCommand) Run(args []string) int {
 			EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials,
 			EnableTemplateTokenCh:        enableTokenCh,
 			Token:                        previousToken,
+			ExitOnError:                  config.AutoAuth.Method.ExitOnError,
 		})
 
 		ss := sink.NewSinkServer(&sink.SinkServerConfig{

command/agent/auth/auth.go

@@ -58,6 +58,7 @@ type AuthHandler struct {
 	minBackoff                   time.Duration
 	enableReauthOnNewCredentials bool
 	enableTemplateTokenCh        bool
+	exitOnError                  bool
 }
 
 type AuthHandlerConfig struct {
@@ -69,6 +70,7 @@ type AuthHandlerConfig struct {
 	Token                        string
 	EnableReauthOnNewCredentials bool
 	EnableTemplateTokenCh        bool
+	ExitOnError                  bool
 }
 
 func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
@@ -86,12 +88,17 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
 		maxBackoff:                   conf.MaxBackoff,
 		enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials,
 		enableTemplateTokenCh:        conf.EnableTemplateTokenCh,
+		exitOnError:                  conf.ExitOnError,
 	}
 
 	return ah
 }
 
-func backoffOrQuit(ctx context.Context, backoff *agentBackoff) {
+func backoff(ctx context.Context, backoff *agentBackoff) bool {
+	if backoff.exitOnErr {
+		return false
+	}
+
 	select {
 	case <-time.After(backoff.current):
 	case <-ctx.Done():
@@ -100,6 +107,7 @@ func backoffOrQuit(ctx context.Context, backoff *agentBackoff) {
 	// Increase exponential backoff for the next time if we don't
 	// successfully auth/renew/etc.
 	backoff.next()
+	return true
 }
 
 func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
@@ -111,9 +119,9 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 		ah.minBackoff = defaultMinBackoff
 	}
 
-	backoff := newAgentBackoff(ah.minBackoff, ah.maxBackoff)
+	backoffCfg := newAgentBackoff(ah.minBackoff, ah.maxBackoff, ah.exitOnError)
 
-	if backoff.min >= backoff.max {
+	if backoffCfg.min >= backoffCfg.max {
 		return errors.New("auth handler: min_backoff cannot be greater than max_backoff")
 	}
 
@@ -167,10 +175,14 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			clientToUse, err = am.(AuthMethodWithClient).AuthClient(ah.client)
 			if err != nil {
 				ah.logger.Error("error creating client for authentication call", "error", err, "backoff", backoff)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+
+				return err
+			}
 		default:
 			clientToUse = ah.client
 		}
@@ -189,11 +201,14 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 
 			secret, err = clientToUse.Auth().Token().LookupSelfWithContext(ctx)
 			if err != nil {
-				ah.logger.Error("could not look up token", "err", err, "backoff", backoff)
+				ah.logger.Error("could not look up token", "err", err, "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 
 			duration, _ := secret.Data["ttl"].(json.Number).Int64()
 			secret.Auth = &api.SecretAuth{
@@ -206,21 +221,27 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 
 			path, header, data, err = am.Authenticate(ctx, ah.client)
 			if err != nil {
-				ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoff)
+				ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 		}
 
 		if ah.wrapTTL > 0 {
 			wrapClient, err := clientToUse.Clone()
 			if err != nil {
-				ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoff)
+				ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 			wrapClient.SetWrappingLookupFunc(func(string, string) string {
 				return ah.wrapTTL.String()
 			})
@@ -238,34 +259,46 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			secret, err = clientToUse.Logical().WriteWithContext(ctx, path, data)
 			// Check errors/sanity
 			if err != nil {
-				ah.logger.Error("error authenticating", "error", err, "backoff", backoff)
+				ah.logger.Error("error authenticating", "error", err, "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 		}
 
 		switch {
 		case ah.wrapTTL > 0:
 			if secret.WrapInfo == nil {
-				ah.logger.Error("authentication returned nil wrap info", "backoff", backoff)
+				ah.logger.Error("authentication returned nil wrap info", "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 			if secret.WrapInfo.Token == "" {
-				ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoff)
+				ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 			wrappedResp, err := jsonutil.EncodeJSON(secret.WrapInfo)
 			if err != nil {
-				ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoff)
+				ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 			ah.logger.Info("authentication successful, sending wrapped token to sinks and pausing")
 			ah.OutputCh <- string(wrappedResp)
 			if ah.enableTemplateTokenCh {
@@ -273,7 +306,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			}
 
 			am.CredSuccess()
-			backoff.reset()
+			backoffCfg.reset()
 
 			select {
 			case <-ctx.Done():
@@ -287,17 +320,23 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 
 		default:
 			if secret == nil || secret.Auth == nil {
-				ah.logger.Error("authentication returned nil auth info", "backoff", backoff)
+				ah.logger.Error("authentication returned nil auth info", "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 			if secret.Auth.ClientToken == "" {
-				ah.logger.Error("authentication returned empty client token", "backoff", backoff)
+				ah.logger.Error("authentication returned empty client token", "backoff", backoffCfg)
-				backoffOrQuit(ctx, backoff)
 				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+				if backoff(ctx, backoffCfg) {
 					continue
 				}
+				return err
+			}
 			ah.logger.Info("authentication successful, sending token to sinks")
 			ah.OutputCh <- secret.Auth.ClientToken
 			if ah.enableTemplateTokenCh {
@@ -305,7 +344,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			}
 
 			am.CredSuccess()
-			backoff.reset()
+			backoffCfg.reset()
 		}
 
 		if watcher != nil {
@@ -316,11 +355,14 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			Secret: secret,
 		})
 		if err != nil {
-			ah.logger.Error("error creating lifetime watcher, backing off and retrying", "error", err, "backoff", backoff)
+			ah.logger.Error("error creating lifetime watcher", "error", err, "backoff", backoffCfg)
-			backoffOrQuit(ctx, backoff)
 			metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+			if backoff(ctx, backoffCfg) {
 				continue
 			}
+			return err
+		}
 
 		// Start the renewal process
 		ah.logger.Info("starting renewal process")
@@ -360,9 +402,10 @@ type agentBackoff struct {
 	min       time.Duration
 	max       time.Duration
 	current   time.Duration
+	exitOnErr bool
 }
 
-func newAgentBackoff(min, max time.Duration) *agentBackoff {
+func newAgentBackoff(min, max time.Duration, exitErr bool) *agentBackoff {
 	if max <= 0 {
 		max = defaultMaxBackoff
 	}
@@ -375,6 +418,7 @@ func newAgentBackoff(min, max time.Duration) *agentBackoff {
 		current:   min,
 		max:       max,
 		min:       min,
+		exitOnErr: exitErr,
 	}
 }
 

command/agent/auth/auth_test.go

@@ -109,7 +109,7 @@ consumption:
 
 func TestAgentBackoff(t *testing.T) {
 	max := 1024 * time.Second
-	backoff := newAgentBackoff(defaultMinBackoff, max)
+	backoff := newAgentBackoff(defaultMinBackoff, max, false)
 
 	// Test initial value
 	if backoff.current != defaultMinBackoff {
@@ -159,7 +159,7 @@ func TestAgentMinBackoffCustom(t *testing.T) {
 
 	for _, test := range tests {
 		max := 1024 * time.Second
-		backoff := newAgentBackoff(test.minBackoff, max)
+		backoff := newAgentBackoff(test.minBackoff, max, false)
 
 		// Test initial value
 		if backoff.current != test.want {

command/agent/config/config.go

@@ -130,6 +130,7 @@ type Method struct {
 	MaxBackoffRaw interface{}   `hcl:"max_backoff"`
 	MaxBackoff    time.Duration `hcl:"-"`
 	Namespace     string        `hcl:"namespace"`
+	ExitOnError   bool          `hcl:"exit_on_err"`
 	Config        map[string]interface{}
 }
 

command/agent/config/config_test.go

@@ -262,6 +262,7 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) {
 			Method: &Method{
 				Type:        "aws",
 				MountPath:   "auth/aws",
+				ExitOnError: false,
 				WrapTTL:     5 * time.Minute,
 				MaxBackoff:  2 * time.Minute,
 				Config: map[string]interface{}{
@@ -304,6 +305,51 @@ func TestLoadConfigFile_Method_InitialBackoff(t *testing.T) {
 			Method: &Method{
 				Type:        "aws",
 				MountPath:   "auth/aws",
+				ExitOnError: false,
+				WrapTTL:     5 * time.Minute,
+				MinBackoff:  5 * time.Second,
+				MaxBackoff:  2 * time.Minute,
+				Config: map[string]interface{}{
+					"role": "foobar",
+				},
+			},
+			Sinks: []*Sink{
+				{
+					Type: "file",
+					Config: map[string]interface{}{
+						"path": "/tmp/file-foo",
+					},
+				},
+			},
+		},
+		Vault: &Vault{
+			Retry: &Retry{
+				NumRetries: 12,
+			},
+		},
+	}
+
+	config.Prune()
+	if diff := deep.Equal(config, expected); diff != nil {
+		t.Fatal(diff)
+	}
+}
+
+func TestLoadConfigFile_Method_ExitOnErr(t *testing.T) {
+	config, err := LoadConfig("./test-fixtures/config-method-exit-on-err.hcl")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		SharedConfig: &configutil.SharedConfig{
+			PidFile: "./pidfile",
+		},
+		AutoAuth: &AutoAuth{
+			Method: &Method{
+				Type:        "aws",
+				MountPath:   "auth/aws",
+				ExitOnError: true,
 				WrapTTL:     5 * time.Minute,
 				MinBackoff:  5 * time.Second,
 				MaxBackoff:  2 * time.Minute,

command/agent/config/test-fixtures/config-method-exit-on-err.hcl

@@ -0,0 +1,21 @@
+pid_file = "./pidfile"
+
+auto_auth {
+	method {
+		type = "aws"
+		wrap_ttl = 300
+        exit_on_err = true
+		config = {
+			role = "foobar"
+		}
+		max_backoff = "2m"
+        min_backoff = "5s"
+	}
+
+	sink {
+		type = "file"
+		config = {
+			path = "/tmp/file-foo"
+		}
+	}
+}

website/content/docs/agent/autoauth/index.mdx

@@ -152,6 +152,11 @@ These are common configuration values that live within the `method` block:
   duration between retries, and **not** the duration that retries will be
   performed before giving up. Uses [duration format strings](/docs/concepts/duration-format).
 
+- `exit_on_err` `(bool: false)` - When set to true, Vault Agent will exit if any 
+  errors occur during authentication. This configurable only affects login attempts 
+  for new tokens (either intial or expired tokens) and will not exit for errors on 
+  valid token renewals.
+
 - `config` `(object: required)` - Configuration of the method itself. See the
   sidebar for information about each method.