VAULT-12564 Add new token_file auto-auth method (#18740)

* VAULT-12564 Work so far on token file auto-auth

* VAULT-12564 remove lifetime watcher struct modifications

* VAULT-12564 add other config items, and clean up

* VAULT-12564 clean-up and more tests

* VAULT-12564 clean-up

* VAULT-12564 lookup-self and some clean-up

* VAULT-12564 safer client usage

* VAULT-12564 some clean-up

* VAULT-12564 changelog

* VAULT-12564 some clean-ups

* VAULT-12564 batch token warning

* VAULT-12564 remove follow_symlink reference

* VAULT-12564 Remove redundant stat, change temp file creation

* VAULT-12564 Remove ability to delete token after auth

changelog/18740.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: Added `token_file` auto-auth configuration to allow using a pre-existing token for Vault Agent.
+```

command/agent.go

@@ -16,6 +16,8 @@ import (
 	"sync"
 	"time"
 
+	token_file "github.com/hashicorp/vault/command/agent/auth/token-file"
+
 	ctconfig "github.com/hashicorp/consul-template/config"
 	"github.com/hashicorp/go-multierror"
 
@@ -368,6 +370,8 @@ func (c *AgentCommand) Run(args []string) int {
 			method, err = kubernetes.NewKubernetesAuthMethod(authConfig)
 		case "approle":
 			method, err = approle.NewApproleAuthMethod(authConfig)
+		case "token_file":
+			method, err = token_file.NewTokenFileAuthMethod(authConfig)
 		case "pcf": // Deprecated.
 			method, err = cf.NewCFAuthMethod(authConfig)
 		default:

command/agent/auth/auth.go

@@ -169,6 +169,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 		var path string
 		var data map[string]interface{}
 		var header http.Header
+		var isTokenFileMethod bool
 
 		switch am.(type) {
 		case AuthMethodWithClient:
@@ -254,9 +255,22 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 		}
 
 		// This should only happen if there's no preloaded token (regular auto-auth login)
-		//  or if a preloaded token has expired and is now switching to auto-auth.
+		// or if a preloaded token has expired and is now switching to auto-auth.
 		if secret.Auth == nil {
-			secret, err = clientToUse.Logical().WriteWithContext(ctx, path, data)
+			isTokenFileMethod = path == "auth/token/lookup-self"
+			if isTokenFileMethod {
+				token, _ := data["token"].(string)
+				lookupSelfClient, err := clientToUse.Clone()
+				if err != nil {
+					ah.logger.Error("failed to clone client to perform token lookup")
+					return err
+				}
+				lookupSelfClient.SetToken(token)
+				secret, err = lookupSelfClient.Auth().Token().LookupSelf()
+			} else {
+				secret, err = clientToUse.Logical().WriteWithContext(ctx, path, data)
+			}
+
 			// Check errors/sanity
 			if err != nil {
 				ah.logger.Error("error authenticating", "error", err, "backoff", backoffCfg)
@@ -269,6 +283,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			}
 		}
 
+		var leaseDuration int
+
 		switch {
 		case ah.wrapTTL > 0:
 			if secret.WrapInfo == nil {
@@ -319,28 +335,77 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			}
 
 		default:
-			if secret == nil || secret.Auth == nil {
+			// We handle the token_file method specially, as it's the only
-				ah.logger.Error("authentication returned nil auth info", "backoff", backoffCfg)
+			// auth method that isn't actually authenticating, i.e. the secret
-				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+			// returned does not have an Auth struct attached
+			isTokenFileMethod := path == "auth/token/lookup-self"
+			if isTokenFileMethod {
+				// We still check the response of the request to ensure the token is valid
+				// i.e. if the token is invalid, we will fail in the authentication step
+				if secret == nil || secret.Data == nil {
+					ah.logger.Error("token file validation failed, token may be invalid", "backoff", backoffCfg)
+					metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
 
-				if backoff(ctx, backoffCfg) {
+					if backoff(ctx, backoffCfg) {
-					continue
+						continue
+					}
+					return err
 				}
-				return err
+				token, ok := secret.Data["id"].(string)
-			}
+				if !ok || token == "" {
-			if secret.Auth.ClientToken == "" {
+					ah.logger.Error("token file validation returned empty client token", "backoff", backoffCfg)
-				ah.logger.Error("authentication returned empty client token", "backoff", backoffCfg)
+					metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
-				metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
 
-				if backoff(ctx, backoffCfg) {
+					if backoff(ctx, backoffCfg) {
-					continue
+						continue
+					}
+					return err
+				}
+
+				duration, _ := secret.Data["ttl"].(json.Number).Int64()
+				leaseDuration = int(duration)
+				renewable, _ := secret.Data["renewable"].(bool)
+				secret.Auth = &api.SecretAuth{
+					ClientToken:   token,
+					LeaseDuration: int(duration),
+					Renewable:     renewable,
+				}
+				ah.logger.Info("authentication successful, sending token to sinks")
+				ah.OutputCh <- token
+				if ah.enableTemplateTokenCh {
+					ah.TemplateTokenCh <- token
+				}
+
+				tokenType := secret.Data["type"].(string)
+				if tokenType == "batch" {
+					ah.logger.Info("note that this token type is batch, and batch tokens cannot be renewed", "ttl", leaseDuration)
+				}
+			} else {
+				if secret == nil || secret.Auth == nil {
+					ah.logger.Error("authentication returned nil auth info", "backoff", backoffCfg)
+					metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+					if backoff(ctx, backoffCfg) {
+						continue
+					}
+					return err
+				}
+				if secret.Auth.ClientToken == "" {
+					ah.logger.Error("authentication returned empty client token", "backoff", backoffCfg)
+					metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
+
+					if backoff(ctx, backoffCfg) {
+						continue
+					}
+					return err
+				}
+
+				leaseDuration = secret.LeaseDuration
+				ah.logger.Info("authentication successful, sending token to sinks")
+				ah.OutputCh <- secret.Auth.ClientToken
+				if ah.enableTemplateTokenCh {
+					ah.TemplateTokenCh <- secret.Auth.ClientToken
 				}
-				return err
-			}
-			ah.logger.Info("authentication successful, sending token to sinks")
-			ah.OutputCh <- secret.Auth.ClientToken
-			if ah.enableTemplateTokenCh {
-				ah.TemplateTokenCh <- secret.Auth.ClientToken
 			}
 
 			am.CredSuccess()
@@ -364,10 +429,15 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 			return err
 		}
 
-		// Start the renewal process
-		ah.logger.Info("starting renewal process")
 		metrics.IncrCounter([]string{"agent", "auth", "success"}, 1)
-		go watcher.Renew()
+		// We don't want to trigger the renewal process for tokens with
+		// unlimited TTL, such as the root token.
+		if leaseDuration == 0 && isTokenFileMethod {
+			ah.logger.Info("not starting token renewal process, as token has unlimited TTL")
+		} else {
+			ah.logger.Info("starting renewal process")
+			go watcher.Renew()
+		}
 
 	LifetimeWatcherLoop:
 		for {

command/agent/auth/token-file/token_file.go

@@ -0,0 +1,83 @@
+package token_file
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/command/agent/auth"
+)
+
+type tokenFileMethod struct {
+	logger    hclog.Logger
+	mountPath string
+
+	cachedToken   string
+	tokenFilePath string
+}
+
+func NewTokenFileAuthMethod(conf *auth.AuthConfig) (auth.AuthMethod, error) {
+	if conf == nil {
+		return nil, errors.New("empty config")
+	}
+	if conf.Config == nil {
+		return nil, errors.New("empty config data")
+	}
+
+	a := &tokenFileMethod{
+		logger:    conf.Logger,
+		mountPath: "auth/token",
+	}
+
+	tokenFilePathRaw, ok := conf.Config["token_file_path"]
+	if !ok {
+		return nil, errors.New("missing 'token_file_path' value")
+	}
+	a.tokenFilePath, ok = tokenFilePathRaw.(string)
+	if !ok {
+		return nil, errors.New("could not convert 'token_file_path' config value to string")
+	}
+	if a.tokenFilePath == "" {
+		return nil, errors.New("'token_file_path' value is empty")
+	}
+
+	return a, nil
+}
+
+func (a *tokenFileMethod) Authenticate(ctx context.Context, client *api.Client) (string, http.Header, map[string]interface{}, error) {
+	token, err := os.ReadFile(a.tokenFilePath)
+	if err != nil {
+		if a.cachedToken == "" {
+			return "", nil, nil, fmt.Errorf("error reading token file and no cached token known: %w", err)
+		}
+		a.logger.Warn("error reading token file", "error", err)
+	}
+	if len(token) == 0 {
+		if a.cachedToken == "" {
+			return "", nil, nil, errors.New("token file empty and no cached token known")
+		}
+		a.logger.Warn("token file exists but read empty value, re-using cached value")
+	} else {
+		a.cachedToken = strings.TrimSpace(string(token))
+	}
+
+	// i.e. auth/token/lookup-self
+	return fmt.Sprintf("%s/lookup-self", a.mountPath), nil, map[string]interface{}{
+		"token": a.cachedToken,
+	}, nil
+}
+
+func (a *tokenFileMethod) NewCreds() chan struct{} {
+	return nil
+}
+
+func (a *tokenFileMethod) CredSuccess() {
+}
+
+func (a *tokenFileMethod) Shutdown() {
+}

command/agent/auth/token-file/token_file_test.go

@@ -0,0 +1,81 @@
+package token_file
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/command/agent/auth"
+	"github.com/hashicorp/vault/sdk/helper/logging"
+)
+
+func TestNewTokenFileAuthMethodEmptyConfig(t *testing.T) {
+	logger := logging.NewVaultLogger(log.Trace)
+	_, err := NewTokenFileAuthMethod(&auth.AuthConfig{
+		Logger: logger.Named("auth.method"),
+		Config: map[string]interface{}{},
+	})
+	if err == nil {
+		t.Fatal("Expected error due to empty config")
+	}
+}
+
+func TestNewTokenFileEmptyFilePath(t *testing.T) {
+	logger := logging.NewVaultLogger(log.Trace)
+	_, err := NewTokenFileAuthMethod(&auth.AuthConfig{
+		Logger: logger.Named("auth.method"),
+		Config: map[string]interface{}{
+			"token_file_path": "",
+		},
+	})
+	if err == nil {
+		t.Fatalf("Expected error when giving empty file path")
+	}
+}
+
+func TestNewTokenFileAuthenticate(t *testing.T) {
+	tokenFile, err := os.Create(filepath.Join(t.TempDir(), "token_file"))
+	tokenFileContents := "super-secret-token"
+	if err != nil {
+		t.Fatal(err)
+	}
+	tokenFileName := tokenFile.Name()
+	tokenFile.Close() // WriteFile doesn't need it open
+	os.WriteFile(tokenFileName, []byte(tokenFileContents), 0o666)
+	defer os.Remove(tokenFileName)
+
+	logger := logging.NewVaultLogger(log.Trace)
+	am, err := NewTokenFileAuthMethod(&auth.AuthConfig{
+		Logger: logger.Named("auth.method"),
+		Config: map[string]interface{}{
+			"token_file_path": tokenFileName,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path, headers, data, err := am.Authenticate(nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if path != "auth/token/lookup-self" {
+		t.Fatalf("Incorrect path, was %s", path)
+	}
+	if headers != nil {
+		t.Fatalf("Expected no headers, instead got %v", headers)
+	}
+	if data == nil {
+		t.Fatal("Data was nil")
+	}
+	tokenDataFromAuthMethod := data["token"].(string)
+	if tokenDataFromAuthMethod != tokenFileContents {
+		t.Fatalf("Incorrect token file contents return by auth method, expected %s, got %s", tokenFileContents, tokenDataFromAuthMethod)
+	}
+
+	_, err = os.Stat(tokenFileName)
+	if err != nil {
+		t.Fatal("Token file removed")
+	}
+}

command/agent/token_file_end_to_end_test.go

@@ -0,0 +1,159 @@
+package agent
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/command/agent/auth"
+	token_file "github.com/hashicorp/vault/command/agent/auth/token-file"
+	"github.com/hashicorp/vault/command/agent/sink"
+	"github.com/hashicorp/vault/command/agent/sink/file"
+	vaulthttp "github.com/hashicorp/vault/http"
+	"github.com/hashicorp/vault/sdk/helper/logging"
+	"github.com/hashicorp/vault/vault"
+)
+
+func TestTokenFileEndToEnd(t *testing.T) {
+	var err error
+	logger := logging.NewVaultLogger(log.Trace)
+	coreConfig := &vault.CoreConfig{
+		DisableMlock: true,
+		DisableCache: true,
+		Logger:       log.NewNullLogger(),
+	}
+
+	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+	})
+
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	cores := cluster.Cores
+
+	vault.TestWaitActive(t, cores[0].Core)
+
+	client := cores[0].Client
+
+	secret, err := client.Auth().Token().Create(nil)
+	if err != nil || secret == nil {
+		t.Fatal(err)
+	}
+
+	tokenFile, err := os.Create(filepath.Join(t.TempDir(), "token_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tokenFileName := tokenFile.Name()
+	tokenFile.Close() // WriteFile doesn't need it open
+	os.WriteFile(tokenFileName, []byte(secret.Auth.ClientToken), 0o666)
+	defer os.Remove(tokenFileName)
+
+	ahConfig := &auth.AuthHandlerConfig{
+		Logger: logger.Named("auth.handler"),
+		Client: client,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+
+	am, err := token_file.NewTokenFileAuthMethod(&auth.AuthConfig{
+		Logger: logger.Named("auth.method"),
+		Config: map[string]interface{}{
+			"token_file_path": tokenFileName,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ah := auth.NewAuthHandler(ahConfig)
+	errCh := make(chan error)
+	go func() {
+		errCh <- ah.Run(ctx, am)
+	}()
+	defer func() {
+		select {
+		case <-ctx.Done():
+		case err := <-errCh:
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}()
+
+	// We close these right away because we're just basically testing
+	// permissions and finding a usable file name
+	sinkFile, err := os.Create(filepath.Join(t.TempDir(), "auth.tokensink.test."))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tokenSinkFileName := sinkFile.Name()
+	sinkFile.Close()
+	os.Remove(tokenSinkFileName)
+	t.Logf("output: %s", tokenSinkFileName)
+
+	config := &sink.SinkConfig{
+		Logger: logger.Named("sink.file"),
+		Config: map[string]interface{}{
+			"path": tokenSinkFileName,
+		},
+		WrapTTL: 10 * time.Second,
+	}
+
+	fs, err := file.NewFileSink(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	config.Sink = fs
+
+	ss := sink.NewSinkServer(&sink.SinkServerConfig{
+		Logger: logger.Named("sink.server"),
+		Client: client,
+	})
+	go func() {
+		errCh <- ss.Run(ctx, ah.OutputCh, []*sink.SinkConfig{config})
+	}()
+	defer func() {
+		select {
+		case <-ctx.Done():
+		case err := <-errCh:
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}()
+
+	// This has to be after the other defers, so it happens first. It allows
+	// successful test runs to immediately cancel all of the runner goroutines
+	// and unblock any of the blocking defer calls by the runner's DoneCh that
+	// comes before this and avoid successful tests from taking the entire
+	// timeout duration.
+	defer cancel()
+
+	if stat, err := os.Lstat(tokenSinkFileName); err == nil {
+		t.Fatalf("expected err but got %s", stat)
+	} else if !os.IsNotExist(err) {
+		t.Fatal("expected notexist err")
+	}
+
+	// Wait 2 seconds for the env variables to be detected and an auth to be generated.
+	time.Sleep(time.Second * 2)
+
+	token, err := readToken(tokenSinkFileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if token.Token == "" {
+		t.Fatal("expected token but didn't receive it")
+	}
+
+	_, err = os.Stat(tokenFileName)
+	if err != nil {
+		t.Fatal("Token file removed")
+	}
+}