luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

VAULT-11510 Vault Agent can start listeners without caching (#18137) 

* VAULT-11510 Vault Agent can start listeners without caching

* VAULT-11510 fix order of imports

* VAULT-11510 changelog

* VAULT-11510 typo and better switch

* VAULT-11510 update name

* VAULT-11510 New api_proxy stanza to configure API proxy

* VAULT-11510 First pass at API Proxy docs

* VAULT-11510 nav data

* VAULT-11510 typo

* VAULT-11510 docs update
This commit is contained in:
Violet Hynes 2022-12-05 10:51:03 -05:00 committed by GitHub
parent 2398634862
commit 398cf38e1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 998 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
changelog/18137.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
agent: Configured Vault Agent listeners now listen without the need for caching to be configured.
```

269
command/agent.go
View File

@ -16,6 +16,8 @@ import (
"sync"
"time"
"github.com/hashicorp/vault/command/agent/sink/inmem"
systemd "github.com/coreos/go-systemd/daemon"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-secure-stdlib/gatedwriter"
@ -39,7 +41,6 @@ import (
agentConfig "github.com/hashicorp/vault/command/agent/config"
"github.com/hashicorp/vault/command/agent/sink"
"github.com/hashicorp/vault/command/agent/sink/file"
"github.com/hashicorp/vault/command/agent/sink/inmem"
"github.com/hashicorp/vault/command/agent/template"
"github.com/hashicorp/vault/command/agent/winsvc"
"github.com/hashicorp/vault/helper/logging"
@ -421,10 +422,37 @@ func (c *AgentCommand) Run(args []string) int {
enforceConsistency := cache.EnforceConsistencyNever
whenInconsistent := cache.WhenInconsistentFail
if config.APIProxy != nil {
switch config.APIProxy.EnforceConsistency {
case "always":
enforceConsistency = cache.EnforceConsistencyAlways
case "never", "":
default:
c.UI.Error(fmt.Sprintf("Unknown api_proxy setting for enforce_consistency: %q", config.APIProxy.EnforceConsistency))
return 1
}
switch config.APIProxy.WhenInconsistent {
case "retry":
whenInconsistent = cache.WhenInconsistentRetry
case "forward":
whenInconsistent = cache.WhenInconsistentForward
case "fail", "":
default:
c.UI.Error(fmt.Sprintf("Unknown api_proxy setting for when_inconsistent: %q", config.APIProxy.WhenInconsistent))
return 1
}
}
// Keep Cache configuration for legacy reasons, but error if defined alongside API Proxy
if config.Cache != nil {
switch config.Cache.EnforceConsistency {
case "always":
enforceConsistency = cache.EnforceConsistencyAlways
if enforceConsistency != cache.EnforceConsistencyNever {
c.UI.Error("enforce_consistency configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.")
return 1
} else {
enforceConsistency = cache.EnforceConsistencyAlways
}
case "never", "":
default:
c.UI.Error(fmt.Sprintf("Unknown cache setting for enforce_consistency: %q", config.Cache.EnforceConsistency))
@ -433,9 +461,19 @@ func (c *AgentCommand) Run(args []string) int {
switch config.Cache.WhenInconsistent {
case "retry":
whenInconsistent = cache.WhenInconsistentRetry
if whenInconsistent != cache.WhenInconsistentFail {
c.UI.Error("when_inconsistent configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.")
return 1
} else {
whenInconsistent = cache.WhenInconsistentRetry
}
case "forward":
whenInconsistent = cache.WhenInconsistentForward
if whenInconsistent != cache.WhenInconsistentFail {
c.UI.Error("when_inconsistent configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.")
return 1
} else {
whenInconsistent = cache.WhenInconsistentForward
}
case "fail", "":
default:
c.UI.Error(fmt.Sprintf("Unknown cache setting for when_inconsistent: %q", config.Cache.WhenInconsistent))
@ -466,36 +504,39 @@ func (c *AgentCommand) Run(args []string) int {
var leaseCache *cache.LeaseCache
var previousToken string
// Parse agent listener configurations
proxyClient, err := client.CloneWithHeaders()
if err != nil {
c.UI.Error(fmt.Sprintf("Error cloning client for proxying: %v", err))
return 1
}
if config.DisableIdleConnsAPIProxy {
proxyClient.SetMaxIdleConnections(-1)
}
if config.DisableKeepAlivesAPIProxy {
proxyClient.SetDisableKeepAlives(true)
}
apiProxyLogger := c.logger.Named("apiproxy")
// The API proxy to be used, if listeners are configured
apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{
Client: proxyClient,
Logger: apiProxyLogger,
EnforceConsistency: enforceConsistency,
WhenInconsistentAction: whenInconsistent,
})
if err != nil {
c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err))
return 1
}
// Parse agent cache configurations
if config.Cache != nil {
cacheLogger := c.logger.Named("cache")
proxyClient, err := client.CloneWithHeaders()
if err != nil {
c.UI.Error(fmt.Sprintf("Error cloning client for caching: %v", err))
return 1
}
if config.DisableIdleConnsCaching {
proxyClient.SetMaxIdleConnections(-1)
}
if config.DisableKeepAlivesCaching {
proxyClient.SetDisableKeepAlives(true)
}
// Create the API proxier
apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{
Client: proxyClient,
Logger: cacheLogger.Named("apiproxy"),
EnforceConsistency: enforceConsistency,
WhenInconsistentAction: whenInconsistent,
})
if err != nil {
c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err))
return 1
}
// Create the lease cache proxier and set its underlying proxier to
// the API proxier.
leaseCache, err = cache.NewLeaseCache(&cache.LeaseCacheConfig{
@ -654,104 +695,106 @@ func (c *AgentCommand) Run(args []string) int {
leaseCache.SetPersistentStorage(ps)
}
}
}
var inmemSink sink.Sink
if config.Cache.UseAutoAuthToken {
cacheLogger.Debug("auto-auth token is allowed to be used; configuring inmem sink")
inmemSink, err = inmem.New(&sink.SinkConfig{
Logger: cacheLogger,
}, leaseCache)
var listeners []net.Listener
// If there are templates, add an in-process listener
if len(config.Templates) > 0 {
config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType})
}
for i, lnConfig := range config.Listeners {
var ln net.Listener
var tlsConf *tls.Config
if lnConfig.Type == listenerutil.BufConnType {
inProcListener := bufconn.Listen(1024 * 1024)
if config.Cache != nil {
config.Cache.InProcDialer = listenerutil.NewBufConnWrapper(inProcListener)
}
ln = inProcListener
} else {
ln, tlsConf, err = cache.StartListener(lnConfig)
if err != nil {
c.UI.Error(fmt.Sprintf("Error creating inmem sink for cache: %v", err))
c.UI.Error(fmt.Sprintf("Error starting listener: %v", err))
return 1
}
sinks = append(sinks, &sink.SinkConfig{
Logger: cacheLogger,
Sink: inmemSink,
})
}
proxyVaultToken := !config.Cache.ForceAutoAuthToken
listeners = append(listeners, ln)
// Create the request handler
cacheHandler := cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken)
var listeners []net.Listener
// If there are templates, add an in-process listener
if len(config.Templates) > 0 {
config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType})
}
for i, lnConfig := range config.Listeners {
var ln net.Listener
var tlsConf *tls.Config
if lnConfig.Type == listenerutil.BufConnType {
inProcListener := bufconn.Listen(1024 * 1024)
config.Cache.InProcDialer = listenerutil.NewBufConnWrapper(inProcListener)
ln = inProcListener
} else {
ln, tlsConf, err = cache.StartListener(lnConfig)
proxyVaultToken := true
var inmemSink sink.Sink
if config.APIProxy != nil {
if config.APIProxy.UseAutoAuthToken {
apiProxyLogger.Debug("auto-auth token is allowed to be used; configuring inmem sink")
inmemSink, err = inmem.New(&sink.SinkConfig{
Logger: apiProxyLogger,
}, leaseCache)
if err != nil {
c.UI.Error(fmt.Sprintf("Error starting listener: %v", err))
c.UI.Error(fmt.Sprintf("Error creating inmem sink for cache: %v", err))
return 1
}
sinks = append(sinks, &sink.SinkConfig{
Logger: apiProxyLogger,
Sink: inmemSink,
})
}
listeners = append(listeners, ln)
// Parse 'require_request_header' listener config option, and wrap
// the request handler if necessary
muxHandler := cacheHandler
if lnConfig.RequireRequestHeader && ("metrics_only" != lnConfig.Role) {
muxHandler = verifyRequestHeader(muxHandler)
}
// Create a muxer and add paths relevant for the lease cache layer
mux := http.NewServeMux()
quitEnabled := lnConfig.AgentAPI != nil && lnConfig.AgentAPI.EnableQuit
mux.Handle(consts.AgentPathMetrics, c.handleMetrics())
if "metrics_only" != lnConfig.Role {
mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
mux.Handle(consts.AgentPathQuit, c.handleQuit(quitEnabled))
mux.Handle("/", muxHandler)
}
scheme := "https://"
if tlsConf == nil {
scheme = "http://"
}
if ln.Addr().Network() == "unix" {
scheme = "unix://"
}
infoKey := fmt.Sprintf("api address %d", i+1)
info[infoKey] = scheme + ln.Addr().String()
infoKeys = append(infoKeys, infoKey)
server := &http.Server{
Addr: ln.Addr().String(),
TLSConfig: tlsConf,
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
IdleTimeout: 5 * time.Minute,
ErrorLog: cacheLogger.StandardLogger(nil),
}
go server.Serve(ln)
proxyVaultToken = !config.APIProxy.ForceAutoAuthToken
}
// Ensure that listeners are closed at all the exits
listenerCloseFunc := func() {
for _, ln := range listeners {
ln.Close()
}
muxHandler := cache.ProxyHandler(ctx, apiProxyLogger, apiProxy, inmemSink, proxyVaultToken)
// Parse 'require_request_header' listener config option, and wrap
// the request handler if necessary
if lnConfig.RequireRequestHeader && ("metrics_only" != lnConfig.Role) {
muxHandler = verifyRequestHeader(muxHandler)
}
defer c.cleanupGuard.Do(listenerCloseFunc)
// Create a muxer and add paths relevant for the lease cache layer
mux := http.NewServeMux()
quitEnabled := lnConfig.AgentAPI != nil && lnConfig.AgentAPI.EnableQuit
mux.Handle(consts.AgentPathMetrics, c.handleMetrics())
if "metrics_only" != lnConfig.Role {
mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
mux.Handle(consts.AgentPathQuit, c.handleQuit(quitEnabled))
mux.Handle("/", muxHandler)
}
scheme := "https://"
if tlsConf == nil {
scheme = "http://"
}
if ln.Addr().Network() == "unix" {
scheme = "unix://"
}
infoKey := fmt.Sprintf("api address %d", i+1)
info[infoKey] = scheme + ln.Addr().String()
infoKeys = append(infoKeys, infoKey)
server := &http.Server{
Addr: ln.Addr().String(),
TLSConfig: tlsConf,
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
IdleTimeout: 5 * time.Minute,
ErrorLog: apiProxyLogger.StandardLogger(nil),
}
go server.Serve(ln)
}
// Ensure that listeners are closed at all the exits
listenerCloseFunc := func() {
for _, ln := range listeners {
ln.Close()
}
}
defer c.cleanupGuard.Do(listenerCloseFunc)
// Inform any tests that the server is ready
if c.startedCh != nil {
close(c.startedCh)

231
command/agent/cache/api_proxy_test.go vendored
View File

@ -1,8 +1,18 @@
package cache
import (
"context"
"fmt"
"net"
"net/http"
"os"
"testing"
"time"
"github.com/hashicorp/vault/builtin/credential/userpass"
vaulthttp "github.com/hashicorp/vault/http"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/vault"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/api"
@ -11,6 +21,12 @@ import (
"github.com/hashicorp/vault/sdk/helper/logging"
)
const policyAdmin = `
path "*" {
capabilities = ["sudo", "create", "read", "update", "delete", "list"]
}
`
func TestAPIProxy(t *testing.T) {
cleanup, client, _, _ := setupClusterAndAgent(namespace.RootContext(nil), t, nil)
defer cleanup()
@ -47,6 +63,42 @@ func TestAPIProxy(t *testing.T) {
}
}
func TestAPIProxyNoCache(t *testing.T) {
cleanup, client, _, _ := setupClusterAndAgentNoCache(namespace.RootContext(nil), t, nil)
defer cleanup()
proxier, err := NewAPIProxy(&APIProxyConfig{
Client: client,
Logger: logging.NewVaultLogger(hclog.Trace),
})
if err != nil {
t.Fatal(err)
}
r := client.NewRequest("GET", "/v1/sys/health")
req, err := r.ToHTTP()
if err != nil {
t.Fatal(err)
}
resp, err := proxier.Send(namespace.RootContext(nil), &SendRequest{
Request: req,
})
if err != nil {
t.Fatal(err)
}
var result api.HealthResponse
err = jsonutil.DecodeJSONFromReader(resp.Response.Body, &result)
if err != nil {
t.Fatal(err)
}
if !result.Initialized || result.Sealed || result.Standby {
t.Fatalf("bad sys/health response: %#v", result)
}
}
func TestAPIProxy_queryParams(t *testing.T) {
// Set up an agent that points to a standby node for this particular test
// since it needs to proxy a /sys/health?standbyok=true request to a standby
@ -93,3 +145,182 @@ func TestAPIProxy_queryParams(t *testing.T) {
t.Fatalf("exptected standby to return 200, got: %v", resp.Response.StatusCode)
}
}
// setupClusterAndAgent is a helper func used to set up a test cluster and
// caching agent against the active node. It returns a cleanup func that should
// be deferred immediately along with two clients, one for direct cluster
// communication and another to talk to the caching agent.
func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) {
return setupClusterAndAgentCommon(ctx, t, coreConfig, false, true)
}
// setupClusterAndAgentNoCache is a helper func used to set up a test cluster and
// proxying agent against the active node. It returns a cleanup func that should
// be deferred immediately along with two clients, one for direct cluster
// communication and another to talk to the caching agent.
func setupClusterAndAgentNoCache(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) {
return setupClusterAndAgentCommon(ctx, t, coreConfig, false, false)
}
// setupClusterAndAgentOnStandby is a helper func used to set up a test cluster
// and caching agent against a standby node. It returns a cleanup func that
// should be deferred immediately along with two clients, one for direct cluster
// communication and another to talk to the caching agent.
func setupClusterAndAgentOnStandby(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) {
return setupClusterAndAgentCommon(ctx, t, coreConfig, true, true)
}
func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig, onStandby bool, useCache bool) (func(), *api.Client, *api.Client, *LeaseCache) {
t.Helper()
if ctx == nil {
ctx = context.Background()
}
// Handle sane defaults
if coreConfig == nil {
coreConfig = &vault.CoreConfig{
DisableMlock: true,
DisableCache: true,
Logger: logging.NewVaultLogger(hclog.Trace),
}
}
// Always set up the userpass backend since we use that to generate an admin
// token for the client that will make proxied requests to through the agent.
if coreConfig.CredentialBackends == nil || coreConfig.CredentialBackends["userpass"] == nil {
coreConfig.CredentialBackends = map[string]logical.Factory{
"userpass": userpass.Factory,
}
}
// Init new test cluster
cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
HandlerFunc: vaulthttp.Handler,
})
cluster.Start()
cores := cluster.Cores
vault.TestWaitActive(t, cores[0].Core)
activeClient := cores[0].Client
standbyClient := cores[1].Client
// clienToUse is the client for the agent to point to.
clienToUse := activeClient
if onStandby {
clienToUse = standbyClient
}
// Add an admin policy
if err := activeClient.Sys().PutPolicy("admin", policyAdmin); err != nil {
t.Fatal(err)
}
// Set up the userpass auth backend and an admin user. Used for getting a token
// for the agent later down in this func.
err := activeClient.Sys().EnableAuthWithOptions("userpass", &api.EnableAuthOptions{
Type: "userpass",
})
if err != nil {
t.Fatal(err)
}
_, err = activeClient.Logical().Write("auth/userpass/users/foo", map[string]interface{}{
"password": "bar",
"policies": []string{"admin"},
})
if err != nil {
t.Fatal(err)
}
// Set up env vars for agent consumption
origEnvVaultAddress := os.Getenv(api.EnvVaultAddress)
os.Setenv(api.EnvVaultAddress, clienToUse.Address())
origEnvVaultCACert := os.Getenv(api.EnvVaultCACert)
os.Setenv(api.EnvVaultCACert, fmt.Sprintf("%s/ca_cert.pem", cluster.TempDir))
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
apiProxyLogger := logging.NewVaultLogger(hclog.Trace).Named("apiproxy")
// Create the API proxier
apiProxy, err := NewAPIProxy(&APIProxyConfig{
Client: clienToUse,
Logger: apiProxyLogger,
})
if err != nil {
t.Fatal(err)
}
// Create a muxer and add paths relevant for the lease cache layer and API proxy layer
mux := http.NewServeMux()
var leaseCache *LeaseCache
if useCache {
cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache")
// Create the lease cache proxier and set its underlying proxier to
// the API proxier.
leaseCache, err = NewLeaseCache(&LeaseCacheConfig{
Client: clienToUse,
BaseContext: ctx,
Proxier: apiProxy,
Logger: cacheLogger.Named("leasecache"),
})
if err != nil {
t.Fatal(err)
}
mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx))
mux.Handle("/", ProxyHandler(ctx, cacheLogger, leaseCache, nil, true))
} else {
mux.Handle("/", ProxyHandler(ctx, apiProxyLogger, apiProxy, nil, true))
}
server := &http.Server{
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
IdleTimeout: 5 * time.Minute,
ErrorLog: apiProxyLogger.StandardLogger(nil),
}
go server.Serve(listener)
// testClient is the client that is used to talk to the agent for proxying/caching behavior.
testClient, err := activeClient.Clone()
if err != nil {
t.Fatal(err)
}
if err := testClient.SetAddress("http://" + listener.Addr().String()); err != nil {
t.Fatal(err)
}
// Login via userpass method to derive a managed token. Set that token as the
// testClient's token
resp, err := testClient.Logical().Write("auth/userpass/login/foo", map[string]interface{}{
"password": "bar",
})
if err != nil {
t.Fatal(err)
}
testClient.SetToken(resp.Auth.ClientToken)
cleanup := func() {
// We wait for a tiny bit for things such as agent renewal to exit properly
time.Sleep(50 * time.Millisecond)
cluster.Cleanup()
os.Setenv(api.EnvVaultAddress, origEnvVaultAddress)
os.Setenv(api.EnvVaultCACert, origEnvVaultCACert)
listener.Close()
}
return cleanup, clienToUse, testClient, leaseCache
}

174
command/agent/cache/cache_test.go vendored
View File

@ -8,7 +8,6 @@ import (
"math/rand"
"net"
"net/http"
"os"
"sync"
"testing"
"time"
@ -17,7 +16,6 @@ import (
"github.com/hashicorp/go-hclog"
kv "github.com/hashicorp/vault-plugin-secrets-kv"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/builtin/credential/userpass"
"github.com/hashicorp/vault/command/agent/cache/cachememdb"
"github.com/hashicorp/vault/command/agent/sink/mock"
"github.com/hashicorp/vault/helper/namespace"
@ -28,174 +26,6 @@ import (
"github.com/hashicorp/vault/vault"
)
const policyAdmin = `
path "*" {
capabilities = ["sudo", "create", "read", "update", "delete", "list"]
}
`
// setupClusterAndAgent is a helper func used to set up a test cluster and
// caching agent against the active node. It returns a cleanup func that should
// be deferred immediately along with two clients, one for direct cluster
// communication and another to talk to the caching agent.
func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) {
return setupClusterAndAgentCommon(ctx, t, coreConfig, false)
}
// setupClusterAndAgentOnStandby is a helper func used to set up a test cluster
// and caching agent against a standby node. It returns a cleanup func that
// should be deferred immediately along with two clients, one for direct cluster
// communication and another to talk to the caching agent.
func setupClusterAndAgentOnStandby(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) {
return setupClusterAndAgentCommon(ctx, t, coreConfig, true)
}
func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig, onStandby bool) (func(), *api.Client, *api.Client, *LeaseCache) {
t.Helper()
if ctx == nil {
ctx = context.Background()
}
// Handle sane defaults
if coreConfig == nil {
coreConfig = &vault.CoreConfig{
DisableMlock: true,
DisableCache: true,
Logger: logging.NewVaultLogger(hclog.Trace),
}
}
// Always set up the userpass backend since we use that to generate an admin
// token for the client that will make proxied requests to through the agent.
if coreConfig.CredentialBackends == nil || coreConfig.CredentialBackends["userpass"] == nil {
coreConfig.CredentialBackends = map[string]logical.Factory{
"userpass": userpass.Factory,
}
}
// Init new test cluster
cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
HandlerFunc: vaulthttp.Handler,
})
cluster.Start()
cores := cluster.Cores
vault.TestWaitActive(t, cores[0].Core)
activeClient := cores[0].Client
standbyClient := cores[1].Client
// clienToUse is the client for the agent to point to.
clienToUse := activeClient
if onStandby {
clienToUse = standbyClient
}
// Add an admin policy
if err := activeClient.Sys().PutPolicy("admin", policyAdmin); err != nil {
t.Fatal(err)
}
// Set up the userpass auth backend and an admin user. Used for getting a token
// for the agent later down in this func.
err := activeClient.Sys().EnableAuthWithOptions("userpass", &api.EnableAuthOptions{
Type: "userpass",
})
if err != nil {
t.Fatal(err)
}
_, err = activeClient.Logical().Write("auth/userpass/users/foo", map[string]interface{}{
"password": "bar",
"policies": []string{"admin"},
})
if err != nil {
t.Fatal(err)
}
// Set up env vars for agent consumption
origEnvVaultAddress := os.Getenv(api.EnvVaultAddress)
os.Setenv(api.EnvVaultAddress, clienToUse.Address())
origEnvVaultCACert := os.Getenv(api.EnvVaultCACert)
os.Setenv(api.EnvVaultCACert, fmt.Sprintf("%s/ca_cert.pem", cluster.TempDir))
cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache")
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
// Create the API proxier
apiProxy, err := NewAPIProxy(&APIProxyConfig{
Client: clienToUse,
Logger: cacheLogger.Named("apiproxy"),
})
if err != nil {
t.Fatal(err)
}
// Create the lease cache proxier and set its underlying proxier to
// the API proxier.
leaseCache, err := NewLeaseCache(&LeaseCacheConfig{
Client: clienToUse,
BaseContext: ctx,
Proxier: apiProxy,
Logger: cacheLogger.Named("leasecache"),
})
if err != nil {
t.Fatal(err)
}
// Create a muxer and add paths relevant for the lease cache layer
mux := http.NewServeMux()
mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx))
mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, nil, true))
server := &http.Server{
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
IdleTimeout: 5 * time.Minute,
ErrorLog: cacheLogger.StandardLogger(nil),
}
go server.Serve(listener)
// testClient is the client that is used to talk to the agent for proxying/caching behavior.
testClient, err := activeClient.Clone()
if err != nil {
t.Fatal(err)
}
if err := testClient.SetAddress("http://" + listener.Addr().String()); err != nil {
t.Fatal(err)
}
// Login via userpass method to derive a managed token. Set that token as the
// testClient's token
resp, err := testClient.Logical().Write("auth/userpass/login/foo", map[string]interface{}{
"password": "bar",
})
if err != nil {
t.Fatal(err)
}
testClient.SetToken(resp.Auth.ClientToken)
cleanup := func() {
// We wait for a tiny bit for things such as agent renewal to exit properly
time.Sleep(50 * time.Millisecond)
cluster.Cleanup()
os.Setenv(api.EnvVaultAddress, origEnvVaultAddress)
os.Setenv(api.EnvVaultCACert, origEnvVaultCACert)
listener.Close()
}
return cleanup, clienToUse, testClient, leaseCache
}
func tokenRevocationValidation(t *testing.T, sampleSpace map[string]string, expected map[string]string, leaseCache *LeaseCache) {
t.Helper()
for val, valType := range sampleSpace {
@ -248,7 +78,7 @@ func TestCache_AutoAuthTokenStripping(t *testing.T) {
mux := http.NewServeMux()
mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true))
mux.Handle("/", ProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true))
server := &http.Server{
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,
@ -337,7 +167,7 @@ func TestCache_AutoAuthClientTokenProxyStripping(t *testing.T) {
mux := http.NewServeMux()
// mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false))
mux.Handle("/", ProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false))
server := &http.Server{
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,

6
command/agent/cache/handler.go vendored
View File

@ -20,7 +20,7 @@ import (
"github.com/hashicorp/vault/sdk/logical"
)
func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler {
func ProxyHandler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
logger.Info("received request", "method", r.Method, "path", r.URL.Path)
@ -36,7 +36,7 @@ func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSin
}
// Parse and reset body.
reqBody, err := ioutil.ReadAll(r.Body)
reqBody, err := io.ReadAll(r.Body)
if err != nil {
logger.Error("failed to read request body")
logical.RespondError(w, http.StatusInternalServerError, errors.New("failed to read request body"))
@ -45,7 +45,7 @@ func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSin
if r.Body != nil {
r.Body.Close()
}
r.Body = ioutil.NopCloser(bytes.NewReader(reqBody))
r.Body = io.NopCloser(bytes.NewReader(reqBody))
req := &SendRequest{
Token: token,
Request: r,

5
command/agent/cache/lease_cache.go vendored
View File

@ -568,6 +568,11 @@ func computeIndexID(req *SendRequest) (string, error) {
// HandleCacheClear returns a handlerFunc that can perform cache clearing operations.
func (c *LeaseCache) HandleCacheClear(ctx context.Context) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// If the cache is not enabled, return a 200
if c == nil {
return
}
// Only handle POST/PUT requests
switch r.Method {
case http.MethodPost:

6
command/agent/cache/proxy.go vendored
View File

@ -3,7 +3,7 @@ package cache
import (
"bytes"
"context"
"io/ioutil"
"io"
"net/http"
"time"
@ -59,7 +59,7 @@ func NewSendResponse(apiResponse *api.Response, responseBody []byte) (*SendRespo
case len(responseBody) > 0:
resp.ResponseBody = responseBody
case apiResponse.Body != nil:
respBody, err := ioutil.ReadAll(apiResponse.Body)
respBody, err := io.ReadAll(apiResponse.Body)
if err != nil {
return nil, err
}
@ -67,7 +67,7 @@ func NewSendResponse(apiResponse *api.Response, responseBody []byte) (*SendRespo
apiResponse.Body.Close()
// Re-set the response body after reading from the Reader
apiResponse.Body = ioutil.NopCloser(bytes.NewReader(respBody))
apiResponse.Body = io.NopCloser(bytes.NewReader(respBody))
resp.ResponseBody = respBody
}

2
command/agent/cache_end_to_end_test.go
View File

@ -315,7 +315,7 @@ func TestCache_UsingAutoAuthToken(t *testing.T) {
mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
// Passing a non-nil inmemsink tells the agent to use the auto-auth token
mux.Handle("/", cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, true))
mux.Handle("/", cache.ProxyHandler(ctx, cacheLogger, leaseCache, inmemSink, true))
server := &http.Server{
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,

110
command/agent/config/config.go
View File

@ -19,22 +19,23 @@ import (
"github.com/mitchellh/mapstructure"
)
// Config is the configuration for the vault server.
// Config is the configuration for Vault Agent.
type Config struct {
*configutil.SharedConfig `hcl:"-"`
AutoAuth *AutoAuth `hcl:"auto_auth"`
ExitAfterAuth bool `hcl:"exit_after_auth"`
Cache *Cache `hcl:"cache"`
APIProxy *APIProxy `hcl:"api_proxy""`
Vault *Vault `hcl:"vault"`
TemplateConfig *TemplateConfig `hcl:"template_config"`
Templates []*ctconfig.TemplateConfig `hcl:"templates"`
DisableIdleConns []string `hcl:"disable_idle_connections"`
DisableIdleConnsCaching bool `hcl:"-"`
DisableIdleConnsAPIProxy bool `hcl:"-"`
DisableIdleConnsTemplating bool `hcl:"-"`
DisableIdleConnsAutoAuth bool `hcl:"-"`
DisableKeepAlives []string `hcl:"disable_keep_alives"`
DisableKeepAlivesCaching bool `hcl:"-"`
DisableKeepAlivesAPIProxy bool `hcl:"-"`
DisableKeepAlivesTemplating bool `hcl:"-"`
DisableKeepAlivesAutoAuth bool `hcl:"-"`
}
@ -88,6 +89,15 @@ type transportDialer interface {
DialContext(ctx context.Context, network, address string) (net.Conn, error)
}
// APIProxy contains any configuration needed for proxy mode
type APIProxy struct {
UseAutoAuthTokenRaw interface{} `hcl:"use_auto_auth_token"`
UseAutoAuthToken bool `hcl:"-"`
ForceAutoAuthToken bool `hcl:"-"`
EnforceConsistency string `hcl:"enforce_consistency"`
WhenInconsistent string `hcl:"when_inconsistent"`
}
// Cache contains any configuration needed for Cache mode
type Cache struct {
UseAutoAuthTokenRaw interface{} `hcl:"use_auto_auth_token"`
@ -222,6 +232,20 @@ func LoadConfig(path string) (*Config, error) {
return nil, fmt.Errorf("error parsing 'cache':%w", err)
}
if err := parseAPIProxy(result, list); err != nil {
return nil, fmt.Errorf("error parsing 'api_proxy':%w", err)
}
if result.APIProxy != nil && result.Cache != nil {
if result.Cache.UseAutoAuthTokenRaw != nil {
if result.APIProxy.UseAutoAuthTokenRaw != nil {
return nil, fmt.Errorf("use_auto_auth_token defined in both api_proxy and cache config. Please remove this configuration from the cache block")
} else {
result.APIProxy.ForceAutoAuthToken = result.Cache.ForceAutoAuthToken
}
}
}
if err := parseTemplateConfig(result, list); err != nil {
return nil, fmt.Errorf("error parsing 'template_config': %w", err)
}
@ -230,6 +254,13 @@ func LoadConfig(path string) (*Config, error) {
return nil, fmt.Errorf("error parsing 'template': %w", err)
}
if result.Cache != nil && result.APIProxy == nil && len(result.Listeners) > 0 {
result.APIProxy = &APIProxy{
UseAutoAuthToken: result.Cache.UseAutoAuthToken,
ForceAutoAuthToken: result.Cache.ForceAutoAuthToken,
}
}
if result.Cache != nil {
if len(result.Listeners) < 1 && len(result.Templates) < 1 {
return nil, fmt.Errorf("enabling the cache requires at least 1 template or 1 listener to be defined")
@ -239,17 +270,32 @@ func LoadConfig(path string) (*Config, error) {
if result.AutoAuth == nil {
return nil, fmt.Errorf("cache.use_auto_auth_token is true but auto_auth not configured")
}
if result.AutoAuth.Method.WrapTTL > 0 {
if result.AutoAuth != nil && result.AutoAuth.Method != nil && result.AutoAuth.Method.WrapTTL > 0 {
return nil, fmt.Errorf("cache.use_auto_auth_token is true and auto_auth uses wrapping")
}
}
}
if result.APIProxy != nil {
if len(result.Listeners) < 1 {
return nil, fmt.Errorf("configuring the api_proxy requires at least 1 listener to be defined")
}
if result.APIProxy.UseAutoAuthToken {
if result.AutoAuth == nil {
return nil, fmt.Errorf("api_proxy.use_auto_auth_token is true but auto_auth not configured")
}
if result.AutoAuth != nil && result.AutoAuth.Method != nil && result.AutoAuth.Method.WrapTTL > 0 {
return nil, fmt.Errorf("api_proxy.use_auto_auth_token is true and auto_auth uses wrapping")
}
}
}
if result.AutoAuth != nil {
if len(result.AutoAuth.Sinks) == 0 &&
(result.Cache == nil || !result.Cache.UseAutoAuthToken) &&
(result.APIProxy == nil || !result.APIProxy.UseAutoAuthToken) &&
len(result.Templates) == 0 {
return nil, fmt.Errorf("auto_auth requires at least one sink or at least one template or cache.use_auto_auth_token=true")
return nil, fmt.Errorf("auto_auth requires at least one sink or at least one template or api_proxy.use_auto_auth_token=true")
}
}
@ -284,8 +330,8 @@ func LoadConfig(path string) (*Config, error) {
switch subsystem {
case "auto-auth":
result.DisableIdleConnsAutoAuth = true
case "caching":
result.DisableIdleConnsCaching = true
case "caching", "proxying":
result.DisableIdleConnsAPIProxy = true
case "templating":
result.DisableIdleConnsTemplating = true
case "":
@ -306,8 +352,8 @@ func LoadConfig(path string) (*Config, error) {
switch subsystem {
case "auto-auth":
result.DisableKeepAlivesAutoAuth = true
case "caching":
result.DisableKeepAlivesCaching = true
case "caching", "proxying":
result.DisableKeepAlivesAPIProxy = true
case "templating":
result.DisableKeepAlivesTemplating = true
case "":
@ -386,6 +432,50 @@ func parseRetry(result *Config, list *ast.ObjectList) error {
return nil
}
func parseAPIProxy(result *Config, list *ast.ObjectList) error {
name := "api_proxy"
apiProxyList := list.Filter(name)
if len(apiProxyList.Items) == 0 {
return nil
}
if len(apiProxyList.Items) > 1 {
return fmt.Errorf("one and only one %q block is required", name)
}
item := apiProxyList.Items[0]
var apiProxy APIProxy
err := hcl.DecodeObject(&apiProxy, item.Val)
if err != nil {
return err
}
if apiProxy.UseAutoAuthTokenRaw != nil {
apiProxy.UseAutoAuthToken, err = parseutil.ParseBool(apiProxy.UseAutoAuthTokenRaw)
if err != nil {
// Could be a value of "force" instead of "true"/"false"
switch apiProxy.UseAutoAuthTokenRaw.(type) {
case string:
v := apiProxy.UseAutoAuthTokenRaw.(string)
if !strings.EqualFold(v, "force") {
return fmt.Errorf("value of 'use_auto_auth_token' can be either true/false/force, %q is an invalid option", apiProxy.UseAutoAuthTokenRaw)
}
apiProxy.UseAutoAuthToken = true
apiProxy.ForceAutoAuthToken = true
default:
return err
}
}
}
result.APIProxy = &apiProxy
return nil
}
func parseCache(result *Config, list *ast.ObjectList) error {
name := "cache"

238
command/agent/config/config_test.go
View File

@ -69,6 +69,10 @@ func TestLoadConfigFile_AgentCache(t *testing.T) {
},
},
},
APIProxy: &APIProxy{
UseAutoAuthToken: true,
ForceAutoAuthToken: false,
},
Cache: &Cache{
UseAutoAuthToken: true,
UseAutoAuthTokenRaw: true,
@ -394,7 +398,8 @@ func TestLoadConfigFile_AgentCache_NoAutoAuth(t *testing.T) {
}
expected := &Config{
Cache: &Cache{},
APIProxy: &APIProxy{},
Cache: &Cache{},
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
Listeners: []*configutil.Listener{
@ -467,6 +472,13 @@ func TestLoadConfigFile_Bad_AgentCache_AutoAuth_Method_wrapping(t *testing.T) {
}
}
func TestLoadConfigFile_Bad_APIProxy_And_Cache_Same_Config(t *testing.T) {
_, err := LoadConfig("./test-fixtures/bad-config-api_proxy-cache.hcl")
if err == nil {
t.Fatal("LoadConfig should return an error when cache and api_proxy try and configure the same value")
}
}
func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-cache-auto_auth-no-sink.hcl")
if err != nil {
@ -493,6 +505,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) {
},
},
},
APIProxy: &APIProxy{
UseAutoAuthToken: true,
ForceAutoAuthToken: false,
},
Cache: &Cache{
UseAutoAuthToken: true,
UseAutoAuthTokenRaw: true,
@ -537,6 +553,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_Force(t *testing.T) {
},
},
},
APIProxy: &APIProxy{
UseAutoAuthToken: true,
ForceAutoAuthToken: true,
},
Cache: &Cache{
UseAutoAuthToken: true,
UseAutoAuthTokenRaw: "force",
@ -581,6 +601,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_True(t *testing.T) {
},
},
},
APIProxy: &APIProxy{
UseAutoAuthToken: true,
ForceAutoAuthToken: false,
},
Cache: &Cache{
UseAutoAuthToken: true,
UseAutoAuthTokenRaw: "true",
@ -599,6 +623,52 @@ func TestLoadConfigFile_AgentCache_AutoAuth_True(t *testing.T) {
}
}
func TestLoadConfigFile_Agent_AutoAuth_APIProxyAllConfig(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl")
if err != nil {
t.Fatalf("err: %s", err)
}
expected := &Config{
SharedConfig: &configutil.SharedConfig{
Listeners: []*configutil.Listener{
{
Type: "tcp",
Address: "127.0.0.1:8300",
TLSDisable: true,
},
},
PidFile: "./pidfile",
},
AutoAuth: &AutoAuth{
Method: &Method{
Type: "aws",
MountPath: "auth/aws",
Config: map[string]interface{}{
"role": "foobar",
},
},
},
APIProxy: &APIProxy{
UseAutoAuthToken: true,
UseAutoAuthTokenRaw: "force",
ForceAutoAuthToken: true,
EnforceConsistency: "always",
WhenInconsistent: "forward",
},
Vault: &Vault{
Retry: &Retry{
NumRetries: 12,
},
},
}
config.Prune()
if diff := deep.Equal(config, expected); diff != nil {
t.Fatal(diff)
}
}
func TestLoadConfigFile_AgentCache_AutoAuth_False(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-cache-auto_auth-false.hcl")
if err != nil {
@ -636,6 +706,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_False(t *testing.T) {
},
},
},
APIProxy: &APIProxy{
UseAutoAuthToken: false,
ForceAutoAuthToken: false,
},
Cache: &Cache{
UseAutoAuthToken: false,
UseAutoAuthTokenRaw: "false",
@ -661,6 +735,7 @@ func TestLoadConfigFile_AgentCache_Persist(t *testing.T) {
}
expected := &Config{
APIProxy: &APIProxy{},
Cache: &Cache{
Persist: &Persist{
Type: "kubernetes",
@ -1075,6 +1150,7 @@ func TestLoadConfigFile_EnforceConsistency(t *testing.T) {
},
PidFile: "",
},
APIProxy: &APIProxy{},
Cache: &Cache{
EnforceConsistency: "always",
WhenInconsistent: "retry",
@ -1092,6 +1168,40 @@ func TestLoadConfigFile_EnforceConsistency(t *testing.T) {
}
}
func TestLoadConfigFile_EnforceConsistency_APIProxy(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-consistency-apiproxy.hcl")
if err != nil {
t.Fatal(err)
}
expected := &Config{
SharedConfig: &configutil.SharedConfig{
Listeners: []*configutil.Listener{
{
Type: "tcp",
Address: "127.0.0.1:8300",
TLSDisable: true,
},
},
PidFile: "",
},
APIProxy: &APIProxy{
EnforceConsistency: "always",
WhenInconsistent: "retry",
},
Vault: &Vault{
Retry: &Retry{
NumRetries: 12,
},
},
}
config.Prune()
if diff := deep.Equal(config, expected); diff != nil {
t.Fatal(diff)
}
}
func TestLoadConfigFile_Disable_Idle_Conns_All(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-disable-idle-connections-all.hcl")
if err != nil {
@ -1102,8 +1212,8 @@ func TestLoadConfigFile_Disable_Idle_Conns_All(t *testing.T) {
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
},
DisableIdleConns: []string{"auto-auth", "caching", "templating"},
DisableIdleConnsCaching: true,
DisableIdleConns: []string{"auto-auth", "caching", "templating", "proxying"},
DisableIdleConnsAPIProxy: true,
DisableIdleConnsAutoAuth: true,
DisableIdleConnsTemplating: true,
AutoAuth: &AutoAuth{
@ -1152,7 +1262,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Auto_Auth(t *testing.T) {
PidFile: "./pidfile",
},
DisableIdleConns: []string{"auto-auth"},
DisableIdleConnsCaching: false,
DisableIdleConnsAPIProxy: false,
DisableIdleConnsAutoAuth: true,
DisableIdleConnsTemplating: false,
AutoAuth: &AutoAuth{
@ -1201,7 +1311,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Templating(t *testing.T) {
PidFile: "./pidfile",
},
DisableIdleConns: []string{"templating"},
DisableIdleConnsCaching: false,
DisableIdleConnsAPIProxy: false,
DisableIdleConnsAutoAuth: false,
DisableIdleConnsTemplating: true,
AutoAuth: &AutoAuth{
@ -1250,7 +1360,56 @@ func TestLoadConfigFile_Disable_Idle_Conns_Caching(t *testing.T) {
PidFile: "./pidfile",
},
DisableIdleConns: []string{"caching"},
DisableIdleConnsCaching: true,
DisableIdleConnsAPIProxy: true,
DisableIdleConnsAutoAuth: false,
DisableIdleConnsTemplating: false,
AutoAuth: &AutoAuth{
Method: &Method{
Type: "aws",
MountPath: "auth/aws",
Namespace: "my-namespace/",
Config: map[string]interface{}{
"role": "foobar",
},
},
Sinks: []*Sink{
{
Type: "file",
DHType: "curve25519",
DHPath: "/tmp/file-foo-dhpath",
AAD: "foobar",
Config: map[string]interface{}{
"path": "/tmp/file-foo",
},
},
},
},
Vault: &Vault{
Address: "http://127.0.0.1:1111",
Retry: &Retry{
ctconfig.DefaultRetryAttempts,
},
},
}
config.Prune()
if diff := deep.Equal(config, expected); diff != nil {
t.Fatal(diff)
}
}
func TestLoadConfigFile_Disable_Idle_Conns_Proxying(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-disable-idle-connections-proxying.hcl")
if err != nil {
t.Fatal(err)
}
expected := &Config{
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
},
DisableIdleConns: []string{"proxying"},
DisableIdleConnsAPIProxy: true,
DisableIdleConnsAutoAuth: false,
DisableIdleConnsTemplating: false,
AutoAuth: &AutoAuth{
@ -1299,7 +1458,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Empty(t *testing.T) {
PidFile: "./pidfile",
},
DisableIdleConns: []string{},
DisableIdleConnsCaching: false,
DisableIdleConnsAPIProxy: false,
DisableIdleConnsAutoAuth: false,
DisableIdleConnsTemplating: false,
AutoAuth: &AutoAuth{
@ -1354,7 +1513,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Env(t *testing.T) {
PidFile: "./pidfile",
},
DisableIdleConns: []string{"auto-auth", "caching", "templating"},
DisableIdleConnsCaching: true,
DisableIdleConnsAPIProxy: true,
DisableIdleConnsAutoAuth: true,
DisableIdleConnsTemplating: true,
AutoAuth: &AutoAuth{
@ -1409,8 +1568,8 @@ func TestLoadConfigFile_Disable_Keep_Alives_All(t *testing.T) {
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
},
DisableKeepAlives: []string{"auto-auth", "caching", "templating"},
DisableKeepAlivesCaching: true,
DisableKeepAlives: []string{"auto-auth", "caching", "templating", "proxying"},
DisableKeepAlivesAPIProxy: true,
DisableKeepAlivesAutoAuth: true,
DisableKeepAlivesTemplating: true,
AutoAuth: &AutoAuth{
@ -1459,7 +1618,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Auto_Auth(t *testing.T) {
PidFile: "./pidfile",
},
DisableKeepAlives: []string{"auto-auth"},
DisableKeepAlivesCaching: false,
DisableKeepAlivesAPIProxy: false,
DisableKeepAlivesAutoAuth: true,
DisableKeepAlivesTemplating: false,
AutoAuth: &AutoAuth{
@ -1508,7 +1667,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Templating(t *testing.T) {
PidFile: "./pidfile",
},
DisableKeepAlives: []string{"templating"},
DisableKeepAlivesCaching: false,
DisableKeepAlivesAPIProxy: false,
DisableKeepAlivesAutoAuth: false,
DisableKeepAlivesTemplating: true,
AutoAuth: &AutoAuth{
@ -1557,7 +1716,56 @@ func TestLoadConfigFile_Disable_Keep_Alives_Caching(t *testing.T) {
PidFile: "./pidfile",
},
DisableKeepAlives: []string{"caching"},
DisableKeepAlivesCaching: true,
DisableKeepAlivesAPIProxy: true,
DisableKeepAlivesAutoAuth: false,
DisableKeepAlivesTemplating: false,
AutoAuth: &AutoAuth{
Method: &Method{
Type: "aws",
MountPath: "auth/aws",
Namespace: "my-namespace/",
Config: map[string]interface{}{
"role": "foobar",
},
},
Sinks: []*Sink{
{
Type: "file",
DHType: "curve25519",
DHPath: "/tmp/file-foo-dhpath",
AAD: "foobar",
Config: map[string]interface{}{
"path": "/tmp/file-foo",
},
},
},
},
Vault: &Vault{
Address: "http://127.0.0.1:1111",
Retry: &Retry{
ctconfig.DefaultRetryAttempts,
},
},
}
config.Prune()
if diff := deep.Equal(config, expected); diff != nil {
t.Fatal(diff)
}
}
func TestLoadConfigFile_Disable_Keep_Alives_Proxying(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-proxying.hcl")
if err != nil {
t.Fatal(err)
}
expected := &Config{
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
},
DisableKeepAlives: []string{"proxying"},
DisableKeepAlivesAPIProxy: true,
DisableKeepAlivesAutoAuth: false,
DisableKeepAlivesTemplating: false,
AutoAuth: &AutoAuth{
@ -1606,7 +1814,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Empty(t *testing.T) {
PidFile: "./pidfile",
},
DisableKeepAlives: []string{},
DisableKeepAlivesCaching: false,
DisableKeepAlivesAPIProxy: false,
DisableKeepAlivesAutoAuth: false,
DisableKeepAlivesTemplating: false,
AutoAuth: &AutoAuth{
@ -1661,7 +1869,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Env(t *testing.T) {
PidFile: "./pidfile",
},
DisableKeepAlives: []string{"auto-auth", "caching", "templating"},
DisableKeepAlivesCaching: true,
DisableKeepAlivesAPIProxy: true,
DisableKeepAlivesAutoAuth: true,
DisableKeepAlivesTemplating: true,
AutoAuth: &AutoAuth{

23
command/agent/config/test-fixtures/bad-config-api_proxy-cache.hcl Normal file
View File

@ -0,0 +1,23 @@
pid_file = "./pidfile"
auto_auth {
method {
type = "aws"
config = {
role = "foobar"
}
}
}
cache {
use_auto_auth_token = true
}
api_proxy {
use_auto_auth_token = "force"
}
listener "tcp" {
address = "127.0.0.1:8300"
tls_disable = true
}

21
command/agent/config/test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl Normal file
View File

@ -0,0 +1,21 @@
pid_file = "./pidfile"
auto_auth {
method {
type = "aws"
config = {
role = "foobar"
}
}
}
api_proxy {
use_auto_auth_token = "force"
enforce_consistency = "always"
when_inconsistent = "forward"
}
listener "tcp" {
address = "127.0.0.1:8300"
tls_disable = true
}

2
command/agent/config/test-fixtures/config-cache-auto_auth-true.hcl
View File

@ -11,10 +11,10 @@ auto_auth {
cache {
use_auto_auth_token = "true"
force_auto_auth_token = false
}
listener "tcp" {
address = "127.0.0.1:8300"
tls_disable = true
}

9
command/agent/config/test-fixtures/config-consistency-apiproxy.hcl Normal file
View File

@ -0,0 +1,9 @@
api_proxy {
enforce_consistency = "always"
when_inconsistent = "retry"
}
listener "tcp" {
address = "127.0.0.1:8300"
tls_disable = true
}

2
command/agent/config/test-fixtures/config-disable-idle-connections-all.hcl
View File

@ -1,5 +1,5 @@
pid_file = "./pidfile"
disable_idle_connections = ["auto-auth","caching","templating"]
disable_idle_connections = ["auto-auth","caching","templating","proxying"]
auto_auth {
method {

27
command/agent/config/test-fixtures/config-disable-idle-connections-proxying.hcl Normal file
View File

@ -0,0 +1,27 @@
pid_file = "./pidfile"
disable_idle_connections = ["proxying"]
auto_auth {
method {
type = "aws"
namespace = "my-namespace/"
config = {
role = "foobar"
}
}
sink {
type = "file"
config = {
path = "/tmp/file-foo"
}
aad = "foobar"
dh_type = "curve25519"
dh_path = "/tmp/file-foo-dhpath"
}
}
vault {
address = "http://127.0.0.1:1111"
}

2
command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl
View File

@ -1,5 +1,5 @@
pid_file = "./pidfile"
disable_keep_alives = ["auto-auth","caching","templating"]
disable_keep_alives = ["auto-auth","caching","templating","proxying"]
auto_auth {
method {

27
command/agent/config/test-fixtures/config-disable-keep-alives-proxying.hcl Normal file
View File

@ -0,0 +1,27 @@
pid_file = "./pidfile"
disable_keep_alives = ["proxying"]
auto_auth {
method {
type = "aws"
namespace = "my-namespace/"
config = {
role = "foobar"
}
}
sink {
type = "file"
config = {
path = "/tmp/file-foo"
}
aad = "foobar"
dh_type = "curve25519"
dh_path = "/tmp/file-foo-dhpath"
}
}
vault {
address = "http://127.0.0.1:1111"
}

87
website/content/docs/agent/apiproxy.mdx Normal file
View File

@ -0,0 +1,87 @@
---
layout: docs
page_title: Vault Agent API Proxy
description: >-
Vault Agent's API Proxy functionality allows you to use Vault Agent's API as a proxy
for Vault's API.
---
# Vault Agent API Proxy
Vault Agent's API Proxy functionality allows you to use Vault Agent's API as a proxy
for Vault's API.
## Functionality
The [`listener` stanza](/docs/agent#listener-stanza) for Vault Agent configures a listener for Vault Agent. If
its `role` is not set to `metrics_only`, it will act as a proxy for the Vault server that
has been configured in the [`vault` stanza](/docs/agent#vault-stanza) stanza of Vault Agent. This enables access to the Vault
API from the Agent API, and can be configured to optionally allow or force the automatic use of
the Auto-Auth token for these requests, as described below.
If a `listener` has been configured alongside a `cache` stanza, the API Proxy will
first attempt to utilize the cache subsystem for qualifying requests, before forwarding the
request to Vault. See the [caching docs](/docs/agent/caching) for more information on caching.
## Using Auto-Auth Token
Vault Agent allows for easy authentication to Vault in a wide variety of
environments using [Auto-Auth](/docs/agent/autoauth). By setting the
`use_auto_auth_token` (see below) configuration, clients will not be required
to provide a Vault token to the requests made to the Agent. When this
configuration is set, if the request doesn't already bear a token, then the
auto-auth token will be used to forward the request to the Vault server. This
configuration will be overridden if the request already has a token attached,
in which case, the token present in the request will be used to forward the
request to the Vault server.
## Forcing Auto-Auth Token
Vault Agent can be configured to force the use of the auto-auth token by using
the value `force` for the `use_auto_auth_token` option. This configuration
overrides the default behavior described above in [Using Auto-Auth
Token](/docs/agent/apiproxy#using-auto-auth-token), and instead ignores any
existing Vault token in the request and instead uses the auto-auth token.
## Configuration (`api_proxy`)
The top level `api_proxy` block has the following configuration entries:
- `use_auto_auth_token` `(bool/string: false)` - If set, the requests made to Agent
without a Vault token will be forwarded to the Vault server with the
auto-auth token attached. If the requests already bear a token, this
configuration will be overridden and the token in the request will be used to
forward the request to the Vault server. If set to `"force"` Agent will use the
auto-auth token, overwriting the attached Vault token if set.
The following two `api_proxy` options are only useful when making requests to a Vault
Enterprise cluster, and are documented as part of its
[Eventual Consistency](/docs/enterprise/consistency#vault-agent-and-consistency-headers)
page.
- `enforce_consistency` `(string: "never")` - Set to one of `"always"`
or `"never"`.
- `when_inconsistent` `(string: optional)` - Set to one of `"fail"`, `"retry"`,
or `"forward"`.
### Example Configuration
Here is an example of a `listener` configuration alongside `api_proxy` configuration to force the use of the auto_auth token
and enforce consistency.
```hcl
# Other Vault Agent configuration blocks
# ...
api_proxy {
use_auto_auth_token = "force"
enforce_consistency = "always"
}
listener "tcp" {
address = "127.0.0.1:8100"
tls_disable = true
}
```

78
website/content/docs/agent/caching/index.mdx
View File

@ -36,26 +36,6 @@ specific scenarios.
that are issued using the tokens managed by the agent, will be cached and
its renewals are taken care of.
## Using Auto-Auth Token
Vault Agent allows for easy authentication to Vault in a wide variety of
environments using [Auto-Auth](/docs/agent/autoauth). By setting the
`use_auto_auth_token` (see below) configuration, clients will not be required
to provide a Vault token to the requests made to the agent. When this
configuration is set, if the request doesn't already bear a token, then the
auto-auth token will be used to forward the request to the Vault server. This
configuration will be overridden if the request already has a token attached,
in which case, the token present in the request will be used to forward the
request to the Vault server.
## Forcing Auto-Auth Token
Vault Agent can be configured to force the use of the auto-auth token by using
the value `force` for the `use_auto_auth_token` option. This configuration
overrides the default behavior described above in [Using Auto-Auth
Token](/docs/agent/caching#using-auto-auth-token), and instead ignores any
existing Vault token in the request and instead uses the auto-auth token.
## Persistent Cache
Vault Agent can restore tokens and leases from a persistent cache file created
@ -134,7 +114,8 @@ belonging to the cached response, the `request_path` that resulted in the
cached response, the `lease` that is attached to the cached response, the
`namespace` to which the cached response belongs to, and a few more. This API
exposes some factors through which associated cache entries are fetched and
evicted.
evicted. For listeners without caching enabled, this API will still be available,
but will do nothing (there is no cache to clear) and will return a `200` response.
| Method | Path | Produces |
| :----- | :---------------------- | :--------------------- |
@ -159,7 +140,7 @@ evicted.
```json
{
"type": "token",
"value": "s.rlNjegSKykWcplOkwsjd8bP9"
"value": "hvs.rlNjegSKykWcplOkwsjd8bP9"
}
```
@ -174,34 +155,24 @@ $ curl \
## Configuration (`cache`)
The top level `cache` block has the following configuration entries:
- `use_auto_auth_token (bool/string: false)` - If set, the requests made to agent
without a Vault token will be forwarded to the Vault server with the
auto-auth token attached. If the requests already bear a token, this
configuration will be overridden and the token in the request will be used to
forward the request to the Vault server. If set to `"force"` Agent will use the
auto-auth token, overwriting the attached Vault token if set.
The presence of the top level `cache` block in any way (including an empty `cache` block) will enable the cache.
The top level `cache` block has the following configuration entry:
- `persist` `(object: optional)` - Configuration for the persistent cache.
The following two `cache` options are only useful when talking to a Vault
Enterprise cluster, and are documented as part of its
[Eventual Consistency](/docs/enterprise/consistency#vault-agent-and-consistency-headers)
page.
- `enforce_consistency` `(string: "never")` - Set to one of `"always"`
or `"never"`.
- `when_inconsistent` `(string: optional)` - Set to one of `"fail"`, `"retry"`,
or `"forward"`.
The `cache` block also supports the `use_auto_auth_token`, `enforce_consistency`, and
`when_inconsistent` configuration values of the `api_proxy` block
[described in the API Proxy documentation](/docs/agent/apiproxy#configuration-api_proxy) only to
maintain backwards compatibility. This configuration **cannot** be specified alongside `api_proxy` equivalents,
should not be preferred over configuring these values in the `api_proxy` block,
and `api_proxy` should be the preferred place to configure these values.
-> **Note:** When the `cache` block is defined, at least one
[template][agent-template] or [listener][agent-listener] must also be defined
in the config, otherwise there is no way to utilize the cache.
[agent-template]: /docs/agent/template
[agent-listener]: /docs/agent/caching#configuration-listener
[agent-listener]: /docs/agent#listener-stanza
### Configuration (Persist)
@ -220,12 +191,17 @@ These are common configuration values that live within the `persist` block:
- `exit_on_err` `(bool: optional)` - When set to true, if any errors occur during
a persistent cache restore, Vault Agent will exit with an error. Defaults to `true`.
- `service_account_token_file` `(string: optional)` - When `type` is set to `kubernetes`,
this configures the path on disk where the Kubernetes service account token can be found.
Defaults to `/var/run/secrets/kubernetes.io/serviceaccount/token`.
## Configuration (`listener`)
- `listener` `(array of objects: required)` - Configuration for the listeners.
There can be one or more `listener` blocks at the top level. These configuration
values are common to both `tcp` and `unix` listener blocks. Blocks of type
There can be one or more `listener` blocks at the top level. Adding a listener enables
the [API Proxy](/docs/agent/apiproxy) and enables the API proxy to use the cache, if configured.
These configuration values are common to both `tcp` and `unix` listener blocks. Blocks of type
`tcp` support the standard `tcp` [listener](/docs/configuration/listener/tcp)
options. Additionally, the `role` string option is available as part of the top level
of the `listener` block, which can be configured to `metrics_only` to serve only metrics,
@ -251,14 +227,26 @@ or the default role, `default`, which serves everything (including metrics).
### Example Configuration
Here is an example of a cache configuration alongside a listener that only serves metrics.
Here is an example of a cache configuration with the optional `persist` block,
alongside a regular listener, and a listener that only serves metrics.
```hcl
# Other Vault Agent configuration blocks
# ...
cache {
use_auto_auth_token = true
persist = {
type = "kubernetes"
path = "/vault/agent-cache/"
keep_after_import = true
exit_on_err = true
service_account_token_file = "/tmp/serviceaccount/token"
}
}
listener "tcp" {
address = "127.0.0.1:8100"
tls_disable = true
}
listener "tcp" {

43
website/content/docs/agent/index.mdx
View File

@ -85,6 +85,8 @@ Vault Agent is a client daemon that provides the following features:
- [Auto-Auth][autoauth] - Automatically authenticate to Vault and manage the
token renewal process for locally-retrieved dynamic secrets.
- [API Proxy][apiproxy] - Allows Vault Agent to act as a proxy for Vault's API,
optionally using (or forcing the use of) the Auto-Auth token.
- [Caching][caching] - Allows client-side caching of responses containing newly
created tokens and responses containing leased secrets generated off of these
newly created tokens. The agent also manages the renewals of the cached tokens and leases.
@ -101,6 +103,16 @@ for information.
Auto-Auth functionality takes place within an `auto_auth` configuration stanza.
## API Proxy
Vault Agent can act as an API proxy for Vault, allowing you to talk to Vault's
API via a listener defined for Agent. It can be configured to optionally allow or force the automatic use of
the Auto-Auth token for these requests. Please see the [API Proxy docs][apiproxy]
for more information.
API Proxy functionality takes place within a defined `listener`, and its behaviour can be configured with an
[`api_proxy` stanza](/docs/agent/apiproxy#configuration-api_proxy).
## Caching
Vault Agent allows client-side caching of responses containing newly created tokens
@ -162,12 +174,14 @@ See the [caching](/docs/agent/caching#api) page for details on the cache API.
### Configuration File Options
These are the currently-available general configuration option:
These are the currently-available general configuration options:
- `vault` <code>([vault][vault]: <optional\>)</code> - Specifies the remote Vault server the Agent connects to.
- `auto_auth` <code>([auto_auth][autoauth]: <optional\>)</code> - Specifies the method and other options used for Auto-Auth functionality.
- `api_proxy` <code>([api_proxy][apiproxy]: <optional\>)</code> - Specifies options used for API Proxy functionality.
- `cache` <code>([cache][caching]: <optional\>)</code> - Specifies options used for Caching functionality.
- `listener` <code>([listener][listener]: <optional\>)</code> - Specifies the addresses and ports on which the Agent will respond to requests.
@ -180,11 +194,13 @@ These are the currently-available general configuration option:
token was retrieved and all sinks successfully wrote it
- `disable_idle_connections` `(string array: [])` - A list of strings that disables idle connections for various features in Vault Agent.
Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS`
Valid values include: `auto-auth`, `caching`, `proxying`, and `templating`. `proxying` configures this for the API proxy, which is
identical in function to `caching` for historical reasons. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS`
environment variable as a comma separated string. This environment variable will override any values found in a configuration file.
- `disable_keep_alives` `(string array: [])` - A list of strings that disables keep alives for various features in Vault Agent.
Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES`
Valid values include: `auto-auth`, `caching`, `proxying`, and `templating`. `proxying` configures this for the API proxy, which is
identical in function to `caching` for historical reasons. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES`
environment variable as a comma separated string. This environment variable will override any values found in a configuration file.
- `template` <code>([template][template]: <optional\>)</code> - Specifies options used for templating Vault secrets to files.
@ -209,7 +225,7 @@ These are the currently-available general configuration option:
### vault Stanza
There can at most be one top level `vault` block and it has the following
There can at most be one top level `vault` block, and it has the following
configuration entries:
- `address` `(string: <optional>)` - The address of the Vault server to
@ -249,7 +265,7 @@ configuration entries:
The `vault` stanza may contain a `retry` stanza that controls how failing Vault
requests are handled, whether these requests are issued in order to render
templates, or are proxied requests coming from the proxy cache subsystem.
templates, or are proxied requests coming from the api proxy subsystem.
Auto-auth, however, has its own notion of retrying and is not affected by this
section.
@ -284,9 +300,10 @@ to address in the future.
### listener Stanza
Vault Agent supports one or more [listener][listener_main] stanzas. In addition
to the standard listener configuration, an Agent's listener configuration also
supports the following:
Vault Agent supports one or more [listener][listener_main] stanzas. Listeners
can be configured with or without [caching][caching], but will use the cache if it
has been configured, and will enable the [API proxy][apiproxy]. In addition to the standard
listener configuration, an Agent's listener configuration also supports the following:
- `require_request_header` `(bool: false)` - Require that all incoming HTTP
requests on this listener must have an `X-Vault-Request: true` header entry.
@ -294,6 +311,11 @@ supports the following:
Request Forgery attacks. Requests on the listener that do not have the proper
`X-Vault-Request` header will fail, with a HTTP response status code of `412: Precondition Failed`.
- `role` `(string: default)` - `role` determines which APIs the listener serves.
It can be configured to `metrics_only` to serve only metrics, or the default role, `default`,
which serves everything (including metrics). The `require_request_header` does not apply
to `metrics_only` listeners.
- `agent_api` <code>([agent_api][agent-api]: <optional\>)</code> - Manages optional Agent API endpoints.
#### agent_api Stanza
@ -380,6 +402,10 @@ auto_auth {
}
cache {
// An empty cache stanza still enables caching
}
api_proxy {
use_auto_auth_token = true
}
@ -411,6 +437,7 @@ template {
[vault]: /docs/agent#vault-stanza
[autoauth]: /docs/agent/autoauth
[caching]: /docs/agent/caching
[apiproxy]: /docs/agent/apiproxy
[persistent-cache]: /docs/agent/caching/persistent-caches
[template]: /docs/agent/template
[template-config]: /docs/agent/template#template-configurations

4
website/content/docs/enterprise/consistency.mdx
View File

@ -175,8 +175,8 @@ to build equivalent functionality into their client library.
### Vault Agent and consistency headers
Vault Agent Caching will proxy incoming requests to Vault. There is
new Agent configuration available in the `cache` stanza that allows making use
When configured, the [Vault Agent API Proxy](/docs/agent/apiproxy) will proxy incoming requests to Vault. There is
Agent configuration available in the `api_proxy` stanza that allows making use
of some of the above mitigations without modifying clients.
By setting `enforce_consistency="always"`, Agent will always provide

4
website/data/docs-nav-data.json
View File

@ -894,6 +894,10 @@
}
]
},
{
"title": "API Proxy",
"path": "agent/apiproxy"
},
{
"title": "Caching",
"routes": [