connect: ensure all vault connect CA tests use limited privilege tokens (#15669)

All of the current integration tests where Vault is the Connect CA now use non-root tokens for the test. This helps us detect privilege changes in the vault model so we can keep our guides up to date.

One larger change was that the RenewIntermediate function got refactored slightly so it could be used from a test, rather than the large duplicated function we were testing in a test which seemed error prone.

.changelog/15669.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+connect: ensure all vault connect CA tests use limited privilege tokens
+```

agent/agent_endpoint_test.go

@@ -7095,7 +7095,12 @@ func TestAgentConnectCALeafCert_Vault_doesNotChurnLeafCertsAtIdle(t *testing.T)
 	t.Parallel()
 
 	testVault := ca.NewTestVaultServer(t)
-	defer testVault.Stop()
+
+	vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
 
 	a := StartTestAgent(t, TestAgent{Overrides: fmt.Sprintf(`
 		connect {
@@ -7108,7 +7113,7 @@ func TestAgentConnectCALeafCert_Vault_doesNotChurnLeafCertsAtIdle(t *testing.T)
 				intermediate_pki_path = "pki-intermediate/"
 			}
 		}
-	`, testVault.Addr, testVault.RootToken)})
+	`, testVault.Addr, vaultToken)})
 	defer a.Shutdown()
 	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
 	testrpc.WaitForActiveCARoot(t, a.RPC, "dc1", nil)

agent/connect/ca/provider_consul_test.go

@@ -375,6 +375,30 @@ func testCrossSignProviders(t *testing.T, provider1, provider2 Provider) {
 	}
 }
 
+func testCrossSignProvidersShouldFail(t *testing.T, provider1, provider2 Provider) {
+	t.Helper()
+
+	// Get the root from the new provider to be cross-signed.
+	root, err := provider2.GenerateRoot()
+	require.NoError(t, err)
+
+	newRoot, err := connect.ParseCert(root.PEM)
+	require.NoError(t, err)
+	requireNotEncoded(t, newRoot.SubjectKeyId)
+	requireNotEncoded(t, newRoot.AuthorityKeyId)
+
+	newInterPEM, err := provider2.ActiveIntermediate()
+	require.NoError(t, err)
+	newIntermediate, err := connect.ParseCert(newInterPEM)
+	require.NoError(t, err)
+	requireNotEncoded(t, newIntermediate.SubjectKeyId)
+	requireNotEncoded(t, newIntermediate.AuthorityKeyId)
+
+	// Have provider1 cross sign our new root cert.
+	_, err = provider1.CrossSignCA(newRoot)
+	require.Error(t, err)
+}
+
 func TestConsulProvider_SignIntermediate(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")

agent/connect/ca/provider_vault.go

@@ -885,14 +885,12 @@ func makePathHelper(namespace, path string) string {
 
 func (v *VaultProvider) readNamespaced(namespace string, resource string) (*vaultapi.Secret, error) {
 	defer v.setNamespace(namespace)()
-	result, err := v.client.Logical().Read(resource)
+	return v.client.Logical().Read(resource)
-	return result, err
 }
 
 func (v *VaultProvider) writeNamespaced(namespace string, resource string, data map[string]interface{}) (*vaultapi.Secret, error) {
 	defer v.setNamespace(namespace)()
-	result, err := v.client.Logical().Write(resource, data)
+	return v.client.Logical().Write(resource, data)
-	return result, err
 }
 
 func (v *VaultProvider) setNamespace(namespace string) func() {

agent/connect/ca/testing.go

@@ -10,8 +10,10 @@ import (
 	"sync"
 
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-uuid"
 	vaultapi "github.com/hashicorp/vault/api"
 	"github.com/mitchellh/go-testing-interface"
+	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/sdk/freeport"
@@ -27,11 +29,7 @@ import (
 //     these types of old CA key.
 //   - SignIntermediate muse bt able to sign all the types of secondary
 //     intermediate CA key with all these types of primary CA key
-var KeyTestCases = []struct {
+var KeyTestCases = []KeyTestCase{
-	Desc    string
-	KeyType string
-	KeyBits int
-}{
 	{
 		Desc:    "Default Key Type (EC 256)",
 		KeyType: connect.DefaultPrivateKeyType,
@@ -44,6 +42,12 @@ var KeyTestCases = []struct {
 	},
 }
 
+type KeyTestCase struct {
+	Desc    string
+	KeyType string
+	KeyBits int
+}
+
 // CASigningKeyTypes is a struct with params for tests that sign one CA CSR with
 // another CA key.
 type CASigningKeyTypes struct {
@@ -106,17 +110,6 @@ func SkipIfVaultNotPresent(t testing.T) {
 }
 
 func NewTestVaultServer(t testing.T) *TestVaultServer {
-	testVault, err := runTestVault(t)
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	testVault.WaitUntilReady(t)
-
-	return testVault
-}
-
-func runTestVault(t testing.T) (*TestVaultServer, error) {
 	vaultBinaryName := os.Getenv("VAULT_BINARY_NAME")
 	if vaultBinaryName == "" {
 		vaultBinaryName = "vault"
@@ -124,7 +117,7 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
 
 	path, err := exec.LookPath(vaultBinaryName)
 	if err != nil || path == "" {
-		return nil, fmt.Errorf("%q not found on $PATH", vaultBinaryName)
+		t.Fatalf("%q not found on $PATH", vaultBinaryName)
 	}
 
 	ports := freeport.GetN(t, 2)
@@ -138,9 +131,7 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
 	client, err := vaultapi.NewClient(&vaultapi.Config{
 		Address: "http://" + clientAddr,
 	})
-	if err != nil {
+	require.NoError(t, err)
-		return nil, err
-	}
 	client.SetToken(token)
 
 	args := []string{
@@ -157,9 +148,7 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
 	cmd := exec.Command(vaultBinaryName, args...)
 	cmd.Stdout = io.Discard
 	cmd.Stderr = io.Discard
-	if err := cmd.Start(); err != nil {
+	require.NoError(t, cmd.Start())
-		return nil, err
-	}
 
 	testVault := &TestVaultServer{
 		RootToken: token,
@@ -173,7 +162,9 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
 		}
 	})
 
-	return testVault, nil
+	testVault.WaitUntilReady(t)
+
+	return testVault
 }
 
 type TestVaultServer struct {
@@ -224,6 +215,9 @@ func (v *TestVaultServer) Stop() error {
 	// wait for the process to exit to be sure that the data dir can be
 	// deleted on all platforms.
 	if err := v.cmd.Wait(); err != nil {
+		if strings.Contains(err.Error(), "exec: Wait was already called") {
+			return nil
+		}
 		return err
 	}
 	return nil
@@ -238,3 +232,127 @@ func requireTrailingNewline(t testing.T, leafPEM string) {
 		t.Fatalf("cert do not end with a new line")
 	}
 }
+
+// The zero value implies unprivileged.
+type VaultTokenAttributes struct {
+	RootPath, IntermediatePath string
+
+	ConsulManaged bool
+	VaultManaged  bool
+	WithSudo      bool
+
+	CustomRules string
+}
+
+func (a *VaultTokenAttributes) DisplayName() string {
+	switch {
+	case a == nil:
+		return "unprivileged"
+	case a.CustomRules != "":
+		return "custom"
+	case a.ConsulManaged:
+		return "consul-managed"
+	case a.VaultManaged:
+		return "vault-managed"
+	default:
+		return "unprivileged"
+	}
+}
+
+func (a *VaultTokenAttributes) Rules(t testing.T) string {
+	switch {
+	case a == nil:
+		return ""
+
+	case a.CustomRules != "":
+		return a.CustomRules
+
+	case a.RootPath == "":
+		t.Fatal("missing required RootPath")
+		return "" // dead code
+
+	case a.IntermediatePath == "":
+		t.Fatal("missing required IntermediatePath")
+		return "" // dead code
+
+	case a.ConsulManaged:
+		// Consul Managed PKI Mounts
+		rules := fmt.Sprintf(`
+path "sys/mounts" {
+  capabilities = [ "read" ]
+}
+
+path "sys/mounts/%[1]s" {
+  capabilities = [ "create", "read", "update", "delete", "list" ]
+}
+
+path "sys/mounts/%[2]s" {
+  capabilities = [ "create", "read", "update", "delete", "list" ]
+}
+
+# Needed for Consul 1.11+
+path "sys/mounts/%[2]s/tune" {
+  capabilities = [ "update" ]
+}
+
+# vault token renewal
+path "auth/token/renew-self" {
+  capabilities = [ "update" ]
+}
+path "auth/token/lookup-self" {
+  capabilities = [ "read" ]
+}
+
+path "%[1]s/*" {
+  capabilities = [ "create", "read", "update", "delete", "list" ]
+}
+
+path "%[2]s/*" {
+  capabilities = [ "create", "read", "update", "delete", "list" ]
+}
+`, a.RootPath, a.IntermediatePath)
+
+		if a.WithSudo {
+			rules += fmt.Sprintf(`
+
+path "%[1]s/root/sign-self-issued" {
+  capabilities = [ "sudo", "update" ]
+}
+`, a.RootPath)
+		}
+
+		return rules
+
+	case a.VaultManaged:
+		// Vault-managed PKI root.
+		t.Fatal("TODO: implement this and use it in tests")
+		return ""
+
+	default:
+		// zero value
+		return ""
+	}
+}
+
+func CreateVaultTokenWithAttrs(t testing.T, client *vaultapi.Client, attr *VaultTokenAttributes) string {
+	policyName, err := uuid.GenerateUUID()
+	require.NoError(t, err)
+
+	rules := attr.Rules(t)
+
+	token := createVaultTokenAndPolicy(t, client, policyName, rules)
+	// t.Logf("created vault token with scope %q: %s", attr.DisplayName(), token)
+	return token
+}
+
+func createVaultTokenAndPolicy(t testing.T, client *vaultapi.Client, policyName, policyRules string) string {
+	require.NoError(t, client.Sys().PutPolicy(policyName, policyRules))
+
+	renew := true
+	tok, err := client.Auth().Token().Create(&vaultapi.TokenCreateRequest{
+		Policies:  []string{policyName},
+		Renewable: &renew,
+	})
+	require.NoError(t, err)
+	return tok.Auth.ClientToken
+}

agent/consul/acl_server.go

@@ -87,7 +87,7 @@ func (s *Server) checkBindingRuleUUID(id string) (bool, error) {
 }
 
 func (s *Server) InPrimaryDatacenter() bool {
-	return s.config.PrimaryDatacenter == "" || s.config.Datacenter == s.config.PrimaryDatacenter
+	return s.config.InPrimaryDatacenter()
 }
 
 func (s *Server) LocalTokensEnabled() bool {

agent/consul/config.go

@@ -415,6 +415,10 @@ type Config struct {
 	*EnterpriseConfig
 }
 
+func (c *Config) InPrimaryDatacenter() bool {
+	return c.PrimaryDatacenter == "" || c.Datacenter == c.PrimaryDatacenter
+}
+
 // CheckProtocolVersion validates the protocol version.
 func (c *Config) CheckProtocolVersion() error {
 	if c.ProtocolVersion < ProtocolVersionMin {

agent/consul/connect_ca_endpoint_test.go

@@ -9,11 +9,10 @@ import (
 	"testing"
 	"time"
 
+	msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
-
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/connect"
 	ca "github.com/hashicorp/consul/agent/connect/ca"
@@ -560,10 +559,23 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 
 	testVault := ca.NewTestVaultServer(t)
 
-	newConfig := func(keyType string, keyBits int) map[string]interface{} {
+	token1 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
-		return map[string]interface{}{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-primary",
+		ConsulManaged:    true,
+		WithSudo:         true,
+	})
+
+	token2 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
+	newConfig := func(token string, keyType string, keyBits int) map[string]any {
+		return map[string]any{
 			"Address":             testVault.Addr,
-			"Token":               testVault.RootToken,
+			"Token":               token,
 			"RootPKIPath":         "pki-root/",
 			"IntermediatePKIPath": "pki-intermediate/",
 			"PrivateKeyType":      keyType,
@@ -574,7 +586,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 	_, s1 := testServerWithConfig(t, func(c *Config) {
 		c.CAConfig = &structs.CAConfiguration{
 			Provider: "vault",
-			Config:   newConfig(connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits),
+			Config:   newConfig(token1, connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits),
 		}
 	})
 	testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
@@ -592,7 +604,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 			configFn: func() *structs.CAConfiguration {
 				return &structs.CAConfiguration{
 					Provider:                 "vault",
-					Config:                   newConfig("rsa", 4096),
+					Config:                   newConfig(token2, "rsa", 4096),
 					ForceWithoutCrossSigning: true,
 				}
 			},
@@ -602,7 +614,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 			configFn: func() *structs.CAConfiguration {
 				return &structs.CAConfiguration{
 					Provider:                 "vault",
-					Config:                   newConfig("rsa", 2048),
+					Config:                   newConfig(token2, "rsa", 2048),
 					ForceWithoutCrossSigning: true,
 				}
 			},
@@ -613,7 +625,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 			configFn: func() *structs.CAConfiguration {
 				return &structs.CAConfiguration{
 					Provider:                 "vault",
-					Config:                   newConfig("ec", 256),
+					Config:                   newConfig(token2, "ec", 256),
 					ForceWithoutCrossSigning: true,
 				}
 			},
@@ -624,7 +636,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 			configFn: func() *structs.CAConfiguration {
 				return &structs.CAConfiguration{
 					Provider:                 "vault",
-					Config:                   newConfig("rsa", 4096),
+					Config:                   newConfig(token2, "rsa", 4096),
 					ForceWithoutCrossSigning: true,
 				}
 			},
@@ -632,7 +644,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
 	}
 
 	for _, tc := range testSteps {
-		t.Run(tc.name, func(t *testing.T) {
+		testutil.RunStep(t, tc.name, func(t *testing.T) {
 			args := &structs.CARequest{
 				Datacenter: "dc1",
 				Config:     tc.configFn(),

agent/consul/leader_connect_ca.go

@@ -16,14 +16,13 @@ import (
 	"golang.org/x/time/rate"
 
 	"github.com/hashicorp/consul/acl"
-	"github.com/hashicorp/consul/lib/semaphore"
-
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/connect/ca"
 	"github.com/hashicorp/consul/agent/consul/state"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/lib/routine"
+	"github.com/hashicorp/consul/lib/semaphore"
 )
 
 type caState string
@@ -1069,7 +1068,7 @@ func (c *CAManager) secondaryRequestNewSigningCert(provider ca.Provider, newActi
 	}
 
 	if err := setLeafSigningCert(newActiveRoot, intermediatePEM); err != nil {
-		return err
+		return fmt.Errorf("Failed to set the leaf signing cert to the intermediate: %w", err)
 	}
 
 	c.logger.Info("received new intermediate certificate from primary datacenter")
@@ -1117,15 +1116,13 @@ func pruneExpiredIntermediates(caRoot *structs.CARoot) error {
 
 // runRenewIntermediate periodically attempts to renew the intermediate cert.
 func (c *CAManager) runRenewIntermediate(ctx context.Context) error {
-	isPrimary := c.serverConf.Datacenter == c.serverConf.PrimaryDatacenter
-
 	for {
 		select {
 		case <-ctx.Done():
 			return nil
 		case <-time.After(structs.IntermediateCertRenewInterval):
 			retryLoopBackoffAbortOnSuccess(ctx, func() error {
-				return c.RenewIntermediate(ctx, isPrimary)
+				return c.RenewIntermediate(ctx)
 			}, func(err error) {
 				c.logger.Error("error renewing intermediate certs",
 					"routine", intermediateCertRenewWatchRoutineName,
@@ -1139,7 +1136,15 @@ func (c *CAManager) runRenewIntermediate(ctx context.Context) error {
 // RenewIntermediate checks the intermediate cert for
 // expiration. If more than half the time a cert is valid has passed,
 // it will try to renew it.
-func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error {
+func (c *CAManager) RenewIntermediate(ctx context.Context) error {
+	return c.renewIntermediate(ctx, false)
+}
+
+func (c *CAManager) renewIntermediateNow(ctx context.Context) error {
+	return c.renewIntermediate(ctx, true)
+}
+
+func (c *CAManager) renewIntermediate(ctx context.Context, forceNow bool) error {
 	// Grab the 'lock' right away so the provider/config can't be changed out while we check
 	// the intermediate.
 	if _, err := c.setState(caStateRenewIntermediate, true); err != nil {
@@ -1147,6 +1152,8 @@ func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error
 	}
 	defer c.setState(caStateInitialized, false)
 
+	isPrimary := c.serverConf.InPrimaryDatacenter()
+
 	provider, _ := c.getCAProvider()
 	if provider == nil {
 		// this happens when leadership is being revoked and this go routine will be stopped
@@ -1184,8 +1191,10 @@ func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error
 		return fmt.Errorf("error parsing active intermediate cert: %v", err)
 	}
 
-	if lessThanHalfTimePassed(c.timeNow(), intermediateCert.NotBefore, intermediateCert.NotAfter) {
+	if !forceNow {
-		return nil
+		if lessThanHalfTimePassed(c.timeNow(), intermediateCert.NotBefore, intermediateCert.NotAfter) {
+			return nil
+		}
 	}
 
 	// Enough time has passed, go ahead with getting a new intermediate.
@@ -1193,19 +1202,27 @@ func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error
 	if !isPrimary {
 		renewalFunc = c.secondaryRequestNewSigningCert
 	}
-	errCh := make(chan error, 1)
-	go func() {
-		errCh <- renewalFunc(provider, activeRoot)
-	}()
 
-	// Wait for the renewal func to return or for the context to be canceled.
+	if forceNow {
-	select {
+		err := renewalFunc(provider, activeRoot)
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-errCh:
 		if err != nil {
 			return err
 		}
+	} else {
+		errCh := make(chan error, 1)
+		go func() {
+			errCh <- renewalFunc(provider, activeRoot)
+		}()
+
+		// Wait for the renewal func to return or for the context to be canceled.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case err := <-errCh:
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	if err := c.persistNewRootAndConfig(provider, activeRoot, nil); err != nil {

agent/consul/leader_connect_ca_test.go

@@ -16,14 +16,13 @@ import (
 	"testing"
 	"time"
 
+	msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
+	"github.com/hashicorp/consul-net-rpc/net/rpc"
 	vaultapi "github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
 
-	msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
-	"github.com/hashicorp/consul-net-rpc/net/rpc"
-
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/connect"
 	ca "github.com/hashicorp/consul/agent/connect/ca"
@@ -48,12 +47,24 @@ func TestCAManager_Initialize_Vault_Secondary_SharedVault(t *testing.T) {
 
 	vault := ca.NewTestVaultServer(t)
 
+	primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-primary",
+		ConsulManaged:    true,
+	})
+
+	secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-secondary",
+		ConsulManaged:    true,
+	})
+
 	_, serverDC1 := testServerWithConfig(t, func(c *Config) {
 		c.CAConfig = &structs.CAConfiguration{
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               primaryVaultToken,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-primary/",
 			},
@@ -81,7 +92,7 @@ func TestCAManager_Initialize_Vault_Secondary_SharedVault(t *testing.T) {
 				Provider: "vault",
 				Config: map[string]interface{}{
 					"Address":             vault.Addr,
-					"Token":               vault.RootToken,
+					"Token":               secondaryVaultToken,
 					"RootPKIPath":         "pki-root/",
 					"IntermediatePKIPath": "pki-secondary/",
 				},
@@ -393,7 +404,7 @@ func TestCAManager_UpdateConfigWhileRenewIntermediate(t *testing.T) {
 	// happen in the expected order.
 	errCh := make(chan error)
 	go func() {
-		errCh <- manager.RenewIntermediate(context.TODO(), false)
+		errCh <- manager.RenewIntermediate(context.TODO())
 	}()
 
 	waitForCh(t, delegate.callbackCh, "provider/GenerateIntermediateCSR")
@@ -570,7 +581,14 @@ func TestCAManager_Initialize_Logging(t *testing.T) {
 
 func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
 	ca.SkipIfVaultNotPresent(t)
+
 	vault := ca.NewTestVaultServer(t)
+	vaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+		WithSudo:         true,
+	})
 
 	_, s1 := testServerWithConfig(t, func(c *Config) {
 		c.PrimaryDatacenter = "dc1"
@@ -578,7 +596,7 @@ func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               vaultToken,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-intermediate/",
 			},
@@ -599,12 +617,18 @@ func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, connect.HexString(cert.SubjectKeyId), origRoot.SigningKeyID)
 
+	vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root-2",
+		IntermediatePath: "pki-intermediate-2",
+		ConsulManaged:    true,
+	})
+
 	err = s1.caManager.UpdateConfiguration(&structs.CARequest{
 		Config: &structs.CAConfiguration{
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               vaultToken2,
 				"RootPKIPath":         "pki-root-2/",
 				"IntermediatePKIPath": "pki-intermediate-2/",
 			},
@@ -636,12 +660,18 @@ func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
 	meshRootPath := "pki-root"
 	primaryCert := setupPrimaryCA(t, vclient, meshRootPath, "")
 
+	primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         meshRootPath,
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
 	_, s1 := testServerWithConfig(t, func(c *Config) {
 		c.CAConfig = &structs.CAConfiguration{
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               primaryVaultToken,
 				"RootPKIPath":         meshRootPath,
 				"IntermediatePKIPath": "pki-intermediate/",
 			},
@@ -665,6 +695,12 @@ func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
 	// TODO: renew primary leaf signing cert
 	// TODO: rotate root
 
+	secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         meshRootPath,
+		IntermediatePath: "pki-secondary",
+		ConsulManaged:    true,
+	})
+
 	testutil.RunStep(t, "run secondary DC", func(t *testing.T) {
 		_, sDC2 := testServerWithConfig(t, func(c *Config) {
 			c.Datacenter = "dc2"
@@ -673,7 +709,7 @@ func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
 				Provider: "vault",
 				Config: map[string]interface{}{
 					"Address":             vault.Addr,
-					"Token":               vault.RootToken,
+					"Token":               secondaryVaultToken,
 					"RootPKIPath":         meshRootPath,
 					"IntermediatePKIPath": "pki-secondary/",
 				},
@@ -704,12 +740,18 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
 
 	vault := ca.NewTestVaultServer(t)
 
+	primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
 	_, sDC1 := testServerWithConfig(t, func(c *Config) {
 		c.CAConfig = &structs.CAConfiguration{
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               primaryVaultToken,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-intermediate/",
 			},
@@ -718,6 +760,12 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
 	defer sDC1.Shutdown()
 	testrpc.WaitForActiveCARoot(t, sDC1.RPC, "dc1", nil)
 
+	secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate-2",
+		ConsulManaged:    true,
+	})
+
 	_, sDC2 := testServerWithConfig(t, func(c *Config) {
 		c.Datacenter = "dc2"
 		c.PrimaryDatacenter = "dc1"
@@ -725,7 +773,7 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               secondaryVaultToken,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-intermediate-2/",
 			},
@@ -740,7 +788,7 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
 	err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", &structs.DCSpecificRequest{}, &configBefore)
 	require.NoError(t, err)
 
-	renewLeafSigningCert(t, sDC1.caManager, sDC1.caManager.primaryRenewIntermediate)
+	require.NoError(t, sDC1.caManager.renewIntermediateNow(context.Background()))
 
 	// Give the secondary some time to notice the update
 	time.Sleep(100 * time.Millisecond)
@@ -778,37 +826,62 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
 
 	vault := ca.NewTestVaultServer(t)
 	vclient := vault.Client()
+
 	rootPEM := generateExternalRootCA(t, vclient)
 
 	primaryCAPath := "pki-primary"
 	primaryCert := setupPrimaryCA(t, vclient, primaryCAPath, rootPEM)
 
+	primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         primaryCAPath,
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+		WithSudo:         true,
+	})
+
 	_, serverDC1 := testServerWithConfig(t, func(c *Config) {
 		c.CAConfig = &structs.CAConfiguration{
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               primaryVaultToken,
 				"RootPKIPath":         primaryCAPath,
 				"IntermediatePKIPath": "pki-intermediate/",
 			},
 		}
 	})
 	testrpc.WaitForTestAgent(t, serverDC1.RPC, "dc1")
+	testrpc.WaitForActiveCARoot(t, serverDC1.RPC, "dc1", nil)
 
-	var origLeaf string
+	var (
+		origLeaf               string
+		primaryLeafSigningCert string
+	)
 	roots := structs.IndexedCARoots{}
 	testutil.RunStep(t, "verify primary DC", func(t *testing.T) {
 		codec := rpcClient(t, serverDC1)
 		err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
 		require.NoError(t, err)
-		require.Len(t, roots.Roots, 1)
+
-		require.Equal(t, primaryCert, roots.Roots[0].RootCert)
+		// Verify CA trust heirarchy is expected.
+		require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
+		require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
 		require.Contains(t, roots.Roots[0].RootCert, rootPEM)
+		require.Len(t, roots.Roots[0].IntermediateCerts, 1, "should just have the primary's intermediate")
+
+		active := roots.Active()
 
 		leafCert := getLeafCert(t, codec, roots.TrustDomain, "dc1")
-		verifyLeafCert(t, roots.Active(), leafCert)
+		verifyLeafCert(t, active, leafCert)
+
 		origLeaf = leafCert
+		primaryLeafSigningCert = serverDC1.caManager.getLeafSigningCertFromRoot(active)
+	})
+
+	secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "should-be-ignored",
+		IntermediatePath: "pki-secondary",
+		ConsulManaged:    true,
 	})
 
 	_, serverDC2 := testServerWithConfig(t, func(c *Config) {
@@ -818,72 +891,91 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             vault.Addr,
-				"Token":               vault.RootToken,
+				"Token":               secondaryVaultToken,
 				"RootPKIPath":         "should-be-ignored",
 				"IntermediatePKIPath": "pki-secondary/",
 			},
 		}
 	})
+	joinWAN(t, serverDC2, serverDC1)
+	testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
 
-	var origLeafSecondary string
+	var (
+		origLeafSecondary        string
+		secondaryLeafSigningCert string
+	)
 	testutil.RunStep(t, "start secondary DC", func(t *testing.T) {
-		joinWAN(t, serverDC2, serverDC1)
-		testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
-
 		codec := rpcClient(t, serverDC2)
 		roots = structs.IndexedCARoots{}
 		err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
 		require.NoError(t, err)
-		require.Len(t, roots.Roots, 1)
+
+		require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
+		require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
+		require.Contains(t, roots.Roots[0].RootCert, rootPEM)
+		require.Len(t, roots.Roots[0].IntermediateCerts, 2, "should have the primary's intermediate and our own")
+
+		active := roots.Active()
 
 		leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
 		verifyLeafCert(t, roots.Roots[0], leafPEM)
+
 		origLeafSecondary = leafPEM
+		secondaryLeafSigningCert = serverDC2.caManager.getLeafSigningCertFromRoot(active)
 	})
 
 	testutil.RunStep(t, "renew leaf signing CA in primary", func(t *testing.T) {
-		previous := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
+		require.NoError(t, serverDC1.caManager.renewIntermediateNow(context.Background()))
-
-		renewLeafSigningCert(t, serverDC1.caManager, serverDC1.caManager.primaryRenewIntermediate)
 
 		codec := rpcClient(t, serverDC1)
 		roots = structs.IndexedCARoots{}
 		err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
 		require.NoError(t, err)
-		require.Len(t, roots.Roots, 1)
+
-		require.Len(t, roots.Roots[0].IntermediateCerts, 2)
+		require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
+		require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
+		require.Contains(t, roots.Roots[0].RootCert, rootPEM)
+		require.Len(t, roots.Roots[0].IntermediateCerts, 2, "we renewed, so we have our old primary and our new primary intermediate")
+
+		active := roots.Active()
 
 		newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
-		require.NotEqual(t, previous, newCert)
+		require.NotEqual(t, primaryLeafSigningCert, newCert)
+		primaryLeafSigningCert = newCert
 
 		leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
-		verifyLeafCert(t, roots.Roots[0], leafPEM)
+		verifyLeafCert(t, active, leafPEM)
 
 		// original certs from old signing cert should still verify
-		verifyLeafCert(t, roots.Roots[0], origLeaf)
+		verifyLeafCert(t, active, origLeaf)
 	})
 
+	var oldSecondaryData *structs.CARoot
 	testutil.RunStep(t, "renew leaf signing CA in secondary", func(t *testing.T) {
-		previous := serverDC2.caManager.getLeafSigningCertFromRoot(roots.Active())
+		require.NoError(t, serverDC2.caManager.renewIntermediateNow(context.Background()))
-
-		renewLeafSigningCert(t, serverDC2.caManager, serverDC2.caManager.secondaryRequestNewSigningCert)
 
 		codec := rpcClient(t, serverDC2)
 		roots = structs.IndexedCARoots{}
 		err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
 		require.NoError(t, err)
-		require.Len(t, roots.Roots, 1)
-		// one intermediate from primary, two from secondary
-		require.Len(t, roots.Roots[0].IntermediateCerts, 3)
 
-		newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
+		require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
-		require.NotEqual(t, previous, newCert)
+		require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
+		require.Contains(t, roots.Roots[0].RootCert, rootPEM)
+		require.Len(t, roots.Roots[0].IntermediateCerts, 3, "one intermediate from primary, two from secondary")
+
+		active := roots.Active()
+		oldSecondaryData = active
+
+		newCert := serverDC2.caManager.getLeafSigningCertFromRoot(active)
+		require.NotEqual(t, secondaryLeafSigningCert, newCert)
+		secondaryLeafSigningCert = newCert
 
 		leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
-		verifyLeafCert(t, roots.Roots[0], leafPEM)
+		verifyLeafCert(t, active, leafPEM)
 
 		// original certs from old signing cert should still verify
-		verifyLeafCert(t, roots.Roots[0], origLeaf)
+		verifyLeafCert(t, active, origLeaf)
 	})
 
 	testutil.RunStep(t, "rotate root by changing the provider", func(t *testing.T) {
@@ -902,20 +994,47 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
 		roots = structs.IndexedCARoots{}
 		err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
 		require.NoError(t, err)
-		require.Len(t, roots.Roots, 2)
+
+		require.Len(t, roots.Roots, 2, "two because we rotated the provider")
+
 		active := roots.Active()
-		require.Len(t, active.IntermediateCerts, 1)
+		require.NotEqual(t, primaryCert, active.RootCert, "should NOT be the offline root, because we switched")
+		require.NotContains(t, active.RootCert, rootPEM)
+		require.Len(t, active.IntermediateCerts, 1, "only one new intermediate in the primary")
 
 		leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
 		verifyLeafCert(t, roots.Active(), leafPEM)
 
+		// wait for secondary to witness it
+		codec2 := rpcClient(t, serverDC2)
+		retry.Run(t, func(r *retry.R) {
+			var reply structs.IndexedCARoots
+			err := msgpackrpc.CallWithCodec(codec2, "ConnectCA.Roots", &structs.DCSpecificRequest{
+				Datacenter: "dc2",
+			}, &reply)
+			require.NoError(r, err)
+
+			require.Len(r, reply.Roots, 2, "primary provider rotated, so secondary gets rekeyed")
+
+			active := reply.Active()
+			require.NotNil(r, active)
+
+			if oldSecondaryData.ID == reply.ActiveRootID {
+				r.Fatal("wait; did not witness primary root rotation yet")
+			}
+
+			newCert := serverDC2.caManager.getLeafSigningCertFromRoot(roots.Active())
+			require.NotEqual(r, secondaryLeafSigningCert, newCert)
+			secondaryLeafSigningCert = newCert
+		})
+
 		// original certs from old root cert should still verify
 		verifyLeafCertWithRoots(t, roots, origLeaf)
 
 		// original certs from secondary should still verify
 		rootsSecondary := structs.IndexedCARoots{}
 		r := &structs.DCSpecificRequest{Datacenter: "dc2"}
-		err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", r, &rootsSecondary)
+		err = msgpackrpc.CallWithCodec(codec2, "ConnectCA.Roots", r, &rootsSecondary)
 		require.NoError(t, err)
 		verifyLeafCertWithRoots(t, rootsSecondary, origLeafSecondary)
 	})
@@ -923,6 +1042,12 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
 	testutil.RunStep(t, "rotate to a different external root", func(t *testing.T) {
 		setupPrimaryCA(t, vclient, "pki-primary-2/", rootPEM)
 
+		primaryVaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
+			RootPath:         "pki-primary-2",
+			IntermediatePath: "pki-intermediate-2",
+			ConsulManaged:    true,
+		})
+
 		codec := rpcClient(t, serverDC1)
 		req := &structs.CARequest{
 			Op: structs.CAOpSetConfig,
@@ -930,7 +1055,7 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
 				Provider: "vault",
 				Config: map[string]interface{}{
 					"Address":             vault.Addr,
-					"Token":               vault.RootToken,
+					"Token":               primaryVaultToken2,
 					"RootPKIPath":         "pki-primary-2/",
 					"IntermediatePKIPath": "pki-intermediate-2/",
 				},
@@ -963,28 +1088,6 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
 	})
 }
 
-// renewLeafSigningCert mimics RenewIntermediate. This is unfortunate, but
-// necessary for now as there is no easy way to invoke that logic unconditionally.
-// Currently, it requires patching values and polling for the operation to
-// complete, which adds a lot of distractions to a test case.
-// With this function we can instead unconditionally rotate the leaf signing cert
-// synchronously.
-func renewLeafSigningCert(t *testing.T, manager *CAManager, fn func(ca.Provider, *structs.CARoot) error) {
-	t.Helper()
-	provider, _ := manager.getCAProvider()
-
-	store := manager.delegate.State()
-	_, root, err := store.CARootActive(nil)
-	require.NoError(t, err)
-
-	activeRoot := root.Clone()
-	err = fn(provider, activeRoot)
-	require.NoError(t, err)
-	err = manager.persistNewRootAndConfig(provider, activeRoot, nil)
-	require.NoError(t, err)
-	manager.setCAProvider(provider, activeRoot)
-}
-
 func generateExternalRootCA(t *testing.T, client *vaultapi.Client) string {
 	t.Helper()
 	err := client.Sys().Mount("corp", &vaultapi.MountInput{
@@ -1046,6 +1149,7 @@ func setupPrimaryCA(t *testing.T, client *vaultapi.Client, path string, rootPEM
 		"certificate": buf.String(),
 	})
 	require.NoError(t, err, "failed to set signed intermediate")
+	// TODO: also fix issuers here?
 	return lib.EnsureTrailingNewline(buf.String())
 }
 

agent/consul/leader_connect_test.go

@@ -328,13 +328,19 @@ func TestCAManager_RenewIntermediate_Vault_Primary(t *testing.T) {
 
 	testVault := ca.NewTestVaultServer(t)
 
+	vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
 	_, s1 := testServerWithConfig(t, func(c *Config) {
 		c.PrimaryDatacenter = "dc1"
 		c.CAConfig = &structs.CAConfiguration{
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             testVault.Addr,
-				"Token":               testVault.RootToken,
+				"Token":               vaultToken,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-intermediate/",
 				"LeafCertTTL":         "2s",
@@ -700,7 +706,6 @@ func TestCAManager_Initialize_Vault_KeepOldRoots_Primary(t *testing.T) {
 	t.Parallel()
 
 	testVault := ca.NewTestVaultServer(t)
-	defer testVault.Stop()
 
 	dir1pre, s1pre := testServer(t)
 	defer os.RemoveAll(dir1pre)
@@ -710,12 +715,18 @@ func TestCAManager_Initialize_Vault_KeepOldRoots_Primary(t *testing.T) {
 
 	testrpc.WaitForLeader(t, s1pre.RPC, "dc1")
 
+	vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
 	// Update the CA config to use Vault - this should force the generation of a new root cert.
 	vaultCAConf := &structs.CAConfiguration{
 		Provider: "vault",
 		Config: map[string]interface{}{
 			"Address":             testVault.Addr,
-			"Token":               testVault.RootToken,
+			"Token":               vaultToken,
 			"RootPKIPath":         "pki-root/",
 			"IntermediatePKIPath": "pki-intermediate/",
 		},
@@ -765,7 +776,12 @@ func TestCAManager_Initialize_Vault_FixesSigningKeyID_Primary(t *testing.T) {
 	t.Parallel()
 
 	testVault := ca.NewTestVaultServer(t)
-	defer testVault.Stop()
+
+	vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
 
 	dir1pre, s1pre := testServerWithConfig(t, func(c *Config) {
 		c.Build = "1.6.0"
@@ -774,7 +790,7 @@ func TestCAManager_Initialize_Vault_FixesSigningKeyID_Primary(t *testing.T) {
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             testVault.Addr,
-				"Token":               testVault.RootToken,
+				"Token":               vaultToken,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-intermediate/",
 			},
@@ -1545,13 +1561,19 @@ func TestCAManager_Initialize_Vault_BadCAConfigDoesNotPreventLeaderEstablishment
 	require.Empty(t, rootsList.Roots)
 	require.Nil(t, activeRoot)
 
+	goodVaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
 	// Now that the leader is up and we have verified that there are no roots / CA init failed,
 	// verify that we can reconfigure away from the bad configuration.
 	newConfig := &structs.CAConfiguration{
 		Provider: "vault",
 		Config: map[string]interface{}{
 			"Address":             testVault.Addr,
-			"Token":               testVault.RootToken,
+			"Token":               goodVaultToken,
 			"RootPKIPath":         "pki-root/",
 			"IntermediatePKIPath": "pki-intermediate/",
 		},
@@ -1675,7 +1697,13 @@ func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T)
 	ca.SkipIfVaultNotPresent(t)
 
 	testVault := ca.NewTestVaultServer(t)
-	defer testVault.Stop()
+
+	vaultToken1 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+		WithSudo:         true,
+	})
 
 	_, s1 := testServerWithConfig(t, func(c *Config) {
 		c.Build = "1.9.1"
@@ -1684,7 +1712,7 @@ func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T)
 			Provider: "vault",
 			Config: map[string]interface{}{
 				"Address":             testVault.Addr,
-				"Token":               testVault.RootToken,
+				"Token":               vaultToken1,
 				"RootPKIPath":         "pki-root/",
 				"IntermediatePKIPath": "pki-intermediate/",
 			},
@@ -1705,13 +1733,19 @@ func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T)
 	require.Len(t, rootList.Roots, 1)
 	oldRoot := rootList.Roots[0]
 
+	vaultToken2 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
+		RootPath:         "pki-root-2",
+		IntermediatePath: "pki-intermediate",
+		ConsulManaged:    true,
+	})
+
 	// Update the provider config to use a new PKI path, which should
 	// cause a rotation.
 	newConfig := &structs.CAConfiguration{
 		Provider: "vault",
 		Config: map[string]interface{}{
 			"Address":             testVault.Addr,
-			"Token":               testVault.RootToken,
+			"Token":               vaultToken2,
 			"RootPKIPath":         "pki-root-2/",
 			"IntermediatePKIPath": "pki-intermediate/",
 		},