luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

connect: ensure all vault connect CA tests use limited privilege tokens (#15669) 

All of the current integration tests where Vault is the Connect CA now use non-root tokens for the test. This helps us detect privilege changes in the vault model so we can keep our guides up to date.

One larger change was that the RenewIntermediate function got refactored slightly so it could be used from a test, rather than the large duplicated function we were testing in a test which seemed error prone.
This commit is contained in:
R.B. Boyer 2022-12-06 10:06:36 -06:00 committed by GitHub
parent a88d1239e3
commit ba6b24babf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 943 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/15669.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
connect: ensure all vault connect CA tests use limited privilege tokens
```

9
agent/agent_endpoint_test.go
View File

@ -7095,7 +7095,12 @@ func TestAgentConnectCALeafCert_Vault_doesNotChurnLeafCertsAtIdle(t *testing.T)
t.Parallel()
testVault := ca.NewTestVaultServer(t)
defer testVault.Stop()
vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
a := StartTestAgent(t, TestAgent{Overrides: fmt.Sprintf(`
connect {
@ -7108,7 +7113,7 @@ func TestAgentConnectCALeafCert_Vault_doesNotChurnLeafCertsAtIdle(t *testing.T)
intermediate_pki_path = "pki-intermediate/"
}
}
`, testVault.Addr, testVault.RootToken)})
`, testVault.Addr, vaultToken)})
defer a.Shutdown()
testrpc.WaitForTestAgent(t, a.RPC, "dc1")
testrpc.WaitForActiveCARoot(t, a.RPC, "dc1", nil)

24
agent/connect/ca/provider_consul_test.go
View File

@ -375,6 +375,30 @@ func testCrossSignProviders(t *testing.T, provider1, provider2 Provider) {
}
}
func testCrossSignProvidersShouldFail(t *testing.T, provider1, provider2 Provider) {
t.Helper()
// Get the root from the new provider to be cross-signed.
root, err := provider2.GenerateRoot()
require.NoError(t, err)
newRoot, err := connect.ParseCert(root.PEM)
require.NoError(t, err)
requireNotEncoded(t, newRoot.SubjectKeyId)
requireNotEncoded(t, newRoot.AuthorityKeyId)
newInterPEM, err := provider2.ActiveIntermediate()
require.NoError(t, err)
newIntermediate, err := connect.ParseCert(newInterPEM)
require.NoError(t, err)
requireNotEncoded(t, newIntermediate.SubjectKeyId)
requireNotEncoded(t, newIntermediate.AuthorityKeyId)
// Have provider1 cross sign our new root cert.
_, err = provider1.CrossSignCA(newRoot)
require.Error(t, err)
}
func TestConsulProvider_SignIntermediate(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")

6
agent/connect/ca/provider_vault.go
View File

@ -885,14 +885,12 @@ func makePathHelper(namespace, path string) string {
func (v *VaultProvider) readNamespaced(namespace string, resource string) (*vaultapi.Secret, error) {
defer v.setNamespace(namespace)()
result, err := v.client.Logical().Read(resource)
return result, err
return v.client.Logical().Read(resource)
}
func (v *VaultProvider) writeNamespaced(namespace string, resource string, data map[string]interface{}) (*vaultapi.Secret, error) {
defer v.setNamespace(namespace)()
result, err := v.client.Logical().Write(resource, data)
return result, err
return v.client.Logical().Write(resource, data)
}
func (v *VaultProvider) setNamespace(namespace string) func() {

765
agent/connect/ca/provider_vault_test.go
View File

File diff suppressed because it is too large Load Diff

166
agent/connect/ca/testing.go
View File

@ -10,8 +10,10 @@ import (
"sync"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-uuid"
vaultapi "github.com/hashicorp/vault/api"
"github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/sdk/freeport"
@ -27,11 +29,7 @@ import (
// these types of old CA key.
// - SignIntermediate muse bt able to sign all the types of secondary
// intermediate CA key with all these types of primary CA key
var KeyTestCases = []struct {
Desc string
KeyType string
KeyBits int
}{
var KeyTestCases = []KeyTestCase{
{
Desc: "Default Key Type (EC 256)",
KeyType: connect.DefaultPrivateKeyType,
@ -44,6 +42,12 @@ var KeyTestCases = []struct {
},
}
type KeyTestCase struct {
Desc string
KeyType string
KeyBits int
}
// CASigningKeyTypes is a struct with params for tests that sign one CA CSR with
// another CA key.
type CASigningKeyTypes struct {
@ -106,17 +110,6 @@ func SkipIfVaultNotPresent(t testing.T) {
}
func NewTestVaultServer(t testing.T) *TestVaultServer {
testVault, err := runTestVault(t)
if err != nil {
t.Fatalf("err: %v", err)
}
testVault.WaitUntilReady(t)
return testVault
}
func runTestVault(t testing.T) (*TestVaultServer, error) {
vaultBinaryName := os.Getenv("VAULT_BINARY_NAME")
if vaultBinaryName == "" {
vaultBinaryName = "vault"
@ -124,7 +117,7 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
path, err := exec.LookPath(vaultBinaryName)
if err != nil || path == "" {
return nil, fmt.Errorf("%q not found on $PATH", vaultBinaryName)
t.Fatalf("%q not found on $PATH", vaultBinaryName)
}
ports := freeport.GetN(t, 2)
@ -138,9 +131,7 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
client, err := vaultapi.NewClient(&vaultapi.Config{
Address: "http://" + clientAddr,
})
if err != nil {
return nil, err
}
require.NoError(t, err)
client.SetToken(token)
args := []string{
@ -157,9 +148,7 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
cmd := exec.Command(vaultBinaryName, args...)
cmd.Stdout = io.Discard
cmd.Stderr = io.Discard
if err := cmd.Start(); err != nil {
return nil, err
}
require.NoError(t, cmd.Start())
testVault := &TestVaultServer{
RootToken: token,
@ -173,7 +162,9 @@ func runTestVault(t testing.T) (*TestVaultServer, error) {
}
})
return testVault, nil
testVault.WaitUntilReady(t)
return testVault
}
type TestVaultServer struct {
@ -224,6 +215,9 @@ func (v *TestVaultServer) Stop() error {
// wait for the process to exit to be sure that the data dir can be
// deleted on all platforms.
if err := v.cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exec: Wait was already called") {
return nil
}
return err
}
return nil
@ -238,3 +232,127 @@ func requireTrailingNewline(t testing.T, leafPEM string) {
t.Fatalf("cert do not end with a new line")
}
}
// The zero value implies unprivileged.
type VaultTokenAttributes struct {
RootPath, IntermediatePath string
ConsulManaged bool
VaultManaged bool
WithSudo bool
CustomRules string
}
func (a *VaultTokenAttributes) DisplayName() string {
switch {
case a == nil:
return "unprivileged"
case a.CustomRules != "":
return "custom"
case a.ConsulManaged:
return "consul-managed"
case a.VaultManaged:
return "vault-managed"
default:
return "unprivileged"
}
}
func (a *VaultTokenAttributes) Rules(t testing.T) string {
switch {
case a == nil:
return ""
case a.CustomRules != "":
return a.CustomRules
case a.RootPath == "":
t.Fatal("missing required RootPath")
return "" // dead code
case a.IntermediatePath == "":
t.Fatal("missing required IntermediatePath")
return "" // dead code
case a.ConsulManaged:
// Consul Managed PKI Mounts
rules := fmt.Sprintf(`
path "sys/mounts" {
capabilities = [ "read" ]
}
path "sys/mounts/%[1]s" {
capabilities = [ "create", "read", "update", "delete", "list" ]
}
path "sys/mounts/%[2]s" {
capabilities = [ "create", "read", "update", "delete", "list" ]
}
# Needed for Consul 1.11+
path "sys/mounts/%[2]s/tune" {
capabilities = [ "update" ]
}
# vault token renewal
path "auth/token/renew-self" {
capabilities = [ "update" ]
}
path "auth/token/lookup-self" {
capabilities = [ "read" ]
}
path "%[1]s/*" {
capabilities = [ "create", "read", "update", "delete", "list" ]
}
path "%[2]s/*" {
capabilities = [ "create", "read", "update", "delete", "list" ]
}
`, a.RootPath, a.IntermediatePath)
if a.WithSudo {
rules += fmt.Sprintf(`
path "%[1]s/root/sign-self-issued" {
capabilities = [ "sudo", "update" ]
}
`, a.RootPath)
}
return rules
case a.VaultManaged:
// Vault-managed PKI root.
t.Fatal("TODO: implement this and use it in tests")
return ""
default:
// zero value
return ""
}
}
func CreateVaultTokenWithAttrs(t testing.T, client *vaultapi.Client, attr *VaultTokenAttributes) string {
policyName, err := uuid.GenerateUUID()
require.NoError(t, err)
rules := attr.Rules(t)
token := createVaultTokenAndPolicy(t, client, policyName, rules)
// t.Logf("created vault token with scope %q: %s", attr.DisplayName(), token)
return token
}
func createVaultTokenAndPolicy(t testing.T, client *vaultapi.Client, policyName, policyRules string) string {
require.NoError(t, client.Sys().PutPolicy(policyName, policyRules))
renew := true
tok, err := client.Auth().Token().Create(&vaultapi.TokenCreateRequest{
Policies: []string{policyName},
Renewable: &renew,
})
require.NoError(t, err)
return tok.Auth.ClientToken
}

2
agent/consul/acl_server.go
View File

@ -87,7 +87,7 @@ func (s *Server) checkBindingRuleUUID(id string) (bool, error) {
}
func (s *Server) InPrimaryDatacenter() bool {
return s.config.PrimaryDatacenter == "" || s.config.Datacenter == s.config.PrimaryDatacenter
return s.config.InPrimaryDatacenter()
}
func (s *Server) LocalTokensEnabled() bool {

4
agent/consul/config.go
View File

@ -415,6 +415,10 @@ type Config struct {
*EnterpriseConfig
}
func (c *Config) InPrimaryDatacenter() bool {
return c.PrimaryDatacenter == "" || c.Datacenter == c.PrimaryDatacenter
}
// CheckProtocolVersion validates the protocol version.
func (c *Config) CheckProtocolVersion() error {
if c.ProtocolVersion < ProtocolVersionMin {

34
agent/consul/connect_ca_endpoint_test.go
View File

@ -9,11 +9,10 @@ import (
"testing"
"time"
msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/connect"
ca "github.com/hashicorp/consul/agent/connect/ca"
@ -560,10 +559,23 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
testVault := ca.NewTestVaultServer(t)
newConfig := func(keyType string, keyBits int) map[string]interface{} {
return map[string]interface{}{
token1 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-primary",
ConsulManaged: true,
WithSudo: true,
})
token2 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
newConfig := func(token string, keyType string, keyBits int) map[string]any {
return map[string]any{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": token,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
"PrivateKeyType": keyType,
@ -574,7 +586,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
_, s1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: newConfig(connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits),
Config: newConfig(token1, connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits),
}
})
testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
@ -592,7 +604,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
configFn: func() *structs.CAConfiguration {
return &structs.CAConfiguration{
Provider: "vault",
Config: newConfig("rsa", 4096),
Config: newConfig(token2, "rsa", 4096),
ForceWithoutCrossSigning: true,
}
},
@ -602,7 +614,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
configFn: func() *structs.CAConfiguration {
return &structs.CAConfiguration{
Provider: "vault",
Config: newConfig("rsa", 2048),
Config: newConfig(token2, "rsa", 2048),
ForceWithoutCrossSigning: true,
}
},
@ -613,7 +625,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
configFn: func() *structs.CAConfiguration {
return &structs.CAConfiguration{
Provider: "vault",
Config: newConfig("ec", 256),
Config: newConfig(token2, "ec", 256),
ForceWithoutCrossSigning: true,
}
},
@ -624,7 +636,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
configFn: func() *structs.CAConfiguration {
return &structs.CAConfiguration{
Provider: "vault",
Config: newConfig("rsa", 4096),
Config: newConfig(token2, "rsa", 4096),
ForceWithoutCrossSigning: true,
}
},
@ -632,7 +644,7 @@ func TestConnectCAConfig_Vault_TriggerRotation_Fails(t *testing.T) {
}
for _, tc := range testSteps {
t.Run(tc.name, func(t *testing.T) {
testutil.RunStep(t, tc.name, func(t *testing.T) {
args := &structs.CARequest{
Datacenter: "dc1",
Config: tc.configFn(),

53
agent/consul/leader_connect_ca.go
View File

@ -16,14 +16,13 @@ import (
"golang.org/x/time/rate"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/lib/semaphore"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/connect/ca"
"github.com/hashicorp/consul/agent/consul/state"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/lib/routine"
"github.com/hashicorp/consul/lib/semaphore"
)
type caState string
@ -1069,7 +1068,7 @@ func (c *CAManager) secondaryRequestNewSigningCert(provider ca.Provider, newActi
}
if err := setLeafSigningCert(newActiveRoot, intermediatePEM); err != nil {
return err
return fmt.Errorf("Failed to set the leaf signing cert to the intermediate: %w", err)
}
c.logger.Info("received new intermediate certificate from primary datacenter")
@ -1117,15 +1116,13 @@ func pruneExpiredIntermediates(caRoot *structs.CARoot) error {
// runRenewIntermediate periodically attempts to renew the intermediate cert.
func (c *CAManager) runRenewIntermediate(ctx context.Context) error {
isPrimary := c.serverConf.Datacenter == c.serverConf.PrimaryDatacenter
for {
select {
case <-ctx.Done():
return nil
case <-time.After(structs.IntermediateCertRenewInterval):
retryLoopBackoffAbortOnSuccess(ctx, func() error {
return c.RenewIntermediate(ctx, isPrimary)
return c.RenewIntermediate(ctx)
}, func(err error) {
c.logger.Error("error renewing intermediate certs",
"routine", intermediateCertRenewWatchRoutineName,
@ -1139,7 +1136,15 @@ func (c *CAManager) runRenewIntermediate(ctx context.Context) error {
// RenewIntermediate checks the intermediate cert for
// expiration. If more than half the time a cert is valid has passed,
// it will try to renew it.
func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error {
func (c *CAManager) RenewIntermediate(ctx context.Context) error {
return c.renewIntermediate(ctx, false)
}
func (c *CAManager) renewIntermediateNow(ctx context.Context) error {
return c.renewIntermediate(ctx, true)
}
func (c *CAManager) renewIntermediate(ctx context.Context, forceNow bool) error {
// Grab the 'lock' right away so the provider/config can't be changed out while we check
// the intermediate.
if _, err := c.setState(caStateRenewIntermediate, true); err != nil {
@ -1147,6 +1152,8 @@ func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error
}
defer c.setState(caStateInitialized, false)
isPrimary := c.serverConf.InPrimaryDatacenter()
provider, _ := c.getCAProvider()
if provider == nil {
// this happens when leadership is being revoked and this go routine will be stopped
@ -1184,8 +1191,10 @@ func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error
return fmt.Errorf("error parsing active intermediate cert: %v", err)
}
if lessThanHalfTimePassed(c.timeNow(), intermediateCert.NotBefore, intermediateCert.NotAfter) {
return nil
if !forceNow {
if lessThanHalfTimePassed(c.timeNow(), intermediateCert.NotBefore, intermediateCert.NotAfter) {
return nil
}
}
// Enough time has passed, go ahead with getting a new intermediate.
@ -1193,19 +1202,27 @@ func (c *CAManager) RenewIntermediate(ctx context.Context, isPrimary bool) error
if !isPrimary {
renewalFunc = c.secondaryRequestNewSigningCert
}
errCh := make(chan error, 1)
go func() {
errCh <- renewalFunc(provider, activeRoot)
}()
// Wait for the renewal func to return or for the context to be canceled.
select {
case <-ctx.Done():
return ctx.Err()
case err := <-errCh:
if forceNow {
err := renewalFunc(provider, activeRoot)
if err != nil {
return err
}
} else {
errCh := make(chan error, 1)
go func() {
errCh <- renewalFunc(provider, activeRoot)
}()
// Wait for the renewal func to return or for the context to be canceled.
select {
case <-ctx.Done():
return ctx.Err()
case err := <-errCh:
if err != nil {
return err
}
}
}
if err := c.persistNewRootAndConfig(provider, activeRoot, nil); err != nil {

240
agent/consul/leader_connect_ca_test.go
View File

@ -16,14 +16,13 @@ import (
"testing"
"time"
msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
"github.com/hashicorp/consul-net-rpc/net/rpc"
vaultapi "github.com/hashicorp/vault/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
"github.com/hashicorp/consul-net-rpc/net/rpc"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/connect"
ca "github.com/hashicorp/consul/agent/connect/ca"
@ -48,12 +47,24 @@ func TestCAManager_Initialize_Vault_Secondary_SharedVault(t *testing.T) {
vault := ca.NewTestVaultServer(t)
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-primary",
ConsulManaged: true,
})
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-secondary",
ConsulManaged: true,
})
_, serverDC1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": primaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-primary/",
},
@ -81,7 +92,7 @@ func TestCAManager_Initialize_Vault_Secondary_SharedVault(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": secondaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-secondary/",
},
@ -393,7 +404,7 @@ func TestCAManager_UpdateConfigWhileRenewIntermediate(t *testing.T) {
// happen in the expected order.
errCh := make(chan error)
go func() {
errCh <- manager.RenewIntermediate(context.TODO(), false)
errCh <- manager.RenewIntermediate(context.TODO())
}()
waitForCh(t, delegate.callbackCh, "provider/GenerateIntermediateCSR")
@ -570,7 +581,14 @@ func TestCAManager_Initialize_Logging(t *testing.T) {
func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
ca.SkipIfVaultNotPresent(t)
vault := ca.NewTestVaultServer(t)
vaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
WithSudo: true,
})
_, s1 := testServerWithConfig(t, func(c *Config) {
c.PrimaryDatacenter = "dc1"
@ -578,7 +596,7 @@ func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": vaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
@ -599,12 +617,18 @@ func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
require.NoError(t, err)
require.Equal(t, connect.HexString(cert.SubjectKeyId), origRoot.SigningKeyID)
vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root-2",
IntermediatePath: "pki-intermediate-2",
ConsulManaged: true,
})
err = s1.caManager.UpdateConfiguration(&structs.CARequest{
Config: &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": vaultToken2,
"RootPKIPath": "pki-root-2/",
"IntermediatePKIPath": "pki-intermediate-2/",
},
@ -636,12 +660,18 @@ func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
meshRootPath := "pki-root"
primaryCert := setupPrimaryCA(t, vclient, meshRootPath, "")
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: meshRootPath,
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
_, s1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": primaryVaultToken,
"RootPKIPath": meshRootPath,
"IntermediatePKIPath": "pki-intermediate/",
},
@ -665,6 +695,12 @@ func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
// TODO: renew primary leaf signing cert
// TODO: rotate root
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: meshRootPath,
IntermediatePath: "pki-secondary",
ConsulManaged: true,
})
testutil.RunStep(t, "run secondary DC", func(t *testing.T) {
_, sDC2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
@ -673,7 +709,7 @@ func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": secondaryVaultToken,
"RootPKIPath": meshRootPath,
"IntermediatePKIPath": "pki-secondary/",
},
@ -704,12 +740,18 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
vault := ca.NewTestVaultServer(t)
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
_, sDC1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": primaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
@ -718,6 +760,12 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
defer sDC1.Shutdown()
testrpc.WaitForActiveCARoot(t, sDC1.RPC, "dc1", nil)
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate-2",
ConsulManaged: true,
})
_, sDC2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc1"
@ -725,7 +773,7 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": secondaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate-2/",
},
@ -740,7 +788,7 @@ func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", &structs.DCSpecificRequest{}, &configBefore)
require.NoError(t, err)
renewLeafSigningCert(t, sDC1.caManager, sDC1.caManager.primaryRenewIntermediate)
require.NoError(t, sDC1.caManager.renewIntermediateNow(context.Background()))
// Give the secondary some time to notice the update
time.Sleep(100 * time.Millisecond)
@ -778,37 +826,62 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
vault := ca.NewTestVaultServer(t)
vclient := vault.Client()
rootPEM := generateExternalRootCA(t, vclient)
primaryCAPath := "pki-primary"
primaryCert := setupPrimaryCA(t, vclient, primaryCAPath, rootPEM)
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: primaryCAPath,
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
WithSudo: true,
})
_, serverDC1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": primaryVaultToken,
"RootPKIPath": primaryCAPath,
"IntermediatePKIPath": "pki-intermediate/",
},
}
})
testrpc.WaitForTestAgent(t, serverDC1.RPC, "dc1")
testrpc.WaitForActiveCARoot(t, serverDC1.RPC, "dc1", nil)
var origLeaf string
var (
origLeaf string
primaryLeafSigningCert string
)
roots := structs.IndexedCARoots{}
testutil.RunStep(t, "verify primary DC", func(t *testing.T) {
codec := rpcClient(t, serverDC1)
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
require.Equal(t, primaryCert, roots.Roots[0].RootCert)
// Verify CA trust heirarchy is expected.
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 1, "should just have the primary's intermediate")
active := roots.Active()
leafCert := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Active(), leafCert)
verifyLeafCert(t, active, leafCert)
origLeaf = leafCert
primaryLeafSigningCert = serverDC1.caManager.getLeafSigningCertFromRoot(active)
})
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "should-be-ignored",
IntermediatePath: "pki-secondary",
ConsulManaged: true,
})
_, serverDC2 := testServerWithConfig(t, func(c *Config) {
@ -818,72 +891,91 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": secondaryVaultToken,
"RootPKIPath": "should-be-ignored",
"IntermediatePKIPath": "pki-secondary/",
},
}
})
joinWAN(t, serverDC2, serverDC1)
testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
var origLeafSecondary string
var (
origLeafSecondary string
secondaryLeafSigningCert string
)
testutil.RunStep(t, "start secondary DC", func(t *testing.T) {
joinWAN(t, serverDC2, serverDC1)
testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
codec := rpcClient(t, serverDC2)
roots = structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 2, "should have the primary's intermediate and our own")
active := roots.Active()
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
verifyLeafCert(t, roots.Roots[0], leafPEM)
origLeafSecondary = leafPEM
secondaryLeafSigningCert = serverDC2.caManager.getLeafSigningCertFromRoot(active)
})
testutil.RunStep(t, "renew leaf signing CA in primary", func(t *testing.T) {
previous := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
renewLeafSigningCert(t, serverDC1.caManager, serverDC1.caManager.primaryRenewIntermediate)
require.NoError(t, serverDC1.caManager.renewIntermediateNow(context.Background()))
codec := rpcClient(t, serverDC1)
roots = structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
require.Len(t, roots.Roots[0].IntermediateCerts, 2)
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 2, "we renewed, so we have our old primary and our new primary intermediate")
active := roots.Active()
newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
require.NotEqual(t, previous, newCert)
require.NotEqual(t, primaryLeafSigningCert, newCert)
primaryLeafSigningCert = newCert
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Roots[0], leafPEM)
verifyLeafCert(t, active, leafPEM)
// original certs from old signing cert should still verify
verifyLeafCert(t, roots.Roots[0], origLeaf)
verifyLeafCert(t, active, origLeaf)
})
var oldSecondaryData *structs.CARoot
testutil.RunStep(t, "renew leaf signing CA in secondary", func(t *testing.T) {
previous := serverDC2.caManager.getLeafSigningCertFromRoot(roots.Active())
renewLeafSigningCert(t, serverDC2.caManager, serverDC2.caManager.secondaryRequestNewSigningCert)
require.NoError(t, serverDC2.caManager.renewIntermediateNow(context.Background()))
codec := rpcClient(t, serverDC2)
roots = structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
// one intermediate from primary, two from secondary
require.Len(t, roots.Roots[0].IntermediateCerts, 3)
newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
require.NotEqual(t, previous, newCert)
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 3, "one intermediate from primary, two from secondary")
active := roots.Active()
oldSecondaryData = active
newCert := serverDC2.caManager.getLeafSigningCertFromRoot(active)
require.NotEqual(t, secondaryLeafSigningCert, newCert)
secondaryLeafSigningCert = newCert
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
verifyLeafCert(t, roots.Roots[0], leafPEM)
verifyLeafCert(t, active, leafPEM)
// original certs from old signing cert should still verify
verifyLeafCert(t, roots.Roots[0], origLeaf)
verifyLeafCert(t, active, origLeaf)
})
testutil.RunStep(t, "rotate root by changing the provider", func(t *testing.T) {
@ -902,20 +994,47 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
roots = structs.IndexedCARoots{}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 2)
require.Len(t, roots.Roots, 2, "two because we rotated the provider")
active := roots.Active()
require.Len(t, active.IntermediateCerts, 1)
require.NotEqual(t, primaryCert, active.RootCert, "should NOT be the offline root, because we switched")
require.NotContains(t, active.RootCert, rootPEM)
require.Len(t, active.IntermediateCerts, 1, "only one new intermediate in the primary")
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Active(), leafPEM)
// wait for secondary to witness it
codec2 := rpcClient(t, serverDC2)
retry.Run(t, func(r *retry.R) {
var reply structs.IndexedCARoots
err := msgpackrpc.CallWithCodec(codec2, "ConnectCA.Roots", &structs.DCSpecificRequest{
Datacenter: "dc2",
}, &reply)
require.NoError(r, err)
require.Len(r, reply.Roots, 2, "primary provider rotated, so secondary gets rekeyed")
active := reply.Active()
require.NotNil(r, active)
if oldSecondaryData.ID == reply.ActiveRootID {
r.Fatal("wait; did not witness primary root rotation yet")
}
newCert := serverDC2.caManager.getLeafSigningCertFromRoot(roots.Active())
require.NotEqual(r, secondaryLeafSigningCert, newCert)
secondaryLeafSigningCert = newCert
})
// original certs from old root cert should still verify
verifyLeafCertWithRoots(t, roots, origLeaf)
// original certs from secondary should still verify
rootsSecondary := structs.IndexedCARoots{}
r := &structs.DCSpecificRequest{Datacenter: "dc2"}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", r, &rootsSecondary)
err = msgpackrpc.CallWithCodec(codec2, "ConnectCA.Roots", r, &rootsSecondary)
require.NoError(t, err)
verifyLeafCertWithRoots(t, rootsSecondary, origLeafSecondary)
})
@ -923,6 +1042,12 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
testutil.RunStep(t, "rotate to a different external root", func(t *testing.T) {
setupPrimaryCA(t, vclient, "pki-primary-2/", rootPEM)
primaryVaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-primary-2",
IntermediatePath: "pki-intermediate-2",
ConsulManaged: true,
})
codec := rpcClient(t, serverDC1)
req := &structs.CARequest{
Op: structs.CAOpSetConfig,
@ -930,7 +1055,7 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vault.RootToken,
"Token": primaryVaultToken2,
"RootPKIPath": "pki-primary-2/",
"IntermediatePKIPath": "pki-intermediate-2/",
},
@ -963,28 +1088,6 @@ func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
})
}
// renewLeafSigningCert mimics RenewIntermediate. This is unfortunate, but
// necessary for now as there is no easy way to invoke that logic unconditionally.
// Currently, it requires patching values and polling for the operation to
// complete, which adds a lot of distractions to a test case.
// With this function we can instead unconditionally rotate the leaf signing cert
// synchronously.
func renewLeafSigningCert(t *testing.T, manager *CAManager, fn func(ca.Provider, *structs.CARoot) error) {
t.Helper()
provider, _ := manager.getCAProvider()
store := manager.delegate.State()
_, root, err := store.CARootActive(nil)
require.NoError(t, err)
activeRoot := root.Clone()
err = fn(provider, activeRoot)
require.NoError(t, err)
err = manager.persistNewRootAndConfig(provider, activeRoot, nil)
require.NoError(t, err)
manager.setCAProvider(provider, activeRoot)
}
func generateExternalRootCA(t *testing.T, client *vaultapi.Client) string {
t.Helper()
err := client.Sys().Mount("corp", &vaultapi.MountInput{
@ -1046,6 +1149,7 @@ func setupPrimaryCA(t *testing.T, client *vaultapi.Client, path string, rootPEM
"certificate": buf.String(),
})
require.NoError(t, err, "failed to set signed intermediate")
// TODO: also fix issuers here?
return lib.EnsureTrailingNewline(buf.String())
}

52
agent/consul/leader_connect_test.go
View File

@ -328,13 +328,19 @@ func TestCAManager_RenewIntermediate_Vault_Primary(t *testing.T) {
testVault := ca.NewTestVaultServer(t)
vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
_, s1 := testServerWithConfig(t, func(c *Config) {
c.PrimaryDatacenter = "dc1"
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": vaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
"LeafCertTTL": "2s",
@ -700,7 +706,6 @@ func TestCAManager_Initialize_Vault_KeepOldRoots_Primary(t *testing.T) {
t.Parallel()
testVault := ca.NewTestVaultServer(t)
defer testVault.Stop()
dir1pre, s1pre := testServer(t)
defer os.RemoveAll(dir1pre)
@ -710,12 +715,18 @@ func TestCAManager_Initialize_Vault_KeepOldRoots_Primary(t *testing.T) {
testrpc.WaitForLeader(t, s1pre.RPC, "dc1")
vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
// Update the CA config to use Vault - this should force the generation of a new root cert.
vaultCAConf := &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": vaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
@ -765,7 +776,12 @@ func TestCAManager_Initialize_Vault_FixesSigningKeyID_Primary(t *testing.T) {
t.Parallel()
testVault := ca.NewTestVaultServer(t)
defer testVault.Stop()
vaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
dir1pre, s1pre := testServerWithConfig(t, func(c *Config) {
c.Build = "1.6.0"
@ -774,7 +790,7 @@ func TestCAManager_Initialize_Vault_FixesSigningKeyID_Primary(t *testing.T) {
Provider: "vault",
Config: map[string]interface{}{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": vaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
@ -1545,13 +1561,19 @@ func TestCAManager_Initialize_Vault_BadCAConfigDoesNotPreventLeaderEstablishment
require.Empty(t, rootsList.Roots)
require.Nil(t, activeRoot)
goodVaultToken := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
// Now that the leader is up and we have verified that there are no roots / CA init failed,
// verify that we can reconfigure away from the bad configuration.
newConfig := &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": goodVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
@ -1675,7 +1697,13 @@ func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T)
ca.SkipIfVaultNotPresent(t)
testVault := ca.NewTestVaultServer(t)
defer testVault.Stop()
vaultToken1 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
WithSudo: true,
})
_, s1 := testServerWithConfig(t, func(c *Config) {
c.Build = "1.9.1"
@ -1684,7 +1712,7 @@ func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T)
Provider: "vault",
Config: map[string]interface{}{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": vaultToken1,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
@ -1705,13 +1733,19 @@ func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T)
require.Len(t, rootList.Roots, 1)
oldRoot := rootList.Roots[0]
vaultToken2 := ca.CreateVaultTokenWithAttrs(t, testVault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root-2",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
// Update the provider config to use a new PKI path, which should
// cause a rotation.
newConfig := &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": testVault.Addr,
"Token": testVault.RootToken,
"Token": vaultToken2,
"RootPKIPath": "pki-root-2/",
"IntermediatePKIPath": "pki-intermediate/",
},