package consul import ( "context" "crypto/x509" "fmt" "io/ioutil" "os" "path/filepath" "reflect" "strings" "testing" "time" msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc" uuid "github.com/hashicorp/go-uuid" "github.com/stretchr/testify/require" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/connect/ca" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/token" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/testrpc" ) func TestConnectCA_ConfigurationSet_ChangeKeyConfig_Primary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } types := []struct { keyType string keyBits int }{ {connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits}, {"ec", 256}, {"ec", 384}, {"rsa", 2048}, {"rsa", 4096}, } for _, src := range types { for _, dst := range types { if src == dst { continue // skip } src := src dst := dst t.Run(fmt.Sprintf("%s-%d to %s-%d", src.keyType, src.keyBits, dst.keyType, dst.keyBits), func(t *testing.T) { t.Parallel() providerState := map[string]string{"foo": "dc1-value"} // Initialize primary as the primary DC dir1, srv := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc1" c.PrimaryDatacenter = "dc1" c.Build = "1.6.0" c.CAConfig.Config["PrivateKeyType"] = src.keyType c.CAConfig.Config["PrivateKeyBits"] = src.keyBits c.CAConfig.Config["test_state"] = providerState }) defer os.RemoveAll(dir1) defer srv.Shutdown() codec := rpcClient(t, srv) defer codec.Close() testrpc.WaitForLeader(t, srv.RPC, "dc1") testrpc.WaitForActiveCARoot(t, srv.RPC, "dc1", nil) var ( provider ca.Provider caRoot *structs.CARoot ) retry.Run(t, func(r *retry.R) { provider, caRoot = getCAProviderWithLock(srv) require.NotNil(r, caRoot) // Sanity check CA is using the correct key type require.Equal(r, src.keyType, caRoot.PrivateKeyType) require.Equal(r, src.keyBits, caRoot.PrivateKeyBits) }) runStep(t, "sign leaf cert and make sure chain is correct", func(t *testing.T) { spiffeService := &connect.SpiffeIDService{ Host: "node1", Namespace: "default", Datacenter: "dc1", Service: "foo", } raw, _ := connect.TestCSR(t, spiffeService) leafCsr, err := connect.ParseCSR(raw) require.NoError(t, err) leafPEM, err := provider.Sign(leafCsr) require.NoError(t, err) // Check that the leaf signed by the new cert can be verified using the // returned cert chain require.NoError(t, connect.ValidateLeaf(caRoot.RootCert, leafPEM, []string{})) }) runStep(t, "verify persisted state is correct", func(t *testing.T) { state := srv.fsm.State() _, caConfig, err := state.CAConfig(nil) require.NoError(t, err) require.Equal(t, providerState, caConfig.State) }) runStep(t, "change roots", func(t *testing.T) { // Update a config value newConfig := &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "PrivateKey": "", "RootCert": "", "PrivateKeyType": dst.keyType, "PrivateKeyBits": dst.keyBits, // This verifies the state persistence for providers although Consul // provider doesn't actually use that mechanism outside of tests. "test_state": providerState, }, } args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) }) var ( newProvider ca.Provider newCaRoot *structs.CARoot ) retry.Run(t, func(r *retry.R) { newProvider, newCaRoot = getCAProviderWithLock(srv) require.NotNil(r, newCaRoot) // Sanity check CA is using the correct key type require.Equal(r, dst.keyType, newCaRoot.PrivateKeyType) require.Equal(r, dst.keyBits, newCaRoot.PrivateKeyBits) }) runStep(t, "sign leaf cert and make sure NEW chain is correct", func(t *testing.T) { spiffeService := &connect.SpiffeIDService{ Host: "node1", Namespace: "default", Datacenter: "dc1", Service: "foo", } raw, _ := connect.TestCSR(t, spiffeService) leafCsr, err := connect.ParseCSR(raw) require.NoError(t, err) leafPEM, err := newProvider.Sign(leafCsr) require.NoError(t, err) // Check that the leaf signed by the new cert can be verified using the // returned cert chain require.NoError(t, connect.ValidateLeaf(newCaRoot.RootCert, leafPEM, []string{})) }) runStep(t, "verify persisted state is still correct", func(t *testing.T) { state := srv.fsm.State() _, caConfig, err := state.CAConfig(nil) require.NoError(t, err) require.Equal(t, providerState, caConfig.State) }) }) } } } func TestCAManager_Initialize_Secondary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() tests := []struct { keyType string keyBits int }{ {connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits}, {"rsa", 2048}, } dc1State := map[string]string{"foo": "dc1-value"} dc2State := map[string]string{"foo": "dc2-value"} for _, tc := range tests { tc := tc t.Run(fmt.Sprintf("%s-%d", tc.keyType, tc.keyBits), func(t *testing.T) { initialManagementToken := "8a85f086-dd95-4178-b128-e10902767c5c" // Initialize primary as the primary DC dir1, s1 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "primary" c.PrimaryDatacenter = "primary" c.Build = "1.6.0" c.ACLsEnabled = true c.ACLInitialManagementToken = initialManagementToken c.ACLResolverSettings.ACLDefaultPolicy = "deny" c.CAConfig.Config["PrivateKeyType"] = tc.keyType c.CAConfig.Config["PrivateKeyBits"] = tc.keyBits c.CAConfig.Config["test_state"] = dc1State }) defer os.RemoveAll(dir1) defer s1.Shutdown() s1.tokens.UpdateAgentToken(initialManagementToken, token.TokenSourceConfig) testrpc.WaitForLeader(t, s1.RPC, "primary") // secondary as a secondary DC dir2, s2 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "secondary" c.PrimaryDatacenter = "primary" c.Build = "1.6.0" c.ACLsEnabled = true c.ACLResolverSettings.ACLDefaultPolicy = "deny" c.ACLTokenReplication = true c.CAConfig.Config["PrivateKeyType"] = tc.keyType c.CAConfig.Config["PrivateKeyBits"] = tc.keyBits c.CAConfig.Config["test_state"] = dc2State }) defer os.RemoveAll(dir2) defer s2.Shutdown() s2.tokens.UpdateAgentToken(initialManagementToken, token.TokenSourceConfig) s2.tokens.UpdateReplicationToken(initialManagementToken, token.TokenSourceConfig) // Create the WAN link joinWAN(t, s2, s1) testrpc.WaitForLeader(t, s2.RPC, "secondary") // Ensure s2 is authoritative. waitForNewACLReplication(t, s2, structs.ACLReplicateTokens, 1, 1, 0) // Wait until the providers are fully bootstrapped. var ( caRoot *structs.CARoot secondaryProvider ca.Provider intermediatePEM string err error ) retry.Run(t, func(r *retry.R) { _, caRoot = getCAProviderWithLock(s1) secondaryProvider, _ = getCAProviderWithLock(s2) intermediatePEM, err = secondaryProvider.ActiveIntermediate() require.NoError(r, err) // Sanity check CA is using the correct key type require.Equal(r, tc.keyType, caRoot.PrivateKeyType) require.Equal(r, tc.keyBits, caRoot.PrivateKeyBits) // Verify the root lists are equal in each DC's state store. state1 := s1.fsm.State() _, roots1, err := state1.CARoots(nil) require.NoError(r, err) state2 := s2.fsm.State() _, roots2, err := state2.CARoots(nil) require.NoError(r, err) require.Len(r, roots1, 1) require.Len(r, roots2, 1) require.Equal(r, roots1[0].ID, roots2[0].ID) require.Equal(r, roots1[0].RootCert, roots2[0].RootCert) require.Empty(r, roots1[0].IntermediateCerts) require.NotEmpty(r, roots2[0].IntermediateCerts) }) // Have secondary sign a leaf cert and make sure the chain is correct. spiffeService := &connect.SpiffeIDService{ Host: "node1", Namespace: "default", Datacenter: "primary", Service: "foo", } raw, _ := connect.TestCSR(t, spiffeService) leafCsr, err := connect.ParseCSR(raw) require.NoError(t, err) leafPEM, err := secondaryProvider.Sign(leafCsr) require.NoError(t, err) // Check that the leaf signed by the new cert can be verified using the // returned cert chain (signed intermediate + remote root). require.NoError(t, connect.ValidateLeaf(caRoot.RootCert, leafPEM, []string{intermediatePEM})) // Verify that both primary and secondary persisted state as expected - // pass through from the config. { state := s1.fsm.State() _, caConfig, err := state.CAConfig(nil) require.NoError(t, err) require.Equal(t, dc1State, caConfig.State) } { state := s2.fsm.State() _, caConfig, err := state.CAConfig(nil) require.NoError(t, err) require.Equal(t, dc2State, caConfig.State) } }) } } func getCAProviderWithLock(s *Server) (ca.Provider, *structs.CARoot) { return s.caManager.getCAProvider() } func TestCAManager_RenewIntermediate_Vault_Primary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } ca.SkipIfVaultNotPresent(t) // no parallel execution because we change globals patchIntermediateCertRenewInterval(t) testVault := ca.NewTestVaultServer(t) _, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.CAConfig = &structs.CAConfiguration{ Provider: "vault", Config: map[string]interface{}{ "Address": testVault.Addr, "Token": testVault.RootToken, "RootPKIPath": "pki-root/", "IntermediatePKIPath": "pki-intermediate/", "LeafCertTTL": "2s", "IntermediateCertTTL": "7s", }, } }) defer func() { s1.Shutdown() s1.leaderRoutineManager.Wait() }() testrpc.WaitForActiveCARoot(t, s1.RPC, "dc1", nil) store := s1.caManager.delegate.State() _, activeRoot, err := store.CARootActive(nil) require.NoError(t, err) t.Log("original SigningKeyID", activeRoot.SigningKeyID) intermediatePEM := s1.caManager.getLeafSigningCertFromRoot(activeRoot) intermediateCert, err := connect.ParseCert(intermediatePEM) require.NoError(t, err) require.Equal(t, connect.HexString(intermediateCert.SubjectKeyId), activeRoot.SigningKeyID) require.Equal(t, intermediatePEM, s1.caManager.getLeafSigningCertFromRoot(activeRoot)) // Wait for dc1's intermediate to be refreshed. retry.Run(t, func(r *retry.R) { store := s1.caManager.delegate.State() _, storedRoot, err := store.CARootActive(nil) r.Check(err) newIntermediatePEM := s1.caManager.getLeafSigningCertFromRoot(storedRoot) if newIntermediatePEM == intermediatePEM { r.Fatal("not a renewed intermediate") } intermediateCert, err = connect.ParseCert(newIntermediatePEM) r.Check(err) intermediatePEM = newIntermediatePEM }) codec := rpcClient(t, s1) roots := structs.IndexedCARoots{} err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots) require.NoError(t, err) require.Len(t, roots.Roots, 1) activeRoot = roots.Active() require.Equal(t, connect.HexString(intermediateCert.SubjectKeyId), activeRoot.SigningKeyID) require.Equal(t, intermediatePEM, s1.caManager.getLeafSigningCertFromRoot(activeRoot)) // Have the new intermediate sign a leaf cert and make sure the chain is correct. spiffeService := &connect.SpiffeIDService{ Host: roots.TrustDomain, Namespace: "default", Datacenter: "dc1", Service: "foo", } csr, _ := connect.TestCSR(t, spiffeService) req := structs.CASignRequest{CSR: csr} cert := structs.IssuedCert{} err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", &req, &cert) require.NoError(t, err) verifyLeafCert(t, activeRoot, cert.CertPEM) } func patchIntermediateCertRenewInterval(t *testing.T) { origInterval := structs.IntermediateCertRenewInterval origMinTTL := structs.MinLeafCertTTL structs.IntermediateCertRenewInterval = 200 * time.Millisecond structs.MinLeafCertTTL = time.Second t.Cleanup(func() { structs.IntermediateCertRenewInterval = origInterval structs.MinLeafCertTTL = origMinTTL }) } func TestCAManager_RenewIntermediate_Secondary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } // no parallel execution because we change globals patchIntermediateCertRenewInterval(t) _, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.6.0" c.CAConfig = &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "PrivateKey": "", "RootCert": "", "LeafCertTTL": "5s", // The retry loop only retries for 7sec max and // the ttl needs to be below so that it // triggers definitely. // Since certs are created so that they are // valid from 1minute in the past, we need to // account for that, otherwise it will be // expired immediately. "IntermediateCertTTL": time.Minute + (5 * time.Second), }, } }) defer func() { s1.Shutdown() s1.leaderRoutineManager.Wait() }() testrpc.WaitForLeader(t, s1.RPC, "dc1") // dc2 as a secondary DC _, s2 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" }) defer func() { s2.Shutdown() s2.leaderRoutineManager.Wait() }() // Create the WAN link joinWAN(t, s2, s1) testrpc.WaitForActiveCARoot(t, s2.RPC, "dc2", nil) store := s2.fsm.State() _, activeRoot, err := store.CARootActive(nil) require.NoError(t, err) t.Log("original SigningKeyID", activeRoot.SigningKeyID) intermediatePEM := s2.caManager.getLeafSigningCertFromRoot(activeRoot) intermediateCert, err := connect.ParseCert(intermediatePEM) require.NoError(t, err) require.Equal(t, intermediatePEM, s2.caManager.getLeafSigningCertFromRoot(activeRoot)) require.Equal(t, connect.HexString(intermediateCert.SubjectKeyId), activeRoot.SigningKeyID) // Wait for dc2's intermediate to be refreshed. retry.Run(t, func(r *retry.R) { store := s2.caManager.delegate.State() _, storedRoot, err := store.CARootActive(nil) r.Check(err) newIntermediatePEM := s2.caManager.getLeafSigningCertFromRoot(storedRoot) if newIntermediatePEM == intermediatePEM { r.Fatal("not a renewed intermediate") } intermediateCert, err = connect.ParseCert(newIntermediatePEM) r.Check(err) intermediatePEM = newIntermediatePEM }) codec := rpcClient(t, s2) roots := structs.IndexedCARoots{} err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots) require.NoError(t, err) require.Len(t, roots.Roots, 1) _, activeRoot, err = store.CARootActive(nil) require.NoError(t, err) require.Equal(t, connect.HexString(intermediateCert.SubjectKeyId), activeRoot.SigningKeyID) require.Equal(t, intermediatePEM, s2.caManager.getLeafSigningCertFromRoot(activeRoot)) // Have dc2 sign a leaf cert and make sure the chain is correct. spiffeService := &connect.SpiffeIDService{ Host: roots.TrustDomain, Namespace: "default", Datacenter: "dc2", Service: "foo", } csr, _ := connect.TestCSR(t, spiffeService) req := structs.CASignRequest{CSR: csr} cert := structs.IssuedCert{} err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", &req, &cert) require.NoError(t, err) verifyLeafCert(t, activeRoot, cert.CertPEM) } func TestConnectCA_ConfigurationSet_RootRotation_Secondary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.6.0" c.PrimaryDatacenter = "dc1" }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForLeader(t, s1.RPC, "dc1") // dc2 as a secondary DC dir2, s2 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.Build = "1.6.0" }) defer os.RemoveAll(dir2) defer s2.Shutdown() // Create the WAN link joinWAN(t, s2, s1) testrpc.WaitForLeader(t, s2.RPC, "dc2") // Get the original intermediate secondaryProvider, _ := getCAProviderWithLock(s2) oldIntermediatePEM, err := secondaryProvider.ActiveIntermediate() require.NoError(t, err) require.NotEmpty(t, oldIntermediatePEM) // Capture the current root var originalRoot *structs.CARoot { rootList, activeRoot, err := getTestRoots(s1, "dc1") require.NoError(t, err) require.Len(t, rootList.Roots, 1) originalRoot = activeRoot } // Wait for current state to be reflected in both datacenters. testrpc.WaitForActiveCARoot(t, s1.RPC, "dc1", originalRoot) testrpc.WaitForActiveCARoot(t, s2.RPC, "dc2", originalRoot) // Update the provider config to use a new private key, which should // cause a rotation. _, newKey, err := connect.GeneratePrivateKey() require.NoError(t, err) newConfig := &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "PrivateKey": newKey, "RootCert": "", "IntermediateCertTTL": 72 * 24 * time.Hour, }, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} require.NoError(t, s1.RPC("ConnectCA.ConfigurationSet", args, &reply)) } var updatedRoot *structs.CARoot { rootList, activeRoot, err := getTestRoots(s1, "dc1") require.NoError(t, err) require.Len(t, rootList.Roots, 2) updatedRoot = activeRoot } testrpc.WaitForActiveCARoot(t, s1.RPC, "dc1", updatedRoot) testrpc.WaitForActiveCARoot(t, s2.RPC, "dc2", updatedRoot) // Wait for dc2's intermediate to be refreshed. var intermediatePEM string retry.Run(t, func(r *retry.R) { intermediatePEM, err = secondaryProvider.ActiveIntermediate() r.Check(err) if intermediatePEM == oldIntermediatePEM { r.Fatal("not a new intermediate") } }) require.NoError(t, err) // Verify the root lists have been rotated in each DC's state store. state1 := s1.fsm.State() _, primaryRoot, err := state1.CARootActive(nil) require.NoError(t, err) state2 := s2.fsm.State() _, roots2, err := state2.CARoots(nil) require.NoError(t, err) require.Equal(t, 2, len(roots2)) newRoot := roots2[0] oldRoot := roots2[1] if roots2[1].Active { newRoot = roots2[1] oldRoot = roots2[0] } require.False(t, oldRoot.Active) require.True(t, newRoot.Active) require.Equal(t, primaryRoot.ID, newRoot.ID) require.Equal(t, primaryRoot.RootCert, newRoot.RootCert) // Get the new root from dc1 and validate a chain of: // dc2 leaf -> dc2 intermediate -> dc1 root _, caRoot := getCAProviderWithLock(s1) // Have dc2 sign a leaf cert and make sure the chain is correct. spiffeService := &connect.SpiffeIDService{ Host: "node1", Namespace: "default", Datacenter: "dc1", Service: "foo", } raw, _ := connect.TestCSR(t, spiffeService) leafCsr, err := connect.ParseCSR(raw) require.NoError(t, err) leafPEM, err := secondaryProvider.Sign(leafCsr) require.NoError(t, err) cert, err := connect.ParseCert(leafPEM) require.NoError(t, err) // Check that the leaf signed by the new intermediate can be verified using the // returned cert chain (signed intermediate + remote root). intermediatePool := x509.NewCertPool() intermediatePool.AppendCertsFromPEM([]byte(intermediatePEM)) rootPool := x509.NewCertPool() rootPool.AppendCertsFromPEM([]byte(caRoot.RootCert)) _, err = cert.Verify(x509.VerifyOptions{ Intermediates: intermediatePool, Roots: rootPool, }) require.NoError(t, err) } func TestCAManager_Initialize_Vault_FixesSigningKeyID_Primary(t *testing.T) { ca.SkipIfVaultNotPresent(t) if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() testVault := ca.NewTestVaultServer(t) defer testVault.Stop() dir1pre, s1pre := testServerWithConfig(t, func(c *Config) { c.Build = "1.6.0" c.PrimaryDatacenter = "dc1" c.CAConfig = &structs.CAConfiguration{ Provider: "vault", Config: map[string]interface{}{ "Address": testVault.Addr, "Token": testVault.RootToken, "RootPKIPath": "pki-root/", "IntermediatePKIPath": "pki-intermediate/", }, } }) defer os.RemoveAll(dir1pre) defer s1pre.Shutdown() testrpc.WaitForLeader(t, s1pre.RPC, "dc1") // Restore the pre-1.9.3/1.8.8/1.7.12 behavior of the SigningKeyID not being derived // from the intermediates in the primary (which only matters for provider=vault). var primaryRootSigningKeyID string { state := s1pre.fsm.State() // Get the highest index idx, activePrimaryRoot, err := state.CARootActive(nil) require.NoError(t, err) require.NotNil(t, activePrimaryRoot) rootCert, err := connect.ParseCert(activePrimaryRoot.RootCert) require.NoError(t, err) // Force this to be derived just from the root, not the intermediate. primaryRootSigningKeyID = connect.EncodeSigningKeyID(rootCert.SubjectKeyId) activePrimaryRoot.SigningKeyID = primaryRootSigningKeyID // Store the root cert in raft _, err = s1pre.raftApply(structs.ConnectCARequestType, &structs.CARequest{ Op: structs.CAOpSetRoots, Index: idx, Roots: []*structs.CARoot{activePrimaryRoot}, }) require.NoError(t, err) } // Shutdown s1pre and restart it to trigger the secondary CA init to correct // the SigningKeyID. s1pre.Shutdown() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.DataDir = s1pre.config.DataDir c.Datacenter = "dc1" c.PrimaryDatacenter = "dc1" c.NodeName = s1pre.config.NodeName c.NodeID = s1pre.config.NodeID }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Retry since it will take some time to init the primary CA fully and there // isn't a super clean way to watch specifically until it's done than polling // the CA provider anyway. retry.Run(t, func(r *retry.R) { // verify that the root is now corrected provider, activeRoot := getCAProviderWithLock(s1) require.NotNil(r, provider) require.NotNil(r, activeRoot) activeIntermediate, err := provider.ActiveIntermediate() require.NoError(r, err) intermediateCert, err := connect.ParseCert(activeIntermediate) require.NoError(r, err) // Force this to be derived just from the root, not the intermediate. expect := connect.EncodeSigningKeyID(intermediateCert.SubjectKeyId) // The in-memory representation was saw the correction via a setCAProvider call. require.Equal(r, expect, activeRoot.SigningKeyID) // The state store saw the correction, too. _, activePrimaryRoot, err := s1.fsm.State().CARootActive(nil) require.NoError(r, err) require.NotNil(r, activePrimaryRoot) require.Equal(r, expect, activePrimaryRoot.SigningKeyID) }) } func TestCAManager_Initialize_FixesSigningKeyID_Secondary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.6.0" c.PrimaryDatacenter = "dc1" }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForLeader(t, s1.RPC, "dc1") // dc2 as a secondary DC dir2pre, s2pre := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.Build = "1.6.0" }) defer os.RemoveAll(dir2pre) defer s2pre.Shutdown() // Create the WAN link joinWAN(t, s2pre, s1) testrpc.WaitForLeader(t, s2pre.RPC, "dc2") // Restore the pre-1.6.1 behavior of the SigningKeyID not being derived // from the intermediates. var secondaryRootSigningKeyID string { state := s2pre.fsm.State() // Get the highest index idx, activeSecondaryRoot, err := state.CARootActive(nil) require.NoError(t, err) require.NotNil(t, activeSecondaryRoot) rootCert, err := connect.ParseCert(activeSecondaryRoot.RootCert) require.NoError(t, err) // Force this to be derived just from the root, not the intermediate. secondaryRootSigningKeyID = connect.EncodeSigningKeyID(rootCert.SubjectKeyId) activeSecondaryRoot.SigningKeyID = secondaryRootSigningKeyID // Store the root cert in raft _, err = s2pre.raftApply(structs.ConnectCARequestType, &structs.CARequest{ Op: structs.CAOpSetRoots, Index: idx, Roots: []*structs.CARoot{activeSecondaryRoot}, }) require.NoError(t, err) } // Shutdown s2pre and restart it to trigger the secondary CA init to correct // the SigningKeyID. s2pre.Shutdown() dir2, s2 := testServerWithConfig(t, func(c *Config) { c.DataDir = s2pre.config.DataDir c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.NodeName = s2pre.config.NodeName c.NodeID = s2pre.config.NodeID }) defer os.RemoveAll(dir2) defer s2.Shutdown() testrpc.WaitForLeader(t, s2.RPC, "dc2") // Retry since it will take some time to init the secondary CA fully and there // isn't a super clean way to watch specifically until it's done than polling // the CA provider anyway. retry.Run(t, func(r *retry.R) { // verify that the root is now corrected provider, activeRoot := getCAProviderWithLock(s2) require.NotNil(r, provider) require.NotNil(r, activeRoot) activeIntermediate, err := provider.ActiveIntermediate() require.NoError(r, err) intermediateCert, err := connect.ParseCert(activeIntermediate) require.NoError(r, err) // Force this to be derived just from the root, not the intermediate. expect := connect.EncodeSigningKeyID(intermediateCert.SubjectKeyId) // The in-memory representation was saw the correction via a setCAProvider call. require.Equal(r, expect, activeRoot.SigningKeyID) // The state store saw the correction, too. _, activeSecondaryRoot, err := s2.fsm.State().CARootActive(nil) require.NoError(r, err) require.NotNil(r, activeSecondaryRoot) require.Equal(r, expect, activeSecondaryRoot.SigningKeyID) }) } func TestCAManager_Initialize_TransitionFromPrimaryToSecondary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() // Initialize dc1 as the primary DC id1, err := uuid.GenerateUUID() require.NoError(t, err) dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.CAConfig.ClusterID = id1 c.Build = "1.6.0" }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForLeader(t, s1.RPC, "dc1") // dc2 as a primary DC initially id2, err := uuid.GenerateUUID() require.NoError(t, err) dir2, s2 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc2" c.CAConfig.ClusterID = id2 c.Build = "1.6.0" }) defer os.RemoveAll(dir2) defer s2.Shutdown() // Get the initial (primary) roots state for the secondary testrpc.WaitForLeader(t, s2.RPC, "dc2") args := structs.DCSpecificRequest{Datacenter: "dc2"} var dc2PrimaryRoots structs.IndexedCARoots require.NoError(t, s2.RPC("ConnectCA.Roots", &args, &dc2PrimaryRoots)) require.Len(t, dc2PrimaryRoots.Roots, 1) // Shutdown s2 and restart it with the dc1 as the primary s2.Shutdown() dir3, s3 := testServerWithConfig(t, func(c *Config) { c.DataDir = s2.config.DataDir c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.NodeName = s2.config.NodeName c.NodeID = s2.config.NodeID }) defer os.RemoveAll(dir3) defer s3.Shutdown() // Create the WAN link joinWAN(t, s3, s1) testrpc.WaitForLeader(t, s3.RPC, "dc2") // Verify the secondary has migrated its TrustDomain and added the new primary's root. retry.Run(t, func(r *retry.R) { args = structs.DCSpecificRequest{Datacenter: "dc1"} var dc1Roots structs.IndexedCARoots require.NoError(r, s1.RPC("ConnectCA.Roots", &args, &dc1Roots)) require.Len(r, dc1Roots.Roots, 1) args = structs.DCSpecificRequest{Datacenter: "dc2"} var dc2SecondaryRoots structs.IndexedCARoots require.NoError(r, s3.RPC("ConnectCA.Roots", &args, &dc2SecondaryRoots)) // dc2's TrustDomain should have changed to the primary's require.Equal(r, dc2SecondaryRoots.TrustDomain, dc1Roots.TrustDomain) require.NotEqual(r, dc2SecondaryRoots.TrustDomain, dc2PrimaryRoots.TrustDomain) // Both roots should be present and correct require.Len(r, dc2SecondaryRoots.Roots, 2) var oldSecondaryRoot *structs.CARoot var newSecondaryRoot *structs.CARoot if dc2SecondaryRoots.Roots[0].ID == dc2PrimaryRoots.Roots[0].ID { oldSecondaryRoot = dc2SecondaryRoots.Roots[0] newSecondaryRoot = dc2SecondaryRoots.Roots[1] } else { oldSecondaryRoot = dc2SecondaryRoots.Roots[1] newSecondaryRoot = dc2SecondaryRoots.Roots[0] } // The old root should have its TrustDomain filled in as the old domain. require.Equal(r, oldSecondaryRoot.ExternalTrustDomain, strings.TrimSuffix(dc2PrimaryRoots.TrustDomain, ".consul")) require.Equal(r, oldSecondaryRoot.ID, dc2PrimaryRoots.Roots[0].ID) require.Equal(r, oldSecondaryRoot.RootCert, dc2PrimaryRoots.Roots[0].RootCert) require.Equal(r, newSecondaryRoot.ID, dc1Roots.Roots[0].ID) require.Equal(r, newSecondaryRoot.RootCert, dc1Roots.Roots[0].RootCert) }) } func TestCAManager_Initialize_SecondaryBeforePrimary(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() // Initialize dc1 as the primary DC dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.Build = "1.3.0" c.MaxQueryTime = 500 * time.Millisecond }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForLeader(t, s1.RPC, "dc1") // dc2 as a secondary DC dir2, s2 := testServerWithConfig(t, func(c *Config) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.Build = "1.6.0" c.MaxQueryTime = 500 * time.Millisecond }) defer os.RemoveAll(dir2) defer s2.Shutdown() // Create the WAN link joinWAN(t, s2, s1) testrpc.WaitForLeader(t, s2.RPC, "dc2") // ensure all the CA initialization stuff would have already been done // this is necessary to ensure that not only has a leader been elected // but that it has also finished its establishLeadership call retry.Run(t, func(r *retry.R) { require.True(r, s1.isReadyForConsistentReads()) require.True(r, s2.isReadyForConsistentReads()) }) // Verify the primary has a root (we faked its version too low but since its the primary it ignores any version checks) retry.Run(t, func(r *retry.R) { state1 := s1.fsm.State() _, roots1, err := state1.CARoots(nil) require.NoError(r, err) require.Len(r, roots1, 1) }) // Verify the secondary does not have a root - defers initialization until the primary has been upgraded. state2 := s2.fsm.State() _, roots2, err := state2.CARoots(nil) require.NoError(t, err) require.Empty(t, roots2) // Update the version on the fly so s2 kicks off the secondary DC transition. tags := s1.config.SerfWANConfig.Tags tags["build"] = "1.6.0" s1.serfWAN.SetTags(tags) // Wait for the secondary transition to happen and then verify the secondary DC // has both roots present. retry.Run(t, func(r *retry.R) { state1 := s1.fsm.State() _, roots1, err := state1.CARoots(nil) require.NoError(r, err) require.Len(r, roots1, 1) state2 := s2.fsm.State() _, roots2, err := state2.CARoots(nil) require.NoError(r, err) require.Len(r, roots2, 1) // ensure the roots are the same require.Equal(r, roots1[0].ID, roots2[0].ID) require.Equal(r, roots1[0].RootCert, roots2[0].RootCert) secondaryProvider, _ := getCAProviderWithLock(s2) inter, err := secondaryProvider.ActiveIntermediate() require.NoError(r, err) require.NotEmpty(r, inter, "should have valid intermediate") }) secondaryProvider, _ := getCAProviderWithLock(s2) intermediatePEM, err := secondaryProvider.ActiveIntermediate() require.NoError(t, err) _, caRoot := getCAProviderWithLock(s1) // Have dc2 sign a leaf cert and make sure the chain is correct. spiffeService := &connect.SpiffeIDService{ Host: "node1", Namespace: "default", Datacenter: "dc1", Service: "foo", } raw, _ := connect.TestCSR(t, spiffeService) leafCsr, err := connect.ParseCSR(raw) require.NoError(t, err) leafPEM, err := secondaryProvider.Sign(leafCsr) require.NoError(t, err) cert, err := connect.ParseCert(leafPEM) require.NoError(t, err) // Check that the leaf signed by the new cert can be verified using the // returned cert chain (signed intermediate + remote root). intermediatePool := x509.NewCertPool() intermediatePool.AppendCertsFromPEM([]byte(intermediatePEM)) rootPool := x509.NewCertPool() rootPool.AppendCertsFromPEM([]byte(caRoot.RootCert)) _, err = cert.Verify(x509.VerifyOptions{ Intermediates: intermediatePool, Roots: rootPool, }) require.NoError(t, err) } func getTestRoots(s *Server, datacenter string) (*structs.IndexedCARoots, *structs.CARoot, error) { rootReq := &structs.DCSpecificRequest{ Datacenter: datacenter, } var rootList structs.IndexedCARoots if err := s.RPC("ConnectCA.Roots", rootReq, &rootList); err != nil { return nil, nil, err } active := rootList.Active() return &rootList, active, nil } func TestLeader_CARootPruning(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } // Can not use t.Parallel(), because this modifies a global. origPruneInterval := caRootPruneInterval caRootPruneInterval = 200 * time.Millisecond t.Cleanup(func() { // Reset the value of the global prune interval so that it doesn't affect other tests caRootPruneInterval = origPruneInterval }) dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") // Get the current root rootReq := &structs.DCSpecificRequest{ Datacenter: "dc1", } var rootList structs.IndexedCARoots require.Nil(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList)) require.Len(t, rootList.Roots, 1) oldRoot := rootList.Roots[0] // Update the provider config to use a new private key, which should // cause a rotation. _, newKey, err := connect.GeneratePrivateKey() require.NoError(t, err) newConfig := &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "LeafCertTTL": "500ms", "PrivateKey": newKey, "RootCert": "", "SkipValidate": true, }, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) } // Should have 2 roots now. _, roots, err := s1.fsm.State().CARoots(nil) require.NoError(t, err) require.Len(t, roots, 2) time.Sleep(2 * time.Second) // Now the old root should be pruned. _, roots, err = s1.fsm.State().CARoots(nil) require.NoError(t, err) require.Len(t, roots, 1) require.True(t, roots[0].Active) require.NotEqual(t, roots[0].ID, oldRoot.ID) } func TestConnectCA_ConfigurationSet_PersistsRoots(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() dir2, s2 := testServerDCBootstrap(t, "dc1", false) defer os.RemoveAll(dir2) defer s2.Shutdown() dir3, s3 := testServerDCBootstrap(t, "dc1", false) defer os.RemoveAll(dir3) defer s3.Shutdown() joinLAN(t, s2, s1) joinLAN(t, s3, s1) testrpc.WaitForLeader(t, s1.RPC, "dc1") // Get the current root rootReq := &structs.DCSpecificRequest{ Datacenter: "dc1", } var rootList structs.IndexedCARoots require.Nil(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList)) require.Len(t, rootList.Roots, 1) // Update the provider config to use a new private key, which should // cause a rotation. _, newKey, err := connect.GeneratePrivateKey() require.NoError(t, err) newConfig := &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "PrivateKey": newKey, "RootCert": "", }, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) } // Get the active root before leader change. _, root := getCAProviderWithLock(s1) require.Len(t, root.IntermediateCerts, 1) // Force a leader change and make sure the root CA values are preserved. s1.Leave() s1.Shutdown() retry.Run(t, func(r *retry.R) { var leader *Server for _, s := range []*Server{s2, s3} { if s.IsLeader() { leader = s break } } if leader == nil { r.Fatal("no leader") } _, newLeaderRoot := getCAProviderWithLock(leader) if !reflect.DeepEqual(newLeaderRoot, root) { r.Fatalf("got %v, want %v", newLeaderRoot, root) } }) } func TestParseCARoot(t *testing.T) { type test struct { name string pem string wantSerial uint64 wantSigningKeyID string wantKeyType string wantKeyBits int wantErr bool } // Test certs generated with // go run connect/certgen/certgen.go -out-dir /tmp/connect-certs -key-type ec -key-bits 384 // for various key types. This does limit the exposure to formats that might // exist in external certificates which can be used as Connect CAs. // Specifically many other certs will have serial numbers that don't fit into // 64 bits but for reasons we truncate down to 64 bits which means our // `SerialNumber` will not match the one reported by openssl. We should // probably fix that at some point as it seems like a big footgun but it would // be a breaking API change to change the type to not be a JSON number and // JSON numbers don't even support the full range of a uint64... tests := []test{ {"no cert", "", 0, "", "", 0, true}, { name: "default cert", // Watchout for indentations they will break PEM format pem: readTestData(t, "cert-with-ec-256-key.pem"), // Based on `openssl x509 -noout -text` report from the cert wantSerial: 8341954965092507701, wantSigningKeyID: "97:4D:17:81:64:F8:B4:AF:05:E8:6C:79:C5:40:3B:0E:3E:8B:C0:AE:38:51:54:8A:2F:05:DB:E3:E8:E4:24:EC", wantKeyType: "ec", wantKeyBits: 256, wantErr: false, }, { name: "ec 384 cert", // Watchout for indentations they will break PEM format pem: readTestData(t, "cert-with-ec-384-key.pem"), // Based on `openssl x509 -noout -text` report from the cert wantSerial: 2935109425518279965, wantSigningKeyID: "0B:A0:88:9B:DC:95:31:51:2E:3D:D4:F9:42:D0:6A:A0:62:46:82:D2:7C:22:E7:29:A9:AA:E8:A5:8C:CF:C7:42", wantKeyType: "ec", wantKeyBits: 384, wantErr: false, }, { name: "rsa 4096 cert", // Watchout for indentations they will break PEM format pem: readTestData(t, "cert-with-rsa-4096-key.pem"), // Based on `openssl x509 -noout -text` report from the cert wantSerial: 5186695743100577491, wantSigningKeyID: "92:FA:CC:97:57:1E:31:84:A2:33:DD:9B:6A:A8:7C:FC:BE:E2:94:CA:AC:B3:33:17:39:3B:B8:67:9B:DC:C1:08", wantKeyType: "rsa", wantKeyBits: 4096, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { root, err := parseCARoot(tt.pem, "consul", "cluster") if tt.wantErr { require.Error(t, err) return } require.NoError(t, err) require.Equal(t, tt.wantSerial, root.SerialNumber) require.Equal(t, strings.ToLower(tt.wantSigningKeyID), root.SigningKeyID) require.Equal(t, tt.wantKeyType, root.PrivateKeyType) require.Equal(t, tt.wantKeyBits, root.PrivateKeyBits) }) } } func readTestData(t *testing.T, name string) string { t.Helper() path := filepath.Join("testdata", name) bs, err := ioutil.ReadFile(path) if err != nil { t.Fatalf("failed reading fixture file %s: %s", name, err) } return string(bs) } func TestLessThanHalfTimePassed(t *testing.T) { now := time.Now() require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(-5*time.Second))) require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now)) require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(5*time.Second))) require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(10*time.Second))) require.True(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(20*time.Second))) } func TestRetryLoopBackoffHandleSuccess(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } type test struct { desc string loopFn func() error abort bool timedOut bool } success := func() error { return nil } failure := func() error { return fmt.Errorf("test error") } tests := []test{ {"loop without error and no abortOnSuccess keeps running", success, false, true}, {"loop with error and no abortOnSuccess keeps running", failure, false, true}, {"loop without error and abortOnSuccess is stopped", success, true, false}, {"loop with error and abortOnSuccess keeps running", failure, true, true}, } for _, tc := range tests { tc := tc t.Run(tc.desc, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() retryLoopBackoffHandleSuccess(ctx, tc.loopFn, func(_ error) {}, tc.abort) select { case <-ctx.Done(): if !tc.timedOut { t.Fatal("should not have timed out") } default: if tc.timedOut { t.Fatal("should have timed out") } } }) } } func TestCAManager_Initialize_Vault_BadCAConfigDoesNotPreventLeaderEstablishment(t *testing.T) { ca.SkipIfVaultNotPresent(t) testVault := ca.NewTestVaultServer(t) defer testVault.Stop() _, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.9.1" c.PrimaryDatacenter = "dc1" c.CAConfig = &structs.CAConfiguration{ Provider: "vault", Config: map[string]interface{}{ "Address": testVault.Addr, "Token": "not-the-root", "RootPKIPath": "pki-root/", "IntermediatePKIPath": "pki-intermediate/", }, } }) defer s1.Shutdown() waitForLeaderEstablishment(t, s1) rootsList, activeRoot, err := getTestRoots(s1, "dc1") require.NoError(t, err) require.Empty(t, rootsList.Roots) require.Nil(t, activeRoot) // Now that the leader is up and we have verified that there are no roots / CA init failed, // verify that we can reconfigure away from the bad configuration. newConfig := &structs.CAConfiguration{ Provider: "vault", Config: map[string]interface{}{ "Address": testVault.Addr, "Token": testVault.RootToken, "RootPKIPath": "pki-root/", "IntermediatePKIPath": "pki-intermediate/", }, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} retry.Run(t, func(r *retry.R) { require.NoError(r, s1.RPC("ConnectCA.ConfigurationSet", args, &reply)) }) } rootsList, activeRoot, err = getTestRoots(s1, "dc1") require.NoError(t, err) require.NotEmpty(t, rootsList.Roots) require.NotNil(t, activeRoot) } func TestCAManager_Initialize_BadCAConfigDoesNotPreventLeaderEstablishment(t *testing.T) { ca.SkipIfVaultNotPresent(t) _, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.9.1" c.PrimaryDatacenter = "dc1" c.CAConfig = &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "RootCert": "garbage", }, } }) defer s1.Shutdown() waitForLeaderEstablishment(t, s1) rootsList, activeRoot, err := getTestRoots(s1, "dc1") require.NoError(t, err) require.Empty(t, rootsList.Roots) require.Nil(t, activeRoot) newConfig := &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{}, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} retry.Run(t, func(r *retry.R) { require.NoError(r, s1.RPC("ConnectCA.ConfigurationSet", args, &reply)) }) } rootsList, activeRoot, err = getTestRoots(s1, "dc1") require.NoError(t, err) require.NotEmpty(t, rootsList.Roots) require.NotNil(t, activeRoot) } func TestConnectCA_ConfigurationSet_ForceWithoutCrossSigning(t *testing.T) { dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() waitForLeaderEstablishment(t, s1) // Get the current root rootReq := &structs.DCSpecificRequest{ Datacenter: "dc1", } var rootList structs.IndexedCARoots require.Nil(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList)) require.Len(t, rootList.Roots, 1) oldRoot := rootList.Roots[0] // Update the provider config to use a new private key, which should // cause a rotation. _, newKey, err := connect.GeneratePrivateKey() require.NoError(t, err) newConfig := &structs.CAConfiguration{ Provider: "consul", Config: map[string]interface{}{ "LeafCertTTL": "500ms", "PrivateKey": newKey, "RootCert": "", "SkipValidate": true, }, ForceWithoutCrossSigning: true, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) } // Old root should no longer be active. _, roots, err := s1.fsm.State().CARoots(nil) require.NoError(t, err) require.Len(t, roots, 2) for _, r := range roots { if r.ID == oldRoot.ID { require.False(t, r.Active) } else { require.True(t, r.Active) } } } func TestConnectCA_ConfigurationSet_Vault_ForceWithoutCrossSigning(t *testing.T) { ca.SkipIfVaultNotPresent(t) testVault := ca.NewTestVaultServer(t) defer testVault.Stop() _, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.9.1" c.PrimaryDatacenter = "dc1" c.CAConfig = &structs.CAConfiguration{ Provider: "vault", Config: map[string]interface{}{ "Address": testVault.Addr, "Token": testVault.RootToken, "RootPKIPath": "pki-root/", "IntermediatePKIPath": "pki-intermediate/", }, } }) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() waitForLeaderEstablishment(t, s1) // Get the current root rootReq := &structs.DCSpecificRequest{ Datacenter: "dc1", } var rootList structs.IndexedCARoots require.Nil(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList)) require.Len(t, rootList.Roots, 1) oldRoot := rootList.Roots[0] // Update the provider config to use a new PKI path, which should // cause a rotation. newConfig := &structs.CAConfiguration{ Provider: "vault", Config: map[string]interface{}{ "Address": testVault.Addr, "Token": testVault.RootToken, "RootPKIPath": "pki-root-2/", "IntermediatePKIPath": "pki-intermediate/", }, ForceWithoutCrossSigning: true, } { args := &structs.CARequest{ Datacenter: "dc1", Config: newConfig, } var reply interface{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) } // Old root should no longer be active. _, roots, err := s1.fsm.State().CARoots(nil) require.NoError(t, err) require.Len(t, roots, 2) for _, r := range roots { if r.ID == oldRoot.ID { require.False(t, r.Active) } else { require.True(t, r.Active) } } }