8654adfc53
Previously we were inconsistently checking the response for errors. This PR moves the response-is-error check into raftApply, so that all callers can look at only the error response, instead of having to know that errors could come from two places. This should expose a few more errors that were previously hidden because in some calls to raftApply we were ignoring the response return value. Also handle errors more consistently. In some cases we would log the error before returning it. This can be very confusing because it can result in the same error being logged multiple times. Instead return a wrapped error.
1534 lines
44 KiB
Go
1534 lines
44 KiB
Go
package consul
|
|
|
|
import (
|
|
"context"
|
|
"crypto/x509"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
uuid "github.com/hashicorp/go-uuid"
|
|
msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/hashicorp/consul/agent/connect"
|
|
"github.com/hashicorp/consul/agent/connect/ca"
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
"github.com/hashicorp/consul/agent/token"
|
|
"github.com/hashicorp/consul/sdk/testutil/retry"
|
|
"github.com/hashicorp/consul/testrpc"
|
|
)
|
|
|
|
func TestLeader_SecondaryCA_Initialize(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
keyType string
|
|
keyBits int
|
|
}{
|
|
{connect.DefaultPrivateKeyType, connect.DefaultPrivateKeyBits},
|
|
{"rsa", 2048},
|
|
}
|
|
|
|
dc1State := map[string]string{"foo": "dc1-value"}
|
|
dc2State := map[string]string{"foo": "dc2-value"}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(fmt.Sprintf("%s-%d", tc.keyType, tc.keyBits), func(t *testing.T) {
|
|
masterToken := "8a85f086-dd95-4178-b128-e10902767c5c"
|
|
|
|
// Initialize primary as the primary DC
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "primary"
|
|
c.ACLDatacenter = "primary"
|
|
c.Build = "1.6.0"
|
|
c.ACLsEnabled = true
|
|
c.ACLMasterToken = masterToken
|
|
c.ACLDefaultPolicy = "deny"
|
|
c.CAConfig.Config["PrivateKeyType"] = tc.keyType
|
|
c.CAConfig.Config["PrivateKeyBits"] = tc.keyBits
|
|
c.CAConfig.Config["test_state"] = dc1State
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
s1.tokens.UpdateAgentToken(masterToken, token.TokenSourceConfig)
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "primary")
|
|
|
|
// secondary as a secondary DC
|
|
dir2, s2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "secondary"
|
|
c.ACLDatacenter = "primary"
|
|
c.Build = "1.6.0"
|
|
c.ACLsEnabled = true
|
|
c.ACLDefaultPolicy = "deny"
|
|
c.ACLTokenReplication = true
|
|
c.CAConfig.Config["PrivateKeyType"] = tc.keyType
|
|
c.CAConfig.Config["PrivateKeyBits"] = tc.keyBits
|
|
c.CAConfig.Config["test_state"] = dc2State
|
|
})
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
s2.tokens.UpdateAgentToken(masterToken, token.TokenSourceConfig)
|
|
s2.tokens.UpdateReplicationToken(masterToken, token.TokenSourceConfig)
|
|
|
|
// Create the WAN link
|
|
joinWAN(t, s2, s1)
|
|
|
|
testrpc.WaitForLeader(t, s2.RPC, "secondary")
|
|
|
|
waitForNewACLs(t, s1)
|
|
waitForNewACLs(t, s2)
|
|
|
|
// Ensure s2 is authoritative.
|
|
waitForNewACLReplication(t, s2, structs.ACLReplicateTokens, 1, 1, 0)
|
|
|
|
// Wait until the providers are fully bootstrapped.
|
|
var (
|
|
caRoot *structs.CARoot
|
|
secondaryProvider ca.Provider
|
|
intermediatePEM string
|
|
err error
|
|
)
|
|
retry.Run(t, func(r *retry.R) {
|
|
_, caRoot = getCAProviderWithLock(s1)
|
|
secondaryProvider, _ = getCAProviderWithLock(s2)
|
|
intermediatePEM, err = secondaryProvider.ActiveIntermediate()
|
|
require.NoError(r, err)
|
|
|
|
// Sanity check CA is using the correct key type
|
|
require.Equal(r, tc.keyType, caRoot.PrivateKeyType)
|
|
require.Equal(r, tc.keyBits, caRoot.PrivateKeyBits)
|
|
|
|
// Verify the root lists are equal in each DC's state store.
|
|
state1 := s1.fsm.State()
|
|
_, roots1, err := state1.CARoots(nil)
|
|
require.NoError(r, err)
|
|
|
|
state2 := s2.fsm.State()
|
|
_, roots2, err := state2.CARoots(nil)
|
|
require.NoError(r, err)
|
|
require.Len(r, roots1, 1)
|
|
require.Len(r, roots2, 1)
|
|
require.Equal(r, roots1[0].ID, roots2[0].ID)
|
|
require.Equal(r, roots1[0].RootCert, roots2[0].RootCert)
|
|
require.Empty(r, roots1[0].IntermediateCerts)
|
|
require.NotEmpty(r, roots2[0].IntermediateCerts)
|
|
})
|
|
|
|
// Have secondary sign a leaf cert and make sure the chain is correct.
|
|
spiffeService := &connect.SpiffeIDService{
|
|
Host: "node1",
|
|
Namespace: "default",
|
|
Datacenter: "primary",
|
|
Service: "foo",
|
|
}
|
|
raw, _ := connect.TestCSR(t, spiffeService)
|
|
|
|
leafCsr, err := connect.ParseCSR(raw)
|
|
require.NoError(t, err)
|
|
|
|
leafPEM, err := secondaryProvider.Sign(leafCsr)
|
|
require.NoError(t, err)
|
|
|
|
// Check that the leaf signed by the new cert can be verified using the
|
|
// returned cert chain (signed intermediate + remote root).
|
|
require.NoError(t, connect.ValidateLeaf(caRoot.RootCert, leafPEM, []string{intermediatePEM}))
|
|
|
|
// Verify that both primary and secondary persisted state as expected -
|
|
// pass through from the config.
|
|
{
|
|
state := s1.fsm.State()
|
|
_, caConfig, err := state.CAConfig(nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, dc1State, caConfig.State)
|
|
}
|
|
{
|
|
state := s2.fsm.State()
|
|
_, caConfig, err := state.CAConfig(nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, dc2State, caConfig.State)
|
|
}
|
|
|
|
})
|
|
}
|
|
}
|
|
|
|
func waitForActiveCARoot(t *testing.T, srv *Server, expect *structs.CARoot) {
|
|
retry.Run(t, func(r *retry.R) {
|
|
_, root := getCAProviderWithLock(srv)
|
|
if root == nil {
|
|
r.Fatal("no root")
|
|
}
|
|
if root.ID != expect.ID {
|
|
r.Fatalf("current active root is %s; waiting for %s", root.ID, expect.ID)
|
|
}
|
|
})
|
|
}
|
|
|
|
func getCAProviderWithLock(s *Server) (ca.Provider, *structs.CARoot) {
|
|
return s.caManager.getCAProvider()
|
|
}
|
|
|
|
func TestLeader_Vault_PrimaryCA_IntermediateRenew(t *testing.T) {
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
// no parallel execution because we change globals
|
|
origInterval := structs.IntermediateCertRenewInterval
|
|
origMinTTL := structs.MinLeafCertTTL
|
|
origDriftBuffer := ca.CertificateTimeDriftBuffer
|
|
defer func() {
|
|
structs.IntermediateCertRenewInterval = origInterval
|
|
structs.MinLeafCertTTL = origMinTTL
|
|
ca.CertificateTimeDriftBuffer = origDriftBuffer
|
|
}()
|
|
|
|
// Vault backdates certs by 30s by default.
|
|
ca.CertificateTimeDriftBuffer = 30 * time.Second
|
|
structs.IntermediateCertRenewInterval = time.Millisecond
|
|
structs.MinLeafCertTTL = time.Second
|
|
require := require.New(t)
|
|
|
|
testVault := ca.NewTestVaultServer(t)
|
|
defer testVault.Stop()
|
|
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.6.0"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": testVault.Addr,
|
|
"Token": testVault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
"LeafCertTTL": "1s",
|
|
// The retry loop only retries for 7sec max and
|
|
// the ttl needs to be below so that it
|
|
// triggers definitely.
|
|
"IntermediateCertTTL": "5s",
|
|
},
|
|
}
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// Capture the current root.
|
|
var originalRoot *structs.CARoot
|
|
{
|
|
rootList, activeRoot, err := getTestRoots(s1, "dc1")
|
|
require.NoError(err)
|
|
require.Len(rootList.Roots, 1)
|
|
originalRoot = activeRoot
|
|
}
|
|
|
|
// Get the original intermediate.
|
|
waitForActiveCARoot(t, s1, originalRoot)
|
|
provider, _ := getCAProviderWithLock(s1)
|
|
intermediatePEM, err := provider.ActiveIntermediate()
|
|
require.NoError(err)
|
|
_, err = connect.ParseCert(intermediatePEM)
|
|
require.NoError(err)
|
|
|
|
// Wait for dc1's intermediate to be refreshed.
|
|
// It is possible that test fails when the blocking query doesn't return.
|
|
retry.Run(t, func(r *retry.R) {
|
|
provider, _ = getCAProviderWithLock(s1)
|
|
newIntermediatePEM, err := provider.ActiveIntermediate()
|
|
r.Check(err)
|
|
_, err = connect.ParseCert(intermediatePEM)
|
|
r.Check(err)
|
|
if newIntermediatePEM == intermediatePEM {
|
|
r.Fatal("not a renewed intermediate")
|
|
}
|
|
intermediatePEM = newIntermediatePEM
|
|
})
|
|
require.NoError(err)
|
|
|
|
// Get the root from dc1 and validate a chain of:
|
|
// dc1 leaf -> dc1 intermediate -> dc1 root
|
|
provider, caRoot := getCAProviderWithLock(s1)
|
|
|
|
// Have the new intermediate sign a leaf cert and make sure the chain is correct.
|
|
spiffeService := &connect.SpiffeIDService{
|
|
Host: "node1",
|
|
Namespace: "default",
|
|
Datacenter: "dc1",
|
|
Service: "foo",
|
|
}
|
|
raw, _ := connect.TestCSR(t, spiffeService)
|
|
|
|
leafCsr, err := connect.ParseCSR(raw)
|
|
require.NoError(err)
|
|
|
|
leafPEM, err := provider.Sign(leafCsr)
|
|
require.NoError(err)
|
|
|
|
cert, err := connect.ParseCert(leafPEM)
|
|
require.NoError(err)
|
|
|
|
// Check that the leaf signed by the new intermediate can be verified using the
|
|
// returned cert chain (signed intermediate + remote root).
|
|
intermediatePool := x509.NewCertPool()
|
|
intermediatePool.AppendCertsFromPEM([]byte(intermediatePEM))
|
|
rootPool := x509.NewCertPool()
|
|
rootPool.AppendCertsFromPEM([]byte(caRoot.RootCert))
|
|
|
|
_, err = cert.Verify(x509.VerifyOptions{
|
|
Intermediates: intermediatePool,
|
|
Roots: rootPool,
|
|
})
|
|
require.NoError(err)
|
|
}
|
|
|
|
func TestLeader_SecondaryCA_IntermediateRenew(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
// no parallel execution because we change globals
|
|
origInterval := structs.IntermediateCertRenewInterval
|
|
origMinTTL := structs.MinLeafCertTTL
|
|
defer func() {
|
|
structs.IntermediateCertRenewInterval = origInterval
|
|
structs.MinLeafCertTTL = origMinTTL
|
|
}()
|
|
|
|
structs.IntermediateCertRenewInterval = time.Millisecond
|
|
structs.MinLeafCertTTL = time.Second
|
|
require := require.New(t)
|
|
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.6.0"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{
|
|
"PrivateKey": "",
|
|
"RootCert": "",
|
|
"RotationPeriod": "2160h",
|
|
"LeafCertTTL": "5s",
|
|
// The retry loop only retries for 7sec max and
|
|
// the ttl needs to be below so that it
|
|
// triggers definitely.
|
|
// Since certs are created so that they are
|
|
// valid from 1minute in the past, we need to
|
|
// account for that, otherwise it will be
|
|
// expired immediately.
|
|
"IntermediateCertTTL": time.Minute + (5 * time.Second),
|
|
},
|
|
}
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// dc2 as a secondary DC
|
|
dir2, s2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
// Create the WAN link
|
|
joinWAN(t, s2, s1)
|
|
testrpc.WaitForLeader(t, s2.RPC, "dc2")
|
|
|
|
// Get the original intermediate
|
|
// TODO: Wait for intermediate instead of wait for leader
|
|
secondaryProvider, _ := getCAProviderWithLock(s2)
|
|
intermediatePEM, err := secondaryProvider.ActiveIntermediate()
|
|
require.NoError(err)
|
|
cert, err := connect.ParseCert(intermediatePEM)
|
|
require.NoError(err)
|
|
currentCertSerialNumber := cert.SerialNumber
|
|
currentCertAuthorityKeyId := cert.AuthorityKeyId
|
|
|
|
// Capture the current root
|
|
var originalRoot *structs.CARoot
|
|
{
|
|
rootList, activeRoot, err := getTestRoots(s1, "dc1")
|
|
require.NoError(err)
|
|
require.Len(rootList.Roots, 1)
|
|
originalRoot = activeRoot
|
|
}
|
|
|
|
waitForActiveCARoot(t, s1, originalRoot)
|
|
waitForActiveCARoot(t, s2, originalRoot)
|
|
|
|
// Wait for dc2's intermediate to be refreshed.
|
|
// It is possible that test fails when the blocking query doesn't return.
|
|
// When https://github.com/hashicorp/consul/pull/3777 is merged
|
|
// however, defaultQueryTime will be configurable and we con lower it
|
|
// so that it returns for sure.
|
|
retry.Run(t, func(r *retry.R) {
|
|
secondaryProvider, _ = getCAProviderWithLock(s2)
|
|
intermediatePEM, err = secondaryProvider.ActiveIntermediate()
|
|
r.Check(err)
|
|
cert, err := connect.ParseCert(intermediatePEM)
|
|
r.Check(err)
|
|
if cert.SerialNumber.Cmp(currentCertSerialNumber) == 0 || !reflect.DeepEqual(cert.AuthorityKeyId, currentCertAuthorityKeyId) {
|
|
currentCertSerialNumber = cert.SerialNumber
|
|
currentCertAuthorityKeyId = cert.AuthorityKeyId
|
|
r.Fatal("not a renewed intermediate")
|
|
}
|
|
})
|
|
require.NoError(err)
|
|
|
|
// Get the root from dc1 and validate a chain of:
|
|
// dc2 leaf -> dc2 intermediate -> dc1 root
|
|
_, caRoot := getCAProviderWithLock(s1)
|
|
|
|
// Have dc2 sign a leaf cert and make sure the chain is correct.
|
|
spiffeService := &connect.SpiffeIDService{
|
|
Host: "node1",
|
|
Namespace: "default",
|
|
Datacenter: "dc1",
|
|
Service: "foo",
|
|
}
|
|
raw, _ := connect.TestCSR(t, spiffeService)
|
|
|
|
leafCsr, err := connect.ParseCSR(raw)
|
|
require.NoError(err)
|
|
|
|
leafPEM, err := secondaryProvider.Sign(leafCsr)
|
|
require.NoError(err)
|
|
|
|
cert, err = connect.ParseCert(leafPEM)
|
|
require.NoError(err)
|
|
|
|
// Check that the leaf signed by the new intermediate can be verified using the
|
|
// returned cert chain (signed intermediate + remote root).
|
|
intermediatePool := x509.NewCertPool()
|
|
intermediatePool.AppendCertsFromPEM([]byte(intermediatePEM))
|
|
rootPool := x509.NewCertPool()
|
|
rootPool.AppendCertsFromPEM([]byte(caRoot.RootCert))
|
|
|
|
_, err = cert.Verify(x509.VerifyOptions{
|
|
Intermediates: intermediatePool,
|
|
Roots: rootPool,
|
|
})
|
|
require.NoError(err)
|
|
}
|
|
|
|
func TestLeader_SecondaryCA_IntermediateRefresh(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// dc2 as a secondary DC
|
|
dir2, s2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
// Create the WAN link
|
|
joinWAN(t, s2, s1)
|
|
testrpc.WaitForLeader(t, s2.RPC, "dc2")
|
|
|
|
// Get the original intermediate
|
|
secondaryProvider, _ := getCAProviderWithLock(s2)
|
|
oldIntermediatePEM, err := secondaryProvider.ActiveIntermediate()
|
|
require.NoError(err)
|
|
require.NotEmpty(oldIntermediatePEM)
|
|
|
|
// Capture the current root
|
|
var originalRoot *structs.CARoot
|
|
{
|
|
rootList, activeRoot, err := getTestRoots(s1, "dc1")
|
|
require.NoError(err)
|
|
require.Len(rootList.Roots, 1)
|
|
originalRoot = activeRoot
|
|
}
|
|
|
|
// Wait for current state to be reflected in both datacenters.
|
|
testrpc.WaitForActiveCARoot(t, s1.RPC, "dc1", originalRoot)
|
|
testrpc.WaitForActiveCARoot(t, s2.RPC, "dc2", originalRoot)
|
|
|
|
// Update the provider config to use a new private key, which should
|
|
// cause a rotation.
|
|
_, newKey, err := connect.GeneratePrivateKey()
|
|
require.NoError(err)
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{
|
|
"PrivateKey": newKey,
|
|
"RootCert": "",
|
|
"RotationPeriod": 90 * 24 * time.Hour,
|
|
"IntermediateCertTTL": 72 * 24 * time.Hour,
|
|
},
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
require.NoError(s1.RPC("ConnectCA.ConfigurationSet", args, &reply))
|
|
}
|
|
|
|
var updatedRoot *structs.CARoot
|
|
{
|
|
rootList, activeRoot, err := getTestRoots(s1, "dc1")
|
|
require.NoError(err)
|
|
require.Len(rootList.Roots, 2)
|
|
updatedRoot = activeRoot
|
|
}
|
|
|
|
testrpc.WaitForActiveCARoot(t, s1.RPC, "dc1", updatedRoot)
|
|
testrpc.WaitForActiveCARoot(t, s2.RPC, "dc2", updatedRoot)
|
|
|
|
// Wait for dc2's intermediate to be refreshed.
|
|
var intermediatePEM string
|
|
retry.Run(t, func(r *retry.R) {
|
|
intermediatePEM, err = secondaryProvider.ActiveIntermediate()
|
|
r.Check(err)
|
|
if intermediatePEM == oldIntermediatePEM {
|
|
r.Fatal("not a new intermediate")
|
|
}
|
|
})
|
|
require.NoError(err)
|
|
|
|
// Verify the root lists have been rotated in each DC's state store.
|
|
state1 := s1.fsm.State()
|
|
_, primaryRoot, err := state1.CARootActive(nil)
|
|
require.NoError(err)
|
|
|
|
state2 := s2.fsm.State()
|
|
_, roots2, err := state2.CARoots(nil)
|
|
require.NoError(err)
|
|
require.Equal(2, len(roots2))
|
|
|
|
newRoot := roots2[0]
|
|
oldRoot := roots2[1]
|
|
if roots2[1].Active {
|
|
newRoot = roots2[1]
|
|
oldRoot = roots2[0]
|
|
}
|
|
require.False(oldRoot.Active)
|
|
require.True(newRoot.Active)
|
|
require.Equal(primaryRoot.ID, newRoot.ID)
|
|
require.Equal(primaryRoot.RootCert, newRoot.RootCert)
|
|
|
|
// Get the new root from dc1 and validate a chain of:
|
|
// dc2 leaf -> dc2 intermediate -> dc1 root
|
|
_, caRoot := getCAProviderWithLock(s1)
|
|
|
|
// Have dc2 sign a leaf cert and make sure the chain is correct.
|
|
spiffeService := &connect.SpiffeIDService{
|
|
Host: "node1",
|
|
Namespace: "default",
|
|
Datacenter: "dc1",
|
|
Service: "foo",
|
|
}
|
|
raw, _ := connect.TestCSR(t, spiffeService)
|
|
|
|
leafCsr, err := connect.ParseCSR(raw)
|
|
require.NoError(err)
|
|
|
|
leafPEM, err := secondaryProvider.Sign(leafCsr)
|
|
require.NoError(err)
|
|
|
|
cert, err := connect.ParseCert(leafPEM)
|
|
require.NoError(err)
|
|
|
|
// Check that the leaf signed by the new intermediate can be verified using the
|
|
// returned cert chain (signed intermediate + remote root).
|
|
intermediatePool := x509.NewCertPool()
|
|
intermediatePool.AppendCertsFromPEM([]byte(intermediatePEM))
|
|
rootPool := x509.NewCertPool()
|
|
rootPool.AppendCertsFromPEM([]byte(caRoot.RootCert))
|
|
|
|
_, err = cert.Verify(x509.VerifyOptions{
|
|
Intermediates: intermediatePool,
|
|
Roots: rootPool,
|
|
})
|
|
require.NoError(err)
|
|
}
|
|
|
|
func TestLeader_Vault_PrimaryCA_FixSigningKeyID_OnRestart(t *testing.T) {
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
testVault := ca.NewTestVaultServer(t)
|
|
defer testVault.Stop()
|
|
|
|
dir1pre, s1pre := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.6.0"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": testVault.Addr,
|
|
"Token": testVault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
defer os.RemoveAll(dir1pre)
|
|
defer s1pre.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1pre.RPC, "dc1")
|
|
|
|
// Restore the pre-1.9.3/1.8.8/1.7.12 behavior of the SigningKeyID not being derived
|
|
// from the intermediates in the primary (which only matters for provider=vault).
|
|
var primaryRootSigningKeyID string
|
|
{
|
|
state := s1pre.fsm.State()
|
|
|
|
// Get the highest index
|
|
idx, activePrimaryRoot, err := state.CARootActive(nil)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, activePrimaryRoot)
|
|
|
|
rootCert, err := connect.ParseCert(activePrimaryRoot.RootCert)
|
|
require.NoError(t, err)
|
|
|
|
// Force this to be derived just from the root, not the intermediate.
|
|
primaryRootSigningKeyID = connect.EncodeSigningKeyID(rootCert.SubjectKeyId)
|
|
activePrimaryRoot.SigningKeyID = primaryRootSigningKeyID
|
|
|
|
// Store the root cert in raft
|
|
_, err = s1pre.raftApply(structs.ConnectCARequestType, &structs.CARequest{
|
|
Op: structs.CAOpSetRoots,
|
|
Index: idx,
|
|
Roots: []*structs.CARoot{activePrimaryRoot},
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// Shutdown s1pre and restart it to trigger the secondary CA init to correct
|
|
// the SigningKeyID.
|
|
s1pre.Shutdown()
|
|
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.DataDir = s1pre.config.DataDir
|
|
c.Datacenter = "dc1"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.NodeName = s1pre.config.NodeName
|
|
c.NodeID = s1pre.config.NodeID
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// Retry since it will take some time to init the primary CA fully and there
|
|
// isn't a super clean way to watch specifically until it's done than polling
|
|
// the CA provider anyway.
|
|
retry.Run(t, func(r *retry.R) {
|
|
// verify that the root is now corrected
|
|
provider, activeRoot := getCAProviderWithLock(s1)
|
|
require.NotNil(r, provider)
|
|
require.NotNil(r, activeRoot)
|
|
|
|
activeIntermediate, err := provider.ActiveIntermediate()
|
|
require.NoError(r, err)
|
|
|
|
intermediateCert, err := connect.ParseCert(activeIntermediate)
|
|
require.NoError(r, err)
|
|
|
|
// Force this to be derived just from the root, not the intermediate.
|
|
expect := connect.EncodeSigningKeyID(intermediateCert.SubjectKeyId)
|
|
|
|
// The in-memory representation was saw the correction via a setCAProvider call.
|
|
require.Equal(r, expect, activeRoot.SigningKeyID)
|
|
|
|
// The state store saw the correction, too.
|
|
_, activePrimaryRoot, err := s1.fsm.State().CARootActive(nil)
|
|
require.NoError(r, err)
|
|
require.NotNil(r, activePrimaryRoot)
|
|
require.Equal(r, expect, activePrimaryRoot.SigningKeyID)
|
|
})
|
|
}
|
|
|
|
func TestLeader_SecondaryCA_FixSigningKeyID_via_IntermediateRefresh(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// dc2 as a secondary DC
|
|
dir2pre, s2pre := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir2pre)
|
|
defer s2pre.Shutdown()
|
|
|
|
// Create the WAN link
|
|
joinWAN(t, s2pre, s1)
|
|
testrpc.WaitForLeader(t, s2pre.RPC, "dc2")
|
|
|
|
// Restore the pre-1.6.1 behavior of the SigningKeyID not being derived
|
|
// from the intermediates.
|
|
var secondaryRootSigningKeyID string
|
|
{
|
|
state := s2pre.fsm.State()
|
|
|
|
// Get the highest index
|
|
idx, activeSecondaryRoot, err := state.CARootActive(nil)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, activeSecondaryRoot)
|
|
|
|
rootCert, err := connect.ParseCert(activeSecondaryRoot.RootCert)
|
|
require.NoError(t, err)
|
|
|
|
// Force this to be derived just from the root, not the intermediate.
|
|
secondaryRootSigningKeyID = connect.EncodeSigningKeyID(rootCert.SubjectKeyId)
|
|
activeSecondaryRoot.SigningKeyID = secondaryRootSigningKeyID
|
|
|
|
// Store the root cert in raft
|
|
_, err = s2pre.raftApply(structs.ConnectCARequestType, &structs.CARequest{
|
|
Op: structs.CAOpSetRoots,
|
|
Index: idx,
|
|
Roots: []*structs.CARoot{activeSecondaryRoot},
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// Shutdown s2pre and restart it to trigger the secondary CA init to correct
|
|
// the SigningKeyID.
|
|
s2pre.Shutdown()
|
|
|
|
dir2, s2 := testServerWithConfig(t, func(c *Config) {
|
|
c.DataDir = s2pre.config.DataDir
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.NodeName = s2pre.config.NodeName
|
|
c.NodeID = s2pre.config.NodeID
|
|
})
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s2.RPC, "dc2")
|
|
|
|
// Retry since it will take some time to init the secondary CA fully and there
|
|
// isn't a super clean way to watch specifically until it's done than polling
|
|
// the CA provider anyway.
|
|
retry.Run(t, func(r *retry.R) {
|
|
// verify that the root is now corrected
|
|
provider, activeRoot := getCAProviderWithLock(s2)
|
|
require.NotNil(r, provider)
|
|
require.NotNil(r, activeRoot)
|
|
|
|
activeIntermediate, err := provider.ActiveIntermediate()
|
|
require.NoError(r, err)
|
|
|
|
intermediateCert, err := connect.ParseCert(activeIntermediate)
|
|
require.NoError(r, err)
|
|
|
|
// Force this to be derived just from the root, not the intermediate.
|
|
expect := connect.EncodeSigningKeyID(intermediateCert.SubjectKeyId)
|
|
|
|
// The in-memory representation was saw the correction via a setCAProvider call.
|
|
require.Equal(r, expect, activeRoot.SigningKeyID)
|
|
|
|
// The state store saw the correction, too.
|
|
_, activeSecondaryRoot, err := s2.fsm.State().CARootActive(nil)
|
|
require.NoError(r, err)
|
|
require.NotNil(r, activeSecondaryRoot)
|
|
require.Equal(r, expect, activeSecondaryRoot.SigningKeyID)
|
|
})
|
|
}
|
|
|
|
func TestLeader_SecondaryCA_TransitionFromPrimary(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
// Initialize dc1 as the primary DC
|
|
id1, err := uuid.GenerateUUID()
|
|
require.NoError(t, err)
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig.ClusterID = id1
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// dc2 as a primary DC initially
|
|
id2, err := uuid.GenerateUUID()
|
|
require.NoError(t, err)
|
|
dir2, s2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc2"
|
|
c.CAConfig.ClusterID = id2
|
|
c.Build = "1.6.0"
|
|
})
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
// Get the initial (primary) roots state for the secondary
|
|
testrpc.WaitForLeader(t, s2.RPC, "dc2")
|
|
args := structs.DCSpecificRequest{Datacenter: "dc2"}
|
|
var dc2PrimaryRoots structs.IndexedCARoots
|
|
require.NoError(t, s2.RPC("ConnectCA.Roots", &args, &dc2PrimaryRoots))
|
|
require.Len(t, dc2PrimaryRoots.Roots, 1)
|
|
|
|
// Shutdown s2 and restart it with the dc1 as the primary
|
|
s2.Shutdown()
|
|
dir3, s3 := testServerWithConfig(t, func(c *Config) {
|
|
c.DataDir = s2.config.DataDir
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.NodeName = s2.config.NodeName
|
|
c.NodeID = s2.config.NodeID
|
|
})
|
|
defer os.RemoveAll(dir3)
|
|
defer s3.Shutdown()
|
|
|
|
// Create the WAN link
|
|
joinWAN(t, s3, s1)
|
|
testrpc.WaitForLeader(t, s3.RPC, "dc2")
|
|
|
|
// Verify the secondary has migrated its TrustDomain and added the new primary's root.
|
|
retry.Run(t, func(r *retry.R) {
|
|
args = structs.DCSpecificRequest{Datacenter: "dc1"}
|
|
var dc1Roots structs.IndexedCARoots
|
|
require.NoError(r, s1.RPC("ConnectCA.Roots", &args, &dc1Roots))
|
|
require.Len(r, dc1Roots.Roots, 1)
|
|
|
|
args = structs.DCSpecificRequest{Datacenter: "dc2"}
|
|
var dc2SecondaryRoots structs.IndexedCARoots
|
|
require.NoError(r, s3.RPC("ConnectCA.Roots", &args, &dc2SecondaryRoots))
|
|
|
|
// dc2's TrustDomain should have changed to the primary's
|
|
require.Equal(r, dc2SecondaryRoots.TrustDomain, dc1Roots.TrustDomain)
|
|
require.NotEqual(r, dc2SecondaryRoots.TrustDomain, dc2PrimaryRoots.TrustDomain)
|
|
|
|
// Both roots should be present and correct
|
|
require.Len(r, dc2SecondaryRoots.Roots, 2)
|
|
var oldSecondaryRoot *structs.CARoot
|
|
var newSecondaryRoot *structs.CARoot
|
|
if dc2SecondaryRoots.Roots[0].ID == dc2PrimaryRoots.Roots[0].ID {
|
|
oldSecondaryRoot = dc2SecondaryRoots.Roots[0]
|
|
newSecondaryRoot = dc2SecondaryRoots.Roots[1]
|
|
} else {
|
|
oldSecondaryRoot = dc2SecondaryRoots.Roots[1]
|
|
newSecondaryRoot = dc2SecondaryRoots.Roots[0]
|
|
}
|
|
|
|
// The old root should have its TrustDomain filled in as the old domain.
|
|
require.Equal(r, oldSecondaryRoot.ExternalTrustDomain, strings.TrimSuffix(dc2PrimaryRoots.TrustDomain, ".consul"))
|
|
|
|
require.Equal(r, oldSecondaryRoot.ID, dc2PrimaryRoots.Roots[0].ID)
|
|
require.Equal(r, oldSecondaryRoot.RootCert, dc2PrimaryRoots.Roots[0].RootCert)
|
|
require.Equal(r, newSecondaryRoot.ID, dc1Roots.Roots[0].ID)
|
|
require.Equal(r, newSecondaryRoot.RootCert, dc1Roots.Roots[0].RootCert)
|
|
})
|
|
}
|
|
|
|
func TestLeader_SecondaryCA_UpgradeBeforePrimary(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
// Initialize dc1 as the primary DC
|
|
dir1, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.Build = "1.3.0"
|
|
c.MaxQueryTime = 500 * time.Millisecond
|
|
})
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// dc2 as a secondary DC
|
|
dir2, s2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.Build = "1.6.0"
|
|
c.MaxQueryTime = 500 * time.Millisecond
|
|
})
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
// Create the WAN link
|
|
joinWAN(t, s2, s1)
|
|
testrpc.WaitForLeader(t, s2.RPC, "dc2")
|
|
|
|
// ensure all the CA initialization stuff would have already been done
|
|
// this is necessary to ensure that not only has a leader been elected
|
|
// but that it has also finished its establishLeadership call
|
|
retry.Run(t, func(r *retry.R) {
|
|
require.True(r, s1.isReadyForConsistentReads())
|
|
require.True(r, s2.isReadyForConsistentReads())
|
|
})
|
|
|
|
// Verify the primary has a root (we faked its version too low but since its the primary it ignores any version checks)
|
|
retry.Run(t, func(r *retry.R) {
|
|
state1 := s1.fsm.State()
|
|
_, roots1, err := state1.CARoots(nil)
|
|
require.NoError(r, err)
|
|
require.Len(r, roots1, 1)
|
|
})
|
|
|
|
// Verify the secondary does not have a root - defers initialization until the primary has been upgraded.
|
|
state2 := s2.fsm.State()
|
|
_, roots2, err := state2.CARoots(nil)
|
|
require.NoError(t, err)
|
|
require.Empty(t, roots2)
|
|
|
|
// Update the version on the fly so s2 kicks off the secondary DC transition.
|
|
tags := s1.config.SerfWANConfig.Tags
|
|
tags["build"] = "1.6.0"
|
|
s1.serfWAN.SetTags(tags)
|
|
|
|
// Wait for the secondary transition to happen and then verify the secondary DC
|
|
// has both roots present.
|
|
secondaryProvider, _ := getCAProviderWithLock(s2)
|
|
retry.Run(t, func(r *retry.R) {
|
|
state1 := s1.fsm.State()
|
|
_, roots1, err := state1.CARoots(nil)
|
|
require.NoError(r, err)
|
|
require.Len(r, roots1, 1)
|
|
|
|
state2 := s2.fsm.State()
|
|
_, roots2, err := state2.CARoots(nil)
|
|
require.NoError(r, err)
|
|
require.Len(r, roots2, 1)
|
|
|
|
// ensure the roots are the same
|
|
require.Equal(r, roots1[0].ID, roots2[0].ID)
|
|
require.Equal(r, roots1[0].RootCert, roots2[0].RootCert)
|
|
|
|
inter, err := secondaryProvider.ActiveIntermediate()
|
|
require.NoError(r, err)
|
|
require.NotEmpty(r, inter, "should have valid intermediate")
|
|
})
|
|
|
|
_, caRoot := getCAProviderWithLock(s1)
|
|
intermediatePEM, err := secondaryProvider.ActiveIntermediate()
|
|
require.NoError(t, err)
|
|
|
|
// Have dc2 sign a leaf cert and make sure the chain is correct.
|
|
spiffeService := &connect.SpiffeIDService{
|
|
Host: "node1",
|
|
Namespace: "default",
|
|
Datacenter: "dc1",
|
|
Service: "foo",
|
|
}
|
|
raw, _ := connect.TestCSR(t, spiffeService)
|
|
|
|
leafCsr, err := connect.ParseCSR(raw)
|
|
require.NoError(t, err)
|
|
|
|
leafPEM, err := secondaryProvider.Sign(leafCsr)
|
|
require.NoError(t, err)
|
|
|
|
cert, err := connect.ParseCert(leafPEM)
|
|
require.NoError(t, err)
|
|
|
|
// Check that the leaf signed by the new cert can be verified using the
|
|
// returned cert chain (signed intermediate + remote root).
|
|
intermediatePool := x509.NewCertPool()
|
|
intermediatePool.AppendCertsFromPEM([]byte(intermediatePEM))
|
|
rootPool := x509.NewCertPool()
|
|
rootPool.AppendCertsFromPEM([]byte(caRoot.RootCert))
|
|
|
|
_, err = cert.Verify(x509.VerifyOptions{
|
|
Intermediates: intermediatePool,
|
|
Roots: rootPool,
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func getTestRoots(s *Server, datacenter string) (*structs.IndexedCARoots, *structs.CARoot, error) {
|
|
rootReq := &structs.DCSpecificRequest{
|
|
Datacenter: datacenter,
|
|
}
|
|
var rootList structs.IndexedCARoots
|
|
if err := s.RPC("ConnectCA.Roots", rootReq, &rootList); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
var active *structs.CARoot
|
|
for _, root := range rootList.Roots {
|
|
if root.Active {
|
|
active = root
|
|
break
|
|
}
|
|
}
|
|
|
|
return &rootList, active, nil
|
|
}
|
|
|
|
func TestLeader_GenerateCASignRequest(t *testing.T) {
|
|
csr := "A"
|
|
s := Server{config: &Config{PrimaryDatacenter: "east"}, tokens: new(token.Store)}
|
|
req := s.generateCASignRequest(csr)
|
|
assert.Equal(t, "east", req.RequestDatacenter())
|
|
}
|
|
|
|
func TestLeader_CARootPruning(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
// Can not use t.Parallel(), because this modifies a global.
|
|
caRootPruneInterval = 200 * time.Millisecond
|
|
|
|
require := require.New(t)
|
|
dir1, s1 := testServer(t)
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
codec := rpcClient(t, s1)
|
|
defer codec.Close()
|
|
|
|
testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
|
|
|
|
// Get the current root
|
|
rootReq := &structs.DCSpecificRequest{
|
|
Datacenter: "dc1",
|
|
}
|
|
var rootList structs.IndexedCARoots
|
|
require.Nil(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList))
|
|
require.Len(rootList.Roots, 1)
|
|
oldRoot := rootList.Roots[0]
|
|
|
|
// Update the provider config to use a new private key, which should
|
|
// cause a rotation.
|
|
_, newKey, err := connect.GeneratePrivateKey()
|
|
require.NoError(err)
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{
|
|
"LeafCertTTL": "500ms",
|
|
"PrivateKey": newKey,
|
|
"RootCert": "",
|
|
"RotationPeriod": "2160h",
|
|
"SkipValidate": true,
|
|
},
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply))
|
|
}
|
|
|
|
// Should have 2 roots now.
|
|
_, roots, err := s1.fsm.State().CARoots(nil)
|
|
require.NoError(err)
|
|
require.Len(roots, 2)
|
|
|
|
time.Sleep(2 * time.Second)
|
|
|
|
// Now the old root should be pruned.
|
|
_, roots, err = s1.fsm.State().CARoots(nil)
|
|
require.NoError(err)
|
|
require.Len(roots, 1)
|
|
require.True(roots[0].Active)
|
|
require.NotEqual(roots[0].ID, oldRoot.ID)
|
|
}
|
|
|
|
func TestLeader_PersistIntermediateCAs(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
dir1, s1 := testServer(t)
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
codec := rpcClient(t, s1)
|
|
defer codec.Close()
|
|
|
|
dir2, s2 := testServerDCBootstrap(t, "dc1", false)
|
|
defer os.RemoveAll(dir2)
|
|
defer s2.Shutdown()
|
|
|
|
dir3, s3 := testServerDCBootstrap(t, "dc1", false)
|
|
defer os.RemoveAll(dir3)
|
|
defer s3.Shutdown()
|
|
|
|
joinLAN(t, s2, s1)
|
|
joinLAN(t, s3, s1)
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// Get the current root
|
|
rootReq := &structs.DCSpecificRequest{
|
|
Datacenter: "dc1",
|
|
}
|
|
var rootList structs.IndexedCARoots
|
|
require.Nil(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList))
|
|
require.Len(rootList.Roots, 1)
|
|
|
|
// Update the provider config to use a new private key, which should
|
|
// cause a rotation.
|
|
_, newKey, err := connect.GeneratePrivateKey()
|
|
require.NoError(err)
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{
|
|
"PrivateKey": newKey,
|
|
"RootCert": "",
|
|
"RotationPeriod": 90 * 24 * time.Hour,
|
|
},
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply))
|
|
}
|
|
|
|
// Get the active root before leader change.
|
|
_, root := getCAProviderWithLock(s1)
|
|
require.Len(root.IntermediateCerts, 1)
|
|
|
|
// Force a leader change and make sure the root CA values are preserved.
|
|
s1.Leave()
|
|
s1.Shutdown()
|
|
|
|
retry.Run(t, func(r *retry.R) {
|
|
var leader *Server
|
|
for _, s := range []*Server{s2, s3} {
|
|
if s.IsLeader() {
|
|
leader = s
|
|
break
|
|
}
|
|
}
|
|
if leader == nil {
|
|
r.Fatal("no leader")
|
|
}
|
|
|
|
_, newLeaderRoot := getCAProviderWithLock(leader)
|
|
if !reflect.DeepEqual(newLeaderRoot, root) {
|
|
r.Fatalf("got %v, want %v", newLeaderRoot, root)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestLeader_ParseCARoot(t *testing.T) {
|
|
type test struct {
|
|
name string
|
|
pem string
|
|
wantSerial uint64
|
|
wantSigningKeyID string
|
|
wantKeyType string
|
|
wantKeyBits int
|
|
wantErr bool
|
|
}
|
|
// Test certs generated with
|
|
// go run connect/certgen/certgen.go -out-dir /tmp/connect-certs -key-type ec -key-bits 384
|
|
// for various key types. This does limit the exposure to formats that might
|
|
// exist in external certificates which can be used as Connect CAs.
|
|
// Specifically many other certs will have serial numbers that don't fit into
|
|
// 64 bits but for reasons we truncate down to 64 bits which means our
|
|
// `SerialNumber` will not match the one reported by openssl. We should
|
|
// probably fix that at some point as it seems like a big footgun but it would
|
|
// be a breaking API change to change the type to not be a JSON number and
|
|
// JSON numbers don't even support the full range of a uint64...
|
|
tests := []test{
|
|
{"no cert", "", 0, "", "", 0, true},
|
|
{
|
|
name: "default cert",
|
|
// Watchout for indentations they will break PEM format
|
|
pem: readTestData(t, "cert-with-ec-256-key.pem"),
|
|
// Based on `openssl x509 -noout -text` report from the cert
|
|
wantSerial: 8341954965092507701,
|
|
wantSigningKeyID: "97:4D:17:81:64:F8:B4:AF:05:E8:6C:79:C5:40:3B:0E:3E:8B:C0:AE:38:51:54:8A:2F:05:DB:E3:E8:E4:24:EC",
|
|
wantKeyType: "ec",
|
|
wantKeyBits: 256,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "ec 384 cert",
|
|
// Watchout for indentations they will break PEM format
|
|
pem: readTestData(t, "cert-with-ec-384-key.pem"),
|
|
// Based on `openssl x509 -noout -text` report from the cert
|
|
wantSerial: 2935109425518279965,
|
|
wantSigningKeyID: "0B:A0:88:9B:DC:95:31:51:2E:3D:D4:F9:42:D0:6A:A0:62:46:82:D2:7C:22:E7:29:A9:AA:E8:A5:8C:CF:C7:42",
|
|
wantKeyType: "ec",
|
|
wantKeyBits: 384,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "rsa 4096 cert",
|
|
// Watchout for indentations they will break PEM format
|
|
pem: readTestData(t, "cert-with-rsa-4096-key.pem"),
|
|
// Based on `openssl x509 -noout -text` report from the cert
|
|
wantSerial: 5186695743100577491,
|
|
wantSigningKeyID: "92:FA:CC:97:57:1E:31:84:A2:33:DD:9B:6A:A8:7C:FC:BE:E2:94:CA:AC:B3:33:17:39:3B:B8:67:9B:DC:C1:08",
|
|
wantKeyType: "rsa",
|
|
wantKeyBits: 4096,
|
|
wantErr: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
require := require.New(t)
|
|
root, err := parseCARoot(tt.pem, "consul", "cluster")
|
|
if tt.wantErr {
|
|
require.Error(err)
|
|
return
|
|
}
|
|
require.NoError(err)
|
|
require.Equal(tt.wantSerial, root.SerialNumber)
|
|
require.Equal(strings.ToLower(tt.wantSigningKeyID), root.SigningKeyID)
|
|
require.Equal(tt.wantKeyType, root.PrivateKeyType)
|
|
require.Equal(tt.wantKeyBits, root.PrivateKeyBits)
|
|
})
|
|
}
|
|
}
|
|
|
|
func readTestData(t *testing.T, name string) string {
|
|
t.Helper()
|
|
path := filepath.Join("testdata", name)
|
|
bs, err := ioutil.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("failed reading fixture file %s: %s", name, err)
|
|
}
|
|
return string(bs)
|
|
}
|
|
|
|
func TestLeader_lessThanHalfTimePassed(t *testing.T) {
|
|
now := time.Now()
|
|
require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(-5*time.Second)))
|
|
require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now))
|
|
require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(5*time.Second)))
|
|
require.False(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(10*time.Second)))
|
|
|
|
require.True(t, lessThanHalfTimePassed(now, now.Add(-10*time.Second), now.Add(20*time.Second)))
|
|
}
|
|
|
|
func TestLeader_retryLoopBackoffHandleSuccess(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
type test struct {
|
|
desc string
|
|
loopFn func() error
|
|
abort bool
|
|
timedOut bool
|
|
}
|
|
success := func() error {
|
|
return nil
|
|
}
|
|
failure := func() error {
|
|
return fmt.Errorf("test error")
|
|
}
|
|
tests := []test{
|
|
{"loop without error and no abortOnSuccess keeps running", success, false, true},
|
|
{"loop with error and no abortOnSuccess keeps running", failure, false, true},
|
|
{"loop without error and abortOnSuccess is stopped", success, true, false},
|
|
{"loop with error and abortOnSuccess keeps running", failure, true, true},
|
|
}
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
defer cancel()
|
|
|
|
retryLoopBackoffHandleSuccess(ctx, tc.loopFn, func(_ error) {}, tc.abort)
|
|
select {
|
|
case <-ctx.Done():
|
|
if !tc.timedOut {
|
|
t.Fatal("should not have timed out")
|
|
}
|
|
default:
|
|
if tc.timedOut {
|
|
t.Fatal("should have timed out")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLeader_Vault_BadCAConfigShouldntPreventLeaderEstablishment(t *testing.T) {
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
testVault := ca.NewTestVaultServer(t)
|
|
defer testVault.Stop()
|
|
|
|
_, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.9.1"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": testVault.Addr,
|
|
"Token": "not-the-root",
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
defer s1.Shutdown()
|
|
|
|
waitForLeaderEstablishment(t, s1)
|
|
|
|
rootsList, activeRoot, err := getTestRoots(s1, "dc1")
|
|
require.NoError(t, err)
|
|
require.Empty(t, rootsList.Roots)
|
|
require.Nil(t, activeRoot)
|
|
|
|
// Now that the leader is up and we have verified that there are no roots / CA init failed,
|
|
// verify that we can reconfigure away from the bad configuration.
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": testVault.Addr,
|
|
"Token": testVault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
retry.Run(t, func(r *retry.R) {
|
|
require.NoError(r, s1.RPC("ConnectCA.ConfigurationSet", args, &reply))
|
|
})
|
|
}
|
|
|
|
rootsList, activeRoot, err = getTestRoots(s1, "dc1")
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, rootsList.Roots)
|
|
require.NotNil(t, activeRoot)
|
|
}
|
|
|
|
func TestLeader_Consul_BadCAConfigShouldntPreventLeaderEstablishment(t *testing.T) {
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
_, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.9.1"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{
|
|
"RootCert": "garbage",
|
|
},
|
|
}
|
|
})
|
|
defer s1.Shutdown()
|
|
|
|
waitForLeaderEstablishment(t, s1)
|
|
|
|
rootsList, activeRoot, err := getTestRoots(s1, "dc1")
|
|
require.NoError(t, err)
|
|
require.Empty(t, rootsList.Roots)
|
|
require.Nil(t, activeRoot)
|
|
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{},
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
retry.Run(t, func(r *retry.R) {
|
|
require.NoError(r, s1.RPC("ConnectCA.ConfigurationSet", args, &reply))
|
|
})
|
|
}
|
|
|
|
rootsList, activeRoot, err = getTestRoots(s1, "dc1")
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, rootsList.Roots)
|
|
require.NotNil(t, activeRoot)
|
|
}
|
|
|
|
func TestLeader_Consul_ForceWithoutCrossSigning(t *testing.T) {
|
|
require := require.New(t)
|
|
dir1, s1 := testServer(t)
|
|
defer os.RemoveAll(dir1)
|
|
defer s1.Shutdown()
|
|
codec := rpcClient(t, s1)
|
|
defer codec.Close()
|
|
|
|
waitForLeaderEstablishment(t, s1)
|
|
|
|
// Get the current root
|
|
rootReq := &structs.DCSpecificRequest{
|
|
Datacenter: "dc1",
|
|
}
|
|
var rootList structs.IndexedCARoots
|
|
require.Nil(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList))
|
|
require.Len(rootList.Roots, 1)
|
|
oldRoot := rootList.Roots[0]
|
|
|
|
// Update the provider config to use a new private key, which should
|
|
// cause a rotation.
|
|
_, newKey, err := connect.GeneratePrivateKey()
|
|
require.NoError(err)
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
Config: map[string]interface{}{
|
|
"LeafCertTTL": "500ms",
|
|
"PrivateKey": newKey,
|
|
"RootCert": "",
|
|
"RotationPeriod": "2160h",
|
|
"SkipValidate": true,
|
|
},
|
|
ForceWithoutCrossSigning: true,
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply))
|
|
}
|
|
|
|
// Old root should no longer be active.
|
|
_, roots, err := s1.fsm.State().CARoots(nil)
|
|
require.NoError(err)
|
|
require.Len(roots, 2)
|
|
for _, r := range roots {
|
|
if r.ID == oldRoot.ID {
|
|
require.False(r.Active)
|
|
} else {
|
|
require.True(r.Active)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLeader_Vault_ForceWithoutCrossSigning(t *testing.T) {
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
require := require.New(t)
|
|
testVault := ca.NewTestVaultServer(t)
|
|
defer testVault.Stop()
|
|
|
|
_, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.Build = "1.9.1"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": testVault.Addr,
|
|
"Token": testVault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
defer s1.Shutdown()
|
|
codec := rpcClient(t, s1)
|
|
defer codec.Close()
|
|
|
|
waitForLeaderEstablishment(t, s1)
|
|
|
|
// Get the current root
|
|
rootReq := &structs.DCSpecificRequest{
|
|
Datacenter: "dc1",
|
|
}
|
|
var rootList structs.IndexedCARoots
|
|
require.Nil(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList))
|
|
require.Len(rootList.Roots, 1)
|
|
oldRoot := rootList.Roots[0]
|
|
|
|
// Update the provider config to use a new PKI path, which should
|
|
// cause a rotation.
|
|
newConfig := &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": testVault.Addr,
|
|
"Token": testVault.RootToken,
|
|
"RootPKIPath": "pki-root-2/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
ForceWithoutCrossSigning: true,
|
|
}
|
|
{
|
|
args := &structs.CARequest{
|
|
Datacenter: "dc1",
|
|
Config: newConfig,
|
|
}
|
|
var reply interface{}
|
|
|
|
require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply))
|
|
}
|
|
|
|
// Old root should no longer be active.
|
|
_, roots, err := s1.fsm.State().CARoots(nil)
|
|
require.NoError(err)
|
|
require.Len(roots, 2)
|
|
for _, r := range roots {
|
|
if r.ID == oldRoot.ID {
|
|
require.False(r.Active)
|
|
} else {
|
|
require.True(r.Active)
|
|
}
|
|
}
|
|
}
|