6ef38eaea7
For mTLS to work between two proxies in peered clusters with different root CAs, proxies need to configure their outbound listener to use different root certificates for validation. Up until peering was introduced proxies would only ever use one set of root certificates to validate all mesh traffic, both inbound and outbound. Now an upstream proxy may have a leaf certificate signed by a CA that's different from the dialing proxy's. This PR makes changes to proxycfg and xds so that the upstream TLS validation uses different root certificates depending on which cluster is being dialed.
1046 lines
33 KiB
Go
1046 lines
33 KiB
Go
package consul
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/rsa"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"math/big"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
vaultapi "github.com/hashicorp/vault/api"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc"
|
|
|
|
msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
|
|
"github.com/hashicorp/consul-net-rpc/net/rpc"
|
|
|
|
"github.com/hashicorp/consul/agent/connect"
|
|
ca "github.com/hashicorp/consul/agent/connect/ca"
|
|
"github.com/hashicorp/consul/agent/consul/fsm"
|
|
"github.com/hashicorp/consul/agent/consul/state"
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
"github.com/hashicorp/consul/agent/token"
|
|
"github.com/hashicorp/consul/lib"
|
|
"github.com/hashicorp/consul/sdk/testutil"
|
|
"github.com/hashicorp/consul/sdk/testutil/retry"
|
|
"github.com/hashicorp/consul/testrpc"
|
|
)
|
|
|
|
// TODO(kyhavlov): replace with t.Deadline()
|
|
const CATestTimeout = 7 * time.Second
|
|
|
|
func TestCAManager_Initialize_Vault_Secondary_SharedVault(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
vault := ca.NewTestVaultServer(t)
|
|
|
|
_, serverDC1 := testServerWithConfig(t, func(c *Config) {
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-primary/",
|
|
},
|
|
}
|
|
})
|
|
|
|
testutil.RunStep(t, "check primary DC", func(t *testing.T) {
|
|
testrpc.WaitForTestAgent(t, serverDC1.RPC, "dc1")
|
|
|
|
codec := rpcClient(t, serverDC1)
|
|
roots := structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
|
|
verifyLeafCert(t, roots.Roots[0], leafPEM)
|
|
})
|
|
|
|
testutil.RunStep(t, "start secondary DC", func(t *testing.T) {
|
|
_, serverDC2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-secondary/",
|
|
},
|
|
}
|
|
})
|
|
joinWAN(t, serverDC2, serverDC1)
|
|
testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
|
|
|
|
codec := rpcClient(t, serverDC2)
|
|
roots := structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
|
|
verifyLeafCert(t, roots.Roots[0], leafPEM)
|
|
})
|
|
}
|
|
|
|
func verifyLeafCert(t *testing.T, root *structs.CARoot, leafCertPEM string) {
|
|
t.Helper()
|
|
roots := structs.IndexedCARoots{
|
|
ActiveRootID: root.ID,
|
|
Roots: []*structs.CARoot{root},
|
|
}
|
|
verifyLeafCertWithRoots(t, roots, leafCertPEM)
|
|
}
|
|
|
|
func verifyLeafCertWithRoots(t *testing.T, roots structs.IndexedCARoots, leafCertPEM string) {
|
|
t.Helper()
|
|
leaf, intermediates, err := connect.ParseLeafCerts(leafCertPEM)
|
|
require.NoError(t, err)
|
|
|
|
pool := x509.NewCertPool()
|
|
for _, r := range roots.Roots {
|
|
ok := pool.AppendCertsFromPEM([]byte(r.RootCert))
|
|
if !ok {
|
|
t.Fatalf("Failed to add root CA PEM to cert pool")
|
|
}
|
|
}
|
|
|
|
// verify with intermediates from leaf CertPEM
|
|
_, err = leaf.Verify(x509.VerifyOptions{
|
|
Roots: pool,
|
|
Intermediates: intermediates,
|
|
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
})
|
|
require.NoError(t, err, "failed to verify using intermediates from leaf cert PEM")
|
|
|
|
// verify with intermediates from the CARoot
|
|
intermediates = x509.NewCertPool()
|
|
for _, r := range roots.Roots {
|
|
for _, intermediate := range r.IntermediateCerts {
|
|
c, err := connect.ParseCert(intermediate)
|
|
require.NoError(t, err)
|
|
intermediates.AddCert(c)
|
|
}
|
|
}
|
|
|
|
_, err = leaf.Verify(x509.VerifyOptions{
|
|
Roots: pool,
|
|
Intermediates: intermediates,
|
|
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
})
|
|
require.NoError(t, err, "failed to verify using intermediates from CARoot list")
|
|
}
|
|
|
|
type mockCAServerDelegate struct {
|
|
t *testing.T
|
|
config *Config
|
|
store *state.Store
|
|
primaryRoot *structs.CARoot
|
|
secondaryIntermediate string
|
|
callbackCh chan string
|
|
}
|
|
|
|
func NewMockCAServerDelegate(t *testing.T, config *Config) *mockCAServerDelegate {
|
|
delegate := &mockCAServerDelegate{
|
|
t: t,
|
|
config: config,
|
|
store: state.NewStateStore(nil),
|
|
primaryRoot: connect.TestCAWithTTL(t, nil, 1*time.Second),
|
|
callbackCh: make(chan string, 0),
|
|
}
|
|
delegate.store.CASetConfig(1, testCAConfig())
|
|
|
|
return delegate
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) State() *state.Store {
|
|
return m.store
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) ProviderState(id string) (*structs.CAConsulProviderState, error) {
|
|
_, s, err := m.store.CAProviderState(id)
|
|
return s, err
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) IsLeader() bool {
|
|
return true
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) ServersSupportMultiDCConnectCA() error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) ApplyCALeafRequest() (uint64, error) {
|
|
return 3, nil
|
|
}
|
|
|
|
// ApplyCARequest mirrors FSM.applyConnectCAOperation because that functionality
|
|
// is not exported.
|
|
func (m *mockCAServerDelegate) ApplyCARequest(req *structs.CARequest) (interface{}, error) {
|
|
idx, _, err := m.store.CAConfig(nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
m.callbackCh <- fmt.Sprintf("raftApply/ConnectCA")
|
|
|
|
result := fsm.ApplyConnectCAOperationFromRequest(m.store, req, idx+1)
|
|
if err, ok := result.(error); ok && err != nil {
|
|
return nil, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) forwardDC(method, dc string, args interface{}, reply interface{}) error {
|
|
switch method {
|
|
case "ConnectCA.Roots":
|
|
roots := reply.(*structs.IndexedCARoots)
|
|
roots.TrustDomain = connect.TestClusterID
|
|
roots.Roots = []*structs.CARoot{m.primaryRoot}
|
|
roots.ActiveRootID = m.primaryRoot.ID
|
|
case "ConnectCA.SignIntermediate":
|
|
r := reply.(*string)
|
|
*r = m.secondaryIntermediate
|
|
default:
|
|
return fmt.Errorf("received call to unsupported method %q", method)
|
|
}
|
|
|
|
m.callbackCh <- fmt.Sprintf("forwardDC/%s", method)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *mockCAServerDelegate) generateCASignRequest(csr string) *structs.CASignRequest {
|
|
return &structs.CASignRequest{
|
|
Datacenter: m.config.PrimaryDatacenter,
|
|
CSR: csr,
|
|
}
|
|
}
|
|
|
|
// mockCAProvider mocks an empty provider implementation with a channel in order to coordinate
|
|
// waiting for certain methods to be called.
|
|
type mockCAProvider struct {
|
|
callbackCh chan string
|
|
rootPEM string
|
|
intermediatePem string
|
|
}
|
|
|
|
func (m *mockCAProvider) Configure(cfg ca.ProviderConfig) error { return nil }
|
|
func (m *mockCAProvider) State() (map[string]string, error) { return nil, nil }
|
|
func (m *mockCAProvider) GenerateRoot() (ca.RootResult, error) {
|
|
return ca.RootResult{PEM: m.rootPEM}, nil
|
|
}
|
|
func (m *mockCAProvider) GenerateIntermediateCSR() (string, error) {
|
|
m.callbackCh <- "provider/GenerateIntermediateCSR"
|
|
return "", nil
|
|
}
|
|
func (m *mockCAProvider) SetIntermediate(intermediatePEM, rootPEM string) error {
|
|
m.callbackCh <- "provider/SetIntermediate"
|
|
return nil
|
|
}
|
|
func (m *mockCAProvider) ActiveIntermediate() (string, error) {
|
|
if m.intermediatePem == "" {
|
|
return m.rootPEM, nil
|
|
}
|
|
return m.intermediatePem, nil
|
|
}
|
|
func (m *mockCAProvider) GenerateIntermediate() (string, error) { return "", nil }
|
|
func (m *mockCAProvider) Sign(*x509.CertificateRequest) (string, error) { return "", nil }
|
|
func (m *mockCAProvider) SignIntermediate(*x509.CertificateRequest) (string, error) { return "", nil }
|
|
func (m *mockCAProvider) CrossSignCA(*x509.Certificate) (string, error) { return "", nil }
|
|
func (m *mockCAProvider) SupportsCrossSigning() (bool, error) { return false, nil }
|
|
func (m *mockCAProvider) Cleanup(_ bool, _ map[string]interface{}) error { return nil }
|
|
|
|
func waitForCh(t *testing.T, ch chan string, expected string) {
|
|
t.Helper()
|
|
select {
|
|
case op := <-ch:
|
|
if op != expected {
|
|
t.Fatalf("got unexpected op %q, wanted %q", op, expected)
|
|
}
|
|
case <-time.After(CATestTimeout):
|
|
t.Fatalf("never got op %q", expected)
|
|
}
|
|
}
|
|
|
|
func waitForEmptyCh(t *testing.T, ch chan string) {
|
|
select {
|
|
case op := <-ch:
|
|
t.Fatalf("got unexpected op %q", op)
|
|
case <-time.After(1 * time.Second):
|
|
}
|
|
}
|
|
|
|
func testCAConfig() *structs.CAConfiguration {
|
|
return &structs.CAConfiguration{
|
|
ClusterID: connect.TestClusterID,
|
|
Provider: "mock",
|
|
Config: map[string]interface{}{
|
|
"LeafCertTTL": "72h",
|
|
"IntermediateCertTTL": "2160h",
|
|
},
|
|
}
|
|
}
|
|
|
|
// initTestManager initializes a CAManager with a mockCAServerDelegate, consuming
|
|
// the ops that come through the channels and returning when initialization has finished.
|
|
func initTestManager(t *testing.T, manager *CAManager, delegate *mockCAServerDelegate) {
|
|
t.Helper()
|
|
initCh := make(chan struct{})
|
|
go func() {
|
|
require.NoError(t, manager.Initialize())
|
|
close(initCh)
|
|
}()
|
|
for i := 0; i < 5; i++ {
|
|
select {
|
|
case <-delegate.callbackCh:
|
|
case <-time.After(CATestTimeout):
|
|
t.Fatal("failed waiting for initialization events")
|
|
}
|
|
}
|
|
select {
|
|
case <-initCh:
|
|
case <-time.After(CATestTimeout):
|
|
t.Fatal("failed waiting for initialization")
|
|
}
|
|
}
|
|
|
|
func TestCAManager_Initialize(t *testing.T) {
|
|
conf := DefaultConfig()
|
|
conf.ConnectEnabled = true
|
|
conf.PrimaryDatacenter = "dc1"
|
|
conf.Datacenter = "dc2"
|
|
delegate := NewMockCAServerDelegate(t, conf)
|
|
delegate.secondaryIntermediate = delegate.primaryRoot.RootCert
|
|
manager := NewCAManager(delegate, nil, testutil.Logger(t), conf)
|
|
|
|
manager.providerShim = &mockCAProvider{
|
|
callbackCh: delegate.callbackCh,
|
|
rootPEM: delegate.primaryRoot.RootCert,
|
|
}
|
|
|
|
// Call Initialize and then confirm the RPCs and provider calls
|
|
// happen in the expected order.
|
|
require.Equal(t, caStateUninitialized, manager.state)
|
|
errCh := make(chan error)
|
|
go func() {
|
|
err := manager.Initialize()
|
|
assert.NoError(t, err)
|
|
errCh <- err
|
|
}()
|
|
|
|
waitForCh(t, delegate.callbackCh, "forwardDC/ConnectCA.Roots")
|
|
require.EqualValues(t, caStateInitializing, manager.state)
|
|
waitForCh(t, delegate.callbackCh, "provider/GenerateIntermediateCSR")
|
|
waitForCh(t, delegate.callbackCh, "forwardDC/ConnectCA.SignIntermediate")
|
|
waitForCh(t, delegate.callbackCh, "provider/SetIntermediate")
|
|
waitForCh(t, delegate.callbackCh, "raftApply/ConnectCA")
|
|
waitForEmptyCh(t, delegate.callbackCh)
|
|
|
|
// Make sure the Initialize call returned successfully.
|
|
select {
|
|
case err := <-errCh:
|
|
require.NoError(t, err)
|
|
case <-time.After(CATestTimeout):
|
|
t.Fatal("never got result from errCh")
|
|
}
|
|
|
|
require.Equal(t, caStateInitialized, manager.state)
|
|
}
|
|
|
|
func TestCAManager_UpdateConfigWhileRenewIntermediate(t *testing.T) {
|
|
|
|
// No parallel execution because we change globals
|
|
patchIntermediateCertRenewInterval(t)
|
|
|
|
conf := DefaultConfig()
|
|
conf.ConnectEnabled = true
|
|
conf.PrimaryDatacenter = "dc1"
|
|
conf.Datacenter = "dc2"
|
|
delegate := NewMockCAServerDelegate(t, conf)
|
|
delegate.secondaryIntermediate = delegate.primaryRoot.RootCert
|
|
manager := NewCAManager(delegate, nil, testutil.Logger(t), conf)
|
|
manager.providerShim = &mockCAProvider{
|
|
callbackCh: delegate.callbackCh,
|
|
rootPEM: delegate.primaryRoot.RootCert,
|
|
}
|
|
initTestManager(t, manager, delegate)
|
|
|
|
// Simulate Wait half the TTL for the cert to need renewing.
|
|
manager.timeNow = func() time.Time {
|
|
return time.Now().Add(500 * time.Millisecond)
|
|
}
|
|
|
|
// Call RenewIntermediate and then confirm the RPCs and provider calls
|
|
// happen in the expected order.
|
|
errCh := make(chan error)
|
|
go func() {
|
|
errCh <- manager.RenewIntermediate(context.TODO(), false)
|
|
}()
|
|
|
|
waitForCh(t, delegate.callbackCh, "provider/GenerateIntermediateCSR")
|
|
|
|
// Call UpdateConfiguration while RenewIntermediate is still in-flight to
|
|
// make sure we get an error about the state being occupied.
|
|
go func() {
|
|
require.EqualValues(t, caStateRenewIntermediate, manager.state)
|
|
require.Error(t, errors.New("already in state"), manager.UpdateConfiguration(&structs.CARequest{}))
|
|
}()
|
|
|
|
waitForCh(t, delegate.callbackCh, "forwardDC/ConnectCA.SignIntermediate")
|
|
waitForCh(t, delegate.callbackCh, "provider/SetIntermediate")
|
|
waitForCh(t, delegate.callbackCh, "raftApply/ConnectCA")
|
|
waitForEmptyCh(t, delegate.callbackCh)
|
|
|
|
// Make sure the RenewIntermediate call returned successfully.
|
|
select {
|
|
case err := <-errCh:
|
|
require.NoError(t, err)
|
|
case <-time.After(CATestTimeout):
|
|
t.Fatal("never got result from errCh")
|
|
}
|
|
|
|
require.EqualValues(t, caStateInitialized, manager.state)
|
|
}
|
|
|
|
func TestCAManager_SignCertificate_WithExpiredCert(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
args := []struct {
|
|
testName string
|
|
notBeforeRoot time.Time
|
|
notAfterRoot time.Time
|
|
notBeforeIntermediate time.Time
|
|
notAfterIntermediate time.Time
|
|
isError bool
|
|
errorMsg string
|
|
}{
|
|
{"intermediate valid", time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), false, ""},
|
|
{"intermediate expired", time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(-2, 0, 0), time.Now().AddDate(0, 0, -1), true, "intermediate expired: certificate expired, expiration date"},
|
|
{"root expired", time.Now().AddDate(-2, 0, 0), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), true, "root expired: certificate expired, expiration date"},
|
|
// a cert that is not yet valid is ok, assume it will be valid soon enough
|
|
{"intermediate in the future", time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(0, 0, 1), time.Now().AddDate(0, 0, 2), false, ""},
|
|
{"root in the future", time.Now().AddDate(0, 0, 1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), false, ""},
|
|
}
|
|
|
|
caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
|
|
require.NoError(t, err, "failed to generate key")
|
|
|
|
for _, arg := range args {
|
|
t.Run(arg.testName, func(t *testing.T) {
|
|
// No parallel execution because we change globals
|
|
// Set the interval and drift buffer low for renewing the cert.
|
|
origInterval := structs.IntermediateCertRenewInterval
|
|
origDriftBuffer := ca.CertificateTimeDriftBuffer
|
|
defer func() {
|
|
structs.IntermediateCertRenewInterval = origInterval
|
|
ca.CertificateTimeDriftBuffer = origDriftBuffer
|
|
}()
|
|
structs.IntermediateCertRenewInterval = time.Millisecond
|
|
ca.CertificateTimeDriftBuffer = 0
|
|
|
|
conf := DefaultConfig()
|
|
conf.ConnectEnabled = true
|
|
conf.PrimaryDatacenter = "dc1"
|
|
conf.Datacenter = "dc2"
|
|
|
|
rootPEM := generateCertPEM(t, caPrivKey, arg.notBeforeRoot, arg.notAfterRoot)
|
|
intermediatePEM := generateCertPEM(t, caPrivKey, arg.notBeforeIntermediate, arg.notAfterIntermediate)
|
|
|
|
delegate := NewMockCAServerDelegate(t, conf)
|
|
delegate.primaryRoot.RootCert = rootPEM
|
|
delegate.secondaryIntermediate = intermediatePEM
|
|
manager := NewCAManager(delegate, nil, testutil.Logger(t), conf)
|
|
|
|
manager.providerShim = &mockCAProvider{
|
|
callbackCh: delegate.callbackCh,
|
|
rootPEM: rootPEM,
|
|
intermediatePem: intermediatePEM,
|
|
}
|
|
initTestManager(t, manager, delegate)
|
|
|
|
// Simulate Wait half the TTL for the cert to need renewing.
|
|
manager.timeNow = func() time.Time {
|
|
return time.Now().UTC().Add(500 * time.Millisecond)
|
|
}
|
|
|
|
// Call RenewIntermediate and then confirm the RPCs and provider calls
|
|
// happen in the expected order.
|
|
|
|
_, err := manager.SignCertificate(&x509.CertificateRequest{}, &connect.SpiffeIDAgent{})
|
|
if arg.isError {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), arg.errorMsg)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func generateCertPEM(t *testing.T, caPrivKey *rsa.PrivateKey, notBefore time.Time, notAfter time.Time) string {
|
|
t.Helper()
|
|
ca := &x509.Certificate{
|
|
SerialNumber: big.NewInt(2019),
|
|
Subject: pkix.Name{
|
|
Organization: []string{"Company, INC."},
|
|
Country: []string{"US"},
|
|
Province: []string{""},
|
|
Locality: []string{"San Francisco"},
|
|
StreetAddress: []string{"Golden Gate Bridge"},
|
|
PostalCode: []string{"94016"},
|
|
},
|
|
NotBefore: notBefore,
|
|
NotAfter: notAfter,
|
|
IsCA: true,
|
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
|
|
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
|
|
BasicConstraintsValid: true,
|
|
URIs: []*url.URL{connect.SpiffeIDAgent{Host: "foo"}.URI()},
|
|
}
|
|
|
|
caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey)
|
|
require.NoError(t, err, "failed to create cert")
|
|
|
|
caPEM := new(bytes.Buffer)
|
|
err = pem.Encode(caPEM, &pem.Block{
|
|
Type: "CERTIFICATE",
|
|
Bytes: caBytes,
|
|
})
|
|
require.NoError(t, err, "failed to encode")
|
|
return caPEM.String()
|
|
}
|
|
|
|
func TestCADelegateWithState_GenerateCASignRequest(t *testing.T) {
|
|
s := Server{config: &Config{PrimaryDatacenter: "east"}, tokens: new(token.Store)}
|
|
d := &caDelegateWithState{Server: &s}
|
|
req := d.generateCASignRequest("A")
|
|
require.Equal(t, "east", req.RequestDatacenter())
|
|
}
|
|
|
|
func TestCAManager_Initialize_Logging(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
_, conf1 := testServerConfig(t)
|
|
|
|
// Setup dummy logger to catch output
|
|
var buf bytes.Buffer
|
|
logger := testutil.LoggerWithOutput(t, &buf)
|
|
|
|
deps := newDefaultDeps(t, conf1)
|
|
deps.Logger = logger
|
|
|
|
s1, err := NewServer(conf1, deps, grpc.NewServer())
|
|
require.NoError(t, err)
|
|
defer s1.Shutdown()
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
// Wait til CA root is setup
|
|
retry.Run(t, func(r *retry.R) {
|
|
var out structs.IndexedCARoots
|
|
r.Check(s1.RPC("ConnectCA.Roots", structs.DCSpecificRequest{
|
|
Datacenter: conf1.Datacenter,
|
|
}, &out))
|
|
})
|
|
|
|
require.Contains(t, buf.String(), "consul CA provider configured")
|
|
}
|
|
|
|
func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
|
|
ca.SkipIfVaultNotPresent(t)
|
|
vault := ca.NewTestVaultServer(t)
|
|
|
|
_, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
defer func() {
|
|
s1.Shutdown()
|
|
s1.leaderRoutineManager.Wait()
|
|
}()
|
|
|
|
testrpc.WaitForLeader(t, s1.RPC, "dc1")
|
|
|
|
_, origRoot, err := s1.fsm.State().CARootActive(nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, origRoot.IntermediateCerts, 1)
|
|
|
|
cert, err := connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(origRoot))
|
|
require.NoError(t, err)
|
|
require.Equal(t, connect.HexString(cert.SubjectKeyId), origRoot.SigningKeyID)
|
|
|
|
err = s1.caManager.UpdateConfiguration(&structs.CARequest{
|
|
Config: &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-root-2/",
|
|
"IntermediatePKIPath": "pki-intermediate-2/",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, newRoot, err := s1.fsm.State().CARootActive(nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, newRoot.IntermediateCerts, 2,
|
|
"expected one cross-sign cert and one local leaf sign cert")
|
|
require.NotEqual(t, origRoot.ID, newRoot.ID)
|
|
|
|
cert, err = connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(newRoot))
|
|
require.NoError(t, err)
|
|
require.Equal(t, connect.HexString(cert.SubjectKeyId), newRoot.SigningKeyID)
|
|
}
|
|
|
|
func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
vault := ca.NewTestVaultServer(t)
|
|
vclient := vault.Client()
|
|
generateExternalRootCA(t, vclient)
|
|
|
|
meshRootPath := "pki-root"
|
|
primaryCert := setupPrimaryCA(t, vclient, meshRootPath, "")
|
|
|
|
_, s1 := testServerWithConfig(t, func(c *Config) {
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": meshRootPath,
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
|
|
testutil.RunStep(t, "check primary DC", func(t *testing.T) {
|
|
testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
|
|
|
|
codec := rpcClient(t, s1)
|
|
roots := structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
require.Equal(t, primaryCert, roots.Roots[0].RootCert)
|
|
|
|
leafCertPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
|
|
verifyLeafCert(t, roots.Roots[0], leafCertPEM)
|
|
})
|
|
|
|
// TODO: renew primary leaf signing cert
|
|
// TODO: rotate root
|
|
|
|
testutil.RunStep(t, "run secondary DC", func(t *testing.T) {
|
|
_, sDC2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": meshRootPath,
|
|
"IntermediatePKIPath": "pki-secondary/",
|
|
},
|
|
}
|
|
})
|
|
defer sDC2.Shutdown()
|
|
joinWAN(t, sDC2, s1)
|
|
testrpc.WaitForActiveCARoot(t, sDC2.RPC, "dc2", nil)
|
|
|
|
codec := rpcClient(t, sDC2)
|
|
roots := structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
|
|
leafCertPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
|
|
verifyLeafCert(t, roots.Roots[0], leafCertPEM)
|
|
|
|
// TODO: renew secondary leaf signing cert
|
|
})
|
|
}
|
|
|
|
func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
vault := ca.NewTestVaultServer(t)
|
|
|
|
_, sDC1 := testServerWithConfig(t, func(c *Config) {
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
defer sDC1.Shutdown()
|
|
testrpc.WaitForActiveCARoot(t, sDC1.RPC, "dc1", nil)
|
|
|
|
_, sDC2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-root/",
|
|
"IntermediatePKIPath": "pki-intermediate-2/",
|
|
},
|
|
}
|
|
})
|
|
defer sDC2.Shutdown()
|
|
joinWAN(t, sDC2, sDC1)
|
|
testrpc.WaitForActiveCARoot(t, sDC2.RPC, "dc2", nil)
|
|
|
|
codec := rpcClient(t, sDC2)
|
|
var configBefore structs.CAConfiguration
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", &structs.DCSpecificRequest{}, &configBefore)
|
|
require.NoError(t, err)
|
|
|
|
renewLeafSigningCert(t, sDC1.caManager, sDC1.caManager.primaryRenewIntermediate)
|
|
|
|
// Give the secondary some time to notice the update
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
var configAfter structs.CAConfiguration
|
|
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", &structs.DCSpecificRequest{}, &configAfter)
|
|
require.NoError(t, err)
|
|
|
|
require.EqualValues(t, configBefore.ModifyIndex, configAfter.ModifyIndex)
|
|
}
|
|
|
|
func getLeafCert(t *testing.T, codec rpc.ClientCodec, trustDomain string, dc string) string {
|
|
pk, _, err := connect.GeneratePrivateKey()
|
|
require.NoError(t, err)
|
|
spiffeID := &connect.SpiffeIDService{
|
|
Host: trustDomain,
|
|
Service: "srv1",
|
|
Datacenter: dc,
|
|
}
|
|
csr, err := connect.CreateCSR(spiffeID, pk, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
req := structs.CASignRequest{CSR: csr}
|
|
cert := structs.IssuedCert{}
|
|
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", &req, &cert)
|
|
require.NoError(t, err)
|
|
return cert.CertPEM
|
|
}
|
|
|
|
func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
ca.SkipIfVaultNotPresent(t)
|
|
|
|
vault := ca.NewTestVaultServer(t)
|
|
vclient := vault.Client()
|
|
rootPEM := generateExternalRootCA(t, vclient)
|
|
|
|
primaryCAPath := "pki-primary"
|
|
primaryCert := setupPrimaryCA(t, vclient, primaryCAPath, rootPEM)
|
|
|
|
_, serverDC1 := testServerWithConfig(t, func(c *Config) {
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": primaryCAPath,
|
|
"IntermediatePKIPath": "pki-intermediate/",
|
|
},
|
|
}
|
|
})
|
|
testrpc.WaitForTestAgent(t, serverDC1.RPC, "dc1")
|
|
|
|
var origLeaf string
|
|
roots := structs.IndexedCARoots{}
|
|
testutil.RunStep(t, "verify primary DC", func(t *testing.T) {
|
|
codec := rpcClient(t, serverDC1)
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
require.Equal(t, primaryCert, roots.Roots[0].RootCert)
|
|
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
|
|
|
|
leafCert := getLeafCert(t, codec, roots.TrustDomain, "dc1")
|
|
verifyLeafCert(t, roots.Active(), leafCert)
|
|
origLeaf = leafCert
|
|
})
|
|
|
|
_, serverDC2 := testServerWithConfig(t, func(c *Config) {
|
|
c.Datacenter = "dc2"
|
|
c.PrimaryDatacenter = "dc1"
|
|
c.CAConfig = &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "should-be-ignored",
|
|
"IntermediatePKIPath": "pki-secondary/",
|
|
},
|
|
}
|
|
})
|
|
|
|
var origLeafSecondary string
|
|
testutil.RunStep(t, "start secondary DC", func(t *testing.T) {
|
|
joinWAN(t, serverDC2, serverDC1)
|
|
testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
|
|
|
|
codec := rpcClient(t, serverDC2)
|
|
roots = structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
|
|
verifyLeafCert(t, roots.Roots[0], leafPEM)
|
|
origLeafSecondary = leafPEM
|
|
})
|
|
|
|
testutil.RunStep(t, "renew leaf signing CA in primary", func(t *testing.T) {
|
|
previous := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
|
|
|
|
renewLeafSigningCert(t, serverDC1.caManager, serverDC1.caManager.primaryRenewIntermediate)
|
|
|
|
codec := rpcClient(t, serverDC1)
|
|
roots = structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
require.Len(t, roots.Roots[0].IntermediateCerts, 2)
|
|
|
|
newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
|
|
require.NotEqual(t, previous, newCert)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
|
|
verifyLeafCert(t, roots.Roots[0], leafPEM)
|
|
|
|
// original certs from old signing cert should still verify
|
|
verifyLeafCert(t, roots.Roots[0], origLeaf)
|
|
})
|
|
|
|
testutil.RunStep(t, "renew leaf signing CA in secondary", func(t *testing.T) {
|
|
previous := serverDC2.caManager.getLeafSigningCertFromRoot(roots.Active())
|
|
|
|
renewLeafSigningCert(t, serverDC2.caManager, serverDC2.caManager.secondaryRequestNewSigningCert)
|
|
|
|
codec := rpcClient(t, serverDC2)
|
|
roots = structs.IndexedCARoots{}
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 1)
|
|
// one intermediate from primary, two from secondary
|
|
require.Len(t, roots.Roots[0].IntermediateCerts, 3)
|
|
|
|
newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
|
|
require.NotEqual(t, previous, newCert)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
|
|
verifyLeafCert(t, roots.Roots[0], leafPEM)
|
|
|
|
// original certs from old signing cert should still verify
|
|
verifyLeafCert(t, roots.Roots[0], origLeaf)
|
|
})
|
|
|
|
testutil.RunStep(t, "rotate root by changing the provider", func(t *testing.T) {
|
|
codec := rpcClient(t, serverDC1)
|
|
req := &structs.CARequest{
|
|
Op: structs.CAOpSetConfig,
|
|
Config: &structs.CAConfiguration{
|
|
Provider: "consul",
|
|
},
|
|
}
|
|
var resp error
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", req, &resp)
|
|
require.NoError(t, err)
|
|
require.Nil(t, resp)
|
|
|
|
roots = structs.IndexedCARoots{}
|
|
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 2)
|
|
active := roots.Active()
|
|
require.Len(t, active.IntermediateCerts, 1)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
|
|
verifyLeafCert(t, roots.Active(), leafPEM)
|
|
|
|
// original certs from old root cert should still verify
|
|
verifyLeafCertWithRoots(t, roots, origLeaf)
|
|
|
|
// original certs from secondary should still verify
|
|
rootsSecondary := structs.IndexedCARoots{}
|
|
r := &structs.DCSpecificRequest{Datacenter: "dc2"}
|
|
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", r, &rootsSecondary)
|
|
require.NoError(t, err)
|
|
verifyLeafCertWithRoots(t, rootsSecondary, origLeafSecondary)
|
|
})
|
|
|
|
testutil.RunStep(t, "rotate to a different external root", func(t *testing.T) {
|
|
setupPrimaryCA(t, vclient, "pki-primary-2/", rootPEM)
|
|
|
|
codec := rpcClient(t, serverDC1)
|
|
req := &structs.CARequest{
|
|
Op: structs.CAOpSetConfig,
|
|
Config: &structs.CAConfiguration{
|
|
Provider: "vault",
|
|
Config: map[string]interface{}{
|
|
"Address": vault.Addr,
|
|
"Token": vault.RootToken,
|
|
"RootPKIPath": "pki-primary-2/",
|
|
"IntermediatePKIPath": "pki-intermediate-2/",
|
|
},
|
|
},
|
|
}
|
|
var resp error
|
|
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", req, &resp)
|
|
require.NoError(t, err)
|
|
require.Nil(t, resp)
|
|
|
|
roots = structs.IndexedCARoots{}
|
|
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
|
|
require.NoError(t, err)
|
|
require.Len(t, roots.Roots, 3)
|
|
active := roots.Active()
|
|
require.Len(t, active.IntermediateCerts, 2)
|
|
|
|
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
|
|
verifyLeafCert(t, roots.Active(), leafPEM)
|
|
|
|
// original certs from old root cert should still verify
|
|
verifyLeafCertWithRoots(t, roots, origLeaf)
|
|
|
|
// original certs from secondary should still verify
|
|
rootsSecondary := structs.IndexedCARoots{}
|
|
r := &structs.DCSpecificRequest{Datacenter: "dc2"}
|
|
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", r, &rootsSecondary)
|
|
require.NoError(t, err)
|
|
verifyLeafCertWithRoots(t, rootsSecondary, origLeafSecondary)
|
|
})
|
|
}
|
|
|
|
// renewLeafSigningCert mimics RenewIntermediate. This is unfortunate, but
|
|
// necessary for now as there is no easy way to invoke that logic unconditionally.
|
|
// Currently, it requires patching values and polling for the operation to
|
|
// complete, which adds a lot of distractions to a test case.
|
|
// With this function we can instead unconditionally rotate the leaf signing cert
|
|
// synchronously.
|
|
func renewLeafSigningCert(t *testing.T, manager *CAManager, fn func(ca.Provider, *structs.CARoot) error) {
|
|
t.Helper()
|
|
provider, _ := manager.getCAProvider()
|
|
|
|
store := manager.delegate.State()
|
|
_, root, err := store.CARootActive(nil)
|
|
require.NoError(t, err)
|
|
|
|
activeRoot := root.Clone()
|
|
err = fn(provider, activeRoot)
|
|
require.NoError(t, err)
|
|
err = manager.persistNewRootAndConfig(provider, activeRoot, nil)
|
|
require.NoError(t, err)
|
|
manager.setCAProvider(provider, activeRoot)
|
|
}
|
|
|
|
func generateExternalRootCA(t *testing.T, client *vaultapi.Client) string {
|
|
t.Helper()
|
|
err := client.Sys().Mount("corp", &vaultapi.MountInput{
|
|
Type: "pki",
|
|
Description: "External root, probably corporate CA",
|
|
Config: vaultapi.MountConfigInput{
|
|
MaxLeaseTTL: "2400h",
|
|
DefaultLeaseTTL: "1h",
|
|
},
|
|
})
|
|
require.NoError(t, err, "failed to mount")
|
|
|
|
resp, err := client.Logical().Write("corp/root/generate/internal", map[string]interface{}{
|
|
"common_name": "corporate CA",
|
|
"ttl": "2400h",
|
|
})
|
|
require.NoError(t, err, "failed to generate root")
|
|
return lib.EnsureTrailingNewline(resp.Data["certificate"].(string))
|
|
}
|
|
|
|
func setupPrimaryCA(t *testing.T, client *vaultapi.Client, path string, rootPEM string) string {
|
|
t.Helper()
|
|
err := client.Sys().Mount(path, &vaultapi.MountInput{
|
|
Type: "pki",
|
|
Description: "primary CA for Consul CA",
|
|
Config: vaultapi.MountConfigInput{
|
|
MaxLeaseTTL: "2200h",
|
|
DefaultLeaseTTL: "1h",
|
|
},
|
|
})
|
|
require.NoError(t, err, "failed to mount")
|
|
|
|
out, err := client.Logical().Write(path+"/intermediate/generate/internal", map[string]interface{}{
|
|
"common_name": "primary CA",
|
|
"ttl": "2200h",
|
|
"key_type": "ec",
|
|
"key_bits": 256,
|
|
})
|
|
require.NoError(t, err, "failed to generate root")
|
|
|
|
intermediate, err := client.Logical().Write("corp/root/sign-intermediate", map[string]interface{}{
|
|
"csr": out.Data["csr"],
|
|
"use_csr_values": true,
|
|
"format": "pem_bundle",
|
|
"ttl": "2200h",
|
|
})
|
|
require.NoError(t, err, "failed to sign intermediate")
|
|
|
|
var buf strings.Builder
|
|
buf.WriteString(lib.EnsureTrailingNewline(intermediate.Data["certificate"].(string)))
|
|
buf.WriteString(lib.EnsureTrailingNewline(rootPEM))
|
|
|
|
_, err = client.Logical().Write(path+"/intermediate/set-signed", map[string]interface{}{
|
|
"certificate": buf.String(),
|
|
})
|
|
require.NoError(t, err, "failed to set signed intermediate")
|
|
return lib.EnsureTrailingNewline(buf.String())
|
|
}
|