open-consul/agent/consul/leader_connect_ca_test.go

1332 lines
42 KiB
Go

package consul
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net/url"
"strings"
"testing"
"time"
msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc"
"github.com/hashicorp/consul-net-rpc/net/rpc"
vaultapi "github.com/hashicorp/vault/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/connect"
ca "github.com/hashicorp/consul/agent/connect/ca"
"github.com/hashicorp/consul/agent/consul/fsm"
"github.com/hashicorp/consul/agent/consul/state"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/token"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/hashicorp/consul/testrpc"
)
// TODO(kyhavlov): replace with t.Deadline()
const CATestTimeout = 7 * time.Second
func TestCAManager_Initialize_Vault_Secondary_SharedVault(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
ca.SkipIfVaultNotPresent(t)
vault := ca.NewTestVaultServer(t)
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-primary",
ConsulManaged: true,
})
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-secondary",
ConsulManaged: true,
})
_, serverDC1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": primaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-primary/",
},
}
})
testutil.RunStep(t, "check primary DC", func(t *testing.T) {
testrpc.WaitForTestAgent(t, serverDC1.RPC, "dc1")
codec := rpcClient(t, serverDC1)
roots := structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Roots[0], leafPEM)
})
testutil.RunStep(t, "start secondary DC", func(t *testing.T) {
_, serverDC2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc1"
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": secondaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-secondary/",
},
}
})
joinWAN(t, serverDC2, serverDC1)
testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
codec := rpcClient(t, serverDC2)
roots := structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
verifyLeafCert(t, roots.Roots[0], leafPEM)
})
}
func verifyLeafCert(t *testing.T, root *structs.CARoot, leafCertPEM string) {
t.Helper()
roots := structs.IndexedCARoots{
ActiveRootID: root.ID,
Roots: []*structs.CARoot{root},
}
verifyLeafCertWithRoots(t, roots, leafCertPEM)
}
func verifyLeafCertWithRoots(t *testing.T, roots structs.IndexedCARoots, leafCertPEM string) {
t.Helper()
leaf, intermediates, err := connect.ParseLeafCerts(leafCertPEM)
require.NoError(t, err)
pool := x509.NewCertPool()
for _, r := range roots.Roots {
ok := pool.AppendCertsFromPEM([]byte(r.RootCert))
if !ok {
t.Fatalf("Failed to add root CA PEM to cert pool")
}
}
// verify with intermediates from leaf CertPEM
_, err = leaf.Verify(x509.VerifyOptions{
Roots: pool,
Intermediates: intermediates,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
require.NoError(t, err, "failed to verify using intermediates from leaf cert PEM")
// verify with intermediates from the CARoot
intermediates = x509.NewCertPool()
for _, r := range roots.Roots {
for _, intermediate := range r.IntermediateCerts {
c, err := connect.ParseCert(intermediate)
require.NoError(t, err)
intermediates.AddCert(c)
}
}
_, err = leaf.Verify(x509.VerifyOptions{
Roots: pool,
Intermediates: intermediates,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
require.NoError(t, err, "failed to verify using intermediates from CARoot list")
}
type mockCAServerDelegate struct {
t *testing.T
config *Config
store *state.Store
primaryRoot *structs.CARoot
secondaryIntermediate string
callbackCh chan string
}
func NewMockCAServerDelegate(t *testing.T, config *Config) *mockCAServerDelegate {
delegate := &mockCAServerDelegate{
t: t,
config: config,
store: state.NewStateStore(nil),
primaryRoot: connect.TestCAWithTTL(t, nil, 1*time.Second),
callbackCh: make(chan string, 0),
}
delegate.store.CASetConfig(1, testCAConfig())
return delegate
}
func (m *mockCAServerDelegate) State() *state.Store {
return m.store
}
func (m *mockCAServerDelegate) ProviderState(id string) (*structs.CAConsulProviderState, error) {
_, s, err := m.store.CAProviderState(id)
return s, err
}
func (m *mockCAServerDelegate) IsLeader() bool {
return true
}
func (m *mockCAServerDelegate) ServersSupportMultiDCConnectCA() error {
return nil
}
func (m *mockCAServerDelegate) ApplyCALeafRequest() (uint64, error) {
return 3, nil
}
// ApplyCARequest mirrors FSM.applyConnectCAOperation because that functionality
// is not exported.
func (m *mockCAServerDelegate) ApplyCARequest(req *structs.CARequest) (interface{}, error) {
idx, _, err := m.store.CAConfig(nil)
if err != nil {
return nil, err
}
m.callbackCh <- fmt.Sprintf("raftApply/ConnectCA")
result := fsm.ApplyConnectCAOperationFromRequest(m.store, req, idx+1)
if err, ok := result.(error); ok && err != nil {
return nil, err
}
return result, nil
}
func (m *mockCAServerDelegate) forwardDC(method, dc string, args interface{}, reply interface{}) error {
switch method {
case "ConnectCA.Roots":
roots := reply.(*structs.IndexedCARoots)
roots.TrustDomain = connect.TestClusterID
roots.Roots = []*structs.CARoot{m.primaryRoot}
roots.ActiveRootID = m.primaryRoot.ID
case "ConnectCA.SignIntermediate":
r := reply.(*string)
*r = m.secondaryIntermediate
default:
return fmt.Errorf("received call to unsupported method %q", method)
}
m.callbackCh <- fmt.Sprintf("forwardDC/%s", method)
return nil
}
func (m *mockCAServerDelegate) generateCASignRequest(csr string) *structs.CASignRequest {
return &structs.CASignRequest{
Datacenter: m.config.PrimaryDatacenter,
CSR: csr,
}
}
// mockCAProvider mocks an empty provider implementation with a channel in order to coordinate
// waiting for certain methods to be called.
type mockCAProvider struct {
callbackCh chan string
rootPEM string
intermediatePem string
}
func (m *mockCAProvider) Configure(cfg ca.ProviderConfig) error { return nil }
func (m *mockCAProvider) State() (map[string]string, error) { return nil, nil }
func (m *mockCAProvider) GenerateRoot() (ca.RootResult, error) {
return ca.RootResult{PEM: m.rootPEM}, nil
}
func (m *mockCAProvider) GenerateIntermediateCSR() (string, string, error) {
m.callbackCh <- "provider/GenerateIntermediateCSR"
return "", "", nil
}
func (m *mockCAProvider) SetIntermediate(intermediatePEM, rootPEM, _ string) error {
m.callbackCh <- "provider/SetIntermediate"
return nil
}
func (m *mockCAProvider) ActiveIntermediate() (string, error) {
if m.intermediatePem == "" {
return m.rootPEM, nil
}
return m.intermediatePem, nil
}
func (m *mockCAProvider) GenerateIntermediate() (string, error) { return "", nil }
func (m *mockCAProvider) Sign(*x509.CertificateRequest) (string, error) { return "", nil }
func (m *mockCAProvider) SignIntermediate(*x509.CertificateRequest) (string, error) { return "", nil }
func (m *mockCAProvider) CrossSignCA(*x509.Certificate) (string, error) { return "", nil }
func (m *mockCAProvider) SupportsCrossSigning() (bool, error) { return false, nil }
func (m *mockCAProvider) Cleanup(_ bool, _ map[string]interface{}) error { return nil }
func waitForCh(t *testing.T, ch chan string, expected string) {
t.Helper()
select {
case op := <-ch:
if op != expected {
t.Fatalf("got unexpected op %q, wanted %q", op, expected)
}
case <-time.After(CATestTimeout):
t.Fatalf("never got op %q", expected)
}
}
func waitForEmptyCh(t *testing.T, ch chan string) {
select {
case op := <-ch:
t.Fatalf("got unexpected op %q", op)
case <-time.After(1 * time.Second):
}
}
func testCAConfig() *structs.CAConfiguration {
return &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: "mock",
Config: map[string]interface{}{
"LeafCertTTL": "72h",
"IntermediateCertTTL": "2160h",
},
}
}
// initTestManager initializes a CAManager with a mockCAServerDelegate, consuming
// the ops that come through the channels and returning when initialization has finished.
func initTestManager(t *testing.T, manager *CAManager, delegate *mockCAServerDelegate) {
t.Helper()
initCh := make(chan struct{})
go func() {
require.NoError(t, manager.Initialize())
close(initCh)
}()
for i := 0; i < 5; i++ {
select {
case <-delegate.callbackCh:
case <-time.After(CATestTimeout):
t.Fatal("failed waiting for initialization events")
}
}
select {
case <-initCh:
case <-time.After(CATestTimeout):
t.Fatal("failed waiting for initialization")
}
}
func TestCAManager_Initialize(t *testing.T) {
conf := DefaultConfig()
conf.ConnectEnabled = true
conf.PrimaryDatacenter = "dc1"
conf.Datacenter = "dc2"
delegate := NewMockCAServerDelegate(t, conf)
delegate.secondaryIntermediate = delegate.primaryRoot.RootCert
manager := NewCAManager(delegate, nil, testutil.Logger(t), conf)
manager.providerShim = &mockCAProvider{
callbackCh: delegate.callbackCh,
rootPEM: delegate.primaryRoot.RootCert,
}
// Call Initialize and then confirm the RPCs and provider calls
// happen in the expected order.
require.Equal(t, caStateUninitialized, manager.state)
errCh := make(chan error)
go func() {
err := manager.Initialize()
assert.NoError(t, err)
errCh <- err
}()
waitForCh(t, delegate.callbackCh, "forwardDC/ConnectCA.Roots")
require.EqualValues(t, caStateInitializing, manager.state)
waitForCh(t, delegate.callbackCh, "provider/GenerateIntermediateCSR")
waitForCh(t, delegate.callbackCh, "forwardDC/ConnectCA.SignIntermediate")
waitForCh(t, delegate.callbackCh, "provider/SetIntermediate")
waitForCh(t, delegate.callbackCh, "raftApply/ConnectCA")
waitForEmptyCh(t, delegate.callbackCh)
// Make sure the Initialize call returned successfully.
select {
case err := <-errCh:
require.NoError(t, err)
case <-time.After(CATestTimeout):
t.Fatal("never got result from errCh")
}
require.Equal(t, caStateInitialized, manager.state)
}
func TestCAManager_UpdateConfigWhileRenewIntermediate(t *testing.T) {
// No parallel execution because we change globals
patchIntermediateCertRenewInterval(t)
conf := DefaultConfig()
conf.ConnectEnabled = true
conf.PrimaryDatacenter = "dc1"
conf.Datacenter = "dc2"
delegate := NewMockCAServerDelegate(t, conf)
delegate.secondaryIntermediate = delegate.primaryRoot.RootCert
manager := NewCAManager(delegate, nil, testutil.Logger(t), conf)
manager.providerShim = &mockCAProvider{
callbackCh: delegate.callbackCh,
rootPEM: delegate.primaryRoot.RootCert,
}
initTestManager(t, manager, delegate)
// Simulate Wait half the TTL for the cert to need renewing.
manager.timeNow = func() time.Time {
return time.Now().Add(500 * time.Millisecond)
}
// Call RenewIntermediate and then confirm the RPCs and provider calls
// happen in the expected order.
errCh := make(chan error)
go func() {
errCh <- manager.RenewIntermediate(context.TODO())
}()
waitForCh(t, delegate.callbackCh, "provider/GenerateIntermediateCSR")
// Call UpdateConfiguration while RenewIntermediate is still in-flight to
// make sure we get an error about the state being occupied.
go func() {
require.EqualValues(t, caStateRenewIntermediate, manager.state)
require.Error(t, errors.New("already in state"), manager.UpdateConfiguration(&structs.CARequest{}))
}()
waitForCh(t, delegate.callbackCh, "forwardDC/ConnectCA.SignIntermediate")
waitForCh(t, delegate.callbackCh, "provider/SetIntermediate")
waitForCh(t, delegate.callbackCh, "raftApply/ConnectCA")
waitForEmptyCh(t, delegate.callbackCh)
// Make sure the RenewIntermediate call returned successfully.
select {
case err := <-errCh:
require.NoError(t, err)
case <-time.After(CATestTimeout):
t.Fatal("never got result from errCh")
}
require.EqualValues(t, caStateInitialized, manager.state)
}
func TestCAManager_SignCertificate_WithExpiredCert(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
args := []struct {
testName string
notBeforeRoot time.Time
notAfterRoot time.Time
notBeforeIntermediate time.Time
notAfterIntermediate time.Time
isError bool
errorMsg string
}{
{"intermediate valid", time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), false, ""},
{"root expired", time.Now().AddDate(-2, 0, 0), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), true, "root expired: certificate expired, expiration date"},
// a cert that is not yet valid is ok, assume it will be valid soon enough
{"intermediate in the future", time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(0, 0, 1), time.Now().AddDate(0, 0, 2), false, ""},
{"root in the future", time.Now().AddDate(0, 0, 1), time.Now().AddDate(0, 0, 2), time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 2), false, ""},
}
caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
require.NoError(t, err, "failed to generate key")
for _, arg := range args {
t.Run(arg.testName, func(t *testing.T) {
// No parallel execution because we change globals
// Set the interval and drift buffer low for renewing the cert.
origInterval := structs.IntermediateCertRenewInterval
origDriftBuffer := ca.CertificateTimeDriftBuffer
defer func() {
structs.IntermediateCertRenewInterval = origInterval
ca.CertificateTimeDriftBuffer = origDriftBuffer
}()
structs.IntermediateCertRenewInterval = time.Millisecond
ca.CertificateTimeDriftBuffer = 0
conf := DefaultConfig()
conf.ConnectEnabled = true
conf.PrimaryDatacenter = "dc1"
conf.Datacenter = "dc2"
rootPEM := generateCertPEM(t, caPrivKey, arg.notBeforeRoot, arg.notAfterRoot)
intermediatePEM := generateCertPEM(t, caPrivKey, arg.notBeforeIntermediate, arg.notAfterIntermediate)
delegate := NewMockCAServerDelegate(t, conf)
delegate.primaryRoot.RootCert = rootPEM
delegate.secondaryIntermediate = intermediatePEM
manager := NewCAManager(delegate, nil, testutil.Logger(t), conf)
manager.providerShim = &mockCAProvider{
callbackCh: delegate.callbackCh,
rootPEM: rootPEM,
intermediatePem: intermediatePEM,
}
initTestManager(t, manager, delegate)
// Simulate Wait half the TTL for the cert to need renewing.
manager.timeNow = func() time.Time {
return time.Now().UTC().Add(500 * time.Millisecond)
}
// Call RenewIntermediate and then confirm the RPCs and provider calls
// happen in the expected order.
_, err := manager.SignCertificate(&x509.CertificateRequest{}, &connect.SpiffeIDAgent{})
if arg.isError {
require.Error(t, err)
require.Contains(t, err.Error(), arg.errorMsg)
} else {
require.NoError(t, err)
}
})
}
}
func generateCertPEM(t *testing.T, caPrivKey *rsa.PrivateKey, notBefore time.Time, notAfter time.Time) string {
t.Helper()
ca := &x509.Certificate{
SerialNumber: big.NewInt(2019),
Subject: pkix.Name{
Organization: []string{"Company, INC."},
Country: []string{"US"},
Province: []string{""},
Locality: []string{"San Francisco"},
StreetAddress: []string{"Golden Gate Bridge"},
PostalCode: []string{"94016"},
},
NotBefore: notBefore,
NotAfter: notAfter,
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
URIs: []*url.URL{connect.SpiffeIDAgent{Host: "foo"}.URI()},
}
caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey)
require.NoError(t, err, "failed to create cert")
caPEM := new(bytes.Buffer)
err = pem.Encode(caPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
require.NoError(t, err, "failed to encode")
return caPEM.String()
}
func TestCADelegateWithState_GenerateCASignRequest(t *testing.T) {
s := Server{config: &Config{PrimaryDatacenter: "east"}, tokens: new(token.Store)}
d := &caDelegateWithState{Server: &s}
req := d.generateCASignRequest("A")
require.Equal(t, "east", req.RequestDatacenter())
}
func TestCAManager_Initialize_Logging(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
t.Parallel()
_, conf1 := testServerConfig(t)
// Setup dummy logger to catch output
var buf bytes.Buffer
logger := testutil.LoggerWithOutput(t, &buf)
deps := newDefaultDeps(t, conf1)
deps.Logger = logger
s1, err := NewServer(conf1, deps, grpc.NewServer())
require.NoError(t, err)
defer s1.Shutdown()
testrpc.WaitForLeader(t, s1.RPC, "dc1")
// Wait til CA root is setup
retry.Run(t, func(r *retry.R) {
var out structs.IndexedCARoots
r.Check(s1.RPC("ConnectCA.Roots", structs.DCSpecificRequest{
Datacenter: conf1.Datacenter,
}, &out))
})
require.Contains(t, buf.String(), "consul CA provider configured")
}
func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) {
ca.SkipIfVaultNotPresent(t)
vault := ca.NewTestVaultServer(t)
vaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
WithSudo: true,
})
_, s1 := testServerWithConfig(t, func(c *Config) {
c.PrimaryDatacenter = "dc1"
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
}
})
defer func() {
s1.Shutdown()
s1.leaderRoutineManager.Wait()
}()
testrpc.WaitForLeader(t, s1.RPC, "dc1")
_, origRoot, err := s1.fsm.State().CARootActive(nil)
require.NoError(t, err)
require.Len(t, origRoot.IntermediateCerts, 1)
cert, err := connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(origRoot))
require.NoError(t, err)
require.Equal(t, connect.HexString(cert.SubjectKeyId), origRoot.SigningKeyID)
vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root-2",
IntermediatePath: "pki-intermediate-2",
ConsulManaged: true,
})
err = s1.caManager.UpdateConfiguration(&structs.CARequest{
Config: &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": vaultToken2,
"RootPKIPath": "pki-root-2/",
"IntermediatePKIPath": "pki-intermediate-2/",
},
},
})
require.NoError(t, err)
_, newRoot, err := s1.fsm.State().CARootActive(nil)
require.NoError(t, err)
require.Len(t, newRoot.IntermediateCerts, 2,
"expected one cross-sign cert and one local leaf sign cert")
require.NotEqual(t, origRoot.ID, newRoot.ID)
cert, err = connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(newRoot))
require.NoError(t, err)
require.Equal(t, connect.HexString(cert.SubjectKeyId), newRoot.SigningKeyID)
}
func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
ca.SkipIfVaultNotPresent(t)
vault := ca.NewTestVaultServer(t)
vclient := vault.Client()
generateExternalRootCA(t, vclient)
meshRootPath := "pki-root"
primaryCert := setupPrimaryCA(t, vclient, meshRootPath, "")
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: meshRootPath,
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
_, s1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": primaryVaultToken,
"RootPKIPath": meshRootPath,
"IntermediatePKIPath": "pki-intermediate/",
},
}
})
testutil.RunStep(t, "check primary DC", func(t *testing.T) {
testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
codec := rpcClient(t, s1)
roots := structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
require.Equal(t, primaryCert, roots.Roots[0].RootCert)
leafCertPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Roots[0], leafCertPEM)
})
// TODO: renew primary leaf signing cert
// TODO: rotate root
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: meshRootPath,
IntermediatePath: "pki-secondary",
ConsulManaged: true,
})
testutil.RunStep(t, "run secondary DC", func(t *testing.T) {
_, sDC2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc1"
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": secondaryVaultToken,
"RootPKIPath": meshRootPath,
"IntermediatePKIPath": "pki-secondary/",
},
}
})
defer sDC2.Shutdown()
joinWAN(t, sDC2, s1)
testrpc.WaitForActiveCARoot(t, sDC2.RPC, "dc2", nil)
codec := rpcClient(t, sDC2)
roots := structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1)
leafCertPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
verifyLeafCert(t, roots.Roots[0], leafCertPEM)
// TODO: renew secondary leaf signing cert
})
}
func TestCAManager_Verify_Vault_NoChangeToSecondaryConfig(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
ca.SkipIfVaultNotPresent(t)
vault := ca.NewTestVaultServer(t)
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
})
_, sDC1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": primaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate/",
},
}
})
defer sDC1.Shutdown()
testrpc.WaitForActiveCARoot(t, sDC1.RPC, "dc1", nil)
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-root",
IntermediatePath: "pki-intermediate-2",
ConsulManaged: true,
})
_, sDC2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc1"
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": secondaryVaultToken,
"RootPKIPath": "pki-root/",
"IntermediatePKIPath": "pki-intermediate-2/",
},
}
})
defer sDC2.Shutdown()
joinWAN(t, sDC2, sDC1)
testrpc.WaitForActiveCARoot(t, sDC2.RPC, "dc2", nil)
codec := rpcClient(t, sDC2)
var configBefore structs.CAConfiguration
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", &structs.DCSpecificRequest{}, &configBefore)
require.NoError(t, err)
require.NoError(t, sDC1.caManager.renewIntermediateNow(context.Background()))
// Give the secondary some time to notice the update
time.Sleep(100 * time.Millisecond)
var configAfter structs.CAConfiguration
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", &structs.DCSpecificRequest{}, &configAfter)
require.NoError(t, err)
require.EqualValues(t, configBefore.ModifyIndex, configAfter.ModifyIndex)
}
func getLeafCert(t *testing.T, codec rpc.ClientCodec, trustDomain string, dc string) string {
pk, _, err := connect.GeneratePrivateKey()
require.NoError(t, err)
spiffeID := &connect.SpiffeIDService{
Host: trustDomain,
Service: "srv1",
Datacenter: dc,
}
csr, err := connect.CreateCSR(spiffeID, pk, nil, nil)
require.NoError(t, err)
req := structs.CASignRequest{CSR: csr}
cert := structs.IssuedCert{}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", &req, &cert)
require.NoError(t, err)
return cert.CertPEM
}
func TestCAManager_Initialize_Vault_WithExternalTrustedCA(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
ca.SkipIfVaultNotPresent(t)
vault := ca.NewTestVaultServer(t)
vclient := vault.Client()
rootPEM := generateExternalRootCA(t, vclient)
primaryCAPath := "pki-primary"
primaryCert := setupPrimaryCA(t, vclient, primaryCAPath, rootPEM)
primaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: primaryCAPath,
IntermediatePath: "pki-intermediate",
ConsulManaged: true,
WithSudo: true,
})
_, serverDC1 := testServerWithConfig(t, func(c *Config) {
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": primaryVaultToken,
"RootPKIPath": primaryCAPath,
"IntermediatePKIPath": "pki-intermediate/",
},
}
})
testrpc.WaitForTestAgent(t, serverDC1.RPC, "dc1")
testrpc.WaitForActiveCARoot(t, serverDC1.RPC, "dc1", nil)
var (
origLeaf string
primaryLeafSigningCert string
)
roots := structs.IndexedCARoots{}
testutil.RunStep(t, "verify primary DC", func(t *testing.T) {
codec := rpcClient(t, serverDC1)
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
// Verify CA trust heirarchy is expected.
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 1, "should just have the primary's intermediate")
active := roots.Active()
leafCert := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, active, leafCert)
origLeaf = leafCert
primaryLeafSigningCert = serverDC1.caManager.getLeafSigningCertFromRoot(active)
})
secondaryVaultToken := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "should-be-ignored",
IntermediatePath: "pki-secondary",
ConsulManaged: true,
})
_, serverDC2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc1"
c.CAConfig = &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": secondaryVaultToken,
"RootPKIPath": "should-be-ignored",
"IntermediatePKIPath": "pki-secondary/",
},
}
})
joinWAN(t, serverDC2, serverDC1)
testrpc.WaitForActiveCARoot(t, serverDC2.RPC, "dc2", nil)
var (
origLeafSecondary string
secondaryLeafSigningCert string
)
testutil.RunStep(t, "start secondary DC", func(t *testing.T) {
codec := rpcClient(t, serverDC2)
roots = structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 2, "should have the primary's intermediate and our own")
active := roots.Active()
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
verifyLeafCert(t, roots.Roots[0], leafPEM)
origLeafSecondary = leafPEM
secondaryLeafSigningCert = serverDC2.caManager.getLeafSigningCertFromRoot(active)
})
testutil.RunStep(t, "renew leaf signing CA in primary", func(t *testing.T) {
require.NoError(t, serverDC1.caManager.renewIntermediateNow(context.Background()))
codec := rpcClient(t, serverDC1)
roots = structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 2, "we renewed, so we have our old primary and our new primary intermediate")
active := roots.Active()
newCert := serverDC1.caManager.getLeafSigningCertFromRoot(roots.Active())
require.NotEqual(t, primaryLeafSigningCert, newCert)
primaryLeafSigningCert = newCert
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, active, leafPEM)
// original certs from old signing cert should still verify
verifyLeafCert(t, active, origLeaf)
})
var oldSecondaryData *structs.CARoot
testutil.RunStep(t, "renew leaf signing CA in secondary", func(t *testing.T) {
require.NoError(t, serverDC2.caManager.renewIntermediateNow(context.Background()))
codec := rpcClient(t, serverDC2)
roots = structs.IndexedCARoots{}
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 1, "should have one because there's no provider rotation yet")
require.Equal(t, primaryCert, roots.Roots[0].RootCert, "should be the offline root")
require.Contains(t, roots.Roots[0].RootCert, rootPEM)
require.Len(t, roots.Roots[0].IntermediateCerts, 3, "one intermediate from primary, two from secondary")
active := roots.Active()
oldSecondaryData = active
newCert := serverDC2.caManager.getLeafSigningCertFromRoot(active)
require.NotEqual(t, secondaryLeafSigningCert, newCert)
secondaryLeafSigningCert = newCert
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc2")
verifyLeafCert(t, active, leafPEM)
// original certs from old signing cert should still verify
verifyLeafCert(t, active, origLeaf)
})
testutil.RunStep(t, "rotate root by changing the provider", func(t *testing.T) {
codec := rpcClient(t, serverDC1)
req := &structs.CARequest{
Op: structs.CAOpSetConfig,
Config: &structs.CAConfiguration{
Provider: "consul",
},
}
var resp error
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", req, &resp)
require.NoError(t, err)
require.Nil(t, resp)
roots = structs.IndexedCARoots{}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 2, "two because we rotated the provider")
active := roots.Active()
require.NotEqual(t, primaryCert, active.RootCert, "should NOT be the offline root, because we switched")
require.NotContains(t, active.RootCert, rootPEM)
require.Len(t, active.IntermediateCerts, 1, "only one new intermediate in the primary")
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Active(), leafPEM)
// wait for secondary to witness it
codec2 := rpcClient(t, serverDC2)
retry.Run(t, func(r *retry.R) {
var reply structs.IndexedCARoots
err := msgpackrpc.CallWithCodec(codec2, "ConnectCA.Roots", &structs.DCSpecificRequest{
Datacenter: "dc2",
}, &reply)
require.NoError(r, err)
require.Len(r, reply.Roots, 2, "primary provider rotated, so secondary gets rekeyed")
active := reply.Active()
require.NotNil(r, active)
if oldSecondaryData.ID == reply.ActiveRootID {
r.Fatal("wait; did not witness primary root rotation yet")
}
newCert := serverDC2.caManager.getLeafSigningCertFromRoot(roots.Active())
require.NotEqual(r, secondaryLeafSigningCert, newCert)
secondaryLeafSigningCert = newCert
})
// original certs from old root cert should still verify
verifyLeafCertWithRoots(t, roots, origLeaf)
// original certs from secondary should still verify
rootsSecondary := structs.IndexedCARoots{}
r := &structs.DCSpecificRequest{Datacenter: "dc2"}
err = msgpackrpc.CallWithCodec(codec2, "ConnectCA.Roots", r, &rootsSecondary)
require.NoError(t, err)
verifyLeafCertWithRoots(t, rootsSecondary, origLeafSecondary)
})
testutil.RunStep(t, "rotate to a different external root", func(t *testing.T) {
setupPrimaryCA(t, vclient, "pki-primary-2/", rootPEM)
primaryVaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{
RootPath: "pki-primary-2",
IntermediatePath: "pki-intermediate-2",
ConsulManaged: true,
})
codec := rpcClient(t, serverDC1)
req := &structs.CARequest{
Op: structs.CAOpSetConfig,
Config: &structs.CAConfiguration{
Provider: "vault",
Config: map[string]interface{}{
"Address": vault.Addr,
"Token": primaryVaultToken2,
"RootPKIPath": "pki-primary-2/",
"IntermediatePKIPath": "pki-intermediate-2/",
},
},
}
var resp error
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", req, &resp)
require.NoError(t, err)
require.Nil(t, resp)
roots = structs.IndexedCARoots{}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(t, err)
require.Len(t, roots.Roots, 3)
active := roots.Active()
require.Len(t, active.IntermediateCerts, 2)
leafPEM := getLeafCert(t, codec, roots.TrustDomain, "dc1")
verifyLeafCert(t, roots.Active(), leafPEM)
// original certs from old root cert should still verify
verifyLeafCertWithRoots(t, roots, origLeaf)
// original certs from secondary should still verify
rootsSecondary := structs.IndexedCARoots{}
r := &structs.DCSpecificRequest{Datacenter: "dc2"}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", r, &rootsSecondary)
require.NoError(t, err)
verifyLeafCertWithRoots(t, rootsSecondary, origLeafSecondary)
})
}
func generateExternalRootCA(t *testing.T, client *vaultapi.Client) string {
t.Helper()
err := client.Sys().Mount("corp", &vaultapi.MountInput{
Type: "pki",
Description: "External root, probably corporate CA",
Config: vaultapi.MountConfigInput{
MaxLeaseTTL: "2400h",
DefaultLeaseTTL: "1h",
},
})
require.NoError(t, err, "failed to mount")
resp, err := client.Logical().Write("corp/root/generate/internal", map[string]interface{}{
"common_name": "corporate CA",
"ttl": "2400h",
})
require.NoError(t, err, "failed to generate root")
return lib.EnsureTrailingNewline(resp.Data["certificate"].(string))
}
func setupPrimaryCA(t *testing.T, client *vaultapi.Client, path string, rootPEM string) string {
t.Helper()
err := client.Sys().Mount(path, &vaultapi.MountInput{
Type: "pki",
Description: "primary CA for Consul CA",
Config: vaultapi.MountConfigInput{
MaxLeaseTTL: "2200h",
DefaultLeaseTTL: "1h",
},
})
require.NoError(t, err, "failed to mount")
out, err := client.Logical().Write(path+"/intermediate/generate/internal", map[string]interface{}{
"common_name": "primary CA",
"ttl": "2200h",
"key_type": "ec",
"key_bits": 256,
})
require.NoError(t, err, "failed to generate root")
intermediate, err := client.Logical().Write("corp/root/sign-intermediate", map[string]interface{}{
"csr": out.Data["csr"],
"use_csr_values": true,
"format": "pem_bundle",
"ttl": "2200h",
})
require.NoError(t, err, "failed to sign intermediate")
cert := intermediate.Data["certificate"].(string)
var buf strings.Builder
buf.WriteString(lib.EnsureTrailingNewline(cert))
if !strings.Contains(strings.TrimSpace(cert), strings.TrimSpace(rootPEM)) {
// Vault < v1.11 included the root in the output of sign-intermediate.
buf.WriteString(lib.EnsureTrailingNewline(rootPEM))
}
_, err = client.Logical().Write(path+"/intermediate/set-signed", map[string]interface{}{
"certificate": buf.String(),
})
require.NoError(t, err, "failed to set signed intermediate")
// TODO: also fix issuers here?
return lib.EnsureTrailingNewline(buf.String())
}
func TestCAManager_Sign_SpiffeIDServer(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
_, s1 := testServerWithConfig(t)
testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
codec := rpcClient(t, s1)
roots := structs.IndexedCARoots{}
retry.Run(t, func(r *retry.R) {
err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", &structs.DCSpecificRequest{}, &roots)
require.NoError(r, err)
require.Len(r, roots.Roots, 1)
})
pk, _, err := connect.GeneratePrivateKey()
require.NoError(t, err)
// Request a leaf certificate for a server.
spiffeID := &connect.SpiffeIDServer{
Host: roots.TrustDomain,
Datacenter: "dc1",
}
csr, err := connect.CreateCSR(spiffeID, pk, nil, nil)
require.NoError(t, err)
req := structs.CASignRequest{CSR: csr}
cert := structs.IssuedCert{}
err = msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", &req, &cert)
require.NoError(t, err)
// Verify the chain of trust.
verifyLeafCert(t, roots.Roots[0], cert.CertPEM)
// Verify the Server's URI.
require.Equal(t, fmt.Sprintf("spiffe://%s/agent/server/dc/dc1", roots.TrustDomain), cert.ServerURI)
}
func TestCAManager_AuthorizeAndSignCertificate(t *testing.T) {
conf := DefaultConfig()
conf.PrimaryDatacenter = "dc1"
conf.Datacenter = "dc2"
manager := NewCAManager(nil, nil, testutil.Logger(t), conf)
agentURL := connect.SpiffeIDAgent{
Agent: "test-agent",
Datacenter: conf.PrimaryDatacenter,
Host: "test-host",
}.URI()
serviceURL := connect.SpiffeIDService{
Datacenter: conf.PrimaryDatacenter,
Namespace: "ns1",
Service: "test-service",
}.URI()
meshURL := connect.SpiffeIDMeshGateway{
Datacenter: conf.PrimaryDatacenter,
Host: "test-host",
Partition: "test-partition",
}.URI()
tests := []struct {
name string
expectErr string
getCSR func() *x509.CertificateRequest
authAllow bool
}{
{
name: "err_not_one_uri",
expectErr: "CSR SAN contains an invalid number of URIs",
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{agentURL, agentURL},
}
},
},
{
name: "err_email",
expectErr: "CSR SAN does not allow specifying email addresses",
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{agentURL},
EmailAddresses: []string{"test@example.com"},
}
},
},
{
name: "err_invalid_spiffe_id",
expectErr: "SPIFFE ID is not in the expected format",
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{connect.SpiffeIDAgent{}.URI()},
}
},
},
{
name: "err_service_write_not_allowed",
expectErr: "Permission denied",
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{serviceURL},
}
},
},
{
name: "err_service_different_dc",
expectErr: "SPIFFE ID in CSR from a different datacenter",
authAllow: true,
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{serviceURL},
}
},
},
{
name: "err_agent_write_not_allowed",
expectErr: "Permission denied",
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{agentURL},
}
},
},
{
name: "err_meshgw_write_not_allowed",
expectErr: "Permission denied",
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{meshURL},
}
},
},
{
name: "err_meshgw_different_dc",
expectErr: "SPIFFE ID in CSR from a different datacenter",
authAllow: true,
getCSR: func() *x509.CertificateRequest {
return &x509.CertificateRequest{
URIs: []*url.URL{meshURL},
}
},
},
{
name: "err_invalid_spiffe_type",
expectErr: "SPIFFE ID in CSR must be a service, mesh-gateway, or agent ID",
getCSR: func() *x509.CertificateRequest {
u := connect.SpiffeIDSigning{
ClusterID: "test-cluster-id",
Domain: "test-domain",
}.URI()
return &x509.CertificateRequest{
URIs: []*url.URL{u},
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
authz := acl.DenyAll()
if tc.authAllow {
authz = acl.AllowAll()
}
cert, err := manager.AuthorizeAndSignCertificate(tc.getCSR(), authz)
if tc.expectErr != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tc.expectErr)
} else {
require.NoError(t, err)
require.NotNil(t, cert)
}
})
}
}