Support vault namespaces in connect CA (#12904)

* Support vault namespaces in connect CA

Follow on to some missed items from #12655

From an internal ticket "Support standard "Vault namespace in the
path" semantics for Connect Vault CA Provider"

Vault allows the namespace to be specified as a prefix in the path of
a PKI definition, but our usage of the Vault API includes calls that
don't support a namespaced key. In particular the sys.* family of
calls simply appends the key, instead of prefixing the namespace in
front of the path.

Unfortunately it is difficult to reliably parse a path with a
namespace; only vault knows what namespaces are present, and the '/'
separator can be inside a key name, as well as separating path
elements. This is in use in the wild; for example
'dc1/intermediate-key' is a relatively common naming schema.

Instead we add two new fields: RootPKINamespace and
IntermediatePKINamespace, which are the absolute namespace paths
'prefixed' in front of the respective PKI Paths.

Signed-off-by: Mark Anderson <manderson@hashicorp.com>

.changelog/12904.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+Support Vault namespaces in Connect CA by adding RootPKINamespace and
+IntermediatePKINamespace fields to the config.
+```

agent/config/builder.go

@@ -697,16 +697,18 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 			"intermediate_cert_ttl": "IntermediateCertTTL",
 
 			// Vault CA config
-			"address":               "Address",
+			"address":                    "Address",
-			"token":                 "Token",
+			"token":                      "Token",
-			"root_pki_path":         "RootPKIPath",
+			"root_pki_path":              "RootPKIPath",
-			"intermediate_pki_path": "IntermediatePKIPath",
+			"root_pki_namespace":         "RootPKINamespace",
-			"ca_file":               "CAFile",
+			"intermediate_pki_path":      "IntermediatePKIPath",
-			"ca_path":               "CAPath",
+			"intermediate_pki_namespace": "IntermediatePKINamespace",
-			"cert_file":             "CertFile",
+			"ca_file":                    "CAFile",
-			"key_file":              "KeyFile",
+			"ca_path":                    "CAPath",
-			"tls_server_name":       "TLSServerName",
+			"cert_file":                  "CertFile",
-			"tls_skip_verify":       "TLSSkipVerify",
+			"key_file":                   "KeyFile",
+			"tls_server_name":            "TLSServerName",
+			"tls_skip_verify":            "TLSSkipVerify",
 
 			// AWS CA config
 			"existing_arn":   "ExistingARN",

agent/connect/ca/provider.go

@@ -94,7 +94,7 @@ type Provider interface {
 	// Sign signs a leaf certificate used by Connect proxies from a CSR. The PEM
 	// returned should include only the leaf certificate as all Intermediates
 	// needed to validate it will be added by Consul based on the active
-	// intemediate and any cross-signed intermediates managed by Consul. Note that
+	// intermediate and any cross-signed intermediates managed by Consul. Note that
 	// providers should return ErrRateLimited if they are unable to complete the
 	// operation due to upstream rate limiting so that clients can intelligently
 	// backoff.

agent/connect/ca/provider_vault.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hashicorp/consul/lib/decode"
@@ -55,7 +56,12 @@ var ErrBackendNotInitialized = fmt.Errorf("backend not initialized")
 
 type VaultProvider struct {
 	config *structs.VaultCAProviderConfig
+
 	client *vaultapi.Client
+	// We modify the namespace on the fly to override default namespace for rootCertificate and intermediateCertificate. Can't guarantee
+	// all operations (specifically Sign) are not called re-entrantly, so we add this for safety.
+	clientMutex   sync.Mutex
+	baseNamespace string
 
 	stopWatcher func()
 
@@ -109,6 +115,7 @@ func (v *VaultProvider) Configure(cfg ProviderConfig) error {
 	// same.
 	if config.Namespace != "" {
 		client.SetNamespace(config.Namespace)
+		v.baseNamespace = config.Namespace
 	}
 
 	if config.AuthMethod != nil {
@@ -282,10 +289,11 @@ func (v *VaultProvider) GenerateRoot() (RootResult, error) {
 	}
 
 	// Set up the root PKI backend if necessary.
-	rootPEM, err := v.getCA(v.config.RootPKIPath)
+	rootPEM, err := v.getCA(v.config.RootPKINamespace, v.config.RootPKIPath)
 	switch err {
 	case ErrBackendNotMounted:
-		err := v.client.Sys().Mount(v.config.RootPKIPath, &vaultapi.MountInput{
+
+		err := v.mountNamespaced(v.config.RootPKINamespace, v.config.RootPKIPath, &vaultapi.MountInput{
 			Type:        "pki",
 			Description: "root CA backend for Consul Connect",
 			Config: vaultapi.MountConfigInput{
@@ -306,7 +314,7 @@ func (v *VaultProvider) GenerateRoot() (RootResult, error) {
 		if err != nil {
 			return RootResult{}, err
 		}
-		resp, err := v.client.Logical().Write(v.config.RootPKIPath+"root/generate/internal", map[string]interface{}{
+		resp, err := v.writeNamespaced(v.config.RootPKINamespace, v.config.RootPKIPath+"root/generate/internal", map[string]interface{}{
 			"common_name": connect.CACN("vault", uid, v.clusterID, v.isPrimary),
 			"uri_sans":    v.spiffeID.URI().String(),
 			"key_type":    v.config.PrivateKeyType,
@@ -327,7 +335,7 @@ func (v *VaultProvider) GenerateRoot() (RootResult, error) {
 		}
 	}
 
-	rootChain, err := v.getCAChain(v.config.RootPKIPath)
+	rootChain, err := v.getCAChain(v.config.RootPKINamespace, v.config.RootPKIPath)
 	if err != nil {
 		return RootResult{}, err
 	}
@@ -358,17 +366,16 @@ func (v *VaultProvider) setupIntermediatePKIPath() error {
 		return nil
 	}
 
-	_, err := v.getCA(v.config.IntermediatePKIPath)
+	_, err := v.getCA(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath)
 	if err != nil {
 		if err == ErrBackendNotMounted {
-			err := v.client.Sys().Mount(v.config.IntermediatePKIPath, &vaultapi.MountInput{
+			err := v.mountNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath, &vaultapi.MountInput{
 				Type:        "pki",
 				Description: "intermediate CA backend for Consul Connect",
 				Config: vaultapi.MountConfigInput{
 					MaxLeaseTTL: v.config.IntermediateCertTTL.String(),
 				},
 			})
-
 			if err != nil {
 				return err
 			}
@@ -379,12 +386,13 @@ func (v *VaultProvider) setupIntermediatePKIPath() error {
 
 	// Create the role for issuing leaf certs if it doesn't exist yet
 	rolePath := v.config.IntermediatePKIPath + "roles/" + VaultCALeafCertRole
-	role, err := v.client.Logical().Read(rolePath)
+	role, err := v.readNamespaced(v.config.IntermediatePKINamespace, rolePath)
+
 	if err != nil {
 		return err
 	}
 	if role == nil {
-		_, err := v.client.Logical().Write(rolePath, map[string]interface{}{
+		_, err := v.writeNamespaced(v.config.IntermediatePKINamespace, rolePath, map[string]interface{}{
 			"allow_any_name":   true,
 			"allowed_uri_sans": "spiffe://*",
 			"key_type":         "any",
@@ -392,6 +400,7 @@ func (v *VaultProvider) setupIntermediatePKIPath() error {
 			"no_store":         true,
 			"require_cn":       false,
 		})
+
 		if err != nil {
 			return err
 		}
@@ -411,7 +420,7 @@ func (v *VaultProvider) generateIntermediateCSR() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := v.client.Logical().Write(v.config.IntermediatePKIPath+"intermediate/generate/internal", map[string]interface{}{
+	data, err := v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"intermediate/generate/internal", map[string]interface{}{
 		"common_name": connect.CACN("vault", uid, v.clusterID, v.isPrimary),
 		"key_type":    v.config.PrivateKeyType,
 		"key_bits":    v.config.PrivateKeyBits,
@@ -443,7 +452,7 @@ func (v *VaultProvider) SetIntermediate(intermediatePEM, rootPEM string) error {
 		return err
 	}
 
-	_, err = v.client.Logical().Write(v.config.IntermediatePKIPath+"intermediate/set-signed", map[string]interface{}{
+	_, err = v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"intermediate/set-signed", map[string]interface{}{
 		"certificate": intermediatePEM,
 	})
 	if err != nil {
@@ -459,7 +468,7 @@ func (v *VaultProvider) ActiveIntermediate() (string, error) {
 		return "", err
 	}
 
-	cert, err := v.getCA(v.config.IntermediatePKIPath)
+	cert, err := v.getCA(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath)
 
 	// This error is expected when calling initializeSecondaryCA for the
 	// first time. It means that the backend is mounted and ready, but
@@ -477,7 +486,9 @@ func (v *VaultProvider) ActiveIntermediate() (string, error) {
 // We have to use the raw NewRequest call here instead of Logical().Read
 // because the endpoint only returns the raw PEM contents of the CA cert
 // and not the typical format of the secrets endpoints.
-func (v *VaultProvider) getCA(path string) (string, error) {
+func (v *VaultProvider) getCA(namespace, path string) (string, error) {
+	defer v.setNamespace(namespace)()
+
 	req := v.client.NewRequest("GET", "/v1/"+path+"/ca/pem")
 	resp, err := v.client.RawRequest(req)
 	if resp != nil {
@@ -504,7 +515,9 @@ func (v *VaultProvider) getCA(path string) (string, error) {
 }
 
 // TODO: refactor to remove duplication with getCA
-func (v *VaultProvider) getCAChain(path string) (string, error) {
+func (v *VaultProvider) getCAChain(namespace, path string) (string, error) {
+	defer v.setNamespace(namespace)()
+
 	req := v.client.NewRequest("GET", "/v1/"+path+"/ca_chain")
 	resp, err := v.client.RawRequest(req)
 	if resp != nil {
@@ -536,7 +549,7 @@ func (v *VaultProvider) GenerateIntermediate() (string, error) {
 	}
 
 	// Sign the CSR with the root backend.
-	intermediate, err := v.client.Logical().Write(v.config.RootPKIPath+"root/sign-intermediate", map[string]interface{}{
+	intermediate, err := v.writeNamespaced(v.config.RootPKINamespace, v.config.RootPKIPath+"root/sign-intermediate", map[string]interface{}{
 		"csr":            csr,
 		"use_csr_values": true,
 		"format":         "pem_bundle",
@@ -550,7 +563,7 @@ func (v *VaultProvider) GenerateIntermediate() (string, error) {
 	}
 
 	// Set the intermediate backend to use the new certificate.
-	_, err = v.client.Logical().Write(v.config.IntermediatePKIPath+"intermediate/set-signed", map[string]interface{}{
+	_, err = v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"intermediate/set-signed", map[string]interface{}{
 		"certificate": intermediate.Data["certificate"],
 	})
 	if err != nil {
@@ -572,7 +585,7 @@ func (v *VaultProvider) Sign(csr *x509.CertificateRequest) (string, error) {
 	}
 
 	// Use the leaf cert role to sign a new cert for this CSR.
-	response, err := v.client.Logical().Write(v.config.IntermediatePKIPath+"sign/"+VaultCALeafCertRole, map[string]interface{}{
+	response, err := v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"sign/"+VaultCALeafCertRole, map[string]interface{}{
 		"csr": pemBuf.String(),
 		"ttl": v.config.LeafCertTTL.String(),
 	})
@@ -605,7 +618,7 @@ func (v *VaultProvider) SignIntermediate(csr *x509.CertificateRequest) (string,
 	}
 
 	// Sign the CSR with the root backend.
-	data, err := v.client.Logical().Write(v.config.RootPKIPath+"root/sign-intermediate", map[string]interface{}{
+	data, err := v.writeNamespaced(v.config.RootPKINamespace, v.config.RootPKIPath+"root/sign-intermediate", map[string]interface{}{
 		"csr":             pemBuf.String(),
 		"use_csr_values":  true,
 		"format":          "pem_bundle",
@@ -630,7 +643,7 @@ func (v *VaultProvider) SignIntermediate(csr *x509.CertificateRequest) (string,
 // CrossSignCA takes a CA certificate and cross-signs it to form a trust chain
 // back to our active root.
 func (v *VaultProvider) CrossSignCA(cert *x509.Certificate) (string, error) {
-	rootPEM, err := v.getCA(v.config.RootPKIPath)
+	rootPEM, err := v.getCA(v.config.RootPKINamespace, v.config.RootPKIPath)
 	if err != nil {
 		return "", err
 	}
@@ -649,7 +662,7 @@ func (v *VaultProvider) CrossSignCA(cert *x509.Certificate) (string, error) {
 	}
 
 	// Have the root PKI backend sign this cert.
-	response, err := v.client.Logical().Write(v.config.RootPKIPath+"root/sign-self-issued", map[string]interface{}{
+	response, err := v.writeNamespaced(v.config.RootPKINamespace, v.config.RootPKIPath+"root/sign-self-issued", map[string]interface{}{
 		"certificate": pemBuf.String(),
 	})
 	if err != nil {
@@ -691,7 +704,7 @@ func (v *VaultProvider) Cleanup(providerTypeChange bool, otherConfig map[string]
 		}
 	}
 
-	err := v.client.Sys().Unmount(v.config.IntermediatePKIPath)
+	err := v.unmountNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath)
 
 	switch err {
 	case ErrBackendNotMounted, ErrBackendNotInitialized:
@@ -709,6 +722,65 @@ func (v *VaultProvider) Stop() {
 
 func (v *VaultProvider) PrimaryUsesIntermediate() {}
 
+// We use raw path here
+func (v *VaultProvider) mountNamespaced(namespace, path string, mountInfo *vaultapi.MountInput) error {
+	defer v.setNamespace(namespace)()
+	r := v.client.NewRequest("POST", fmt.Sprintf("/v1/sys/mounts/%s", path))
+	if err := r.SetJSONBody(mountInfo); err != nil {
+		return err
+	}
+	resp, err := v.client.RawRequest(r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	return err
+}
+
+func (v *VaultProvider) unmountNamespaced(namespace, path string) error {
+	defer v.setNamespace(namespace)()
+	r := v.client.NewRequest("DELETE", fmt.Sprintf("/v1/sys/mounts/%s", path))
+	resp, err := v.client.RawRequest(r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	return err
+}
+
+func makePathHelper(namespace, path string) string {
+	var fullPath string
+	if namespace != "" {
+		fullPath = fmt.Sprintf("/v1/%s/sys/mounts/%s", namespace, path)
+	} else {
+		fullPath = fmt.Sprintf("/v1/sys/mounts/%s", path)
+	}
+	return fullPath
+}
+
+func (v *VaultProvider) readNamespaced(namespace string, resource string) (*vaultapi.Secret, error) {
+	defer v.setNamespace(namespace)()
+	result, err := v.client.Logical().Read(resource)
+	return result, err
+}
+
+func (v *VaultProvider) writeNamespaced(namespace string, resource string, data map[string]interface{}) (*vaultapi.Secret, error) {
+	defer v.setNamespace(namespace)()
+	result, err := v.client.Logical().Write(resource, data)
+	return result, err
+}
+
+func (v *VaultProvider) setNamespace(namespace string) func() {
+	if namespace != "" {
+		v.clientMutex.Lock()
+		v.client.SetNamespace(namespace)
+		return func() {
+			v.client.SetNamespace(v.baseNamespace)
+			v.clientMutex.Unlock()
+		}
+	} else {
+		return func() {}
+	}
+}
+
 func ParseVaultCAConfig(raw map[string]interface{}) (*structs.VaultCAProviderConfig, error) {
 	config := structs.VaultCAProviderConfig{
 		CommonCAProviderConfig: defaultCommonConfig(),

agent/structs/connect_ca.go

@@ -517,11 +517,13 @@ type CAConsulProviderState struct {
 type VaultCAProviderConfig struct {
 	CommonCAProviderConfig `mapstructure:",squash"`
 
-	Address             string
+	Address                  string
-	Token               string
+	Token                    string
-	RootPKIPath         string
+	RootPKIPath              string
-	IntermediatePKIPath string
+	RootPKINamespace         string
-	Namespace           string
+	IntermediatePKIPath      string
+	IntermediatePKINamespace string
+	Namespace                string
 
 	CAFile        string
 	CAPath        string

website/content/docs/connect/ca/vault.mdx

@@ -136,6 +136,9 @@ The configuration options are listed below.
   must contain a valid chain, where each certificate is followed by the certificate
   that authorized it.
 
+- `RootPKINamespace` / `root_pki_namespace` (`string: <optional>`) - The absolute namespace
+  that the `RootPKIPath` is in. Setting this overrides the `Namespace` option for the `RootPKIPath`. Introduced in 1.12.1
+
 - `IntermediatePKIPath` / `intermediate_pki_path` (`string: <required>`) -
   The path to a PKI secrets engine for the generated intermediate certificate.
   This certificate will be signed by the configured root PKI path. If this
@@ -145,6 +148,9 @@ The configuration options are listed below.
   When WAN Federation is enabled, every secondary
   datacenter must specify a unique `intermediate_pki_path`.
 
+- `IntermediatePKINamespace` / `intermedial_pki_namespace` (`string: <optional>`) - The absolute namespace
+  that the `IntermediatePKIPath` is in. Setting this overrides the `Namespace` option for the `IntermediatePKIPath`. Introduced in 1.12.1
+
 - `CAFile` / `ca_file` (`string: ""`) - Specifies an optional path to the CA
   certificate used for Vault communication. If unspecified, this will fallback
   to the default system CA bundle, which varies by OS and version.