From d8f4cc55376a02c67e1f6815e6676359e20e9a1a Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Tue, 26 Apr 2022 13:46:29 -0700 Subject: [PATCH 1/7] Add x-forwarded-client-cert headers Description Add x-fowarded-client-cert information on trusted incoming connections. Envoy provides support forwarding and annotating the x-forwarded-client-cert header via the forward_client_cert_details set_current_client_cert_details filter fields. It would be helpful for consul to support this directly in its config. The escape hatches are a bit cumbersome for this purpose. This has been implemented on incoming connections to envoy. Outgoing (from the local service through the sidecar) will not have a certificate, and so are left alone. A service on an incoming connection will now get headers something like this: ``` X-Forwarded-Client-Cert:[By=spiffe://efad7282-d9b2-3298-f6d8-38b37fb58df3.consul/ns/default/dc/dc1/svc/counting;Hash=61ad5cbdfcb50f5a3ec0ca60923d61613c149a9d4495010a64175c05a0268ab2;Cert="-----BEGIN%20CERTIFICATE-----%0AMIICHDCCAcOgAwIBAgIBCDAKBggqhkjOPQQDAjAxMS8wLQYDVQQDEyZwcmktMTli%0AYXdyb2YuY29uc3VsLmNhLmVmYWQ3MjgyLmNvbnN1bDAeFw0yMjA0MjkwMzE0NTBa%0AFw0yMjA1MDIwMzE0NTBaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARVIZ7Y%0AZEXfbOGBfxGa7Vuok1MIng%2FuzLQK2xLVlSTIPDbO5hstTGP%2B%2FGx182PYFP3jYqk5%0Aq6rYWe1wiPNMA30Io4H8MIH5MA4GA1UdDwEB%2FwQEAwIDuDAdBgNVHSUEFjAUBggr%0ABgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH%2FBAIwADApBgNVHQ4EIgQgrp4q50oX%0AHHghMbxz5Bk8OJFWMdfgH0Upr350WlhyxvkwKwYDVR0jBCQwIoAgUe6uERAIj%2FLM%0AyuFzDc3Wbp9TGAKBJYAwyhF14ToOQCMwYgYDVR0RAQH%2FBFgwVoZUc3BpZmZlOi8v%0AZWZhZDcyODItZDliMi0zMjk4LWY2ZDgtMzhiMzdmYjU4ZGYzLmNvbnN1bC9ucy9k%0AZWZhdWx0L2RjL2RjMS9zdmMvZGFzaGJvYXJkMAoGCCqGSM49BAMCA0cAMEQCIDwb%0AFlchufggNTijnQ5SUcvTZrWlZyq%2FrdVC20nbbmWLAiAVshNNv1xBqJI1NmY2HI9n%0AgRMfb8aEPVSuxEHhqy57eQ%3D%3D%0A-----END%20CERTIFICATE-----%0A";Chain="-----BEGIN%20CERTIFICATE-----%0AMIICHDCCAcOgAwIBAgIBCDAKBggqhkjOPQQDAjAxMS8wLQYDVQQDEyZwcmktMTli%0AYXdyb2YuY29uc3VsLmNhLmVmYWQ3MjgyLmNvbnN1bDAeFw0yMjA0MjkwMzE0NTBa%0AFw0yMjA1MDIwMzE0NTBaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARVIZ7Y%0AZEXfbOGBfxGa7Vuok1MIng%2FuzLQK2xLVlSTIPDbO5hstTGP%2B%2FGx182PYFP3jYqk5%0Aq6rYWe1wiPNMA30Io4H8MIH5MA4GA1UdDwEB%2FwQEAwIDuDAdBgNVHSUEFjAUBggr%0ABgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH%2FBAIwADApBgNVHQ4EIgQgrp4q50oX%0AHHghMbxz5Bk8OJFWMdfgH0Upr350WlhyxvkwKwYDVR0jBCQwIoAgUe6uERAIj%2FLM%0AyuFzDc3Wbp9TGAKBJYAwyhF14ToOQCMwYgYDVR0RAQH%2FBFgwVoZUc3BpZmZlOi8v%0AZWZhZDcyODItZDliMi0zMjk4LWY2ZDgtMzhiMzdmYjU4ZGYzLmNvbnN1bC9ucy9k%0AZWZhdWx0L2RjL2RjMS9zdmMvZGFzaGJvYXJkMAoGCCqGSM49BAMCA0cAMEQCIDwb%0AFlchufggNTijnQ5SUcvTZrWlZyq%2FrdVC20nbbmWLAiAVshNNv1xBqJI1NmY2HI9n%0AgRMfb8aEPVSuxEHhqy57eQ%3D%3D%0A-----END%20CERTIFICATE-----%0A";Subject="";URI=spiffe://efad7282-d9b2-3298-f6d8-38b37fb58df3.consul/ns/default/dc/dc1/svc/dashboard] ``` Closes #12852 --- .changelog/12878.txt | 3 + agent/structs/config_entry_mesh.go | 6 + agent/xds/listeners.go | 45 ++++-- agent/xds/listeners_test.go | 21 +++ .../http-listener-with-timeouts.latest.golden | 8 + ...http-public-listener-no-xfcc.latest.golden | 151 ++++++++++++++++++ .../http-public-listener.latest.golden | 8 + ...ting-gateway-service-subsets.latest.golden | 24 +++ ...teway-with-service-resolvers.latest.golden | 24 +++ .../lambda-terminating-gateway.latest.golden | 8 + 10 files changed, 288 insertions(+), 10 deletions(-) create mode 100644 .changelog/12878.txt create mode 100644 agent/xds/testdata/listeners/http-public-listener-no-xfcc.latest.golden diff --git a/.changelog/12878.txt b/.changelog/12878.txt new file mode 100644 index 000000000..e740998b9 --- /dev/null +++ b/.changelog/12878.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: Envoy now inserts x-forwarded-client-cert for incoming proxy connections +``` \ No newline at end of file diff --git a/agent/structs/config_entry_mesh.go b/agent/structs/config_entry_mesh.go index 2d983eb82..be0273549 100644 --- a/agent/structs/config_entry_mesh.go +++ b/agent/structs/config_entry_mesh.go @@ -15,6 +15,8 @@ type MeshConfigEntry struct { TLS *MeshTLSConfig `json:",omitempty"` + HTTP *MeshHTTPConfig `json:",omitempty"` + Meta map[string]string `json:",omitempty"` acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` RaftIndex @@ -33,6 +35,10 @@ type MeshTLSConfig struct { Outgoing *MeshDirectionalTLSConfig `json:",omitempty"` } +type MeshHTTPConfig struct { + SanitizeXForwardedClientCert bool `json:",omitempty"` +} + type MeshDirectionalTLSConfig struct { TLSMinVersion types.TLSVersion `json:",omitempty" alias:"tls_min_version"` TLSMaxVersion types.TLSVersion `json:",omitempty" alias:"tls_max_version"` diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 5fcc83a91..c04a47487 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -859,6 +859,10 @@ func (s *ResourceGenerator) makeInboundListener(cfgSnap *proxycfg.ConfigSnapshot if err != nil { return nil, err } + if meshConfig := cfgSnap.MeshConfig(); meshConfig == nil || meshConfig.HTTP == nil || !meshConfig.HTTP.SanitizeXForwardedClientCert { + filterOpts.forwardClientDetails = true + filterOpts.forwardClientPolicy = envoy_http_v3.HttpConnectionManager_APPEND_FORWARD + } } filter, err := makeListenerFilter(filterOpts) if err != nil { @@ -1146,6 +1150,13 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway( opts.cluster = "" opts.useRDS = true + + if meshConfig := cfgSnap.MeshConfig(); meshConfig == nil || meshConfig.HTTP == nil || !meshConfig.HTTP.SanitizeXForwardedClientCert { + opts.forwardClientDetails = true + // Note: filter Connection may not be mTLS, so then ALWAYS_FORWARD_ONLY. For mTLS connections we might want APPEND_FORWARD. + // Open question; how do I determine if this is mTLS or not? + opts.forwardClientPolicy = envoy_http_v3.HttpConnectionManager_ALWAYS_FORWARD_ONLY + } } filter, err := makeListenerFilter(opts) @@ -1366,16 +1377,18 @@ func (s *ResourceGenerator) getAndModifyUpstreamConfigForListener( } type listenerFilterOpts struct { - useRDS bool - protocol string - filterName string - routeName string - cluster string - statPrefix string - routePath string - requestTimeoutMs *int - ingressGateway bool - httpAuthzFilter *envoy_http_v3.HttpFilter + useRDS bool + protocol string + filterName string + routeName string + cluster string + statPrefix string + routePath string + requestTimeoutMs *int + ingressGateway bool + httpAuthzFilter *envoy_http_v3.HttpFilter + forwardClientDetails bool + forwardClientPolicy envoy_http_v3.HttpConnectionManager_ForwardClientCertDetails } func makeListenerFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) { @@ -1513,6 +1526,18 @@ func makeHTTPFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) cfg.Http2ProtocolOptions = &envoy_core_v3.Http2ProtocolOptions{} } + // Note the default leads to setting HttpConnectionManager_SANITIZE + if opts.forwardClientDetails { + cfg.ForwardClientCertDetails = opts.forwardClientPolicy + cfg.SetCurrentClientCertDetails = &envoy_http_v3.HttpConnectionManager_SetCurrentClientCertDetails{ + Subject: &wrappers.BoolValue{Value: true}, + Cert: true, + Chain: true, + Dns: true, + Uri: true, + } + } + // Like injectConnectFilters for L4, here we ensure that the first filter // (other than the "envoy.grpc_http1_bridge" filter) in the http filter // chain of a public listener is the authz filter to prevent unauthorized diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index d80cde7b1..dcdb37552 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -166,6 +166,27 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, }, + { + name: "http-public-listener-no-xfcc", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshot(t, + func(ns *structs.NodeService) { + ns.Proxy.Config["protocol"] = "http" + }, + []cache.UpdateEvent{ + { + CorrelationID: "mesh", + Result: &structs.ConfigEntryResponse{ + Entry: &structs.MeshConfigEntry{ + HTTP: &structs.MeshHTTPConfig{ + SanitizeXForwardedClientCert: true, + }, + }, + }, + }, + }) + }, + }, { name: "http-listener-with-timeouts", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/listeners/http-listener-with-timeouts.latest.golden b/agent/xds/testdata/listeners/http-listener-with-timeouts.latest.golden index 0cd9b242b..679b07105 100644 --- a/agent/xds/testdata/listeners/http-listener-with-timeouts.latest.golden +++ b/agent/xds/testdata/listeners/http-listener-with-timeouts.latest.golden @@ -67,6 +67,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "APPEND_FORWARD", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "public_listener", "routeConfig": { "name": "public_listener", diff --git a/agent/xds/testdata/listeners/http-public-listener-no-xfcc.latest.golden b/agent/xds/testdata/listeners/http-public-listener-no-xfcc.latest.golden new file mode 100644 index 000000000..d0a676eff --- /dev/null +++ b/agent/xds/testdata/listeners/http-public-listener-no-xfcc.latest.golden @@ -0,0 +1,151 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "public_listener", + "routeConfig": { + "name": "public_listener", + "virtualHosts": [ + { + "name": "public_listener", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "local_app" + } + } + ] + } + ] + }, + "httpFilters": [ + { + "name": "envoy.filters.http.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "rules": { + + } + } + }, + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": { + + } + } + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/http-public-listener.latest.golden b/agent/xds/testdata/listeners/http-public-listener.latest.golden index d0a676eff..66db47bb2 100644 --- a/agent/xds/testdata/listeners/http-public-listener.latest.golden +++ b/agent/xds/testdata/listeners/http-public-listener.latest.golden @@ -67,6 +67,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "APPEND_FORWARD", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "public_listener", "routeConfig": { "name": "public_listener", diff --git a/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden b/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden index c2ce2223b..3d7cabb88 100644 --- a/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden +++ b/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden @@ -184,6 +184,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { @@ -258,6 +266,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { @@ -332,6 +348,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { diff --git a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden index 32cfda120..51ca730bc 100644 --- a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden +++ b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden @@ -130,6 +130,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { @@ -212,6 +220,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { @@ -348,6 +364,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { diff --git a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden index f415e40ab..ad93b113d 100644 --- a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden +++ b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden @@ -184,6 +184,14 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "setCurrentClientCertDetails": { + "cert": true, + "chain": true, + "dns": true, + "subject": true, + "uri": true + }, "statPrefix": "upstream.web.default.default.dc1", "rds": { "configSource": { From 33bc0a8cb366cddf3d82e61da589a26858d540c6 Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Fri, 29 Apr 2022 18:23:21 -0700 Subject: [PATCH 2/7] Add some docs Signed-off-by: Mark Anderson --- .../content/docs/connect/config-entries/mesh.mdx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/website/content/docs/connect/config-entries/mesh.mdx b/website/content/docs/connect/config-entries/mesh.mdx index a72a54aa0..2cbfe3f58 100644 --- a/website/content/docs/connect/config-entries/mesh.mdx +++ b/website/content/docs/connect/config-entries/mesh.mdx @@ -273,7 +273,7 @@ Note that the Kubernetes example does not include a `partition` field. Configura name: 'Incoming', yaml: false, type: 'TLSDirectionConfig: ', - description: `TLS configuration for inbound mTLS connections targeting + description: `TLS configuration for inbound mTLS connections targeting the public listener on \`connect-proxy\` and \`terminating-gateway\` proxy kinds.`, children: [ @@ -359,6 +359,20 @@ Note that the Kubernetes example does not include a `partition` field. Configura }, ], }, + { + name: 'HTTP', + type: 'HTTPConfig: ', + description: 'HTTP configuration for the service mesh.', + children: [ + { + name: 'SanitizeXForwardedClientCert', + yaml: false, + type: 'bool: ', + description: `Set the envoy forwardClientCertDetails to SANITIZE everywhere. Ordinarily Consul will configure Envoy to + insert x-forwarded-client-cert headers where appropriate. This returns Consul to the pre 1.12.1 behavior`, + }, + ], + }, ]} /> From c6dbc3417268fbe650142be82ad52a8b065bd695 Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Mon, 2 May 2022 09:35:25 -0700 Subject: [PATCH 3/7] Fixup missed config entry Signed-off-by: Mark Anderson --- agent/structs/config_entry_mesh.go | 4 ---- api/config_entry_mesh.go | 10 +++++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/agent/structs/config_entry_mesh.go b/agent/structs/config_entry_mesh.go index be0273549..980b81ca5 100644 --- a/agent/structs/config_entry_mesh.go +++ b/agent/structs/config_entry_mesh.go @@ -35,10 +35,6 @@ type MeshTLSConfig struct { Outgoing *MeshDirectionalTLSConfig `json:",omitempty"` } -type MeshHTTPConfig struct { - SanitizeXForwardedClientCert bool `json:",omitempty"` -} - type MeshDirectionalTLSConfig struct { TLSMinVersion types.TLSVersion `json:",omitempty" alias:"tls_min_version"` TLSMaxVersion types.TLSVersion `json:",omitempty" alias:"tls_max_version"` diff --git a/api/config_entry_mesh.go b/api/config_entry_mesh.go index 30fab166c..406e87dfc 100644 --- a/api/config_entry_mesh.go +++ b/api/config_entry_mesh.go @@ -1,6 +1,8 @@ package api -import "encoding/json" +import ( + "encoding/json" +) // MeshConfigEntry manages the global configuration for all service mesh // proxies. @@ -19,6 +21,8 @@ type MeshConfigEntry struct { TLS *MeshTLSConfig `json:",omitempty"` + HTTP *MeshHTTPConfig `json:",omitempty"` + Meta map[string]string `json:",omitempty"` // CreateIndex is the Raft index this entry was created at. This is a @@ -46,6 +50,10 @@ type MeshDirectionalTLSConfig struct { CipherSuites []string `json:",omitempty" alias:"cipher_suites"` } +type MeshHTTPConfig struct { + SanitizeXForwardedClientCert bool `alias:"sanitize_x_forwarded_client_cert"` +} + func (e *MeshConfigEntry) GetKind() string { return MeshConfig } func (e *MeshConfigEntry) GetName() string { return MeshConfigMesh } func (e *MeshConfigEntry) GetPartition() string { return e.Partition } From e6282c7c644a21062ede75f89dffcf2313deaae6 Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Mon, 2 May 2022 09:35:34 -0700 Subject: [PATCH 4/7] Docs and changelog edits Signed-off-by: Mark Anderson --- .changelog/12878.txt | 4 ++-- agent/structs/config_entry_mesh.go | 4 ++++ website/content/docs/connect/config-entries/mesh.mdx | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.changelog/12878.txt b/.changelog/12878.txt index e740998b9..8d06d3440 100644 --- a/.changelog/12878.txt +++ b/.changelog/12878.txt @@ -1,3 +1,3 @@ ```release-note:improvement -agent: Envoy now inserts x-forwarded-client-cert for incoming proxy connections -``` \ No newline at end of file +xds: Envoy now inserts x-forwarded-client-cert for incoming proxy connections +``` diff --git a/agent/structs/config_entry_mesh.go b/agent/structs/config_entry_mesh.go index 980b81ca5..868c07a9f 100644 --- a/agent/structs/config_entry_mesh.go +++ b/agent/structs/config_entry_mesh.go @@ -44,6 +44,10 @@ type MeshDirectionalTLSConfig struct { CipherSuites []types.TLSCipherSuite `json:",omitempty" alias:"cipher_suites"` } +type MeshHTTPConfig struct { + SanitizeXForwardedClientCert bool `alias:"sanitize_x_forwarded_client_cert"` +} + func (e *MeshConfigEntry) GetKind() string { return MeshConfig } diff --git a/website/content/docs/connect/config-entries/mesh.mdx b/website/content/docs/connect/config-entries/mesh.mdx index 2cbfe3f58..a9da7a1ad 100644 --- a/website/content/docs/connect/config-entries/mesh.mdx +++ b/website/content/docs/connect/config-entries/mesh.mdx @@ -368,8 +368,9 @@ Note that the Kubernetes example does not include a `partition` field. Configura name: 'SanitizeXForwardedClientCert', yaml: false, type: 'bool: ', - description: `Set the envoy forwardClientCertDetails to SANITIZE everywhere. Ordinarily Consul will configure Envoy to - insert x-forwarded-client-cert headers where appropriate. This returns Consul to the pre 1.12.1 behavior`, + description: `Set the envoy \`forward_client_cert_details\` option to \`SANITIZE\` for all proxies. This + configures Envoy to not send the \`x-forwarded-client-cert\` header to the next hop. If + unspecified or \`false\`, the XFCC header is propagated to upstream applications.`, }, ], }, From db0c61303f18f6c113cac47b2aec0b7ba64d1ded Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Tue, 3 May 2022 13:29:45 -0700 Subject: [PATCH 5/7] Update mesh config tests Signed-off-by: Mark Anderson --- agent/structs/config_entry_test.go | 9 +++++++++ api/config_entry_test.go | 6 ++++++ command/config/write/config_write_test.go | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/agent/structs/config_entry_test.go b/agent/structs/config_entry_test.go index 661a87ce9..3814dd4c5 100644 --- a/agent/structs/config_entry_test.go +++ b/agent/structs/config_entry_test.go @@ -1694,6 +1694,9 @@ func TestDecodeConfigEntry(t *testing.T) { ] } } + http { + sanitize_x_forwarded_client_cert = true + } `, camel: ` Kind = "mesh" @@ -1722,6 +1725,9 @@ func TestDecodeConfigEntry(t *testing.T) { ] } } + HTTP { + SanitizeXForwardedClientCert = true + } `, expect: &MeshConfigEntry{ Meta: map[string]string{ @@ -1749,6 +1755,9 @@ func TestDecodeConfigEntry(t *testing.T) { }, }, }, + HTTP: &MeshHTTPConfig{ + SanitizeXForwardedClientCert: true, + }, }, }, { diff --git a/api/config_entry_test.go b/api/config_entry_test.go index 0f38f62cd..2f28dcd75 100644 --- a/api/config_entry_test.go +++ b/api/config_entry_test.go @@ -1278,6 +1278,9 @@ func TestDecodeConfigEntry(t *testing.T) { "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" ] } + }, + "HTTP": { + "SanitizeXForwardedClientCert": true } } `, @@ -1307,6 +1310,9 @@ func TestDecodeConfigEntry(t *testing.T) { }, }, }, + HTTP: &MeshHTTPConfig{ + SanitizeXForwardedClientCert: true, + }, }, }, } { diff --git a/command/config/write/config_write_test.go b/command/config/write/config_write_test.go index 679a3b77e..fc297aa09 100644 --- a/command/config/write/config_write_test.go +++ b/command/config/write/config_write_test.go @@ -126,6 +126,9 @@ meta { transparent_proxy { mesh_destinations_only = true } +http { + sanitize_x_forwarded_client_cert = true +} `) ui := cli.NewMockUi() @@ -143,6 +146,9 @@ transparent_proxy { proxy, ok := entry.(*api.MeshConfigEntry) require.True(t, ok) require.Equal(t, map[string]string{"foo": "bar", "gir": "zim"}, proxy.Meta) + require.True(t, proxy.TransparentProxy.MeshDestinationsOnly) + + require.True(t, proxy.HTTP.SanitizeXForwardedClientCert) }) } From 69c129c73fd923d252ec9a7a94950759c9320065 Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Tue, 3 May 2022 13:57:57 -0700 Subject: [PATCH 6/7] Change to use APPEND_FORWARD for terminating gateway Signed-off-by: Mark Anderson --- agent/xds/listeners.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index c04a47487..672d667ad 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -1153,9 +1153,8 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway( if meshConfig := cfgSnap.MeshConfig(); meshConfig == nil || meshConfig.HTTP == nil || !meshConfig.HTTP.SanitizeXForwardedClientCert { opts.forwardClientDetails = true - // Note: filter Connection may not be mTLS, so then ALWAYS_FORWARD_ONLY. For mTLS connections we might want APPEND_FORWARD. - // Open question; how do I determine if this is mTLS or not? - opts.forwardClientPolicy = envoy_http_v3.HttpConnectionManager_ALWAYS_FORWARD_ONLY + // This assumes that we have a client cert (mTLS) (implied by the context of this function) + opts.forwardClientPolicy = envoy_http_v3.HttpConnectionManager_APPEND_FORWARD } } From 13f5a1f6a81b8aec4bac290822ae0125c7df0996 Mon Sep 17 00:00:00 2001 From: Mark Anderson Date: Tue, 3 May 2022 14:48:51 -0700 Subject: [PATCH 7/7] Fix tests for APPEND_FORWARD change Signed-off-by: Mark Anderson --- .../terminating-gateway-service-subsets.latest.golden | 6 +++--- ...terminating-gateway-with-service-resolvers.latest.golden | 6 +++--- .../listeners/lambda-terminating-gateway.latest.golden | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden b/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden index 3d7cabb88..fea8b4775 100644 --- a/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden +++ b/agent/xds/testdata/listeners/terminating-gateway-service-subsets.latest.golden @@ -184,7 +184,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true, @@ -266,7 +266,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true, @@ -348,7 +348,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true, diff --git a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden index 51ca730bc..158ea619c 100644 --- a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden +++ b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway-with-service-resolvers.latest.golden @@ -130,7 +130,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true, @@ -220,7 +220,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true, @@ -364,7 +364,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true, diff --git a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden index ad93b113d..f30e7e875 100644 --- a/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden +++ b/agent/xds/testdata/serverless_plugin/listeners/lambda-terminating-gateway.latest.golden @@ -184,7 +184,7 @@ "name": "envoy.filters.network.http_connection_manager", "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "forwardClientCertDetails": "ALWAYS_FORWARD_ONLY", + "forwardClientCertDetails": "APPEND_FORWARD", "setCurrentClientCertDetails": { "cert": true, "chain": true,