From 5427a1465c9515bf262efb102314756913876177 Mon Sep 17 00:00:00 2001 From: Freddy Date: Mon, 3 May 2021 14:15:22 -0600 Subject: [PATCH] Only consider virtual IPs for transparent proxies (#10162) Initially we were loading every potential upstream address into Envoy and then routing traffic to the logical upstream service. The downside of this behavior is that traffic meant to go to a specific instance would be load balanced across ALL instances. Traffic to specific instance IPs should be forwarded to the original destination and if it's a destination in the mesh then we should ensure the appropriate certificates are used. This PR makes transparent proxying a Kubernetes-only feature for now since support for other environments requires generating virtual IPs, and Consul does not do that at the moment. --- .changelog/10162.txt | 3 + agent/xds/listeners.go | 26 ++--- agent/xds/listeners_test.go | 8 ++ ...alog-destinations-only.envoy-1-18-x.golden | 2 +- ...inations-only.v2compat.envoy-1-16-x.golden | 2 +- .../transparent-proxy.envoy-1-18-x.golden | 2 +- ...sparent-proxy.v2compat.envoy-1-16-x.golden | 2 +- .../docs/connect/transparent-proxy.mdx | 95 +------------------ 8 files changed, 26 insertions(+), 114 deletions(-) create mode 100644 .changelog/10162.txt diff --git a/.changelog/10162.txt b/.changelog/10162.txt new file mode 100644 index 000000000..b28a735f0 --- /dev/null +++ b/.changelog/10162.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: restrict transparent proxy mode to only match on the tagged virtual IP address. +``` \ No newline at end of file diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 85046c090..bd939b429 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -32,6 +32,8 @@ import ( "github.com/hashicorp/consul/sdk/iptables" ) +const virtualIPTag = "virtual" + // listenersFromSnapshot returns the xDS API representation of the "listeners" in the snapshot. func (s *ResourceGenerator) listenersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { if cfgSnap == nil { @@ -144,30 +146,16 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. return nil, err } - // For filter chains used by the transparent proxy, we need to match on multiple destination addresses. - // These might be: the ClusterIP in k8s, or any of the service instance addresses. endpoints := cfgSnap.ConnectProxy.WatchedUpstreamEndpoints[id] uniqueAddrs := make(map[string]struct{}) for _, t := range chain.Targets { - // Store all the possible IP addresses that might be used to dial this endpoint + // Match on the virtual IP for the upstream service. + // We do not match on all endpoints here since it would lead to load balancing across + // all instances when any instance address is dialed. for _, e := range endpoints[t.ID] { - if e.Service.Address != "" { - uniqueAddrs[e.Service.Address] = struct{}{} - } - if e.Node.Address != "" { - uniqueAddrs[e.Node.Address] = struct{}{} - } - - for _, tagged := range e.Node.TaggedAddresses { - if tagged != "" { - uniqueAddrs[tagged] = struct{}{} - } - } - for _, tagged := range e.Service.TaggedAddresses { - if tagged.Address != "" { - uniqueAddrs[tagged.Address] = struct{}{} - } + if vip := e.Service.TaggedAddresses[virtualIPTag]; vip.Address != "" { + uniqueAddrs[vip.Address] = struct{}{} } } } diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 28b51fd17..fa77ffbbe 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -500,7 +500,11 @@ func TestListenersFromSnapshot(t *testing.T) { }, Service: &structs.NodeService{ Service: "google", + Address: "9.9.9.9", Port: 9090, + TaggedAddresses: map[string]structs.ServiceAddress{ + "virtual": {Address: "10.0.0.1"}, + }, }, }, }, @@ -540,7 +544,11 @@ func TestListenersFromSnapshot(t *testing.T) { }, Service: &structs.NodeService{ Service: "google", + Address: "9.9.9.9", Port: 9090, + TaggedAddresses: map[string]structs.ServiceAddress{ + "virtual": {Address: "10.0.0.1"}, + }, }, }, }, diff --git a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden index cd38c0d06..a3c52dc4f 100644 --- a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden +++ b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden @@ -40,7 +40,7 @@ "filterChainMatch": { "prefixRanges": [ { - "addressPrefix": "8.8.8.8", + "addressPrefix": "10.0.0.1", "prefixLen": 32 } ] diff --git a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden index f63b43684..db881a384 100644 --- a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden +++ b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden @@ -40,7 +40,7 @@ "filterChainMatch": { "prefixRanges": [ { - "addressPrefix": "8.8.8.8", + "addressPrefix": "10.0.0.1", "prefixLen": 32 } ] diff --git a/agent/xds/testdata/listeners/transparent-proxy.envoy-1-18-x.golden b/agent/xds/testdata/listeners/transparent-proxy.envoy-1-18-x.golden index 15538eeb6..deec35536 100644 --- a/agent/xds/testdata/listeners/transparent-proxy.envoy-1-18-x.golden +++ b/agent/xds/testdata/listeners/transparent-proxy.envoy-1-18-x.golden @@ -40,7 +40,7 @@ "filterChainMatch": { "prefixRanges": [ { - "addressPrefix": "8.8.8.8", + "addressPrefix": "10.0.0.1", "prefixLen": 32 } ] diff --git a/agent/xds/testdata/listeners/transparent-proxy.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/listeners/transparent-proxy.v2compat.envoy-1-16-x.golden index 22c49ff16..8a3bb4b5d 100644 --- a/agent/xds/testdata/listeners/transparent-proxy.v2compat.envoy-1-16-x.golden +++ b/agent/xds/testdata/listeners/transparent-proxy.v2compat.envoy-1-16-x.golden @@ -40,7 +40,7 @@ "filterChainMatch": { "prefixRanges": [ { - "addressPrefix": "8.8.8.8", + "addressPrefix": "10.0.0.1", "prefixLen": 32 } ] diff --git a/website/content/docs/connect/transparent-proxy.mdx b/website/content/docs/connect/transparent-proxy.mdx index 2e8410cee..69b800eab 100644 --- a/website/content/docs/connect/transparent-proxy.mdx +++ b/website/content/docs/connect/transparent-proxy.mdx @@ -28,33 +28,19 @@ the traffic redirection command is automatically set up via an init container. ## Prerequisites -Transparent proxy requires Consul >= `1.10.0`. - ### Kubernetes * To use transparent proxy on Kubernetes, Consul-helm >= `0.32.0` and Consul-k8s >= `0.26.0` are required in addition to -the Consul version requirements. +the Consul >= `1.10.0`. * If the default policy for ACLs is "deny", then Service Intentions should be set up to allow intended services to connect to each other. Otherwise, all Connect services can talk to all other services. The Kubernetes integration takes care of registering Kubernetes services with Consul, injecting a sidecar proxy, and enabling traffic redirection. -### VMs - -* For a service on a VM to be a part of the service mesh, it needs to run a Connect sidecar proxy. -* The [`consul connect redirect-traffic`](/commands/connect/redirect-traffic) command needs to be run on the VM to -set it up to redirect all inbound and outbound traffic to that VM through the sidecar proxy. Note that this will modify -iptables rules on the host which can affect reachability of the VM unless the command is run within a network namespace. -* Services need to be registered with Consul. -* If the default policy for ACLs is "deny", then Service Intentions should be set up to allow intended services to connect to each other. -Otherwise, all Connect services can talk to all other services. - ## Configuration -### Kubernetes - Transparent proxy can be enabled in Kubernetes on the whole cluster via the Helm value: ```yaml @@ -114,75 +100,13 @@ spec: serviceAccountName: static-server ``` -### VMs - -In other environments, transparent proxy can be enabled via Proxy Defaults and Service Defaults config entries, or via -the proxy service registration: -```json -# Proxy defaults apply to all proxies. -kind = "proxy-defaults" -name = "global" - -mode = "transparent" -transparent_proxy { - outbound_listener_port = 15001 -} -``` -```json -# Service defaults apply to all instances of the web service. -kind = "service-defaults" -name = "web" - -mode = "transparent" -transparent_proxy { - outbound_listener_port = 15001 -} -``` -```json -# Proxy service registrations apply to a single proxy instance. -name = "web-sidecar-proxy" -kind = "connect-proxy" -proxy { - mode = "transparent" - transparent_proxy { - outbound_listener_port = 15001 - } - destination_service_name = "web" - local_service_port = 8080 -} -port = 20000 -``` - -Similar to `mesh_gateway.mode`, the new proxy mode will have the following string values: -* "" - The empty string represents the default value for the feature, and allows for the mode to be overridden by -central configuration, like “service-defaults”. -* "direct" - Explicitly disables configuring transparent proxy, falling back to only configuring explicit upstreams. -* "transparent" - Explicitly enables configuring transparent proxy. - -Additionally, the new Cluster config entry is scoped to the set of federated Consul datacenters and can be used to allow or block -traffic to external destinations. This example shows blocking traffic to external destinations (outside of Consul's catalog): - -```json -kind = "cluster" -name = "cluster" - -transparent_proxy { - catalog_destinations_only = true -} -``` - -## Known Limitations - -* For services on VMs, transparent proxy only supports one service per VM, or per network namespace. This is -because the traffic redirection rules are applicable to the entire namespace (including the default namespace) and will -direct all outbound traffic from the service to it’s sidecar proxy. -* Currently transparent proxy is only supported for services within a single Consul datacenter. +## Known Beta Limitations +* There is no first class support for transparent proxying on VMs. +* Traffic can only be transparently proxied within a Consul datacenter. ## Using Transparent Proxy -### Kubernetes - In Kubernetes, services can reach other services via their [KubeDNS](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/) address or via Pod IPs, and that traffic will be transparently sent through the proxy. Connect services in Kubernetes are required to have a Kubernetes @@ -225,14 +149,3 @@ it can dial `sample-app.default.svc.cluster.local`, using If ACLs with default "deny" policy are enabled, it also needs a [ServiceIntention](/docs/connect/config-entries/service-intentions) allowing it to talk to `sample-app`. - -### VMs -To use transparent proxy on VMs, the service needs to be registered with Consul and a connect proxy needs to be added to -the mesh on the VM. Then, traffic redirection rules need to be set up to direct inbound and outbound traffic through the -sidecar connect proxy. Then, to enable transparent proxy mode to reach this service, you can set apply a service defaults -config entry to configure the mode to be transparent as shown above in the [Configuration section](#configuration). - -Now, once Service Intentions are set up, other services can reach this service's address via an address known to Consul, -and the traffic will go through the proxy. - -~> **Note** Only one service is supported per VM, or per network namespace. See [Known Limitations](#known-limitations)