diff --git a/.changelog/14800.txt b/.changelog/14800.txt index 5a06b00a8..22f0b7126 100644 --- a/.changelog/14800.txt +++ b/.changelog/14800.txt @@ -1,3 +1,3 @@ ```release-note:improvement -connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on upstream connections to mesh gateways in remote datacenters. +connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on terminating gateway upstreams and mesh gateways in remote datacenters. ``` \ No newline at end of file diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 8add00ba2..d6165569a 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -1534,8 +1534,9 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op useEDS = false } - // Set TCP keepalive settings on the upstream gateway cluster if enabled. - if opts.isRemote && cfg.TcpKeepaliveEnable { + // TCP keepalive settings can be enabled for terminating gateway upstreams or remote mesh gateways. + remoteUpstream := opts.isRemote || snap.Kind == structs.ServiceKindTerminatingGateway + if remoteUpstream && cfg.TcpKeepaliveEnable { cluster.UpstreamConnectionOptions = &envoy_cluster_v3.UpstreamConnectionOptions{ TcpKeepalive: &envoy_core_v3.TcpKeepalive{}, } diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index d1a978dd1..b9355b44d 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -416,10 +416,10 @@ func TestClustersFromSnapshot(t *testing.T) { name: "mesh-gateway-tcp-keepalives", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "default", func(ns *structs.NodeService) { - ns.Proxy.Config["envoy_mesh_gateway_tcp_enable_keepalive"] = true - ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_time"] = 120 - ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_interval"] = 60 - ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_probes"] = 7 + ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true + ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 120 + ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 60 + ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7 }, nil) }, }, @@ -674,6 +674,20 @@ func TestClustersFromSnapshot(t *testing.T) { name: "terminating-gateway-lb-config", create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies, }, + { + name: "terminating-gateway-tcp-keepalives", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, func(ns *structs.NodeService) { + if ns.Proxy.Config == nil { + ns.Proxy.Config = map[string]interface{}{} + } + ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true + ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 133 + ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 27 + ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5 + }, nil) + }, + }, { name: "ingress-multiple-listeners-duplicate-service", create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, diff --git a/agent/xds/config.go b/agent/xds/config.go index 9db93714a..1d75e0a90 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -133,12 +133,12 @@ type GatewayConfig struct { // connection to this upstream. Defaults to 5000 (5 seconds) if not set. ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"` - // TCP keepalive settings for connections between remote mesh gateways. + // TCP keepalive settings for remote gateway upstreams (mesh gateways and terminating gateway upstreams). // See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive - TcpKeepaliveEnable bool `mapstructure:"envoy_mesh_gateway_tcp_enable_keepalive"` - TcpKeepaliveTime int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_time"` - TcpKeepaliveInterval int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_interval"` - TcpKeepaliveProbes int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_probes"` + TcpKeepaliveEnable bool `mapstructure:"envoy_gateway_remote_tcp_enable_keepalive"` + TcpKeepaliveTime int `mapstructure:"envoy_gateway_remote_tcp_keepalive_time"` + TcpKeepaliveInterval int `mapstructure:"envoy_gateway_remote_tcp_keepalive_interval"` + TcpKeepaliveProbes int `mapstructure:"envoy_gateway_remote_tcp_keepalive_probes"` } // ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an diff --git a/agent/xds/testdata/clusters/terminating-gateway-tcp-keepalives.latest.golden b/agent/xds/testdata/clusters/terminating-gateway-tcp-keepalives.latest.golden new file mode 100644 index 000000000..1ab63951c --- /dev/null +++ b/agent/xds/testdata/clusters/terminating-gateway-tcp-keepalives.latest.golden @@ -0,0 +1,190 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "filename": "api.cert.pem" + }, + "privateKey": { + "filename": "api.key.pem" + } + } + ], + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + } + }, + "upstreamConnectionOptions": { + "tcpKeepalive": { + "keepaliveProbes": 5, + "keepaliveTime": 133, + "keepaliveInterval": 27 + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "cache.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "upstreamConnectionOptions": { + "tcpKeepalive": { + "keepaliveProbes": 5, + "keepaliveTime": 133, + "keepaliveInterval": 27 + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "db.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "upstreamConnectionOptions": { + "tcpKeepalive": { + "keepaliveProbes": 5, + "keepaliveTime": 133, + "keepaliveInterval": 27 + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + } + }, + "upstreamConnectionOptions": { + "tcpKeepalive": { + "keepaliveProbes": 5, + "keepaliveTime": 133, + "keepaliveInterval": 27 + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index c000702e9..e95e25ccf 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -441,24 +441,25 @@ will continue to be supported. addressed by a hostname, such as a managed database. It also applies to mesh gateways, such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname. -- `envoy_mesh_gateway_tcp_enable_keepalive` - Enables TCP keepalive settings on mesh gateway upstream connections - to remote datacenters. Defaults to `false`. Must be one of `true` or `false`. Details for this feature are available in - the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). +- `envoy_gateway_remote_tcp_enable_keepalive` - Enables TCP keepalive settings on remote + upstream connections for mesh and terminating gateways. Defaults to `false`. Must be one + of `true` or `false`. Details for this feature are available in the + [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). -- `envoy_mesh_gateway_tcp_keepalive_time` - The number of seconds a connection needs to +- `envoy_gateway_remote_tcp_keepalive_time` - The number of seconds a connection needs to be idle before keep-alive probes start being sent. For more information, see the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). - This option only applies to mesh gateway upstream connections to remote datacenters. + This option only applies to remote upstream connections for mesh and terminating gateways. -- `envoy_mesh_gateway_tcp_keepalive_interval` - The number of seconds between keep-alive probes. +- `envoy_gateway_remote_tcp_keepalive_interval` - The number of seconds between keep-alive probes. For more information, see the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). - This option only applies to mesh gateway upstream connections to remote datacenters. + This option only applies to remote upstream connections for mesh and terminating gateways. -- `envoy_mesh_gateway_tcp_keepalive_probes` - Maximum number of keepalive probes to send without +- `envoy_gateway_remote_tcp_keepalive_probes` - Maximum number of keepalive probes to send without response before deciding the connection is dead. For more information, see the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). - This option only applies to mesh gateway upstream connections to remote datacenters. + This option only applies to remote upstream connections for mesh and terminating gateways. ## Advanced Configuration