luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Extend tcp keepalive settings to work for terminating gateways as well

This commit is contained in:
Kyle Havlovitz 2022-10-11 12:22:55 -07:00
parent f8e745315f
commit 096ca5e4b0
6 changed files with 227 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.changelog/14800.txt
View File

@ -1,3 +1,3 @@
```release-note:improvement ```release-note:improvement
connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on upstream connections to mesh gateways in remote datacenters. connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on terminating gateway upstreams and mesh gateways in remote datacenters.
``` ```

5
agent/xds/clusters.go
View File

@ -1534,8 +1534,9 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op
useEDS = false useEDS = false
} }
// Set TCP keepalive settings on the upstream gateway cluster if enabled. // TCP keepalive settings can be enabled for terminating gateway upstreams or remote mesh gateways.
if opts.isRemote && cfg.TcpKeepaliveEnable { remoteUpstream := opts.isRemote || snap.Kind == structs.ServiceKindTerminatingGateway
if remoteUpstream && cfg.TcpKeepaliveEnable {
cluster.UpstreamConnectionOptions = &envoy_cluster_v3.UpstreamConnectionOptions{ cluster.UpstreamConnectionOptions = &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{}, TcpKeepalive: &envoy_core_v3.TcpKeepalive{},
} }

22
agent/xds/clusters_test.go
View File

@ -416,10 +416,10 @@ func TestClustersFromSnapshot(t *testing.T) {
name: "mesh-gateway-tcp-keepalives", name: "mesh-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotMeshGateway(t, "default", func(ns *structs.NodeService) { return proxycfg.TestConfigSnapshotMeshGateway(t, "default", func(ns *structs.NodeService) {
ns.Proxy.Config["envoy_mesh_gateway_tcp_enable_keepalive"] = true ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_time"] = 120 ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 120
ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_interval"] = 60 ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 60
ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_probes"] = 7 ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7
}, nil) }, nil)
}, },
}, },
@ -674,6 +674,20 @@ func TestClustersFromSnapshot(t *testing.T) {
name: "terminating-gateway-lb-config", name: "terminating-gateway-lb-config",
create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies, create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies,
}, },
{
name: "terminating-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, func(ns *structs.NodeService) {
if ns.Proxy.Config == nil {
ns.Proxy.Config = map[string]interface{}{}
}
ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 133
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 27
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5
}, nil)
},
},
{ {
name: "ingress-multiple-listeners-duplicate-service", name: "ingress-multiple-listeners-duplicate-service",
create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService,

10
agent/xds/config.go
View File

@ -133,12 +133,12 @@ type GatewayConfig struct {
// connection to this upstream. Defaults to 5000 (5 seconds) if not set. // connection to this upstream. Defaults to 5000 (5 seconds) if not set.
ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"` ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"`
// TCP keepalive settings for connections between remote mesh gateways. // TCP keepalive settings for remote gateway upstreams (mesh gateways and terminating gateway upstreams).
// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive // See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive
TcpKeepaliveEnable bool `mapstructure:"envoy_mesh_gateway_tcp_enable_keepalive"` TcpKeepaliveEnable bool `mapstructure:"envoy_gateway_remote_tcp_enable_keepalive"`
TcpKeepaliveTime int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_time"` TcpKeepaliveTime int `mapstructure:"envoy_gateway_remote_tcp_keepalive_time"`
TcpKeepaliveInterval int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_interval"` TcpKeepaliveInterval int `mapstructure:"envoy_gateway_remote_tcp_keepalive_interval"`
TcpKeepaliveProbes int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_probes"` TcpKeepaliveProbes int `mapstructure:"envoy_gateway_remote_tcp_keepalive_probes"`
} }
// ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an // ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an

190
agent/xds/testdata/clusters/terminating-gateway-tcp-keepalives.latest.golden vendored Normal file
View File

@ -0,0 +1,190 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
}
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "cache.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "db.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
}
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

19
website/content/docs/connect/proxies/envoy.mdx
View File

@ -441,24 +441,25 @@ will continue to be supported.
addressed by a hostname, such as a managed database. It also applies to mesh gateways, addressed by a hostname, such as a managed database. It also applies to mesh gateways,
such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname. such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname.
- `envoy_mesh_gateway_tcp_enable_keepalive` - Enables TCP keepalive settings on mesh gateway upstream connections - `envoy_gateway_remote_tcp_enable_keepalive` - Enables TCP keepalive settings on remote
to remote datacenters. Defaults to `false`. Must be one of `true` or `false`. Details for this feature are available in upstream connections for mesh and terminating gateways. Defaults to `false`. Must be one
the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). of `true` or `false`. Details for this feature are available in the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
- `envoy_mesh_gateway_tcp_keepalive_time` - The number of seconds a connection needs to - `envoy_gateway_remote_tcp_keepalive_time` - The number of seconds a connection needs to
be idle before keep-alive probes start being sent. For more information, see the be idle before keep-alive probes start being sent. For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to mesh gateway upstream connections to remote datacenters. This option only applies to remote upstream connections for mesh and terminating gateways.
- `envoy_mesh_gateway_tcp_keepalive_interval` - The number of seconds between keep-alive probes. - `envoy_gateway_remote_tcp_keepalive_interval` - The number of seconds between keep-alive probes.
For more information, see the For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to mesh gateway upstream connections to remote datacenters. This option only applies to remote upstream connections for mesh and terminating gateways.
- `envoy_mesh_gateway_tcp_keepalive_probes` - Maximum number of keepalive probes to send without - `envoy_gateway_remote_tcp_keepalive_probes` - Maximum number of keepalive probes to send without
response before deciding the connection is dead. For more information, see the response before deciding the connection is dead. For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive). [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to mesh gateway upstream connections to remote datacenters. This option only applies to remote upstream connections for mesh and terminating gateways.
## Advanced Configuration ## Advanced Configuration