luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge pull request #14800 from hashicorp/mgw-tcp-keepalives 

Add TCP keepalive settings to proxy config for mesh gateways
This commit is contained in:
Kyle Havlovitz 2022-10-14 19:01:02 -07:00 committed by GitHub
parent 5fbb4aaac0 096ca5e4b0
commit 73d252c6d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 405 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/14800.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on terminating gateway upstreams and mesh gateways in remote datacenters.
```

17
agent/xds/clusters.go
View File

@ -1534,6 +1534,23 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op
useEDS = false
}
// TCP keepalive settings can be enabled for terminating gateway upstreams or remote mesh gateways.
remoteUpstream := opts.isRemote || snap.Kind == structs.ServiceKindTerminatingGateway
if remoteUpstream && cfg.TcpKeepaliveEnable {
cluster.UpstreamConnectionOptions = &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{},
}
if cfg.TcpKeepaliveTime != 0 {
cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = makeUint32Value(cfg.TcpKeepaliveTime)
}
if cfg.TcpKeepaliveInterval != 0 {
cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = makeUint32Value(cfg.TcpKeepaliveInterval)
}
if cfg.TcpKeepaliveProbes != 0 {
cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = makeUint32Value(cfg.TcpKeepaliveProbes)
}
}
// If none of the service instances are addressed by a hostname we provide the endpoint IP addresses via EDS
if useEDS {
cluster.ClusterDiscoveryType = &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}

25
agent/xds/clusters_test.go
View File

@ -412,6 +412,17 @@ func TestClustersFromSnapshot(t *testing.T) {
return proxycfg.TestConfigSnapshotMeshGateway(t, "hash-lb-ignored", nil, nil)
},
},
{
name: "mesh-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotMeshGateway(t, "default", func(ns *structs.NodeService) {
ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 120
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 60
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7
}, nil)
},
},
{
name: "ingress-gateway",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
@ -663,6 +674,20 @@ func TestClustersFromSnapshot(t *testing.T) {
name: "terminating-gateway-lb-config",
create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies,
},
{
name: "terminating-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, func(ns *structs.NodeService) {
if ns.Proxy.Config == nil {
ns.Proxy.Config = map[string]interface{}{}
}
ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 133
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 27
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5
}, nil)
},
},
{
name: "ingress-multiple-listeners-duplicate-service",
create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService,

7
agent/xds/config.go
View File

@ -132,6 +132,13 @@ type GatewayConfig struct {
// ConnectTimeoutMs is the number of milliseconds to timeout making a new
// connection to this upstream. Defaults to 5000 (5 seconds) if not set.
ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"`
// TCP keepalive settings for remote gateway upstreams (mesh gateways and terminating gateway upstreams).
// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive
TcpKeepaliveEnable bool `mapstructure:"envoy_gateway_remote_tcp_enable_keepalive"`
TcpKeepaliveTime int `mapstructure:"envoy_gateway_remote_tcp_keepalive_time"`
TcpKeepaliveInterval int `mapstructure:"envoy_gateway_remote_tcp_keepalive_interval"`
TcpKeepaliveProbes int `mapstructure:"envoy_gateway_remote_tcp_keepalive_probes"`
}
// ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an

143
agent/xds/testdata/clusters/mesh-gateway-tcp-keepalives.latest.golden vendored Normal file
View File

@ -0,0 +1,143 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 7,
"keepaliveTime": 120,
"keepaliveInterval": 60
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 7,
"keepaliveTime": 120,
"keepaliveInterval": 60
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-east-1.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 7,
"keepaliveTime": 120,
"keepaliveInterval": 60
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

190
agent/xds/testdata/clusters/terminating-gateway-tcp-keepalives.latest.golden vendored Normal file
View File

@ -0,0 +1,190 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
}
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "cache.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "db.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
}
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

20
website/content/docs/connect/proxies/envoy.mdx
View File

@ -441,6 +441,26 @@ will continue to be supported.
addressed by a hostname, such as a managed database. It also applies to mesh gateways,
such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname.
- `envoy_gateway_remote_tcp_enable_keepalive` - Enables TCP keepalive settings on remote
upstream connections for mesh and terminating gateways. Defaults to `false`. Must be one
of `true` or `false`. Details for this feature are available in the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
- `envoy_gateway_remote_tcp_keepalive_time` - The number of seconds a connection needs to
be idle before keep-alive probes start being sent. For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to remote upstream connections for mesh and terminating gateways.
- `envoy_gateway_remote_tcp_keepalive_interval` - The number of seconds between keep-alive probes.
For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to remote upstream connections for mesh and terminating gateways.
- `envoy_gateway_remote_tcp_keepalive_probes` - Maximum number of keepalive probes to send without
response before deciding the connection is dead. For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to remote upstream connections for mesh and terminating gateways.
## Advanced Configuration
To support more flexibility when configuring Envoy, several "lower-level" options exist