diff --git a/.changelog/14238.txt b/.changelog/14238.txt new file mode 100644 index 000000000..43c570915 --- /dev/null +++ b/.changelog/14238.txt @@ -0,0 +1,3 @@ +```release-note:improvement +envoy: adds additional Envoy outlier ejection parameters to passive health check configurations. +``` \ No newline at end of file diff --git a/agent/consul/config_endpoint_test.go b/agent/consul/config_endpoint_test.go index 3f79b3d1b..aaf7cba94 100644 --- a/agent/consul/config_endpoint_test.go +++ b/agent/consul/config_endpoint_test.go @@ -1399,8 +1399,9 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Protocol: "http", MeshGateway: structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeRemote}, PassiveHealthCheck: &structs.PassiveHealthCheck{ - Interval: 10, - MaxFailures: 2, + Interval: 10, + MaxFailures: 2, + EnforcingConsecutive5xx: uintPointer(60), }, }, Overrides: []*structs.UpstreamConfig{ @@ -1432,8 +1433,9 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Upstream: wildcard, Config: map[string]interface{}{ "passive_health_check": map[string]interface{}{ - "Interval": int64(10), - "MaxFailures": int64(2), + "Interval": int64(10), + "MaxFailures": int64(2), + "EnforcingConsecutive5xx": int64(60), }, "mesh_gateway": map[string]interface{}{ "Mode": "remote", @@ -1445,8 +1447,9 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Upstream: mysql, Config: map[string]interface{}{ "passive_health_check": map[string]interface{}{ - "Interval": int64(10), - "MaxFailures": int64(2), + "Interval": int64(10), + "MaxFailures": int64(2), + "EnforcingConsecutive5xx": int64(60), }, "mesh_gateway": map[string]interface{}{ "Mode": "local", @@ -2507,3 +2510,7 @@ func Test_gateWriteToSecondary_AllowedKinds(t *testing.T) { }) } } + +func uintPointer(v uint32) *uint32 { + return &v +} diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 88c523a15..23c5c8e47 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -969,6 +969,11 @@ type PassiveHealthCheck struct { // MaxFailures is the count of consecutive failures that results in a host // being removed from the pool. MaxFailures uint32 `json:",omitempty" alias:"max_failures"` + + // EnforcingConsecutive5xx is the % chance that a host will be actually ejected + // when an outlier status is detected through consecutive 5xx. + // This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100. + EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"` } func (chk *PassiveHealthCheck) Clone() *PassiveHealthCheck { diff --git a/agent/structs/config_entry_test.go b/agent/structs/config_entry_test.go index a9e113f21..c08e82399 100644 --- a/agent/structs/config_entry_test.go +++ b/agent/structs/config_entry_test.go @@ -2754,8 +2754,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { MaxConcurrentRequests: intPointer(12), }, "passive_health_check": &PassiveHealthCheck{ - MaxFailures: 13, - Interval: 14 * time.Second, + MaxFailures: 13, + Interval: 14 * time.Second, + EnforcingConsecutive5xx: uintPointer(80), }, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, }, @@ -2770,8 +2771,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { MaxConcurrentRequests: intPointer(12), }, "passive_health_check": &PassiveHealthCheck{ - MaxFailures: 13, - Interval: 14 * time.Second, + MaxFailures: 13, + Interval: 14 * time.Second, + EnforcingConsecutive5xx: uintPointer(80), }, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, }, @@ -3067,3 +3069,7 @@ func testConfigEntryNormalizeAndValidate(t *testing.T, cases map[string]configEn }) } } + +func uintPointer(v uint32) *uint32 { + return &v +} diff --git a/agent/structs/testing_connect_proxy_config.go b/agent/structs/testing_connect_proxy_config.go index ad918927a..fdee3f693 100644 --- a/agent/structs/testing_connect_proxy_config.go +++ b/agent/structs/testing_connect_proxy_config.go @@ -26,7 +26,7 @@ func TestUpstreams(t testing.T) Upstreams { Config: map[string]interface{}{ // Float because this is how it is decoded by JSON decoder so this // enables the value returned to be compared directly to a decoded JSON - // response without spurios type loss. + // response without spurious type loss. "connect_timeout_ms": float64(1000), }, }, diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 26087dd1d..5efd5029c 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -169,6 +169,18 @@ func TestClustersFromSnapshot(t *testing.T) { }, nil) }, }, + { + name: "custom-passive-healthcheck", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) { + ns.Proxy.Upstreams[0].Config["passive_health_check"] = map[string]interface{}{ + "enforcing_consecutive_5xx": float64(80), + "max_failures": float64(5), + "interval": float64(10), + } + }, nil) + }, + }, { name: "custom-max-inbound-connections", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/config.go b/agent/xds/config.go index cfbd23e07..0736fb44c 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -174,5 +174,10 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck) *envoy_cluster_v3.Outlier if p.MaxFailures != 0 { od.Consecutive_5Xx = &wrappers.UInt32Value{Value: p.MaxFailures} } + + if p.EnforcingConsecutive5xx != nil { + od.EnforcingConsecutive_5Xx = &wrappers.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } + return od } diff --git a/agent/xds/testdata/clusters/custom-passive-healthcheck.latest.golden b/agent/xds/testdata/clusters/custom-passive-healthcheck.latest.golden new file mode 100644 index 000000000..41bc16f6e --- /dev/null +++ b/agent/xds/testdata/clusters/custom-passive-healthcheck.latest.golden @@ -0,0 +1,147 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + "consecutive5xx": 5, + "interval": "0.000000010s", + "enforcingConsecutive5xx": 80 + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + } + ] + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" + }, + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" + } + ] + } + }, + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/api/config_entry.go b/api/config_entry.go index da685b786..7fe128958 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -196,6 +196,11 @@ type PassiveHealthCheck struct { // MaxFailures is the count of consecutive failures that results in a host // being removed from the pool. MaxFailures uint32 `alias:"max_failures"` + + // EnforcingConsecutive5xx is the % chance that a host will be actually ejected + // when an outlier status is detected through consecutive 5xx. + // This setting can be used to disable ejection or to ramp it up slowly. + EnforcingConsecutive5xx uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"` } // UpstreamLimits describes the limits that are associated with a specific diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index b431e4345..2ce58c271 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -503,6 +503,16 @@ represents a location outside the Consul cluster. They can be dialed directly wh description: `The number of consecutive failures which cause a host to be removed from the load balancer.`, }, + { + name: 'EnforcingConsecutive5xx', + type: 'int: 100', + description: { + hcl: `The % chance that a host will be actually ejected + when an outlier status is detected through consecutive 5xx.`, + yaml: `The % chance that a host will be actually ejected + when an outlier status is detected through consecutive 5xx.`, + }, + }, ], }, ], @@ -635,6 +645,16 @@ represents a location outside the Consul cluster. They can be dialed directly wh description: `The number of consecutive failures which cause a host to be removed from the load balancer.`, }, + { + name: 'EnforcingConsecutive5xx', + type: 'int: 100', + description: { + hcl: `The % chance that a host will be actually ejected + when an outlier status is detected through consecutive 5xx.`, + yaml: `The % chance that a host will be actually ejected + when an outlier status is detected through consecutive 5xx.`, + }, + }, ], }, ], diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 812adff17..d6222e898 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -309,6 +309,8 @@ definition](/docs/connect/registration/service-registration) or load balancer. - `max_failures` - The number of consecutive failures which cause a host to be removed from the load balancer. + - `enforcing_consecutive_5xx` - The % chance that a host will be actually ejected + when an outlier status is detected through consecutive 5xx. ### Gateway Options