Support Envoy's MaxEjectionPercent and BaseEjectionTime config entries for passive health checks (#15979)

* Add MaxEjectionPercent to config entry

* Add BaseEjectionTime to config entry

* Add MaxEjectionPercent and BaseEjectionTime to protobufs

* Add MaxEjectionPercent and BaseEjectionTime to api

* Fix integration test breakage

* Verify MaxEjectionPercent and BaseEjectionTime in integration test upstream confings

* Website docs for MaxEjectionPercent and BaseEjection time

* Add `make docs` to browse docs at http://localhost:3000

* Changelog entry

* so that is the difference between consul-docker and dev-docker

* blah

* update proto funcs

* update proto

---------

Co-authored-by: Maliz <maliheh.monshizadeh@hashicorp.com>
This commit is contained in:
Semir Patel 2023-04-26 17:59:48 -05:00 committed by GitHub
parent f24e5ed1d6
commit 406c1afc04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 786 additions and 504 deletions

3
.changelog/15979.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs.
```

View File

@ -405,6 +405,7 @@ ui-build-image:
@echo "Building UI build container" @echo "Building UI build container"
@docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile @docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile
# Builds consul in a docker container and then dumps executable into ./pkg/bin/...
consul-docker: go-build-image consul-docker: go-build-image
@$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul
@ -538,6 +539,11 @@ envoy-regen:
@find "command/connect/envoy/testdata" -name '*.golden' -delete @find "command/connect/envoy/testdata" -name '*.golden' -delete
@go test -tags '$(GOTAGS)' ./command/connect/envoy -update @go test -tags '$(GOTAGS)' ./command/connect/envoy -update
# Point your web browser to http://localhost:3000/consul to live render docs from ./website/
.PHONY: docs
docs:
make -C website
.PHONY: help .PHONY: help
help: help:
$(info available make targets) $(info available make targets)

View File

@ -1484,6 +1484,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
Interval: 10, Interval: 10,
MaxFailures: 2, MaxFailures: 2,
EnforcingConsecutive5xx: uintPointer(60), EnforcingConsecutive5xx: uintPointer(60),
MaxEjectionPercent: uintPointer(61),
BaseEjectionTime: durationPointer(62 * time.Second),
}, },
}, },
Overrides: []*structs.UpstreamConfig{ Overrides: []*structs.UpstreamConfig{
@ -1518,6 +1520,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
"Interval": int64(10), "Interval": int64(10),
"MaxFailures": int64(2), "MaxFailures": int64(2),
"EnforcingConsecutive5xx": int64(60), "EnforcingConsecutive5xx": int64(60),
"MaxEjectionPercent": int64(61),
"BaseEjectionTime": uint64(62 * time.Second),
}, },
"mesh_gateway": map[string]interface{}{ "mesh_gateway": map[string]interface{}{
"Mode": "none", "Mode": "none",
@ -1532,6 +1536,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
"Interval": int64(10), "Interval": int64(10),
"MaxFailures": int64(2), "MaxFailures": int64(2),
"EnforcingConsecutive5xx": int64(60), "EnforcingConsecutive5xx": int64(60),
"MaxEjectionPercent": int64(61),
"BaseEjectionTime": uint64(62 * time.Second),
}, },
"mesh_gateway": map[string]interface{}{ "mesh_gateway": map[string]interface{}{
"Mode": "local", "Mode": "local",
@ -2639,3 +2645,7 @@ func Test_gateWriteToSecondary_AllowedKinds(t *testing.T) {
func uintPointer(v uint32) *uint32 { func uintPointer(v uint32) *uint32 {
return &v return &v
} }
func durationPointer(d time.Duration) *time.Duration {
return &d
}

View File

@ -7,6 +7,7 @@ import (
"github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/proto/private/pbpeering" "github.com/hashicorp/consul/proto/private/pbpeering"
"github.com/hashicorp/consul/types" "github.com/hashicorp/consul/types"
"time"
) )
// DeepCopy generates a deep copy of *ConfigSnapshot // DeepCopy generates a deep copy of *ConfigSnapshot
@ -452,6 +453,14 @@ func (o *configSnapshotIngressGateway) DeepCopy() *configSnapshotIngressGateway
cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32) cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
*cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx *cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx
} }
if o.Defaults.PassiveHealthCheck.MaxEjectionPercent != nil {
cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = new(uint32)
*cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = *o.Defaults.PassiveHealthCheck.MaxEjectionPercent
}
if o.Defaults.PassiveHealthCheck.BaseEjectionTime != nil {
cp.Defaults.PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
*cp.Defaults.PassiveHealthCheck.BaseEjectionTime = *o.Defaults.PassiveHealthCheck.BaseEjectionTime
}
} }
return &cp return &cp
} }

View File

@ -1101,6 +1101,16 @@ type PassiveHealthCheck struct {
// when an outlier status is detected through consecutive 5xx. // when an outlier status is detected through consecutive 5xx.
// This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100. // This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100.
EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"` EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
// The maximum % of an upstream cluster that can be ejected due to outlier detection.
// Defaults to 10% but will eject at least one host regardless of the value.
// TODO: remove me
MaxEjectionPercent *uint32 `json:",omitempty" alias:"max_ejection_percent"`
// The base time that a host is ejected for. The real time is equal to the base time
// multiplied by the number of times the host has been ejected and is capped by
// max_ejection_time (Default 300s). Defaults to 30000ms or 30s.
BaseEjectionTime *time.Duration `json:",omitempty" alias:"base_ejection_time"`
} }
func (chk *PassiveHealthCheck) Clone() *PassiveHealthCheck { func (chk *PassiveHealthCheck) Clone() *PassiveHealthCheck {
@ -1120,6 +1130,15 @@ func (chk PassiveHealthCheck) Validate() error {
if chk.Interval < 0*time.Second { if chk.Interval < 0*time.Second {
return fmt.Errorf("passive health check interval cannot be negative") return fmt.Errorf("passive health check interval cannot be negative")
} }
if chk.EnforcingConsecutive5xx != nil && *chk.EnforcingConsecutive5xx > 100 {
return fmt.Errorf("passive health check enforcing_consecutive_5xx must be a percentage between 0 and 100")
}
if chk.MaxEjectionPercent != nil && *chk.MaxEjectionPercent > 100 {
return fmt.Errorf("passive health check max_ejection_percent must be a percentage between 0 and 100")
}
if chk.BaseEjectionTime != nil && *chk.BaseEjectionTime < 0*time.Second {
return fmt.Errorf("passive health check base_ejection_time cannot be negative")
}
return nil return nil
} }

View File

@ -416,6 +416,9 @@ func TestDecodeConfigEntry(t *testing.T) {
passive_health_check { passive_health_check {
interval = "2s" interval = "2s"
max_failures = 3 max_failures = 3
enforcing_consecutive_5xx = 4
max_ejection_percent = 5
base_ejection_time = "6s"
} }
}, },
{ {
@ -460,6 +463,9 @@ func TestDecodeConfigEntry(t *testing.T) {
PassiveHealthCheck { PassiveHealthCheck {
MaxFailures = 3 MaxFailures = 3
Interval = "2s" Interval = "2s"
EnforcingConsecutive5xx = 4
MaxEjectionPercent = 5
BaseEjectionTime = "6s"
} }
}, },
{ {
@ -502,8 +508,11 @@ func TestDecodeConfigEntry(t *testing.T) {
{ {
Name: "redis", Name: "redis",
PassiveHealthCheck: &PassiveHealthCheck{ PassiveHealthCheck: &PassiveHealthCheck{
MaxFailures: 3, MaxFailures: 3,
Interval: 2 * time.Second, Interval: 2 * time.Second,
EnforcingConsecutive5xx: uintPointer(4),
MaxEjectionPercent: uintPointer(5),
BaseEjectionTime: durationPointer(6 * time.Second),
}, },
}, },
{ {
@ -2351,6 +2360,39 @@ func TestPassiveHealthCheck_Validate(t *testing.T) {
wantErr: true, wantErr: true,
wantMsg: "cannot be negative", wantMsg: "cannot be negative",
}, },
{
name: "valid enforcing_consecutive_5xx",
input: PassiveHealthCheck{EnforcingConsecutive5xx: uintPointer(100)},
wantErr: false,
},
{
name: "invalid enforcing_consecutive_5xx",
input: PassiveHealthCheck{EnforcingConsecutive5xx: uintPointer(101)},
wantErr: true,
wantMsg: "must be a percentage",
},
{
name: "valid max_ejection_percent",
input: PassiveHealthCheck{MaxEjectionPercent: uintPointer(100)},
wantErr: false,
},
{
name: "invalid max_ejection_percent",
input: PassiveHealthCheck{MaxEjectionPercent: uintPointer(101)},
wantErr: true,
wantMsg: "must be a percentage",
},
{
name: "valid base_ejection_time",
input: PassiveHealthCheck{BaseEjectionTime: durationPointer(0 * time.Second)},
wantErr: false,
},
{
name: "negative base_ejection_time",
input: PassiveHealthCheck{BaseEjectionTime: durationPointer(-1 * time.Second)},
wantErr: true,
wantMsg: "cannot be negative",
},
} }
for _, tc := range tt { for _, tc := range tt {
@ -2890,8 +2932,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
MaxConcurrentRequests: intPointer(5), MaxConcurrentRequests: intPointer(5),
}, },
PassiveHealthCheck: &PassiveHealthCheck{ PassiveHealthCheck: &PassiveHealthCheck{
MaxFailures: 3, Interval: 2 * time.Second,
Interval: 2 * time.Second, MaxFailures: 3,
EnforcingConsecutive5xx: uintPointer(4),
MaxEjectionPercent: uintPointer(5),
BaseEjectionTime: durationPointer(6 * time.Second),
}, },
MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote}, MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
}, },
@ -2908,8 +2953,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
MaxConcurrentRequests: intPointer(5), MaxConcurrentRequests: intPointer(5),
}, },
"passive_health_check": &PassiveHealthCheck{ "passive_health_check": &PassiveHealthCheck{
MaxFailures: 3, Interval: 2 * time.Second,
Interval: 2 * time.Second, MaxFailures: 3,
EnforcingConsecutive5xx: uintPointer(4),
MaxEjectionPercent: uintPointer(5),
BaseEjectionTime: durationPointer(6 * time.Second),
}, },
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote}, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote},
}, },
@ -2928,8 +2976,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
MaxConcurrentRequests: intPointer(5), MaxConcurrentRequests: intPointer(5),
}, },
PassiveHealthCheck: &PassiveHealthCheck{ PassiveHealthCheck: &PassiveHealthCheck{
MaxFailures: 3, Interval: 2 * time.Second,
Interval: 2 * time.Second, MaxFailures: 3,
EnforcingConsecutive5xx: uintPointer(4),
MaxEjectionPercent: uintPointer(5),
BaseEjectionTime: durationPointer(6 * time.Second),
}, },
MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote}, MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
}, },
@ -2945,8 +2996,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
MaxConcurrentRequests: intPointer(12), MaxConcurrentRequests: intPointer(12),
}, },
"passive_health_check": &PassiveHealthCheck{ "passive_health_check": &PassiveHealthCheck{
MaxFailures: 13, MaxFailures: 13,
Interval: 14 * time.Second, Interval: 14 * time.Second,
EnforcingConsecutive5xx: uintPointer(15),
MaxEjectionPercent: uintPointer(16),
BaseEjectionTime: durationPointer(17 * time.Second),
}, },
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
}, },
@ -2962,8 +3016,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
MaxConcurrentRequests: intPointer(5), MaxConcurrentRequests: intPointer(5),
}, },
"passive_health_check": &PassiveHealthCheck{ "passive_health_check": &PassiveHealthCheck{
MaxFailures: 3, Interval: 2 * time.Second,
Interval: 2 * time.Second, MaxFailures: 3,
EnforcingConsecutive5xx: uintPointer(4),
MaxEjectionPercent: uintPointer(5),
BaseEjectionTime: durationPointer(6 * time.Second),
}, },
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote}, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote},
}, },
@ -2985,7 +3042,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
"passive_health_check": &PassiveHealthCheck{ "passive_health_check": &PassiveHealthCheck{
MaxFailures: 13, MaxFailures: 13,
Interval: 14 * time.Second, Interval: 14 * time.Second,
EnforcingConsecutive5xx: uintPointer(80), EnforcingConsecutive5xx: uintPointer(15),
MaxEjectionPercent: uintPointer(16),
BaseEjectionTime: durationPointer(17 * time.Second),
}, },
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
}, },
@ -3003,7 +3062,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
"passive_health_check": &PassiveHealthCheck{ "passive_health_check": &PassiveHealthCheck{
MaxFailures: 13, MaxFailures: 13,
Interval: 14 * time.Second, Interval: 14 * time.Second,
EnforcingConsecutive5xx: uintPointer(80), EnforcingConsecutive5xx: uintPointer(15),
MaxEjectionPercent: uintPointer(16),
BaseEjectionTime: durationPointer(17 * time.Second),
}, },
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
}, },
@ -3138,15 +3199,21 @@ func TestParseUpstreamConfig(t *testing.T) {
name: "passive health check map", name: "passive health check map",
input: map[string]interface{}{ input: map[string]interface{}{
"passive_health_check": map[string]interface{}{ "passive_health_check": map[string]interface{}{
"interval": "22s", "interval": "22s",
"max_failures": 7, "max_failures": 7,
"enforcing_consecutive_5xx": 8,
"max_ejection_percent": 9,
"base_ejection_time": "10s",
}, },
}, },
want: UpstreamConfig{ want: UpstreamConfig{
ConnectTimeoutMs: 5000, ConnectTimeoutMs: 5000,
PassiveHealthCheck: &PassiveHealthCheck{ PassiveHealthCheck: &PassiveHealthCheck{
Interval: 22 * time.Second, Interval: 22 * time.Second,
MaxFailures: 7, MaxFailures: 7,
EnforcingConsecutive5xx: uintPointer(8),
MaxEjectionPercent: uintPointer(9),
BaseEjectionTime: durationPointer(10 * time.Second),
}, },
Protocol: "tcp", Protocol: "tcp",
}, },
@ -3273,10 +3340,6 @@ func requireContainsLower(t *testing.T, haystack, needle string) {
require.Contains(t, strings.ToLower(haystack), strings.ToLower(needle)) require.Contains(t, strings.ToLower(haystack), strings.ToLower(needle))
} }
func intPointer(i int) *int {
return &i
}
func TestConfigEntryQuery_CacheInfoKey(t *testing.T) { func TestConfigEntryQuery_CacheInfoKey(t *testing.T) {
assertCacheInfoKeyIsComplete(t, &ConfigEntryQuery{}) assertCacheInfoKeyIsComplete(t, &ConfigEntryQuery{})
} }
@ -3371,6 +3434,14 @@ func testConfigEntryNormalizeAndValidate(t *testing.T, cases map[string]configEn
} }
} }
func intPointer(i int) *int {
return &i
}
func uintPointer(v uint32) *uint32 { func uintPointer(v uint32) *uint32 {
return &v return &v
} }
func durationPointer(d time.Duration) *time.Duration {
return &d
}

View File

@ -529,6 +529,14 @@ func (o *IngressListener) DeepCopy() *IngressListener {
cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32) cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
*cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx *cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx
} }
if o.Services[i2].PassiveHealthCheck.MaxEjectionPercent != nil {
cp.Services[i2].PassiveHealthCheck.MaxEjectionPercent = new(uint32)
*cp.Services[i2].PassiveHealthCheck.MaxEjectionPercent = *o.Services[i2].PassiveHealthCheck.MaxEjectionPercent
}
if o.Services[i2].PassiveHealthCheck.BaseEjectionTime != nil {
cp.Services[i2].PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
*cp.Services[i2].PassiveHealthCheck.BaseEjectionTime = *o.Services[i2].PassiveHealthCheck.BaseEjectionTime
}
} }
if o.Services[i2].Meta != nil { if o.Services[i2].Meta != nil {
cp.Services[i2].Meta = make(map[string]string, len(o.Services[i2].Meta)) cp.Services[i2].Meta = make(map[string]string, len(o.Services[i2].Meta))
@ -1134,6 +1142,14 @@ func (o *UpstreamConfiguration) DeepCopy() *UpstreamConfiguration {
cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32) cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
*cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx *cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx
} }
if o.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent != nil {
cp.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent = new(uint32)
*cp.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent = *o.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent
}
if o.Overrides[i2].PassiveHealthCheck.BaseEjectionTime != nil {
cp.Overrides[i2].PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
*cp.Overrides[i2].PassiveHealthCheck.BaseEjectionTime = *o.Overrides[i2].PassiveHealthCheck.BaseEjectionTime
}
} }
} }
} }
@ -1164,6 +1180,14 @@ func (o *UpstreamConfiguration) DeepCopy() *UpstreamConfiguration {
cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32) cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
*cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx *cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx
} }
if o.Defaults.PassiveHealthCheck.MaxEjectionPercent != nil {
cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = new(uint32)
*cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = *o.Defaults.PassiveHealthCheck.MaxEjectionPercent
}
if o.Defaults.PassiveHealthCheck.BaseEjectionTime != nil {
cp.Defaults.PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
*cp.Defaults.PassiveHealthCheck.BaseEjectionTime = *o.Defaults.PassiveHealthCheck.BaseEjectionTime
}
} }
} }
return &cp return &cp

View File

@ -3026,6 +3026,26 @@ func DurationFromProto(d *durationpb.Duration) time.Duration {
return d.AsDuration() return d.AsDuration()
} }
// This should only be used for conversions generated by MOG
func DurationPointerToProto(d *time.Duration) *durationpb.Duration {
if d == nil {
return nil
}
return durationpb.New(*d)
}
// This should only be used for conversions generated by MOG
func DurationPointerFromProto(d *durationpb.Duration) *time.Duration {
if d == nil {
return nil
}
return DurationPointer(d.AsDuration())
}
func DurationPointer(d time.Duration) *time.Duration {
return &d
}
// This should only be used for conversions generated by MOG // This should only be used for conversions generated by MOG
func TimeFromProto(s *timestamppb.Timestamp) time.Time { func TimeFromProto(s *timestamppb.Timestamp) time.Time {
return s.AsTime() return s.AsTime()

View File

@ -9,6 +9,7 @@ import (
"sort" "sort"
"testing" "testing"
"text/template" "text/template"
"time"
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
"github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/agent/xds/testcommon"
@ -270,7 +271,9 @@ func TestClustersFromSnapshot(t *testing.T) {
ns.Proxy.Upstreams[0].Config["passive_health_check"] = map[string]interface{}{ ns.Proxy.Upstreams[0].Config["passive_health_check"] = map[string]interface{}{
"enforcing_consecutive_5xx": float64(80), "enforcing_consecutive_5xx": float64(80),
"max_failures": float64(5), "max_failures": float64(5),
"interval": float64(10), "interval": float64(10 * time.Second),
"max_ejection_percent": float64(100),
"base_ejection_time": float64(10 * time.Second),
} }
}, nil) }, nil)
}, },

View File

@ -178,7 +178,7 @@ func ParseGatewayConfig(m map[string]interface{}) (GatewayConfig, error) {
return cfg, err return cfg, err
} }
// Return an envoy.OutlierDetection populated by the values from structs.PassiveHealthChec. // Return an envoy.OutlierDetection populated by the values from structs.PassiveHealthCheck.
// If all values are zero a default empty OutlierDetection will be returned to // If all values are zero a default empty OutlierDetection will be returned to
// enable outlier detection with default values. // enable outlier detection with default values.
// - If override is not nil, it will overwrite the values from p, e.g., ingress gateway defaults // - If override is not nil, it will overwrite the values from p, e.g., ingress gateway defaults
@ -197,13 +197,20 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passive
} }
if p.EnforcingConsecutive5xx != nil { if p.EnforcingConsecutive5xx != nil {
// NOTE: EnforcingConsecutive5xx must be great than 0 for ingress-gateway // NOTE: EnforcingConsecutive5xx must be greater than 0 for ingress-gateway
if *p.EnforcingConsecutive5xx != 0 { if *p.EnforcingConsecutive5xx != 0 {
od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
} else if allowZero { } else if allowZero {
od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
} }
} }
if p.MaxEjectionPercent != nil {
od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *p.MaxEjectionPercent}
}
if p.BaseEjectionTime != nil {
od.BaseEjectionTime = durationpb.New(*p.BaseEjectionTime)
}
} }
if override == nil { if override == nil {
@ -227,5 +234,12 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passive
} }
} }
if override.MaxEjectionPercent != nil {
od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *override.MaxEjectionPercent}
}
if override.BaseEjectionTime != nil {
od.BaseEjectionTime = durationpb.New(*override.BaseEjectionTime)
}
return od return od
} }

View File

@ -20,8 +20,10 @@
}, },
"outlierDetection": { "outlierDetection": {
"consecutive5xx": 5, "consecutive5xx": 5,
"interval": "0.000000010s", "interval": "10s",
"enforcingConsecutive5xx": 80 "enforcingConsecutive5xx": 80,
"maxEjectionPercent": 100,
"baseEjectionTime": "10s"
}, },
"commonLbConfig": { "commonLbConfig": {
"healthyPanicThreshold": { "healthyPanicThreshold": {

View File

@ -277,6 +277,15 @@ type PassiveHealthCheck struct {
// when an outlier status is detected through consecutive 5xx. // when an outlier status is detected through consecutive 5xx.
// This setting can be used to disable ejection or to ramp it up slowly. // This setting can be used to disable ejection or to ramp it up slowly.
EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"` EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
// The maximum % of an upstream cluster that can be ejected due to outlier detection.
// Defaults to 10% but will eject at least one host regardless of the value.
MaxEjectionPercent *uint32 `json:",omitempty" alias:"max_ejection_percent"`
// The base time that a host is ejected for. The real time is equal to the base time
// multiplied by the number of times the host has been ejected and is capped by
// max_ejection_time (Default 300s). Defaults to 30000ms or 30s.
BaseEjectionTime *time.Duration `json:",omitempty" alias:"base_ejection_time"`
} }
// UpstreamLimits describes the limits that are associated with a specific // UpstreamLimits describes the limits that are associated with a specific

View File

@ -484,7 +484,9 @@ func TestDecodeConfigEntry(t *testing.T) {
"PassiveHealthCheck": { "PassiveHealthCheck": {
"MaxFailures": 3, "MaxFailures": 3,
"Interval": "2s", "Interval": "2s",
"EnforcingConsecutive5xx": 60 "EnforcingConsecutive5xx": 60,
"MaxEjectionPercent": 4,
"BaseEjectionTime": "5s"
}, },
"BalanceOutboundConnections": "exact_balance" "BalanceOutboundConnections": "exact_balance"
}, },
@ -507,7 +509,11 @@ func TestDecodeConfigEntry(t *testing.T) {
}, },
"PassiveHealthCheck": { "PassiveHealthCheck": {
"MaxFailures": 5, "MaxFailures": 5,
"Interval": "4s" "Interval": "4s",
"EnforcingConsecutive5xx": 61,
"MaxEjectionPercent": 5,
"BaseEjectionTime": "6s"
} }
} }
} }
@ -539,6 +545,8 @@ func TestDecodeConfigEntry(t *testing.T) {
MaxFailures: 3, MaxFailures: 3,
Interval: 2 * time.Second, Interval: 2 * time.Second,
EnforcingConsecutive5xx: uint32Pointer(60), EnforcingConsecutive5xx: uint32Pointer(60),
MaxEjectionPercent: uint32Pointer(4),
BaseEjectionTime: durationPointer(5 * time.Second),
}, },
BalanceOutboundConnections: "exact_balance", BalanceOutboundConnections: "exact_balance",
}, },
@ -558,8 +566,11 @@ func TestDecodeConfigEntry(t *testing.T) {
MaxConcurrentRequests: intPointer(5), MaxConcurrentRequests: intPointer(5),
}, },
PassiveHealthCheck: &PassiveHealthCheck{ PassiveHealthCheck: &PassiveHealthCheck{
MaxFailures: 5, MaxFailures: 5,
Interval: 4 * time.Second, Interval: 4 * time.Second,
EnforcingConsecutive5xx: uint32Pointer(61),
MaxEjectionPercent: uint32Pointer(5),
BaseEjectionTime: durationPointer(6 * time.Second),
}, },
}, },
}, },
@ -1429,3 +1440,7 @@ func intPointer(v int) *int {
func uint32Pointer(v uint32) *uint32 { func uint32Pointer(v uint32) *uint32 {
return &v return &v
} }
func durationPointer(d time.Duration) *time.Duration {
return &d
}

View File

@ -1243,6 +1243,8 @@ func PassiveHealthCheckToStructs(s *PassiveHealthCheck, t *structs.PassiveHealth
t.Interval = structs.DurationFromProto(s.Interval) t.Interval = structs.DurationFromProto(s.Interval)
t.MaxFailures = s.MaxFailures t.MaxFailures = s.MaxFailures
t.EnforcingConsecutive5xx = pointerToUint32FromUint32(s.EnforcingConsecutive5Xx) t.EnforcingConsecutive5xx = pointerToUint32FromUint32(s.EnforcingConsecutive5Xx)
t.MaxEjectionPercent = pointerToUint32FromUint32(s.MaxEjectionPercent)
t.BaseEjectionTime = structs.DurationPointerFromProto(s.BaseEjectionTime)
} }
func PassiveHealthCheckFromStructs(t *structs.PassiveHealthCheck, s *PassiveHealthCheck) { func PassiveHealthCheckFromStructs(t *structs.PassiveHealthCheck, s *PassiveHealthCheck) {
if s == nil { if s == nil {
@ -1251,6 +1253,8 @@ func PassiveHealthCheckFromStructs(t *structs.PassiveHealthCheck, s *PassiveHeal
s.Interval = structs.DurationToProto(t.Interval) s.Interval = structs.DurationToProto(t.Interval)
s.MaxFailures = t.MaxFailures s.MaxFailures = t.MaxFailures
s.EnforcingConsecutive5Xx = uint32FromPointerToUint32(t.EnforcingConsecutive5xx) s.EnforcingConsecutive5Xx = uint32FromPointerToUint32(t.EnforcingConsecutive5xx)
s.MaxEjectionPercent = uint32FromPointerToUint32(t.MaxEjectionPercent)
s.BaseEjectionTime = structs.DurationPointerToProto(t.BaseEjectionTime)
} }
func PeeringMeshConfigToStructs(s *PeeringMeshConfig, t *structs.PeeringMeshConfig) { func PeeringMeshConfigToStructs(s *PeeringMeshConfig, t *structs.PeeringMeshConfig) {
if s == nil { if s == nil {

File diff suppressed because it is too large Load Diff

View File

@ -633,6 +633,10 @@ message PassiveHealthCheck {
uint32 MaxFailures = 2; uint32 MaxFailures = 2;
// mog: target=EnforcingConsecutive5xx func-to=pointerToUint32FromUint32 func-from=uint32FromPointerToUint32 // mog: target=EnforcingConsecutive5xx func-to=pointerToUint32FromUint32 func-from=uint32FromPointerToUint32
uint32 EnforcingConsecutive5xx = 3; uint32 EnforcingConsecutive5xx = 3;
// mog: func-to=pointerToUint32FromUint32 func-from=uint32FromPointerToUint32
uint32 MaxEjectionPercent = 4;
// mog: func-to=structs.DurationPointerFromProto func-from=structs.DurationPointerToProto
google.protobuf.Duration BaseEjectionTime = 5;
} }
// mog annotation: // mog annotation:

View File

@ -76,6 +76,8 @@ load helpers
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.interval')" = "22s" ] [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.interval')" = "22s" ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.consecutive_5xx')" = null ] [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.consecutive_5xx')" = null ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.enforcing_consecutive_5xx')" = null ] [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.enforcing_consecutive_5xx')" = null ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.max_ejection_percent')" = null ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.base_ejection_time')" = null ]
} }
@test "s2 proxy should have been configured with http rbac filters" { @test "s2 proxy should have been configured with http rbac filters" {

View File

@ -20,6 +20,9 @@ services {
passive_health_check { passive_health_check {
interval = "22s" interval = "22s"
max_failures = 4 max_failures = 4
enforcing_consecutive_5xx = 99
max_ejection_percent = 50
base_ejection_time = "60s"
} }
} }
} }

View File

@ -49,4 +49,7 @@ load helpers
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.consecutive_5xx')" = "4" ] [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.consecutive_5xx')" = "4" ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.interval')" = "22s" ] [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.interval')" = "22s" ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.enforcing_consecutive_5xx')" = "99" ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.max_ejection_percent')" = "50" ]
[ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.base_ejection_time')" = "60s" ]
} }

View File

@ -1746,6 +1746,20 @@ represents a location outside the Consul cluster. Services can dial destinations
after a passive health check detects an outlier status through consecutive 5xx.`, after a passive health check detects an outlier status through consecutive 5xx.`,
}, },
}, },
{
name: 'MaxEjectionPercent',
type: 'int: 10',
description: `Measured in percent (%), the maximum percentage of hosts that can be ejected
from a upstream cluster due to passive health check failures. If not specified, inherits
Envoy's default of 10% or at least one host.`,
},
{
name: 'BaseEjectionTime',
type: 'duration: 30s',
description: `The base time that a host is ejected for. The real time is equal to the base
time multiplied by the number of times the host has been ejected and is capped by
max_ejection_time (Default 300s). If not speficied, inherits Envoy's default value of 30s.`,
},
], ],
}, },
], ],
@ -1897,6 +1911,20 @@ represents a location outside the Consul cluster. Services can dial destinations
after a passive health check detects an outlier status through consecutive 5xx.`, after a passive health check detects an outlier status through consecutive 5xx.`,
}, },
}, },
{
name: 'MaxEjectionPercent',
type: 'int: 10',
description: `Measured in percent (%), the maximum percentage of hosts that can be ejected
from a upstream cluster due to passive health check failures. If not specified, inherits
Envoy's default of 10% or at least one host.`,
},
{
name: 'BaseEjectionTime',
type: 'duration: 30s',
description: `The base time that a host is ejected for. The real time is equal to the base
time multiplied by the number of times the host has been ejected and is capped by
max_ejection_time (Default 300s). If not speficied, inherits Envoy's default value of 30s.`,
},
], ],
}, },
], ],

View File

@ -412,6 +412,13 @@ definition](/consul/docs/connect/registration/service-registration) or
host will be actually ejected when the proxy detects an outlier status host will be actually ejected when the proxy detects an outlier status
through consecutive errors in the 500 code range. If not specified, Consul through consecutive errors in the 500 code range. If not specified, Consul
uses the proxy's default value. For example, `100` for Envoy proxy. uses the proxy's default value. For example, `100` for Envoy proxy.
- `max_ejection_percent` - The maximum percentage of hosts that can be ejected
from a upstream cluster due to passive health check failures. If not specified,
inherits Envoy's default of 10% or at least one host.
- `base_ejection_time` - The base time that a host is ejected for. The real
time is equal to the base time multiplied by the number of times the host has
been ejected and is capped by max_ejection_time (Default 300s). If not
speficied, inherits Envoy's default value of 30s.
- `balance_outbound_connections` - Specifies the strategy for balancing outbound connections - `balance_outbound_connections` - Specifies the strategy for balancing outbound connections
across Envoy worker threads. Consul service mesh Envoy integration supports the across Envoy worker threads. Consul service mesh Envoy integration supports the