From d9e42b0f1cb373933b24d37e2733c89388ef6fc0 Mon Sep 17 00:00:00 2001 From: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Date: Mon, 26 Sep 2022 11:29:06 -0500 Subject: [PATCH] Add envoy connection balancing. (#14616) Add envoy connection balancing config. --- .changelog/14616.txt | 3 + agent/configentry/resolve.go | 39 +++--- agent/configentry/resolve_test.go | 20 +++ agent/structs/config_entry.go | 48 +++++-- agent/structs/config_entry_test.go | 110 ++++++++++++---- agent/xds/config.go | 4 + agent/xds/config_test.go | 11 ++ agent/xds/listeners.go | 18 +++ agent/xds/listeners_test.go | 16 +++ ...-balance-inbound-connections.latest.golden | 122 ++++++++++++++++++ ...tbound-connections-bind-port.latest.golden | 122 ++++++++++++++++++ api/config_entry.go | 41 +++--- api/config_entry_test.go | 14 +- .../config-entries/service-defaults.mdx | 27 ++++ .../content/docs/connect/proxies/envoy.mdx | 16 +++ 15 files changed, 528 insertions(+), 83 deletions(-) create mode 100644 .changelog/14616.txt create mode 100644 agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden create mode 100644 agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden diff --git a/.changelog/14616.txt b/.changelog/14616.txt new file mode 100644 index 000000000..979f4fad4 --- /dev/null +++ b/.changelog/14616.txt @@ -0,0 +1,3 @@ +```release-note:feature +connect: Add Envoy connection balancing configuration fields. +``` diff --git a/agent/configentry/resolve.go b/agent/configentry/resolve.go index f6090e98f..e3f7e54fb 100644 --- a/agent/configentry/resolve.go +++ b/agent/configentry/resolve.go @@ -53,6 +53,7 @@ func ComputeResolvedServiceConfig( structs.NewServiceID(args.Name, &args.EnterpriseMeta), ) if serviceConf != nil { + if serviceConf.Expose.Checks { thisReply.Expose.Checks = true } @@ -62,12 +63,6 @@ func ComputeResolvedServiceConfig( if serviceConf.MeshGateway.Mode != structs.MeshGatewayModeDefault { thisReply.MeshGateway.Mode = serviceConf.MeshGateway.Mode } - if serviceConf.Protocol != "" { - if thisReply.ProxyConfig == nil { - thisReply.ProxyConfig = make(map[string]interface{}) - } - thisReply.ProxyConfig["protocol"] = serviceConf.Protocol - } if serviceConf.TransparentProxy.OutboundListenerPort != 0 { thisReply.TransparentProxy.OutboundListenerPort = serviceConf.TransparentProxy.OutboundListenerPort } @@ -81,25 +76,29 @@ func ComputeResolvedServiceConfig( thisReply.Destination = *serviceConf.Destination } + // Populate values for the proxy config map + proxyConf := thisReply.ProxyConfig + if proxyConf == nil { + proxyConf = make(map[string]interface{}) + } + if serviceConf.Protocol != "" { + proxyConf["protocol"] = serviceConf.Protocol + } + if serviceConf.BalanceInboundConnections != "" { + proxyConf["balance_inbound_connections"] = serviceConf.BalanceInboundConnections + } if serviceConf.MaxInboundConnections > 0 { - if thisReply.ProxyConfig == nil { - thisReply.ProxyConfig = map[string]interface{}{} - } - thisReply.ProxyConfig["max_inbound_connections"] = serviceConf.MaxInboundConnections + proxyConf["max_inbound_connections"] = serviceConf.MaxInboundConnections } - if serviceConf.LocalConnectTimeoutMs > 0 { - if thisReply.ProxyConfig == nil { - thisReply.ProxyConfig = map[string]interface{}{} - } - thisReply.ProxyConfig["local_connect_timeout_ms"] = serviceConf.LocalConnectTimeoutMs + proxyConf["local_connect_timeout_ms"] = serviceConf.LocalConnectTimeoutMs } - if serviceConf.LocalRequestTimeoutMs > 0 { - if thisReply.ProxyConfig == nil { - thisReply.ProxyConfig = map[string]interface{}{} - } - thisReply.ProxyConfig["local_request_timeout_ms"] = serviceConf.LocalRequestTimeoutMs + proxyConf["local_request_timeout_ms"] = serviceConf.LocalRequestTimeoutMs + } + // Add the proxy conf to the response if any fields were populated + if len(proxyConf) > 0 { + thisReply.ProxyConfig = proxyConf } thisReply.Meta = serviceConf.Meta diff --git a/agent/configentry/resolve_test.go b/agent/configentry/resolve_test.go index 301472c1c..a023dca40 100644 --- a/agent/configentry/resolve_test.go +++ b/agent/configentry/resolve_test.go @@ -24,6 +24,26 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { args args want *structs.ServiceConfigResponse }{ + { + name: "proxy with balanceinboundconnections", + args: args{ + scReq: &structs.ServiceConfigRequest{ + Name: "sid", + }, + entries: &ResolvedServiceConfigSet{ + ServiceDefaults: map[structs.ServiceID]*structs.ServiceConfigEntry{ + sid: { + BalanceInboundConnections: "exact_balance", + }, + }, + }, + }, + want: &structs.ServiceConfigResponse{ + ProxyConfig: map[string]interface{}{ + "balance_inbound_connections": "exact_balance", + }, + }, + }, { name: "proxy with maxinboundsconnections", args: args{ diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 72efbffce..5be0a5089 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -38,6 +38,8 @@ const ( MeshConfigMesh string = "mesh" DefaultServiceProtocol = "tcp" + + ConnectionExactBalance = "exact_balance" ) var AllConfigEntryKinds = []string{ @@ -98,19 +100,20 @@ type WarningConfigEntry interface { // ServiceConfiguration is the top-level struct for the configuration of a service // across the entire cluster. type ServiceConfigEntry struct { - Kind string - Name string - Protocol string - Mode ProxyMode `json:",omitempty"` - TransparentProxy TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"` - MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"` - Expose ExposeConfig `json:",omitempty"` - ExternalSNI string `json:",omitempty" alias:"external_sni"` - UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"` - Destination *DestinationConfig `json:",omitempty"` - MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"` - LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"` - LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"` + Kind string + Name string + Protocol string + Mode ProxyMode `json:",omitempty"` + TransparentProxy TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"` + MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"` + Expose ExposeConfig `json:",omitempty"` + ExternalSNI string `json:",omitempty" alias:"external_sni"` + UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"` + Destination *DestinationConfig `json:",omitempty"` + MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"` + LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"` + LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"` + BalanceInboundConnections string `json:",omitempty" alias:"balance_inbound_connections"` Meta map[string]string `json:",omitempty"` acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` @@ -183,6 +186,10 @@ func (e *ServiceConfigEntry) Validate() error { validationErr := validateConfigEntryMeta(e.Meta) + if !isValidConnectionBalance(e.BalanceInboundConnections) { + validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_inbound_connections: %v", e.BalanceInboundConnections)) + } + // External endpoints are invalid with an existing service's upstream configuration if e.UpstreamConfig != nil && e.Destination != nil { validationErr = multierror.Append(validationErr, errors.New("UpstreamConfig and Destination are mutually exclusive for service defaults")) @@ -800,6 +807,10 @@ type UpstreamConfig struct { // MeshGatewayConfig controls how Mesh Gateways are configured and used MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway" ` + + // BalanceOutboundConnections indicates how the proxy should attempt to distribute + // connections across worker threads. Only used by envoy proxies. + BalanceOutboundConnections string `json:",omitempty" alias:"balance_outbound_connections"` } func (cfg UpstreamConfig) Clone() UpstreamConfig { @@ -848,6 +859,9 @@ func (cfg UpstreamConfig) MergeInto(dst map[string]interface{}) { if cfg.PassiveHealthCheck != nil { dst["passive_health_check"] = cfg.PassiveHealthCheck } + if cfg.BalanceOutboundConnections != "" { + dst["balance_outbound_connections"] = cfg.BalanceOutboundConnections + } } func (cfg *UpstreamConfig) NormalizeWithoutName() error { @@ -917,6 +931,10 @@ func (cfg UpstreamConfig) validate(named bool) error { } } + if !isValidConnectionBalance(cfg.BalanceOutboundConnections) { + validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_outbound_connections: %v", cfg.BalanceOutboundConnections)) + } + return validationErr } @@ -1222,3 +1240,7 @@ func validateConfigEntryMeta(meta map[string]string) error { type ConfigEntryDeleteResponse struct { Deleted bool } + +func isValidConnectionBalance(s string) bool { + return s == "" || s == ConnectionExactBalance +} diff --git a/agent/structs/config_entry_test.go b/agent/structs/config_entry_test.go index 7a699417f..6aca9af4e 100644 --- a/agent/structs/config_entry_test.go +++ b/agent/structs/config_entry_test.go @@ -340,6 +340,7 @@ func TestDecodeConfigEntry(t *testing.T) { "moreconfig" { "moar" = "config" } + "balance_inbound_connections" = "exact_balance" } mesh_gateway { mode = "remote" @@ -358,6 +359,7 @@ func TestDecodeConfigEntry(t *testing.T) { "moreconfig" { "moar" = "config" } + "balance_inbound_connections" = "exact_balance" } MeshGateway { Mode = "remote" @@ -376,6 +378,7 @@ func TestDecodeConfigEntry(t *testing.T) { "moreconfig": map[string]interface{}{ "moar": "config", }, + "balance_inbound_connections": "exact_balance", }, MeshGateway: MeshGatewayConfig{ Mode: MeshGatewayModeRemote, @@ -396,6 +399,7 @@ func TestDecodeConfigEntry(t *testing.T) { mesh_gateway { mode = "remote" } + balance_inbound_connections = "exact_balance" upstream_config { overrides = [ { @@ -415,6 +419,7 @@ func TestDecodeConfigEntry(t *testing.T) { defaults { connect_timeout_ms = 5 protocol = "http" + balance_outbound_connections = "exact_balance" envoy_listener_json = "foo" envoy_cluster_json = "bar" limits { @@ -437,6 +442,7 @@ func TestDecodeConfigEntry(t *testing.T) { MeshGateway { Mode = "remote" } + BalanceInboundConnections = "exact_balance" UpstreamConfig { Overrides = [ { @@ -463,6 +469,7 @@ func TestDecodeConfigEntry(t *testing.T) { MaxPendingRequests = 4 MaxConcurrentRequests = 5 } + BalanceOutboundConnections = "exact_balance" } } `, @@ -478,6 +485,7 @@ func TestDecodeConfigEntry(t *testing.T) { MeshGateway: MeshGatewayConfig{ Mode: MeshGatewayModeRemote, }, + BalanceInboundConnections: "exact_balance", UpstreamConfig: &UpstreamConfiguration{ Overrides: []*UpstreamConfig{ { @@ -502,6 +510,7 @@ func TestDecodeConfigEntry(t *testing.T) { MaxPendingRequests: intPointer(4), MaxConcurrentRequests: intPointer(5), }, + BalanceOutboundConnections: "exact_balance", }, }, }, @@ -2651,6 +2660,44 @@ func TestServiceConfigEntry(t *testing.T) { }, validateErr: "Duplicate address", }, + "validate: invalid inbound connection balance": { + entry: &ServiceConfigEntry{ + Kind: ServiceDefaults, + Name: "external", + Protocol: "http", + BalanceInboundConnections: "invalid", + }, + validateErr: "invalid value for balance_inbound_connections", + }, + "validate: invalid default outbound connection balance": { + entry: &ServiceConfigEntry{ + Kind: ServiceDefaults, + Name: "external", + Protocol: "http", + UpstreamConfig: &UpstreamConfiguration{ + Defaults: &UpstreamConfig{ + BalanceOutboundConnections: "invalid", + }, + }, + }, + validateErr: "invalid value for balance_outbound_connections", + }, + "validate: invalid override outbound connection balance": { + entry: &ServiceConfigEntry{ + Kind: ServiceDefaults, + Name: "external", + Protocol: "http", + UpstreamConfig: &UpstreamConfiguration{ + Overrides: []*UpstreamConfig{ + { + Name: "upstream", + BalanceOutboundConnections: "invalid", + }, + }, + }, + }, + validateErr: "invalid value for balance_outbound_connections", + }, } testConfigEntryNormalizeAndValidate(t, cases) } @@ -2665,10 +2712,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { { name: "kitchen sink", source: UpstreamConfig{ - EnvoyListenerJSON: "foo", - EnvoyClusterJSON: "bar", - ConnectTimeoutMs: 5, - Protocol: "http", + BalanceOutboundConnections: "exact_balance", + EnvoyListenerJSON: "foo", + EnvoyClusterJSON: "bar", + ConnectTimeoutMs: 5, + Protocol: "http", Limits: &UpstreamLimits{ MaxConnections: intPointer(3), MaxPendingRequests: intPointer(4), @@ -2682,10 +2730,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { }, destination: make(map[string]interface{}), want: map[string]interface{}{ - "envoy_listener_json": "foo", - "envoy_cluster_json": "bar", - "connect_timeout_ms": 5, - "protocol": "http", + "balance_outbound_connections": "exact_balance", + "envoy_listener_json": "foo", + "envoy_cluster_json": "bar", + "connect_timeout_ms": 5, + "protocol": "http", "limits": &UpstreamLimits{ MaxConnections: intPointer(3), MaxPendingRequests: intPointer(4), @@ -2701,10 +2750,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { { name: "kitchen sink override of destination", source: UpstreamConfig{ - EnvoyListenerJSON: "foo", - EnvoyClusterJSON: "bar", - ConnectTimeoutMs: 5, - Protocol: "http", + BalanceOutboundConnections: "exact_balance", + EnvoyListenerJSON: "foo", + EnvoyClusterJSON: "bar", + ConnectTimeoutMs: 5, + Protocol: "http", Limits: &UpstreamLimits{ MaxConnections: intPointer(3), MaxPendingRequests: intPointer(4), @@ -2717,10 +2767,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote}, }, destination: map[string]interface{}{ - "envoy_listener_json": "zip", - "envoy_cluster_json": "zap", - "connect_timeout_ms": 10, - "protocol": "grpc", + "balance_outbound_connections": "", + "envoy_listener_json": "zip", + "envoy_cluster_json": "zap", + "connect_timeout_ms": 10, + "protocol": "grpc", "limits": &UpstreamLimits{ MaxConnections: intPointer(10), MaxPendingRequests: intPointer(11), @@ -2733,10 +2784,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, }, want: map[string]interface{}{ - "envoy_listener_json": "foo", - "envoy_cluster_json": "bar", - "connect_timeout_ms": 5, - "protocol": "http", + "balance_outbound_connections": "exact_balance", + "envoy_listener_json": "foo", + "envoy_cluster_json": "bar", + "connect_timeout_ms": 5, + "protocol": "http", "limits": &UpstreamLimits{ MaxConnections: intPointer(3), MaxPendingRequests: intPointer(4), @@ -2753,10 +2805,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { name: "empty source leaves destination intact", source: UpstreamConfig{}, destination: map[string]interface{}{ - "envoy_listener_json": "zip", - "envoy_cluster_json": "zap", - "connect_timeout_ms": 10, - "protocol": "grpc", + "balance_outbound_connections": "exact_balance", + "envoy_listener_json": "zip", + "envoy_cluster_json": "zap", + "connect_timeout_ms": 10, + "protocol": "grpc", "limits": &UpstreamLimits{ MaxConnections: intPointer(10), MaxPendingRequests: intPointer(11), @@ -2770,10 +2823,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) { "mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal}, }, want: map[string]interface{}{ - "envoy_listener_json": "zip", - "envoy_cluster_json": "zap", - "connect_timeout_ms": 10, - "protocol": "grpc", + "balance_outbound_connections": "exact_balance", + "envoy_listener_json": "zip", + "envoy_cluster_json": "zap", + "connect_timeout_ms": 10, + "protocol": "grpc", "limits": &UpstreamLimits{ MaxConnections: intPointer(10), MaxPendingRequests: intPointer(11), diff --git a/agent/xds/config.go b/agent/xds/config.go index 0736fb44c..8f3bc9380 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -68,6 +68,10 @@ type ProxyConfig struct { // MaxInboundConnections is the maximum number of inbound connections to // the proxy. If not set, the default is 0 (no limit). MaxInboundConnections int `mapstructure:"max_inbound_connections"` + + // BalanceInboundConnections indicates how the proxy should attempt to distribute + // connections across worker threads. Only used by envoy proxies. + BalanceInboundConnections string `json:",omitempty" alias:"balance_inbound_connections"` } // ParseProxyConfig returns the ProxyConfig parsed from the an opaque map. If an diff --git a/agent/xds/config_test.go b/agent/xds/config_test.go index d683e61d9..574449f1e 100644 --- a/agent/xds/config_test.go +++ b/agent/xds/config_test.go @@ -157,6 +157,17 @@ func TestParseProxyConfig(t *testing.T) { Protocol: "tcp", }, }, + { + name: "balance inbound connections override, string", + input: map[string]interface{}{ + "balance_inbound_connections": "exact_balance", + }, + want: ProxyConfig{ + LocalConnectTimeoutMs: 5000, + Protocol: "tcp", + BalanceInboundConnections: "exact_balance", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index cfea25cbc..9f9136bf3 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -190,6 +190,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. } upstreamListener := makeListener(uid.EnvoyID(), upstreamCfg, envoy_core_v3.TrafficDirection_OUTBOUND) + s.injectConnectionBalanceConfig(cfg.BalanceOutboundConnections, upstreamListener) upstreamListener.FilterChains = []*envoy_listener_v3.FilterChain{ filterChain, } @@ -385,6 +386,8 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. } upstreamListener := makeListener(uid.EnvoyID(), upstreamCfg, envoy_core_v3.TrafficDirection_OUTBOUND) + s.injectConnectionBalanceConfig(cfg.BalanceOutboundConnections, upstreamListener) + upstreamListener.FilterChains = []*envoy_listener_v3.FilterChain{ filterChain, } @@ -559,6 +562,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. } upstreamListener := makeListener(uid.EnvoyID(), u, envoy_core_v3.TrafficDirection_OUTBOUND) + s.injectConnectionBalanceConfig(cfg.BalanceOutboundConnections, upstreamListener) filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ // TODO (SNI partition) add partition for upstream SNI @@ -905,6 +909,19 @@ func makeListenerFromUserConfig(configJSON string) (*envoy_listener_v3.Listener, return &l, nil } +func (s *ResourceGenerator) injectConnectionBalanceConfig(balanceType string, listener *envoy_listener_v3.Listener) { + switch balanceType { + case "": + // Default with no balancing. + case structs.ConnectionExactBalance: + listener.ConnectionBalanceConfig = &envoy_listener_v3.Listener_ConnectionBalanceConfig{ + BalanceType: &envoy_listener_v3.Listener_ConnectionBalanceConfig_ExactBalance_{}, + } + default: + s.Logger.Warn("ignoring invalid connection balance option", "value", balanceType) + } +} + // Ensure that the first filter in each filter chain of a public listener is // the authz filter to prevent unauthorized access. func (s *ResourceGenerator) injectConnectFilters(cfgSnap *proxycfg.ConfigSnapshot, listener *envoy_listener_v3.Listener) error { @@ -1221,6 +1238,7 @@ func (s *ResourceGenerator) makeInboundListener(cfgSnap *proxycfg.ConfigSnapshot } l = makePortListener(name, addr, port, envoy_core_v3.TrafficDirection_INBOUND) + s.injectConnectionBalanceConfig(cfg.BalanceInboundConnections, l) var tracing *envoy_http_v3.HttpConnectionManager_Tracing if cfg.ListenerTracingJSON != "" { diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 1112222f3..39ac2eac0 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -160,6 +160,22 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, }, + { + name: "listener-balance-inbound-connections", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) { + ns.Proxy.Config["balance_inbound_connections"] = "exact_balance" + }, nil) + }, + }, + { + name: "listener-balance-outbound-connections-bind-port", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) { + ns.Proxy.Upstreams[0].Config["balance_outbound_connections"] = "exact_balance" + }, nil) + }, + }, { name: "http-public-listener", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden b/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden new file mode 100644 index 000000000..9c8b0a581 --- /dev/null +++ b/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden @@ -0,0 +1,122 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND", + "connectionBalanceConfig": { + "exactBalance": {} + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden b/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden new file mode 100644 index 000000000..1181ff019 --- /dev/null +++ b/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden @@ -0,0 +1,122 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND", + "connectionBalanceConfig": { + "exactBalance": {} + } + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/api/config_entry.go b/api/config_entry.go index acdb5bfa8..b1827fb59 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -177,6 +177,10 @@ type UpstreamConfig struct { // MeshGatewayConfig controls how Mesh Gateways are configured and used MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway" ` + + // BalanceOutboundConnections indicates that the proxy should attempt to evenly distribute + // outbound connections across worker threads. Only used by envoy proxies. + BalanceOutboundConnections string `json:",omitempty" alias:"balance_outbound_connections"` } // DestinationConfig represents a virtual service, i.e. one that is external to Consul @@ -223,24 +227,25 @@ type UpstreamLimits struct { } type ServiceConfigEntry struct { - Kind string - Name string - Partition string `json:",omitempty"` - Namespace string `json:",omitempty"` - Protocol string `json:",omitempty"` - Mode ProxyMode `json:",omitempty"` - TransparentProxy *TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"` - MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"` - Expose ExposeConfig `json:",omitempty"` - ExternalSNI string `json:",omitempty" alias:"external_sni"` - UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"` - Destination *DestinationConfig `json:",omitempty"` - MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"` - LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"` - LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"` - Meta map[string]string `json:",omitempty"` - CreateIndex uint64 - ModifyIndex uint64 + Kind string + Name string + Partition string `json:",omitempty"` + Namespace string `json:",omitempty"` + Protocol string `json:",omitempty"` + Mode ProxyMode `json:",omitempty"` + TransparentProxy *TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"` + MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"` + Expose ExposeConfig `json:",omitempty"` + ExternalSNI string `json:",omitempty" alias:"external_sni"` + UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"` + Destination *DestinationConfig `json:",omitempty"` + MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"` + LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"` + LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"` + BalanceInboundConnections string `json:",omitempty" alias:"balance_inbound_connections"` + Meta map[string]string `json:",omitempty"` + CreateIndex uint64 + ModifyIndex uint64 } func (s *ServiceConfigEntry) GetKind() string { return s.Kind } diff --git a/api/config_entry_test.go b/api/config_entry_test.go index 1502111d8..376ad6182 100644 --- a/api/config_entry_test.go +++ b/api/config_entry_test.go @@ -104,9 +104,10 @@ func TestAPI_ConfigEntries(t *testing.T) { "foo": "bar", "gir": "zim", }, - MaxInboundConnections: 5, - LocalConnectTimeoutMs: 5000, - LocalRequestTimeoutMs: 7000, + MaxInboundConnections: 5, + BalanceInboundConnections: "exact_balance", + LocalConnectTimeoutMs: 5000, + LocalRequestTimeoutMs: 7000, } dest := &DestinationConfig{ @@ -148,6 +149,7 @@ func TestAPI_ConfigEntries(t *testing.T) { require.Equal(t, service.Meta, readService.Meta) require.Equal(t, service.Meta, readService.GetMeta()) require.Equal(t, service.MaxInboundConnections, readService.MaxInboundConnections) + require.Equal(t, service.BalanceInboundConnections, readService.BalanceInboundConnections) require.Equal(t, service.LocalConnectTimeoutMs, readService.LocalConnectTimeoutMs) require.Equal(t, service.LocalRequestTimeoutMs, readService.LocalRequestTimeoutMs) @@ -446,6 +448,7 @@ func TestDecodeConfigEntry(t *testing.T) { "OutboundListenerPort": 808, "DialedDirectly": true }, + "BalanceInboundConnections": "exact_balance", "UpstreamConfig": { "Overrides": [ { @@ -454,7 +457,8 @@ func TestDecodeConfigEntry(t *testing.T) { "MaxFailures": 3, "Interval": "2s", "EnforcingConsecutive5xx": 60 - } + }, + "BalanceOutboundConnections": "exact_balance" }, { "Name": "finance--billing", @@ -498,6 +502,7 @@ func TestDecodeConfigEntry(t *testing.T) { OutboundListenerPort: 808, DialedDirectly: true, }, + BalanceInboundConnections: "exact_balance", UpstreamConfig: &UpstreamConfiguration{ Overrides: []*UpstreamConfig{ { @@ -507,6 +512,7 @@ func TestDecodeConfigEntry(t *testing.T) { Interval: 2 * time.Second, EnforcingConsecutive5xx: uint32Pointer(60), }, + BalanceOutboundConnections: "exact_balance", }, { Name: "finance--billing", diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index 17920ca97..0ba3d56f5 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -355,6 +355,15 @@ represents a location outside the Consul cluster. They can be dialed directly wh [\`service-intentions\`](/docs/connect/config-entries/service-intentions). Supported values are one of \`tcp\`, \`http\`, \`http2\`, or \`grpc\`.`, }, + { + name: 'BalanceInboundConnections', + type: `string: ""`, + description: `Sets the strategy for allocating inbound connections to the service across proxy threads. + The only supported value is \`exact_balance\`. By default, no connection balancing is used. + Refer to the + [Envoy Connection Balance config](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) + for details.` + }, { name: 'Mode', type: `string: ""`, @@ -445,6 +454,15 @@ represents a location outside the Consul cluster. They can be dialed directly wh }, ], }, + { + name: 'BalanceOutboundConnections', + type: `string: ""`, + description: `Sets the strategy for allocating outbound connections from the upstream across proxy threads. + The only supported value is \`exact_balance\`. By default, no connection balancing is used. + Refer to the + [Envoy Connection Balance config](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) + for details.` + }, { name: 'Limits', type: 'Limits: ', @@ -587,6 +605,15 @@ represents a location outside the Consul cluster. They can be dialed directly wh }, ], }, + { + name: 'BalanceOutboundConnections', + type: `string: ""`, + description: `Sets the strategy for allocating outbound connections from the upstream across proxy threads. + The only supported value is \`exact_balance\`. By default, no connection balancing is used. + Refer to the + [Envoy Connection Balance config](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) + for details.` + }, { name: 'Limits', type: 'Limits: ', diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 4c6acef39..86856a6f9 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -254,6 +254,14 @@ defaults that are inherited by all services. specified, inherits the Envoy default for route timeouts (15s). A value of 0 will disable request timeouts. +- `balance_inbound_connections` - The strategy used for balancing inbound connections + across Envoy worker threads. Consul service mesh Envoy integration supports the + following `balance_inbound_connections` values: + + - `""` - Empty string (default). No connection balancing strategy is used. Consul does not balance inbound connections. + - `exact_balance` - Inbound connections to the service use the + [Envoy Exact Balance Strategy.](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig-exactbalance) + ### Proxy Upstream Config Options The following configuration items may be overridden directly in the @@ -313,6 +321,14 @@ definition](/docs/connect/registration/service-registration) or - `enforcing_consecutive_5xx` - The % chance that a host will be actually ejected when an outlier status is detected through consecutive 5xx. +- `balance_outbound_connections` - Specifies the strategy for balancing outbound connections + across Envoy worker threads. Consul service mesh Envoy integration supports the + following `balance_outbound_connections` values: + + - `""` - Empty string (default). No connection balancing strategy is used. Consul does not balance outbound connections. + - `exact_balance` - Outbound connections from the upstream use the + [Envoy Exact Balance Strategy.](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig-exactbalance) + ### Gateway Options These fields may also be overridden explicitly in the [proxy service