luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Add envoy connection balancing. (#14616) 

Add envoy connection balancing config.
This commit is contained in:
Derek Menteer 2022-09-26 11:29:06 -05:00 committed by GitHub
parent 3eb708e964
commit d9e42b0f1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 528 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/14616.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:feature
connect: Add Envoy connection balancing configuration fields.
```

39
agent/configentry/resolve.go
View File

@ -53,6 +53,7 @@ func ComputeResolvedServiceConfig(
structs.NewServiceID(args.Name, &args.EnterpriseMeta),
)
if serviceConf != nil {
if serviceConf.Expose.Checks {
thisReply.Expose.Checks = true
}
@ -62,12 +63,6 @@ func ComputeResolvedServiceConfig(
if serviceConf.MeshGateway.Mode != structs.MeshGatewayModeDefault {
thisReply.MeshGateway.Mode = serviceConf.MeshGateway.Mode
}
if serviceConf.Protocol != "" {
if thisReply.ProxyConfig == nil {
thisReply.ProxyConfig = make(map[string]interface{})
}
thisReply.ProxyConfig["protocol"] = serviceConf.Protocol
}
if serviceConf.TransparentProxy.OutboundListenerPort != 0 {
thisReply.TransparentProxy.OutboundListenerPort = serviceConf.TransparentProxy.OutboundListenerPort
}
@ -81,25 +76,29 @@ func ComputeResolvedServiceConfig(
thisReply.Destination = *serviceConf.Destination
}
// Populate values for the proxy config map
proxyConf := thisReply.ProxyConfig
if proxyConf == nil {
proxyConf = make(map[string]interface{})
}
if serviceConf.Protocol != "" {
proxyConf["protocol"] = serviceConf.Protocol
}
if serviceConf.BalanceInboundConnections != "" {
proxyConf["balance_inbound_connections"] = serviceConf.BalanceInboundConnections
}
if serviceConf.MaxInboundConnections > 0 {
if thisReply.ProxyConfig == nil {
thisReply.ProxyConfig = map[string]interface{}{}
}
thisReply.ProxyConfig["max_inbound_connections"] = serviceConf.MaxInboundConnections
proxyConf["max_inbound_connections"] = serviceConf.MaxInboundConnections
}
if serviceConf.LocalConnectTimeoutMs > 0 {
if thisReply.ProxyConfig == nil {
thisReply.ProxyConfig = map[string]interface{}{}
}
thisReply.ProxyConfig["local_connect_timeout_ms"] = serviceConf.LocalConnectTimeoutMs
proxyConf["local_connect_timeout_ms"] = serviceConf.LocalConnectTimeoutMs
}
if serviceConf.LocalRequestTimeoutMs > 0 {
if thisReply.ProxyConfig == nil {
thisReply.ProxyConfig = map[string]interface{}{}
}
thisReply.ProxyConfig["local_request_timeout_ms"] = serviceConf.LocalRequestTimeoutMs
proxyConf["local_request_timeout_ms"] = serviceConf.LocalRequestTimeoutMs
}
// Add the proxy conf to the response if any fields were populated
if len(proxyConf) > 0 {
thisReply.ProxyConfig = proxyConf
}
thisReply.Meta = serviceConf.Meta

20
agent/configentry/resolve_test.go
View File

@ -24,6 +24,26 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) {
args args
want *structs.ServiceConfigResponse
}{
{
name: "proxy with balanceinboundconnections",
args: args{
scReq: &structs.ServiceConfigRequest{
Name: "sid",
},
entries: &ResolvedServiceConfigSet{
ServiceDefaults: map[structs.ServiceID]*structs.ServiceConfigEntry{
sid: {
BalanceInboundConnections: "exact_balance",
},
},
},
},
want: &structs.ServiceConfigResponse{
ProxyConfig: map[string]interface{}{
"balance_inbound_connections": "exact_balance",
},
},
},
{
name: "proxy with maxinboundsconnections",
args: args{

48
agent/structs/config_entry.go
View File

@ -38,6 +38,8 @@ const (
MeshConfigMesh string = "mesh"
DefaultServiceProtocol = "tcp"
ConnectionExactBalance = "exact_balance"
)
var AllConfigEntryKinds = []string{
@ -98,19 +100,20 @@ type WarningConfigEntry interface {
// ServiceConfiguration is the top-level struct for the configuration of a service
// across the entire cluster.
type ServiceConfigEntry struct {
Kind string
Name string
Protocol string
Mode ProxyMode `json:",omitempty"`
TransparentProxy TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"`
Expose ExposeConfig `json:",omitempty"`
ExternalSNI string `json:",omitempty" alias:"external_sni"`
UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"`
Destination *DestinationConfig `json:",omitempty"`
MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"`
LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"`
LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"`
Kind string
Name string
Protocol string
Mode ProxyMode `json:",omitempty"`
TransparentProxy TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"`
Expose ExposeConfig `json:",omitempty"`
ExternalSNI string `json:",omitempty" alias:"external_sni"`
UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"`
Destination *DestinationConfig `json:",omitempty"`
MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"`
LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"`
LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"`
BalanceInboundConnections string `json:",omitempty" alias:"balance_inbound_connections"`
Meta map[string]string `json:",omitempty"`
acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
@ -183,6 +186,10 @@ func (e *ServiceConfigEntry) Validate() error {
validationErr := validateConfigEntryMeta(e.Meta)
if !isValidConnectionBalance(e.BalanceInboundConnections) {
validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_inbound_connections: %v", e.BalanceInboundConnections))
}
// External endpoints are invalid with an existing service's upstream configuration
if e.UpstreamConfig != nil && e.Destination != nil {
validationErr = multierror.Append(validationErr, errors.New("UpstreamConfig and Destination are mutually exclusive for service defaults"))
@ -800,6 +807,10 @@ type UpstreamConfig struct {
// MeshGatewayConfig controls how Mesh Gateways are configured and used
MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway" `
// BalanceOutboundConnections indicates how the proxy should attempt to distribute
// connections across worker threads. Only used by envoy proxies.
BalanceOutboundConnections string `json:",omitempty" alias:"balance_outbound_connections"`
}
func (cfg UpstreamConfig) Clone() UpstreamConfig {
@ -848,6 +859,9 @@ func (cfg UpstreamConfig) MergeInto(dst map[string]interface{}) {
if cfg.PassiveHealthCheck != nil {
dst["passive_health_check"] = cfg.PassiveHealthCheck
}
if cfg.BalanceOutboundConnections != "" {
dst["balance_outbound_connections"] = cfg.BalanceOutboundConnections
}
}
func (cfg *UpstreamConfig) NormalizeWithoutName() error {
@ -917,6 +931,10 @@ func (cfg UpstreamConfig) validate(named bool) error {
}
}
if !isValidConnectionBalance(cfg.BalanceOutboundConnections) {
validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_outbound_connections: %v", cfg.BalanceOutboundConnections))
}
return validationErr
}
@ -1222,3 +1240,7 @@ func validateConfigEntryMeta(meta map[string]string) error {
type ConfigEntryDeleteResponse struct {
Deleted bool
}
func isValidConnectionBalance(s string) bool {
return s == "" || s == ConnectionExactBalance
}

110
agent/structs/config_entry_test.go
View File

@ -340,6 +340,7 @@ func TestDecodeConfigEntry(t *testing.T) {
"moreconfig" {
"moar" = "config"
}
"balance_inbound_connections" = "exact_balance"
}
mesh_gateway {
mode = "remote"
@ -358,6 +359,7 @@ func TestDecodeConfigEntry(t *testing.T) {
"moreconfig" {
"moar" = "config"
}
"balance_inbound_connections" = "exact_balance"
}
MeshGateway {
Mode = "remote"
@ -376,6 +378,7 @@ func TestDecodeConfigEntry(t *testing.T) {
"moreconfig": map[string]interface{}{
"moar": "config",
},
"balance_inbound_connections": "exact_balance",
},
MeshGateway: MeshGatewayConfig{
Mode: MeshGatewayModeRemote,
@ -396,6 +399,7 @@ func TestDecodeConfigEntry(t *testing.T) {
mesh_gateway {
mode = "remote"
}
balance_inbound_connections = "exact_balance"
upstream_config {
overrides = [
{
@ -415,6 +419,7 @@ func TestDecodeConfigEntry(t *testing.T) {
defaults {
connect_timeout_ms = 5
protocol = "http"
balance_outbound_connections = "exact_balance"
envoy_listener_json = "foo"
envoy_cluster_json = "bar"
limits {
@ -437,6 +442,7 @@ func TestDecodeConfigEntry(t *testing.T) {
MeshGateway {
Mode = "remote"
}
BalanceInboundConnections = "exact_balance"
UpstreamConfig {
Overrides = [
{
@ -463,6 +469,7 @@ func TestDecodeConfigEntry(t *testing.T) {
MaxPendingRequests = 4
MaxConcurrentRequests = 5
}
BalanceOutboundConnections = "exact_balance"
}
}
`,
@ -478,6 +485,7 @@ func TestDecodeConfigEntry(t *testing.T) {
MeshGateway: MeshGatewayConfig{
Mode: MeshGatewayModeRemote,
},
BalanceInboundConnections: "exact_balance",
UpstreamConfig: &UpstreamConfiguration{
Overrides: []*UpstreamConfig{
{
@ -502,6 +510,7 @@ func TestDecodeConfigEntry(t *testing.T) {
MaxPendingRequests: intPointer(4),
MaxConcurrentRequests: intPointer(5),
},
BalanceOutboundConnections: "exact_balance",
},
},
},
@ -2651,6 +2660,44 @@ func TestServiceConfigEntry(t *testing.T) {
},
validateErr: "Duplicate address",
},
"validate: invalid inbound connection balance": {
entry: &ServiceConfigEntry{
Kind: ServiceDefaults,
Name: "external",
Protocol: "http",
BalanceInboundConnections: "invalid",
},
validateErr: "invalid value for balance_inbound_connections",
},
"validate: invalid default outbound connection balance": {
entry: &ServiceConfigEntry{
Kind: ServiceDefaults,
Name: "external",
Protocol: "http",
UpstreamConfig: &UpstreamConfiguration{
Defaults: &UpstreamConfig{
BalanceOutboundConnections: "invalid",
},
},
},
validateErr: "invalid value for balance_outbound_connections",
},
"validate: invalid override outbound connection balance": {
entry: &ServiceConfigEntry{
Kind: ServiceDefaults,
Name: "external",
Protocol: "http",
UpstreamConfig: &UpstreamConfiguration{
Overrides: []*UpstreamConfig{
{
Name: "upstream",
BalanceOutboundConnections: "invalid",
},
},
},
},
validateErr: "invalid value for balance_outbound_connections",
},
}
testConfigEntryNormalizeAndValidate(t, cases)
}
@ -2665,10 +2712,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
{
name: "kitchen sink",
source: UpstreamConfig{
EnvoyListenerJSON: "foo",
EnvoyClusterJSON: "bar",
ConnectTimeoutMs: 5,
Protocol: "http",
BalanceOutboundConnections: "exact_balance",
EnvoyListenerJSON: "foo",
EnvoyClusterJSON: "bar",
ConnectTimeoutMs: 5,
Protocol: "http",
Limits: &UpstreamLimits{
MaxConnections: intPointer(3),
MaxPendingRequests: intPointer(4),
@ -2682,10 +2730,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
},
destination: make(map[string]interface{}),
want: map[string]interface{}{
"envoy_listener_json": "foo",
"envoy_cluster_json": "bar",
"connect_timeout_ms": 5,
"protocol": "http",
"balance_outbound_connections": "exact_balance",
"envoy_listener_json": "foo",
"envoy_cluster_json": "bar",
"connect_timeout_ms": 5,
"protocol": "http",
"limits": &UpstreamLimits{
MaxConnections: intPointer(3),
MaxPendingRequests: intPointer(4),
@ -2701,10 +2750,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
{
name: "kitchen sink override of destination",
source: UpstreamConfig{
EnvoyListenerJSON: "foo",
EnvoyClusterJSON: "bar",
ConnectTimeoutMs: 5,
Protocol: "http",
BalanceOutboundConnections: "exact_balance",
EnvoyListenerJSON: "foo",
EnvoyClusterJSON: "bar",
ConnectTimeoutMs: 5,
Protocol: "http",
Limits: &UpstreamLimits{
MaxConnections: intPointer(3),
MaxPendingRequests: intPointer(4),
@ -2717,10 +2767,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
},
destination: map[string]interface{}{
"envoy_listener_json": "zip",
"envoy_cluster_json": "zap",
"connect_timeout_ms": 10,
"protocol": "grpc",
"balance_outbound_connections": "",
"envoy_listener_json": "zip",
"envoy_cluster_json": "zap",
"connect_timeout_ms": 10,
"protocol": "grpc",
"limits": &UpstreamLimits{
MaxConnections: intPointer(10),
MaxPendingRequests: intPointer(11),
@ -2733,10 +2784,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
},
want: map[string]interface{}{
"envoy_listener_json": "foo",
"envoy_cluster_json": "bar",
"connect_timeout_ms": 5,
"protocol": "http",
"balance_outbound_connections": "exact_balance",
"envoy_listener_json": "foo",
"envoy_cluster_json": "bar",
"connect_timeout_ms": 5,
"protocol": "http",
"limits": &UpstreamLimits{
MaxConnections: intPointer(3),
MaxPendingRequests: intPointer(4),
@ -2753,10 +2805,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
name: "empty source leaves destination intact",
source: UpstreamConfig{},
destination: map[string]interface{}{
"envoy_listener_json": "zip",
"envoy_cluster_json": "zap",
"connect_timeout_ms": 10,
"protocol": "grpc",
"balance_outbound_connections": "exact_balance",
"envoy_listener_json": "zip",
"envoy_cluster_json": "zap",
"connect_timeout_ms": 10,
"protocol": "grpc",
"limits": &UpstreamLimits{
MaxConnections: intPointer(10),
MaxPendingRequests: intPointer(11),
@ -2770,10 +2823,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
},
want: map[string]interface{}{
"envoy_listener_json": "zip",
"envoy_cluster_json": "zap",
"connect_timeout_ms": 10,
"protocol": "grpc",
"balance_outbound_connections": "exact_balance",
"envoy_listener_json": "zip",
"envoy_cluster_json": "zap",
"connect_timeout_ms": 10,
"protocol": "grpc",
"limits": &UpstreamLimits{
MaxConnections: intPointer(10),
MaxPendingRequests: intPointer(11),

4
agent/xds/config.go
View File

@ -68,6 +68,10 @@ type ProxyConfig struct {
// MaxInboundConnections is the maximum number of inbound connections to
// the proxy. If not set, the default is 0 (no limit).
MaxInboundConnections int `mapstructure:"max_inbound_connections"`
// BalanceInboundConnections indicates how the proxy should attempt to distribute
// connections across worker threads. Only used by envoy proxies.
BalanceInboundConnections string `json:",omitempty" alias:"balance_inbound_connections"`
}
// ParseProxyConfig returns the ProxyConfig parsed from the an opaque map. If an

11
agent/xds/config_test.go
View File

@ -157,6 +157,17 @@ func TestParseProxyConfig(t *testing.T) {
Protocol: "tcp",
},
},
{
name: "balance inbound connections override, string",
input: map[string]interface{}{
"balance_inbound_connections": "exact_balance",
},
want: ProxyConfig{
LocalConnectTimeoutMs: 5000,
Protocol: "tcp",
BalanceInboundConnections: "exact_balance",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

18
agent/xds/listeners.go
View File

@ -190,6 +190,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
}
upstreamListener := makeListener(uid.EnvoyID(), upstreamCfg, envoy_core_v3.TrafficDirection_OUTBOUND)
s.injectConnectionBalanceConfig(cfg.BalanceOutboundConnections, upstreamListener)
upstreamListener.FilterChains = []*envoy_listener_v3.FilterChain{
filterChain,
}
@ -385,6 +386,8 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
}
upstreamListener := makeListener(uid.EnvoyID(), upstreamCfg, envoy_core_v3.TrafficDirection_OUTBOUND)
s.injectConnectionBalanceConfig(cfg.BalanceOutboundConnections, upstreamListener)
upstreamListener.FilterChains = []*envoy_listener_v3.FilterChain{
filterChain,
}
@ -559,6 +562,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
}
upstreamListener := makeListener(uid.EnvoyID(), u, envoy_core_v3.TrafficDirection_OUTBOUND)
s.injectConnectionBalanceConfig(cfg.BalanceOutboundConnections, upstreamListener)
filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{
// TODO (SNI partition) add partition for upstream SNI
@ -905,6 +909,19 @@ func makeListenerFromUserConfig(configJSON string) (*envoy_listener_v3.Listener,
return &l, nil
}
func (s *ResourceGenerator) injectConnectionBalanceConfig(balanceType string, listener *envoy_listener_v3.Listener) {
switch balanceType {
case "":
// Default with no balancing.
case structs.ConnectionExactBalance:
listener.ConnectionBalanceConfig = &envoy_listener_v3.Listener_ConnectionBalanceConfig{
BalanceType: &envoy_listener_v3.Listener_ConnectionBalanceConfig_ExactBalance_{},
}
default:
s.Logger.Warn("ignoring invalid connection balance option", "value", balanceType)
}
}
// Ensure that the first filter in each filter chain of a public listener is
// the authz filter to prevent unauthorized access.
func (s *ResourceGenerator) injectConnectFilters(cfgSnap *proxycfg.ConfigSnapshot, listener *envoy_listener_v3.Listener) error {
@ -1221,6 +1238,7 @@ func (s *ResourceGenerator) makeInboundListener(cfgSnap *proxycfg.ConfigSnapshot
}
l = makePortListener(name, addr, port, envoy_core_v3.TrafficDirection_INBOUND)
s.injectConnectionBalanceConfig(cfg.BalanceInboundConnections, l)
var tracing *envoy_http_v3.HttpConnectionManager_Tracing
if cfg.ListenerTracingJSON != "" {

16
agent/xds/listeners_test.go
View File

@ -160,6 +160,22 @@ func TestListenersFromSnapshot(t *testing.T) {
}, nil)
},
},
{
name: "listener-balance-inbound-connections",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) {
ns.Proxy.Config["balance_inbound_connections"] = "exact_balance"
}, nil)
},
},
{
name: "listener-balance-outbound-connections-bind-port",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) {
ns.Proxy.Upstreams[0].Config["balance_outbound_connections"] = "exact_balance"
}, nil)
},
},
{
name: "http-public-listener",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {

122
agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden vendored Normal file
View File

@ -0,0 +1,122 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "db:127.0.0.1:9191",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9191
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.db.default.default.dc1",
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "prepared_query:geo-cache:127.10.10.10:8181",
"address": {
"socketAddress": {
"address": "127.10.10.10",
"portValue": 8181
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.prepared_query_geo-cache",
"cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:0.0.0.0:9999",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 9999
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "public_listener",
"cluster": "local_app"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
}
],
"trafficDirection": "INBOUND",
"connectionBalanceConfig": {
"exactBalance": {}
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

122
agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden vendored Normal file
View File

@ -0,0 +1,122 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "db:127.0.0.1:9191",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9191
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.db.default.default.dc1",
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND",
"connectionBalanceConfig": {
"exactBalance": {}
}
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "prepared_query:geo-cache:127.10.10.10:8181",
"address": {
"socketAddress": {
"address": "127.10.10.10",
"portValue": 8181
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.prepared_query_geo-cache",
"cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:0.0.0.0:9999",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 9999
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "public_listener",
"cluster": "local_app"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
}
],
"trafficDirection": "INBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

41
api/config_entry.go
View File

@ -177,6 +177,10 @@ type UpstreamConfig struct {
// MeshGatewayConfig controls how Mesh Gateways are configured and used
MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway" `
// BalanceOutboundConnections indicates that the proxy should attempt to evenly distribute
// outbound connections across worker threads. Only used by envoy proxies.
BalanceOutboundConnections string `json:",omitempty" alias:"balance_outbound_connections"`
}
// DestinationConfig represents a virtual service, i.e. one that is external to Consul
@ -223,24 +227,25 @@ type UpstreamLimits struct {
}
type ServiceConfigEntry struct {
Kind string
Name string
Partition string `json:",omitempty"`
Namespace string `json:",omitempty"`
Protocol string `json:",omitempty"`
Mode ProxyMode `json:",omitempty"`
TransparentProxy *TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"`
Expose ExposeConfig `json:",omitempty"`
ExternalSNI string `json:",omitempty" alias:"external_sni"`
UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"`
Destination *DestinationConfig `json:",omitempty"`
MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"`
LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"`
LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"`
Meta map[string]string `json:",omitempty"`
CreateIndex uint64
ModifyIndex uint64
Kind string
Name string
Partition string `json:",omitempty"`
Namespace string `json:",omitempty"`
Protocol string `json:",omitempty"`
Mode ProxyMode `json:",omitempty"`
TransparentProxy *TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"`
Expose ExposeConfig `json:",omitempty"`
ExternalSNI string `json:",omitempty" alias:"external_sni"`
UpstreamConfig *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"`
Destination *DestinationConfig `json:",omitempty"`
MaxInboundConnections int `json:",omitempty" alias:"max_inbound_connections"`
LocalConnectTimeoutMs int `json:",omitempty" alias:"local_connect_timeout_ms"`
LocalRequestTimeoutMs int `json:",omitempty" alias:"local_request_timeout_ms"`
BalanceInboundConnections string `json:",omitempty" alias:"balance_inbound_connections"`
Meta map[string]string `json:",omitempty"`
CreateIndex uint64
ModifyIndex uint64
}
func (s *ServiceConfigEntry) GetKind() string { return s.Kind }

14
api/config_entry_test.go
View File

@ -104,9 +104,10 @@ func TestAPI_ConfigEntries(t *testing.T) {
"foo": "bar",
"gir": "zim",
},
MaxInboundConnections: 5,
LocalConnectTimeoutMs: 5000,
LocalRequestTimeoutMs: 7000,
MaxInboundConnections: 5,
BalanceInboundConnections: "exact_balance",
LocalConnectTimeoutMs: 5000,
LocalRequestTimeoutMs: 7000,
}
dest := &DestinationConfig{
@ -148,6 +149,7 @@ func TestAPI_ConfigEntries(t *testing.T) {
require.Equal(t, service.Meta, readService.Meta)
require.Equal(t, service.Meta, readService.GetMeta())
require.Equal(t, service.MaxInboundConnections, readService.MaxInboundConnections)
require.Equal(t, service.BalanceInboundConnections, readService.BalanceInboundConnections)
require.Equal(t, service.LocalConnectTimeoutMs, readService.LocalConnectTimeoutMs)
require.Equal(t, service.LocalRequestTimeoutMs, readService.LocalRequestTimeoutMs)
@ -446,6 +448,7 @@ func TestDecodeConfigEntry(t *testing.T) {
"OutboundListenerPort": 808,
"DialedDirectly": true
},
"BalanceInboundConnections": "exact_balance",
"UpstreamConfig": {
"Overrides": [
{
@ -454,7 +457,8 @@ func TestDecodeConfigEntry(t *testing.T) {
"MaxFailures": 3,
"Interval": "2s",
"EnforcingConsecutive5xx": 60
}
},
"BalanceOutboundConnections": "exact_balance"
},
{
"Name": "finance--billing",
@ -498,6 +502,7 @@ func TestDecodeConfigEntry(t *testing.T) {
OutboundListenerPort: 808,
DialedDirectly: true,
},
BalanceInboundConnections: "exact_balance",
UpstreamConfig: &UpstreamConfiguration{
Overrides: []*UpstreamConfig{
{
@ -507,6 +512,7 @@ func TestDecodeConfigEntry(t *testing.T) {
Interval: 2 * time.Second,
EnforcingConsecutive5xx: uint32Pointer(60),
},
BalanceOutboundConnections: "exact_balance",
},
{
Name: "finance--billing",

27
website/content/docs/connect/config-entries/service-defaults.mdx
View File

@ -355,6 +355,15 @@ represents a location outside the Consul cluster. They can be dialed directly wh
[\`service-intentions\`](/docs/connect/config-entries/service-intentions).
Supported values are one of \`tcp\`, \`http\`, \`http2\`, or \`grpc\`.`,
},
{
name: 'BalanceInboundConnections',
type: `string: ""`,
description: `Sets the strategy for allocating inbound connections to the service across proxy threads.
The only supported value is \`exact_balance\`. By default, no connection balancing is used.
Refer to the
[Envoy Connection Balance config](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig)
for details.`
},
{
name: 'Mode',
type: `string: ""`,
@ -445,6 +454,15 @@ represents a location outside the Consul cluster. They can be dialed directly wh
},
],
},
{
name: 'BalanceOutboundConnections',
type: `string: ""`,
description: `Sets the strategy for allocating outbound connections from the upstream across proxy threads.
The only supported value is \`exact_balance\`. By default, no connection balancing is used.
Refer to the
[Envoy Connection Balance config](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig)
for details.`
},
{
name: 'Limits',
type: 'Limits: <optional>',
@ -587,6 +605,15 @@ represents a location outside the Consul cluster. They can be dialed directly wh
},
],
},
{
name: 'BalanceOutboundConnections',
type: `string: ""`,
description: `Sets the strategy for allocating outbound connections from the upstream across proxy threads.
The only supported value is \`exact_balance\`. By default, no connection balancing is used.
Refer to the
[Envoy Connection Balance config](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig)
for details.`
},
{
name: 'Limits',
type: 'Limits: <optional>',

16
website/content/docs/connect/proxies/envoy.mdx
View File

@ -254,6 +254,14 @@ defaults that are inherited by all services.
specified, inherits the Envoy default for route timeouts (15s). A value of 0 will
disable request timeouts.
- `balance_inbound_connections` - The strategy used for balancing inbound connections
across Envoy worker threads. Consul service mesh Envoy integration supports the
following `balance_inbound_connections` values:
- `""` - Empty string (default). No connection balancing strategy is used. Consul does not balance inbound connections.
- `exact_balance` - Inbound connections to the service use the
[Envoy Exact Balance Strategy.](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig-exactbalance)
### Proxy Upstream Config Options
The following configuration items may be overridden directly in the
@ -313,6 +321,14 @@ definition](/docs/connect/registration/service-registration) or
- `enforcing_consecutive_5xx` - The % chance that a host will be actually ejected
when an outlier status is detected through consecutive 5xx.
- `balance_outbound_connections` - Specifies the strategy for balancing outbound connections
across Envoy worker threads. Consul service mesh Envoy integration supports the
following `balance_outbound_connections` values:
- `""` - Empty string (default). No connection balancing strategy is used. Consul does not balance outbound connections.
- `exact_balance` - Outbound connections from the upstream use the
[Envoy Exact Balance Strategy.](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig-exactbalance)
### Gateway Options
These fields may also be overridden explicitly in the [proxy service