diff --git a/agent/cache-types/discovery_chain_test.go b/agent/cache-types/discovery_chain_test.go index 9bc365b83..444d38256 100644 --- a/agent/cache-types/discovery_chain_test.go +++ b/agent/cache-types/discovery_chain_test.go @@ -17,7 +17,7 @@ func TestCompiledDiscoveryChain(t *testing.T) { // just do the default chain entries := structs.NewDiscoveryChainConfigEntries() - chain := discoverychain.TestCompileConfigEntries(t, "web", "default", "dc1") + chain := discoverychain.TestCompileConfigEntries(t, "web", "default", "dc1", nil) // Expect the proper RPC call. This also sets the expected value // since that is return-by-pointer in the arguments. diff --git a/agent/consul/config_endpoint.go b/agent/consul/config_endpoint.go index 680c16535..1ba7a62e9 100644 --- a/agent/consul/config_endpoint.go +++ b/agent/consul/config_endpoint.go @@ -333,7 +333,16 @@ func (c *ConfigEntry) ReadDiscoveryChain(args *structs.DiscoveryChainRequest, re return fmt.Errorf("Must provide service name") } - const currentNamespace = "default" + evalDC := args.EvaluateInDatacenter + if evalDC == "" { + evalDC = c.srv.config.Datacenter + } + + evalNS := args.EvaluateInNamespace + if evalNS == "" { + // TODO(namespaces) pull from something else? + evalNS = "default" + } return c.srv.blockingQuery( &args.QueryOptions, @@ -346,11 +355,14 @@ func (c *ConfigEntry) ReadDiscoveryChain(args *structs.DiscoveryChainRequest, re // Then we compile it into something useful. chain, err := discoverychain.Compile(discoverychain.CompileRequest{ - ServiceName: args.Name, - CurrentNamespace: currentNamespace, - CurrentDatacenter: c.srv.config.Datacenter, - InferDefaults: true, - Entries: entries, + ServiceName: args.Name, + CurrentNamespace: evalNS, + CurrentDatacenter: evalDC, + OverrideMeshGateway: args.OverrideMeshGateway, + OverrideProtocol: args.OverrideProtocol, + OverrideConnectTimeout: args.OverrideConnectTimeout, + InferDefaults: true, + Entries: entries, }) if err != nil { return err diff --git a/agent/consul/discoverychain/compile.go b/agent/consul/discoverychain/compile.go index aa7eee8c7..1aa538249 100644 --- a/agent/consul/discoverychain/compile.go +++ b/agent/consul/discoverychain/compile.go @@ -6,6 +6,7 @@ import ( "time" "github.com/hashicorp/consul/agent/structs" + "github.com/mitchellh/hashstructure" "github.com/mitchellh/mapstructure" ) @@ -13,8 +14,27 @@ type CompileRequest struct { ServiceName string CurrentNamespace string CurrentDatacenter string - InferDefaults bool // TODO(rb): remove this? - Entries *structs.DiscoveryChainConfigEntries + + // OverrideMeshGateway allows for the setting to be overridden for any + // resolver in the compiled chain. + OverrideMeshGateway structs.MeshGatewayConfig + + // OverrideProtocol allows for the final protocol for the chain to be + // altered. + // + // - If the chain ordinarily would be TCP and an L7 protocol is passed here + // the chain will not include Routers or Splitters. + // + // - If the chain ordinarily would be L7 and TCP is passed here the chain + // will not include Routers or Splitters. + OverrideProtocol string + + // OverrideConnectTimeout allows for the ConnectTimeout setting to be + // overridden for any resolver in the compiled chain. + OverrideConnectTimeout time.Duration + + InferDefaults bool // TODO(rb): remove this? + Entries *structs.DiscoveryChainConfigEntries } // Compile assembles a discovery chain in the form of a graph of nodes using @@ -53,11 +73,14 @@ func Compile(req CompileRequest) (*structs.CompiledDiscoveryChain, error) { } c := &compiler{ - serviceName: serviceName, - currentNamespace: currentNamespace, - currentDatacenter: currentDatacenter, - inferDefaults: inferDefaults, - entries: entries, + serviceName: serviceName, + currentNamespace: currentNamespace, + currentDatacenter: currentDatacenter, + overrideMeshGateway: req.OverrideMeshGateway, + overrideProtocol: req.OverrideProtocol, + overrideConnectTimeout: req.OverrideConnectTimeout, + inferDefaults: inferDefaults, + entries: entries, splitterNodes: make(map[string]*structs.DiscoveryGraphNode), groupResolverNodes: make(map[structs.DiscoveryTarget]*structs.DiscoveryGraphNode), @@ -67,6 +90,10 @@ func Compile(req CompileRequest) (*structs.CompiledDiscoveryChain, error) { targets: make(map[structs.DiscoveryTarget]struct{}), } + if req.OverrideProtocol != "" { + c.disableAdvancedRoutingFeatures = !enableAdvancedRoutingForProtocol(req.OverrideProtocol) + } + // Clone this resolver map to avoid mutating the input map during compilation. if len(entries.Resolvers) > 0 { for k, v := range entries.Resolvers { @@ -80,10 +107,13 @@ func Compile(req CompileRequest) (*structs.CompiledDiscoveryChain, error) { // compiler is a single-use struct for handling intermediate state necessary // for assembling a discovery chain from raw config entries. type compiler struct { - serviceName string - currentNamespace string - currentDatacenter string - inferDefaults bool + serviceName string + currentNamespace string + currentDatacenter string + overrideMeshGateway structs.MeshGatewayConfig + overrideProtocol string + overrideConnectTimeout time.Duration + inferDefaults bool // config entries that are being compiled (will be mutated during compilation) // @@ -98,6 +128,15 @@ type compiler struct { // or splitting appear in the compiled chain usesAdvancedRoutingFeatures bool + // disableAdvancedRoutingFeatures is set to true if overrideProtocol is set to tcp + disableAdvancedRoutingFeatures bool + + // customizedBy indicates which override values customized how the + // compilation behaved. + // + // This is an OUTPUT field. + customizedBy customizationMarkers + // topNode is computed inside of assembleChain() // // This is an OUTPUT field. @@ -125,6 +164,16 @@ type compiler struct { targets map[structs.DiscoveryTarget]struct{} } +type customizationMarkers struct { + MeshGateway bool + Protocol bool + ConnectTimeout bool +} + +func (m *customizationMarkers) IsZero() bool { + return !m.MeshGateway && !m.Protocol && !m.ConnectTimeout +} + func (c *compiler) recordServiceProtocol(serviceName string) error { if serviceDefault := c.entries.GetService(serviceName); serviceDefault != nil { return c.recordProtocol(serviceName, serviceDefault.Protocol) @@ -209,10 +258,42 @@ func (c *compiler) compile() (*structs.CompiledDiscoveryChain, error) { } structs.DiscoveryTargets(targets).Sort() + if c.overrideProtocol != "" { + if c.overrideProtocol != c.protocol { + c.protocol = c.overrideProtocol + c.customizedBy.Protocol = true + } + } + + var customizationHash string + if !c.customizedBy.IsZero() { + var customization struct { + OverrideMeshGateway structs.MeshGatewayConfig + OverrideProtocol string + OverrideConnectTimeout time.Duration + } + + if c.customizedBy.MeshGateway { + customization.OverrideMeshGateway = c.overrideMeshGateway + } + if c.customizedBy.Protocol { + customization.OverrideProtocol = c.overrideProtocol + } + if c.customizedBy.ConnectTimeout { + customization.OverrideConnectTimeout = c.overrideConnectTimeout + } + v, err := hashstructure.Hash(customization, nil) + if err != nil { + return nil, fmt.Errorf("cannot create customization hash key: %v", err) + } + customizationHash = fmt.Sprintf("%x", v)[0:8] + } + return &structs.CompiledDiscoveryChain{ ServiceName: c.serviceName, Namespace: c.currentNamespace, Datacenter: c.currentDatacenter, + CustomizationHash: customizationHash, Protocol: c.protocol, Node: c.topNode, Resolvers: c.resolvers, @@ -291,6 +372,11 @@ func (c *compiler) assembleChain() error { // The only router we consult is the one for the service name at the top of // the chain. router := c.entries.GetRouter(c.serviceName) + if router != nil && c.disableAdvancedRoutingFeatures { + router = nil + c.customizedBy.Protocol = true + } + if router == nil { // If no router is configured, move on down the line to the next hop of // the chain. @@ -401,6 +487,7 @@ func (c *compiler) getSplitterOrGroupResolverNode(target structs.DiscoveryTarget } func (c *compiler) getSplitterNode(name string) (*structs.DiscoveryGraphNode, error) { + // Do we already have the node? if prev, ok := c.splitterNodes[name]; ok { return prev, nil @@ -408,6 +495,10 @@ func (c *compiler) getSplitterNode(name string) (*structs.DiscoveryGraphNode, er // Fetch the config entry. splitter := c.entries.GetSplitter(name) + if splitter != nil && c.disableAdvancedRoutingFeatures { + splitter = nil + c.customizedBy.Protocol = true + } if splitter == nil { return nil, nil } @@ -520,6 +611,13 @@ RESOLVE_AGAIN: connectTimeout = 5 * time.Second } + if c.overrideConnectTimeout > 0 { + if connectTimeout != c.overrideConnectTimeout { + connectTimeout = c.overrideConnectTimeout + c.customizedBy.ConnectTimeout = true + } + } + // Build node. groupResolverNode := &structs.DiscoveryGraphNode{ Type: structs.DiscoveryGraphNodeTypeGroupResolver, @@ -542,6 +640,13 @@ RESOLVE_AGAIN: groupResolver.MeshGateway.Mode = c.entries.GlobalProxy.MeshGateway.Mode } + if c.overrideMeshGateway.Mode != structs.MeshGatewayModeDefault { + if groupResolver.MeshGateway.Mode != c.overrideMeshGateway.Mode { + groupResolver.MeshGateway.Mode = c.overrideMeshGateway.Mode + c.customizedBy.MeshGateway = true + } + } + // Retain this target even if we may not retain the group resolver. c.targets[target] = struct{}{} diff --git a/agent/consul/discoverychain/compile_test.go b/agent/consul/discoverychain/compile_test.go index 1abe0c47d..5e64a6e9e 100644 --- a/agent/consul/discoverychain/compile_test.go +++ b/agent/consul/discoverychain/compile_test.go @@ -24,10 +24,12 @@ func TestCompile_NoEntries_NoInferDefaults(t *testing.T) { type compileTestCase struct { entries *structs.DiscoveryChainConfigEntries + setup func(req *CompileRequest) // expect: the GroupResolverNodes map should have nil values expect *structs.CompiledDiscoveryChain // expectIsDefault tests behavior of CompiledDiscoveryChain.IsDefault() expectIsDefault bool + expectCustom bool expectErr string expectGraphErr bool } @@ -72,6 +74,11 @@ func TestCompile(t *testing.T) { "failover crosses protocols": testcase_FailoverCrossesProtocols(), "redirect crosses protocols": testcase_RedirectCrossesProtocols(), "redirect to missing subset": testcase_RedirectToMissingSubset(), + + // overrides + "resolver with protocol from override": testcase_ResolverProtocolOverride(), + "resolver with protocol from override ignored": testcase_ResolverProtocolOverrideIgnored(), + "router ignored due to protocol override": testcase_RouterIgnored_ResolverProtocolOverride(), } for name, tc := range cases { @@ -93,13 +100,18 @@ func TestCompile(t *testing.T) { require.NoError(t, entry.Validate()) } - res, err := Compile(CompileRequest{ + req := CompileRequest{ ServiceName: "main", CurrentNamespace: "default", CurrentDatacenter: "dc1", InferDefaults: true, Entries: tc.entries, - }) + } + if tc.setup != nil { + tc.setup(&req) + } + + res, err := Compile(req) if tc.expectErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.expectErr) @@ -128,6 +140,13 @@ func TestCompile(t *testing.T) { } } + if tc.expectCustom { + require.NotEmpty(t, res.CustomizationHash) + res.CustomizationHash = "" + } else { + require.Empty(t, res.CustomizationHash) + } + require.Equal(t, tc.expect, res) require.Equal(t, tc.expectIsDefault, res.IsDefault()) } @@ -1885,6 +1904,122 @@ func testcase_RedirectToMissingSubset() compileTestCase { } } +func testcase_ResolverProtocolOverride() compileTestCase { + entries := newEntries() + setServiceProtocol(entries, "main", "grpc") + + resolver := newDefaultServiceResolver("main") + + expect := &structs.CompiledDiscoveryChain{ + Protocol: "http2", + Node: &structs.DiscoveryGraphNode{ + Type: structs.DiscoveryGraphNodeTypeGroupResolver, + Name: "main", + GroupResolver: &structs.DiscoveryGroupResolver{ + Definition: resolver, + Default: true, + ConnectTimeout: 5 * time.Second, + Target: newTarget("main", "", "default", "dc1"), + }, + }, + Resolvers: map[string]*structs.ServiceResolverConfigEntry{ + "main": resolver, + }, + Targets: []structs.DiscoveryTarget{ + newTarget("main", "", "default", "dc1"), + }, + GroupResolverNodes: map[structs.DiscoveryTarget]*structs.DiscoveryGraphNode{ + newTarget("main", "", "default", "dc1"): nil, + }, + } + return compileTestCase{entries: entries, expect: expect, expectIsDefault: true, + expectCustom: true, + setup: func(req *CompileRequest) { + req.OverrideProtocol = "http2" + }, + } +} + +func testcase_ResolverProtocolOverrideIgnored() compileTestCase { + // This shows that if you try to override the protocol to its current value + // the override is completely ignored. + entries := newEntries() + setServiceProtocol(entries, "main", "http2") + + resolver := newDefaultServiceResolver("main") + + expect := &structs.CompiledDiscoveryChain{ + Protocol: "http2", + Node: &structs.DiscoveryGraphNode{ + Type: structs.DiscoveryGraphNodeTypeGroupResolver, + Name: "main", + GroupResolver: &structs.DiscoveryGroupResolver{ + Definition: resolver, + Default: true, + ConnectTimeout: 5 * time.Second, + Target: newTarget("main", "", "default", "dc1"), + }, + }, + Resolvers: map[string]*structs.ServiceResolverConfigEntry{ + "main": resolver, + }, + Targets: []structs.DiscoveryTarget{ + newTarget("main", "", "default", "dc1"), + }, + GroupResolverNodes: map[structs.DiscoveryTarget]*structs.DiscoveryGraphNode{ + newTarget("main", "", "default", "dc1"): nil, + }, + } + return compileTestCase{entries: entries, expect: expect, expectIsDefault: true, + setup: func(req *CompileRequest) { + req.OverrideProtocol = "http2" + }, + } +} + +func testcase_RouterIgnored_ResolverProtocolOverride() compileTestCase { + entries := newEntries() + setServiceProtocol(entries, "main", "grpc") + + entries.AddRouters( + &structs.ServiceRouterConfigEntry{ + Kind: "service-router", + Name: "main", + }, + ) + + resolver := newDefaultServiceResolver("main") + + expect := &structs.CompiledDiscoveryChain{ + Protocol: "tcp", + Node: &structs.DiscoveryGraphNode{ + Type: structs.DiscoveryGraphNodeTypeGroupResolver, + Name: "main", + GroupResolver: &structs.DiscoveryGroupResolver{ + Definition: resolver, + Default: true, + ConnectTimeout: 5 * time.Second, + Target: newTarget("main", "", "default", "dc1"), + }, + }, + Resolvers: map[string]*structs.ServiceResolverConfigEntry{ + "main": resolver, + }, + Targets: []structs.DiscoveryTarget{ + newTarget("main", "", "default", "dc1"), + }, + GroupResolverNodes: map[structs.DiscoveryTarget]*structs.DiscoveryGraphNode{ + newTarget("main", "", "default", "dc1"): nil, + }, + } + return compileTestCase{entries: entries, expect: expect, expectIsDefault: true, + expectCustom: true, + setup: func(req *CompileRequest) { + req.OverrideProtocol = "tcp" + }, + } +} + func newSimpleRoute(name string, muts ...func(*structs.ServiceRoute)) structs.ServiceRoute { r := structs.ServiceRoute{ Match: &structs.ServiceRouteMatch{ diff --git a/agent/consul/discoverychain/testing.go b/agent/consul/discoverychain/testing.go index 2925d76b8..0c05c44da 100644 --- a/agent/consul/discoverychain/testing.go +++ b/agent/consul/discoverychain/testing.go @@ -11,19 +11,25 @@ func TestCompileConfigEntries( serviceName string, currentNamespace string, currentDatacenter string, + setup func(req *CompileRequest), entries ...structs.ConfigEntry, ) *structs.CompiledDiscoveryChain { set := structs.NewDiscoveryChainConfigEntries() set.AddEntries(entries...) - chain, err := Compile(CompileRequest{ + req := CompileRequest{ ServiceName: serviceName, CurrentNamespace: currentNamespace, CurrentDatacenter: currentDatacenter, InferDefaults: true, Entries: set, - }) + } + if setup != nil { + setup(&req) + } + + chain, err := Compile(req) require.NoError(t, err) return chain } diff --git a/agent/proxycfg/manager_test.go b/agent/proxycfg/manager_test.go index 4ea99708f..225a3e04a 100644 --- a/agent/proxycfg/manager_test.go +++ b/agent/proxycfg/manager_test.go @@ -64,6 +64,11 @@ func TestManager_BasicLifecycle(t *testing.T) { } dbDefaultChain := func() *structs.CompiledDiscoveryChain { return discoverychain.TestCompileConfigEntries(t, "db", "default", "dc1", + func(req *discoverychain.CompileRequest) { + // This is because structs.TestUpstreams uses an opaque config + // to override connect timeouts. + req.OverrideConnectTimeout = 1 * time.Second + }, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", @@ -72,6 +77,11 @@ func TestManager_BasicLifecycle(t *testing.T) { } dbSplitChain := func() *structs.CompiledDiscoveryChain { return discoverychain.TestCompileConfigEntries(t, "db", "default", "dc1", + func(req *discoverychain.CompileRequest) { + // This is because structs.TestUpstreams uses an opaque config + // to override connect timeouts. + req.OverrideConnectTimeout = 1 * time.Second + }, &structs.ProxyConfigEntry{ Kind: structs.ProxyDefaults, Name: structs.ProxyConfigGlobal, @@ -142,9 +152,14 @@ func TestManager_BasicLifecycle(t *testing.T) { }) dbChainCacheKey := testGenCacheKey(&structs.DiscoveryChainRequest{ - Datacenter: "dc1", - QueryOptions: structs.QueryOptions{Token: "my-token"}, - Name: "db", + Name: "db", + EvaluateInDatacenter: "dc1", + EvaluateInNamespace: "default", + // This is because structs.TestUpstreams uses an opaque config + // to override connect timeouts. + OverrideConnectTimeout: 1 * time.Second, + Datacenter: "dc1", + QueryOptions: structs.QueryOptions{Token: "my-token"}, }) dbHealthCacheKey := testGenCacheKey(&structs.ServiceSpecificRequest{ diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 233d9e2eb..36fe2f4a2 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -13,6 +13,7 @@ import ( cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/structs" "github.com/mitchellh/copystructure" + "github.com/mitchellh/mapstructure" ) type CacheNotifier interface { @@ -224,6 +225,9 @@ func (s *state) initWatchesConnectProxy() error { return err } + // TODO(namespaces): pull this from something like s.source.Namespace? + currentNamespace := "default" + // Watch for updates to service endpoints for all upstreams for _, u := range s.proxyCfg.Upstreams { dc := s.source.Datacenter @@ -232,6 +236,20 @@ func (s *state) initWatchesConnectProxy() error { dc = u.Datacenter } + ns := currentNamespace + if u.DestinationNamespace != "" { + ns = u.DestinationNamespace + } + + cfg, err := parseReducedUpstreamConfig(u.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. We'll fall back on + // the plain discovery chain if there is an error so it's safe to + // continue. + s.logger.Printf("[WARN] envoy: failed to parse Upstream[%s].Config: %s", + u.Identifier(), err) + } + switch u.DestinationType { case structs.UpstreamDestTypePreparedQuery: err = s.cache.Notify(s.ctx, cachetype.PreparedQueryName, &structs.PreparedQueryExecuteRequest{ @@ -241,50 +259,23 @@ func (s *state) initWatchesConnectProxy() error { Connect: true, Source: *s.source, }, "upstream:"+u.Identifier(), s.ch) + case structs.UpstreamDestTypeService: fallthrough + case "": // Treat unset as the default Service type - - // Determine if this should use a discovery chain. - // - // TODO(rb): reduce this list of exceptions - var shouldUseDiscoveryChain bool - if dc != s.source.Datacenter { - shouldUseDiscoveryChain = false - } else if u.DestinationNamespace != "" && u.DestinationNamespace != "default" { - shouldUseDiscoveryChain = false - } else { - shouldUseDiscoveryChain = true - } - - if shouldUseDiscoveryChain { - // Watch for discovery chain configuration updates - err = s.cache.Notify(s.ctx, cachetype.CompiledDiscoveryChainName, &structs.DiscoveryChainRequest{ - Datacenter: dc, - QueryOptions: structs.QueryOptions{Token: s.token}, - Name: u.DestinationName, - }, "discovery-chain:"+u.Identifier(), s.ch) - if err != nil { - return err - } - } else { - meshGateway := structs.MeshGatewayModeNone - - // TODO (mesh-gateway)- maybe allow using a gateway within a datacenter at some point - if dc != s.source.Datacenter { - meshGateway = u.MeshGateway.Mode - } - - if err := s.watchConnectProxyService( - s.ctx, - "upstream:"+serviceIDPrefix+u.Identifier(), - u.DestinationName, - dc, - "", - meshGateway, - ); err != nil { - return err - } + err = s.cache.Notify(s.ctx, cachetype.CompiledDiscoveryChainName, &structs.DiscoveryChainRequest{ + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + Name: u.DestinationName, + EvaluateInDatacenter: dc, + EvaluateInNamespace: ns, + OverrideMeshGateway: s.proxyCfg.MeshGateway.OverlayWith(u.MeshGateway), + OverrideProtocol: cfg.Protocol, + OverrideConnectTimeout: cfg.ConnectTimeout(), + }, "discovery-chain:"+u.Identifier(), s.ch) + if err != nil { + return err } default: @@ -294,6 +285,26 @@ func (s *state) initWatchesConnectProxy() error { return nil } +// reducedProxyConfig represents the basic opaque config values that are now +// managed with the discovery chain but for backwards compatibility reasons +// should still affect how the proxy is configured. +// +// The full-blown config is agent/xds.UpstreamConfig +type reducedUpstreamConfig struct { + Protocol string `mapstructure:"protocol"` + ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"` +} + +func (c *reducedUpstreamConfig) ConnectTimeout() time.Duration { + return time.Duration(c.ConnectTimeoutMs) * time.Millisecond +} + +func parseReducedUpstreamConfig(m map[string]interface{}) (reducedUpstreamConfig, error) { + var cfg reducedUpstreamConfig + err := mapstructure.WeakDecode(m, &cfg) + return cfg, err +} + // initWatchesMeshGateway sets up the watches needed based on the current mesh gateway registration func (s *state) initWatchesMeshGateway() error { // Watch for root changes @@ -625,13 +636,10 @@ func (s *state) resetWatchesFromChain( ctx, cancel := context.WithCancel(s.ctx) + // TODO (mesh-gateway)- maybe allow using a gateway within a datacenter at some point meshGateway := structs.MeshGatewayModeDefault if target.Datacenter != s.source.Datacenter { meshGateway = meshGatewayModes[target] - - if meshGateway == structs.MeshGatewayModeDefault { - meshGateway = s.proxyCfg.MeshGateway.Mode - } } // if the default mode diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index ee24664f8..92b481a60 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -233,14 +233,13 @@ func genVerifyPreparedQueryWatch(expectedName string, expectedDatacenter string) } } -func genVerifyDiscoveryChainWatch(expectedName string, expectedDatacenter string) verifyWatchRequest { +func genVerifyDiscoveryChainWatch(expected *structs.DiscoveryChainRequest) verifyWatchRequest { return func(t testing.TB, cacheType string, request cache.Request) { require.Equal(t, cachetype.CompiledDiscoveryChainName, cacheType) reqReal, ok := request.(*structs.DiscoveryChainRequest) require.True(t, ok) - require.Equal(t, expectedDatacenter, reqReal.Datacenter) - require.Equal(t, expectedName, reqReal.Name) + require.Equal(t, expected, reqReal) } } @@ -305,6 +304,203 @@ func TestState_WatchesAndUpdates(t *testing.T) { stages []verificationStage } + newConnectProxyCase := func(meshGatewayProxyConfigValue structs.MeshGatewayMode) testCase { + ns := structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "web-sidecar-proxy", + Service: "web-sidecar-proxy", + Address: "10.0.1.1", + Port: 443, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "web", + Upstreams: structs.Upstreams{ + structs.Upstream{ + DestinationType: structs.UpstreamDestTypePreparedQuery, + DestinationName: "query", + LocalBindPort: 10001, + }, + structs.Upstream{ + DestinationType: structs.UpstreamDestTypeService, + DestinationName: "api", + LocalBindPort: 10002, + }, + structs.Upstream{ + DestinationType: structs.UpstreamDestTypeService, + DestinationName: "api-failover-remote", + Datacenter: "dc2", + LocalBindPort: 10003, + MeshGateway: structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeRemote, + }, + }, + structs.Upstream{ + DestinationType: structs.UpstreamDestTypeService, + DestinationName: "api-failover-local", + Datacenter: "dc2", + LocalBindPort: 10004, + MeshGateway: structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeLocal, + }, + }, + structs.Upstream{ + DestinationType: structs.UpstreamDestTypeService, + DestinationName: "api-failover-direct", + Datacenter: "dc2", + LocalBindPort: 10005, + MeshGateway: structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeNone, + }, + }, + structs.Upstream{ + DestinationType: structs.UpstreamDestTypeService, + DestinationName: "api-dc2", + LocalBindPort: 10006, + }, + }, + }, + } + + if meshGatewayProxyConfigValue != structs.MeshGatewayModeDefault { + ns.Proxy.MeshGateway.Mode = meshGatewayProxyConfigValue + } + + stage0 := verificationStage{ + requiredWatches: map[string]verifyWatchRequest{ + rootsWatchID: genVerifyRootsWatch("dc1"), + leafWatchID: genVerifyLeafWatch("web", "dc1"), + intentionsWatchID: genVerifyIntentionWatch("web", "dc1"), + "upstream:prepared_query:query": genVerifyPreparedQueryWatch("query", "dc1"), + "discovery-chain:api": genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ + Name: "api", + EvaluateInDatacenter: "dc1", + EvaluateInNamespace: "default", + Datacenter: "dc1", + OverrideMeshGateway: structs.MeshGatewayConfig{ + Mode: meshGatewayProxyConfigValue, + }, + }), + "discovery-chain:api-failover-remote?dc=dc2": genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ + Name: "api-failover-remote", + EvaluateInDatacenter: "dc2", + EvaluateInNamespace: "default", + Datacenter: "dc1", + OverrideMeshGateway: structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeRemote, + }, + }), + "discovery-chain:api-failover-local?dc=dc2": genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ + Name: "api-failover-local", + EvaluateInDatacenter: "dc2", + EvaluateInNamespace: "default", + Datacenter: "dc1", + OverrideMeshGateway: structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeLocal, + }, + }), + "discovery-chain:api-failover-direct?dc=dc2": genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ + Name: "api-failover-direct", + EvaluateInDatacenter: "dc2", + EvaluateInNamespace: "default", + Datacenter: "dc1", + OverrideMeshGateway: structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeNone, + }, + }), + "discovery-chain:api-dc2": genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ + Name: "api-dc2", + EvaluateInDatacenter: "dc1", + EvaluateInNamespace: "default", + Datacenter: "dc1", + OverrideMeshGateway: structs.MeshGatewayConfig{ + Mode: meshGatewayProxyConfigValue, + }, + }), + }, + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: "discovery-chain:api", + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, "api", "default", "dc1", + func(req *discoverychain.CompileRequest) { + req.OverrideMeshGateway.Mode = meshGatewayProxyConfigValue + }), + }, + Err: nil, + }, + cache.UpdateEvent{ + CorrelationID: "discovery-chain:api-failover-remote?dc=dc2", + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, "api-failover-remote", "default", "dc2", + func(req *discoverychain.CompileRequest) { + req.OverrideMeshGateway.Mode = structs.MeshGatewayModeRemote + }), + }, + Err: nil, + }, + cache.UpdateEvent{ + CorrelationID: "discovery-chain:api-failover-local?dc=dc2", + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, "api-failover-local", "default", "dc2", + func(req *discoverychain.CompileRequest) { + req.OverrideMeshGateway.Mode = structs.MeshGatewayModeLocal + }), + }, + Err: nil, + }, + cache.UpdateEvent{ + CorrelationID: "discovery-chain:api-failover-direct?dc=dc2", + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, "api-failover-direct", "default", "dc2", + func(req *discoverychain.CompileRequest) { + req.OverrideMeshGateway.Mode = structs.MeshGatewayModeNone + }), + }, + Err: nil, + }, + cache.UpdateEvent{ + CorrelationID: "discovery-chain:api-dc2", + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, "api-dc2", "default", "dc1", + func(req *discoverychain.CompileRequest) { + req.OverrideMeshGateway.Mode = meshGatewayProxyConfigValue + }, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "api-dc2", + Redirect: &structs.ServiceResolverRedirect{ + Service: "api", + Datacenter: "dc2", + }, + }, + ), + }, + Err: nil, + }, + }, + } + + stage1 := verificationStage{ + requiredWatches: map[string]verifyWatchRequest{ + "upstream-target:api,,,dc1:api": genVerifyServiceWatch("api", "", "dc1", true), + "upstream-target:api-failover-remote,,,dc2:api-failover-remote?dc=dc2": genVerifyGatewayWatch("dc2"), + "upstream-target:api-failover-local,,,dc2:api-failover-local?dc=dc2": genVerifyGatewayWatch("dc1"), + "upstream-target:api-failover-direct,,,dc2:api-failover-direct?dc=dc2": genVerifyServiceWatch("api-failover-direct", "", "dc2", true), + }, + } + + if meshGatewayProxyConfigValue == structs.MeshGatewayModeDefault { + stage1.requiredWatches["upstream-target:api,,,dc2:api-dc2"] = genVerifyServiceWatch("api", "", "dc2", true) + } else { + stage1.requiredWatches["upstream-target:api,,,dc2:api-dc2"] = genVerifyGatewayWatch("dc1") + } + + return testCase{ + ns: ns, + sourceDC: "dc1", + stages: []verificationStage{stage0, stage1}, + } + } + cases := map[string]testCase{ "initial-gateway": testCase{ ns: structs.NodeService{ @@ -325,115 +521,8 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, }, }, - "connect-proxy": testCase{ - ns: structs.NodeService{ - Kind: structs.ServiceKindConnectProxy, - ID: "web-sidecar-proxy", - Service: "web-sidecar-proxy", - Address: "10.0.1.1", - Port: 443, - Proxy: structs.ConnectProxyConfig{ - DestinationServiceName: "web", - Upstreams: structs.Upstreams{ - structs.Upstream{ - DestinationType: structs.UpstreamDestTypePreparedQuery, - DestinationName: "query", - LocalBindPort: 10001, - }, - structs.Upstream{ - DestinationType: structs.UpstreamDestTypeService, - DestinationName: "api", - LocalBindPort: 10002, - }, - structs.Upstream{ - DestinationType: structs.UpstreamDestTypeService, - DestinationName: "api-failover-remote", - Datacenter: "dc2", - LocalBindPort: 10003, - MeshGateway: structs.MeshGatewayConfig{ - Mode: structs.MeshGatewayModeRemote, - }, - }, - structs.Upstream{ - DestinationType: structs.UpstreamDestTypeService, - DestinationName: "api-failover-local", - Datacenter: "dc2", - LocalBindPort: 10004, - MeshGateway: structs.MeshGatewayConfig{ - Mode: structs.MeshGatewayModeLocal, - }, - }, - structs.Upstream{ - DestinationType: structs.UpstreamDestTypeService, - DestinationName: "api-failover-direct", - Datacenter: "dc2", - LocalBindPort: 10005, - MeshGateway: structs.MeshGatewayConfig{ - Mode: structs.MeshGatewayModeNone, - }, - }, - structs.Upstream{ - DestinationType: structs.UpstreamDestTypeService, - DestinationName: "api-dc2", - LocalBindPort: 10006, - MeshGateway: structs.MeshGatewayConfig{ - Mode: structs.MeshGatewayModeLocal, - }, - }, - }, - MeshGateway: structs.MeshGatewayConfig{ - Mode: structs.MeshGatewayModeLocal, - }, - }, - }, - sourceDC: "dc1", - stages: []verificationStage{ - verificationStage{ - requiredWatches: map[string]verifyWatchRequest{ - rootsWatchID: genVerifyRootsWatch("dc1"), - leafWatchID: genVerifyLeafWatch("web", "dc1"), - intentionsWatchID: genVerifyIntentionWatch("web", "dc1"), - "upstream:prepared_query:query": genVerifyPreparedQueryWatch("query", "dc1"), - "discovery-chain:api": genVerifyDiscoveryChainWatch("api", "dc1"), - "upstream:" + serviceIDPrefix + "api-failover-remote?dc=dc2": genVerifyGatewayWatch("dc2"), - "upstream:" + serviceIDPrefix + "api-failover-local?dc=dc2": genVerifyGatewayWatch("dc1"), - "upstream:" + serviceIDPrefix + "api-failover-direct?dc=dc2": genVerifyServiceWatch("api-failover-direct", "", "dc2", true), - "discovery-chain:api-dc2": genVerifyDiscoveryChainWatch("api-dc2", "dc1"), - }, - events: []cache.UpdateEvent{ - cache.UpdateEvent{ - CorrelationID: "discovery-chain:api", - Result: &structs.DiscoveryChainResponse{ - Chain: discoverychain.TestCompileConfigEntries(t, "api", "default", "dc1"), - }, - Err: nil, - }, - cache.UpdateEvent{ - CorrelationID: "discovery-chain:api", - Result: &structs.DiscoveryChainResponse{ - Chain: discoverychain.TestCompileConfigEntries(t, "api-dc2", "default", "dc1", - &structs.ServiceResolverConfigEntry{ - Kind: structs.ServiceResolver, - Name: "api-dc2", - Redirect: &structs.ServiceResolverRedirect{ - Service: "api", - Datacenter: "dc2", - }, - }, - ), - }, - Err: nil, - }, - }, - }, - verificationStage{ - requiredWatches: map[string]verifyWatchRequest{ - "upstream-target:api,,,dc1:api": genVerifyServiceWatch("api", "", "dc1", true), - "upstream-target:api,,,dc2:api": genVerifyGatewayWatch("dc1"), - }, - }, - }, - }, + "connect-proxy": newConnectProxyCase(structs.MeshGatewayModeDefault), + "connect-proxy-mesh-gateway-local": newConnectProxyCase(structs.MeshGatewayModeLocal), } for name, tc := range cases { diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index acebcaa68..82856a108 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -433,6 +433,10 @@ func TestConfigSnapshotDiscoveryChain(t testing.T) *ConfigSnapshot { return testConfigSnapshotDiscoveryChain(t, "simple") } +func TestConfigSnapshotDiscoveryChainWithOverrides(t testing.T) *ConfigSnapshot { + return testConfigSnapshotDiscoveryChain(t, "simple-with-overrides") +} + func TestConfigSnapshotDiscoveryChainWithFailover(t testing.T) *ConfigSnapshot { return testConfigSnapshotDiscoveryChain(t, "failover") } @@ -449,8 +453,18 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE roots, leaf := TestCerts(t) // Compile a chain. - var entries []structs.ConfigEntry + var ( + entries []structs.ConfigEntry + compileSetup func(req *discoverychain.CompileRequest) + ) switch variation { + case "simple-with-overrides": + compileSetup = func(req *discoverychain.CompileRequest) { + req.OverrideMeshGateway.Mode = structs.MeshGatewayModeLocal + req.OverrideProtocol = "grpc" + req.OverrideConnectTimeout = 66 * time.Second + } + fallthrough case "simple": entries = append(entries, &structs.ServiceResolverConfigEntry{ @@ -529,7 +543,7 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE entries = append(entries, additionalEntries...) } - dbChain := discoverychain.TestCompileConfigEntries(t, "db", "default", "dc1", entries...) + dbChain := discoverychain.TestCompileConfigEntries(t, "db", "default", "dc1", compileSetup, entries...) dbTarget := structs.DiscoveryTarget{ Service: "db", @@ -574,6 +588,7 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE } switch variation { + case "simple-with-overrides": case "simple": case "failover": snap.ConnectProxy.WatchedUpstreamEndpoints["db"][failTarget] = diff --git a/agent/structs/config_entry_discoverychain.go b/agent/structs/config_entry_discoverychain.go index 935e7c873..5a6c1c873 100644 --- a/agent/structs/config_entry_discoverychain.go +++ b/agent/structs/config_entry_discoverychain.go @@ -986,10 +986,29 @@ func (e *DiscoveryChainConfigEntries) IsChainEmpty() bool { // DiscoveryChainRequest is used when requesting the discovery chain for a // service. type DiscoveryChainRequest struct { - Name string - Datacenter string - // Source QuerySource + Name string + EvaluateInDatacenter string + EvaluateInNamespace string + // OverrideMeshGateway allows for the mesh gateway setting to be overridden + // for any resolver in the compiled chain. + OverrideMeshGateway MeshGatewayConfig + + // OverrideProtocol allows for the final protocol for the chain to be + // altered. + // + // - If the chain ordinarily would be TCP and an L7 protocol is passed here + // the chain will not include Routers or Splitters. + // + // - If the chain ordinarily would be L7 and TCP is passed here the chain + // will not include Routers or Splitters. + OverrideProtocol string + + // OverrideConnectTimeout allows for the ConnectTimeout setting to be + // overridden for any resolver in the compiled chain. + OverrideConnectTimeout time.Duration + + Datacenter string // where to route the RPC QueryOptions } @@ -1008,9 +1027,19 @@ func (r *DiscoveryChainRequest) CacheInfo() cache.RequestInfo { } v, err := hashstructure.Hash(struct { - Name string + Name string + EvaluateInDatacenter string + EvaluateInNamespace string + OverrideMeshGateway MeshGatewayConfig + OverrideProtocol string + OverrideConnectTimeout time.Duration }{ - Name: r.Name, + Name: r.Name, + EvaluateInDatacenter: r.EvaluateInDatacenter, + EvaluateInNamespace: r.EvaluateInNamespace, + OverrideMeshGateway: r.OverrideMeshGateway, + OverrideProtocol: r.OverrideProtocol, + OverrideConnectTimeout: r.OverrideConnectTimeout, }, nil) if err == nil { // If there is an error, we don't set the key. A blank key forces diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 60c13f5bf..78c4cb936 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -37,6 +37,19 @@ type MeshGatewayConfig struct { Mode MeshGatewayMode `json:",omitempty"` } +func (c *MeshGatewayConfig) IsZero() bool { + zeroVal := MeshGatewayConfig{} + return *c == zeroVal +} + +func (base *MeshGatewayConfig) OverlayWith(overlay MeshGatewayConfig) MeshGatewayConfig { + out := *base + if overlay.Mode != MeshGatewayModeDefault { + out.Mode = overlay.Mode + } + return out +} + func ValidateMeshGatewayMode(mode string) (MeshGatewayMode, error) { switch MeshGatewayMode(mode) { case MeshGatewayModeNone: diff --git a/agent/structs/connect_proxy_config_test.go b/agent/structs/connect_proxy_config_test.go index 7c3f9ef98..bc1dd0f50 100644 --- a/agent/structs/connect_proxy_config_test.go +++ b/agent/structs/connect_proxy_config_test.go @@ -2,6 +2,7 @@ package structs import ( "encoding/json" + "fmt" "testing" "github.com/hashicorp/consul/api" @@ -193,3 +194,79 @@ func TestUpstream_UnmarshalJSON(t *testing.T) { }) } } + +func TestMeshGatewayConfig_OverlayWith(t *testing.T) { + var ( + D = MeshGatewayConfig{Mode: MeshGatewayModeDefault} + N = MeshGatewayConfig{Mode: MeshGatewayModeNone} + R = MeshGatewayConfig{Mode: MeshGatewayModeRemote} + L = MeshGatewayConfig{Mode: MeshGatewayModeLocal} + ) + + type testCase struct { + base, overlay, expect MeshGatewayConfig + } + cases := []testCase{ + {D, D, D}, + {D, N, N}, + {D, R, R}, + {D, L, L}, + {N, D, N}, + {N, N, N}, + {N, R, R}, + {N, L, L}, + {R, D, R}, + {R, N, N}, + {R, R, R}, + {R, L, L}, + {L, D, L}, + {L, N, N}, + {L, R, R}, + {L, L, L}, + } + + for _, tc := range cases { + tc := tc + + t.Run(fmt.Sprintf("%s overlaid with %s", tc.base.Mode, tc.overlay.Mode), + func(t *testing.T) { + got := tc.base.OverlayWith(tc.overlay) + require.Equal(t, tc.expect, got) + }) + } +} + +func TestValidateMeshGatewayMode(t *testing.T) { + for _, tc := range []struct { + modeConstant string + modeExplicit string + expect MeshGatewayMode + ok bool + }{ + {string(MeshGatewayModeNone), "none", MeshGatewayModeNone, true}, + {string(MeshGatewayModeDefault), "", MeshGatewayModeDefault, true}, + {string(MeshGatewayModeLocal), "local", MeshGatewayModeLocal, true}, + {string(MeshGatewayModeRemote), "remote", MeshGatewayModeRemote, true}, + } { + tc := tc + + t.Run(tc.modeConstant+" (constant)", func(t *testing.T) { + got, err := ValidateMeshGatewayMode(tc.modeConstant) + if tc.ok { + require.NoError(t, err) + require.Equal(t, tc.expect, got) + } else { + require.Error(t, err) + } + }) + t.Run(tc.modeExplicit+" (explicit)", func(t *testing.T) { + got, err := ValidateMeshGatewayMode(tc.modeExplicit) + if tc.ok { + require.NoError(t, err) + require.Equal(t, tc.expect, got) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/agent/structs/discovery_chain.go b/agent/structs/discovery_chain.go index 6565aabf7..5eb25cc70 100644 --- a/agent/structs/discovery_chain.go +++ b/agent/structs/discovery_chain.go @@ -18,7 +18,17 @@ type CompiledDiscoveryChain struct { Namespace string // the namespace that the chain was compiled within Datacenter string // the datacenter that the chain was compiled within - Protocol string // overall protocol shared by everything in the chain + // CustomizationHash is a unique hash of any data that affects the + // compilation of the discovery chain other than config entries or the + // name/namespace/datacenter evaluation criteria. + // + // If set, this value should be used to prefix/suffix any generated load + // balancer data plane objects to avoid sharing customized and + // non-customized versions. + CustomizationHash string + + // Protocol is the overall protocol shared by everything in the chain. + Protocol string // Node is the top node in the chain. // @@ -49,6 +59,7 @@ func (c *CompiledDiscoveryChain) IsDefault() bool { if c.Node == nil { return true } + // TODO(rb): include CustomizationHash here? return c.Node.Name == c.ServiceName && c.Node.Type == DiscoveryGraphNodeTypeGroupResolver && c.Node.GroupResolver.Default diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 0a128c234..eeb781350 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -58,7 +58,6 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } if chain == nil { - // Either old-school upstream or prepared query. upstreamCluster, err := s.makeUpstreamCluster(u, cfgSnap) if err != nil { return nil, err @@ -255,10 +254,12 @@ func (s *Server) makeUpstreamClustersForDiscoveryChain( groupResolver := node.GroupResolver sni := TargetSNI(target, cfgSnap) - s.Logger.Printf("[DEBUG] xds.clusters - generating cluster for %s", sni) + clusterName := CustomizeClusterName(sni, chain) + + s.Logger.Printf("[DEBUG] xds.clusters - generating cluster for %s", clusterName) c := &envoy.Cluster{ - Name: sni, - AltStatName: sni, // TODO(rb): change this? + Name: clusterName, + AltStatName: clusterName, ConnectTimeout: groupResolver.ConnectTimeout, ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_EDS}, CommonLbConfig: &envoy.Cluster_CommonLbConfig{ diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 714dbdb73..c11181fd9 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -105,6 +105,11 @@ func TestClustersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotDiscoveryChain, setup: nil, }, + { + name: "connect-proxy-with-chain-and-overrides", + create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides, + setup: nil, + }, { name: "connect-proxy-with-chain-and-failover", create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailover, diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 152b1a711..5b9785fa3 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -51,11 +51,11 @@ func (s *Server) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps if chain == nil { // We ONLY want this branch for prepared queries. - sni := UpstreamSNI(&u, "", cfgSnap) + clusterName := UpstreamSNI(&u, "", cfgSnap) endpoints, ok := cfgSnap.ConnectProxy.UpstreamEndpoints[id] if ok { la := makeLoadAssignment( - sni, + clusterName, 0, []loadAssignmentEndpointGroup{ {Endpoints: endpoints}, @@ -122,9 +122,10 @@ func (s *Server) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps } sni := TargetSNI(target, cfgSnap) + clusterName := CustomizeClusterName(sni, chain) la := makeLoadAssignment( - sni, + clusterName, overprovisioningFactor, endpointGroups, cfgSnap.Datacenter, diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index 72e0fed27..41a5456e9 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -245,6 +245,11 @@ func Test_endpointsFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotDiscoveryChain, setup: nil, }, + { + name: "connect-proxy-with-chain-and-overrides", + create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides, + setup: nil, + }, { name: "connect-proxy-with-chain-and-failover", create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailover, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index ddc78ece9..3db8a4cdb 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -61,7 +61,7 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps var upstreamListener proto.Message if chain == nil || chain.IsDefault() { - upstreamListener, err = s.makeUpstreamListener(&u, cfgSnap) + upstreamListener, err = s.makeUpstreamListenerIgnoreDiscoveryChain(&u, chain, cfgSnap) } else { upstreamListener, err = s.makeUpstreamListenerForDiscoveryChain(&u, chain, cfgSnap) } @@ -269,7 +269,12 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri return l, err } -func (s *Server) makeUpstreamListener(u *structs.Upstream, cfgSnap *proxycfg.ConfigSnapshot) (proto.Message, error) { +// makeUpstreamListenerIgnoreDiscoveryChain counterintuitively takes an (optional) chain +func (s *Server) makeUpstreamListenerIgnoreDiscoveryChain( + u *structs.Upstream, + chain *structs.CompiledDiscoveryChain, + cfgSnap *proxycfg.ConfigSnapshot, +) (proto.Message, error) { cfg, err := ParseUpstreamConfig(u.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns @@ -288,7 +293,8 @@ func (s *Server) makeUpstreamListener(u *structs.Upstream, cfgSnap *proxycfg.Con upstreamID := u.Identifier() - clusterName := UpstreamSNI(u, "", cfgSnap) + sni := UpstreamSNI(u, "", cfgSnap) + clusterName := CustomizeClusterName(sni, chain) l := makeListener(upstreamID, addr, u.LocalBindPort) filter, err := makeListenerFilter(false, cfg.Protocol, upstreamID, clusterName, "upstream_", false) diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index fbfbee5ea..acf599830 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -184,6 +184,11 @@ func TestListenersFromSnapshot(t *testing.T) { }, setup: nil, }, + { + name: "connect-proxy-with-chain-and-overrides", + create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides, + setup: nil, + }, { name: "mesh-gateway", create: proxycfg.TestConfigSnapshotMeshGateway, diff --git a/agent/xds/routes.go b/agent/xds/routes.go index ee836d5b3..f8c089df8 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -87,14 +87,14 @@ func makeUpstreamRouteForDiscoveryChain( next := discoveryRoute.DestinationNode if next.Type == structs.DiscoveryGraphNodeTypeSplitter { - routeAction, err = makeRouteActionForSplitter(next.Splits, cfgSnap) + routeAction, err = makeRouteActionForSplitter(next.Splits, chain, cfgSnap) if err != nil { return nil, err } } else if next.Type == structs.DiscoveryGraphNodeTypeGroupResolver { groupResolver := next.GroupResolver - routeAction = makeRouteActionForSingleCluster(groupResolver.Target, cfgSnap) + routeAction = makeRouteActionForSingleCluster(groupResolver.Target, chain, cfgSnap) } else { return nil, fmt.Errorf("unexpected graph node after route %q", next.Type) @@ -142,7 +142,7 @@ func makeUpstreamRouteForDiscoveryChain( } case structs.DiscoveryGraphNodeTypeSplitter: - routeAction, err := makeRouteActionForSplitter(chain.Node.Splits, cfgSnap) + routeAction, err := makeRouteActionForSplitter(chain.Node.Splits, chain, cfgSnap) if err != nil { return nil, err } @@ -157,7 +157,7 @@ func makeUpstreamRouteForDiscoveryChain( case structs.DiscoveryGraphNodeTypeGroupResolver: groupResolver := chain.Node.GroupResolver - routeAction := makeRouteActionForSingleCluster(groupResolver.Target, cfgSnap) + routeAction := makeRouteActionForSingleCluster(groupResolver.Target, chain, cfgSnap) defaultRoute := envoyroute.Route{ Match: makeDefaultRouteMatch(), @@ -304,8 +304,9 @@ func makeDefaultRouteMatch() envoyroute.RouteMatch { } } -func makeRouteActionForSingleCluster(target structs.DiscoveryTarget, cfgSnap *proxycfg.ConfigSnapshot) *envoyroute.Route_Route { - clusterName := TargetSNI(target, cfgSnap) +func makeRouteActionForSingleCluster(target structs.DiscoveryTarget, chain *structs.CompiledDiscoveryChain, cfgSnap *proxycfg.ConfigSnapshot) *envoyroute.Route_Route { + sni := TargetSNI(target, cfgSnap) + clusterName := CustomizeClusterName(sni, chain) return &envoyroute.Route_Route{ Route: &envoyroute.RouteAction{ @@ -316,7 +317,7 @@ func makeRouteActionForSingleCluster(target structs.DiscoveryTarget, cfgSnap *pr } } -func makeRouteActionForSplitter(splits []*structs.DiscoverySplit, cfgSnap *proxycfg.ConfigSnapshot) (*envoyroute.Route_Route, error) { +func makeRouteActionForSplitter(splits []*structs.DiscoverySplit, chain *structs.CompiledDiscoveryChain, cfgSnap *proxycfg.ConfigSnapshot) (*envoyroute.Route_Route, error) { clusters := make([]*envoyroute.WeightedCluster_ClusterWeight, 0, len(splits)) for _, split := range splits { if split.Node.Type != structs.DiscoveryGraphNodeTypeGroupResolver { @@ -324,7 +325,9 @@ func makeRouteActionForSplitter(splits []*structs.DiscoverySplit, cfgSnap *proxy } groupResolver := split.Node.GroupResolver target := groupResolver.Target - clusterName := TargetSNI(target, cfgSnap) + + sni := TargetSNI(target, cfgSnap) + clusterName := CustomizeClusterName(sni, chain) // The smallest representable weight is 1/10000 or .01% but envoy // deals with integers so scale everything up by 100x. diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index 436d0135d..e9bb5dc3c 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -50,6 +50,11 @@ func TestRoutesFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotDiscoveryChain, setup: nil, }, + { + name: "connect-proxy-with-chain-and-overrides", + create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides, + setup: nil, + }, { name: "splitter-with-resolver-redirect", create: proxycfg.TestConfigSnapshotDiscoveryChain_SplitterWithResolverRedirectMultiDC, diff --git a/agent/xds/sni.go b/agent/xds/sni.go index 62cdb6e8a..509824291 100644 --- a/agent/xds/sni.go +++ b/agent/xds/sni.go @@ -45,3 +45,12 @@ func QuerySNI(service string, datacenter string, cfgSnap *proxycfg.ConfigSnapsho func TargetSNI(target structs.DiscoveryTarget, cfgSnap *proxycfg.ConfigSnapshot) string { return ServiceSNI(target.Service, target.ServiceSubset, target.Namespace, target.Datacenter, cfgSnap) } + +func CustomizeClusterName(sni string, chain *structs.CompiledDiscoveryChain) string { + if chain == nil || chain.CustomizationHash == "" { + return sni + } + // Use a colon to delimit this prefix instead of a dot to avoid a + // theoretical collision problem with subsets. + return fmt.Sprintf("%s:%s", chain.CustomizationHash, sni) +} diff --git a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-overrides.golden new file mode 100644 index 000000000..7105cbefe --- /dev/null +++ b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-overrides.golden @@ -0,0 +1,119 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "78ebd528:db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "78ebd528:db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "66s", + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + }, + "http2ProtocolOptions": { + + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + }, + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-overrides.golden new file mode 100644 index 000000000..4388b199e --- /dev/null +++ b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-overrides.golden @@ -0,0 +1,41 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "78ebd528:db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden new file mode 100644 index 000000000..a911cd189 --- /dev/null +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden @@ -0,0 +1,139 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http2_protocol_options": { + }, + "http_filters": [ + { + "config": { + }, + "name": "envoy.grpc_http1_bridge" + }, + { + "name": "envoy.router" + } + ], + "rds": { + "config_source": { + "ads": { + } + }, + "route_config_name": "db" + }, + "stat_prefix": "upstream_db_grpc", + "tracing": { + "operation_name": "EGRESS", + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "upstream_prepared_query_geo-cache_tcp" + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.ext_authz", + "config": { + "grpc_service": { + "envoy_grpc": { + "cluster_name": "local_agent" + }, + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "my-token" + } + ] + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener_tcp" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.golden new file mode 100644 index 000000000..6f7af1958 --- /dev/null +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.golden @@ -0,0 +1,30 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "db", + "virtualHosts": [ + { + "name": "db", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "78ebd528:db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats b/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats index bb72bfba8..9afca9ced 100644 --- a/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats +++ b/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats @@ -15,7 +15,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2" { @@ -47,7 +48,7 @@ load helpers } @test "s1 proxy should be adding cluster name as a tag" { - run retry_default must_match_in_statsd_logs '[#,]envoy.cluster_name:s2(,|$)' primary + run retry_default must_match_in_statsd_logs '[#,]envoy.cluster_name:1a47f6e1_s2(,|$)' primary echo "OUTPUT: $output" diff --git a/test/integration/connect/envoy/case-gateways-local/primary/verify.bats b/test/integration/connect/envoy/case-gateways-local/primary/verify.bats index c8492177d..82face136 100644 --- a/test/integration/connect/envoy/case-gateways-local/primary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-local/primary/verify.bats @@ -19,7 +19,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.secondary HEALTHY 1 + # mesh gateway mode is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 c225dc1c:s2.default.secondary HEALTHY 1 } @test "gateway-primary should have healthy endpoints for secondary" { @@ -33,9 +34,9 @@ load helpers } @test "s1 upstream made 1 connection" { - assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.secondary.*cx_total" 1 + assert_envoy_metric 127.0.0.1:19000 "cluster.c225dc1c_s2.default.secondary.*cx_total" 1 } @test "gateway-primary is used for the upstream connection" { assert_envoy_metric 127.0.0.1:19002 "cluster.secondary.*cx_total" 1 -} \ No newline at end of file +} diff --git a/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats b/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats index 56ef8fa2f..ce4b64dba 100644 --- a/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats @@ -15,7 +15,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.secondary HEALTHY 1 + # mesh gateway mode is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 dd412229:s2.default.secondary HEALTHY 1 } @test "s1 upstream should be able to connect to s2" { @@ -25,5 +26,5 @@ load helpers } @test "s1 upstream made 1 connection" { - assert_envoy_metric 127.0.0.1:19000 "cluster.s2.default.secondary.*cx_total" 1 -} \ No newline at end of file + assert_envoy_metric 127.0.0.1:19000 "cluster.dd412229_s2.default.secondary.*cx_total" 1 +} diff --git a/test/integration/connect/envoy/case-grpc/verify.bats b/test/integration/connect/envoy/case-grpc/verify.bats index 52f29c3fb..1c9acde87 100644 --- a/test/integration/connect/envoy/case-grpc/verify.bats +++ b/test/integration/connect/envoy/case-grpc/verify.bats @@ -15,7 +15,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 ef15b5b5:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2 via grpc" { diff --git a/test/integration/connect/envoy/case-http-badauthz/verify.bats b/test/integration/connect/envoy/case-http-badauthz/verify.bats index ecf5e3ef8..d62269849 100644 --- a/test/integration/connect/envoy/case-http-badauthz/verify.bats +++ b/test/integration/connect/envoy/case-http-badauthz/verify.bats @@ -23,7 +23,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1:s2.default.primary HEALTHY 1 } @test "s1 upstream should NOT be able to connect to s2" { diff --git a/test/integration/connect/envoy/case-http/verify.bats b/test/integration/connect/envoy/case-http/verify.bats index 09bbf34a9..e0a75ca0b 100644 --- a/test/integration/connect/envoy/case-http/verify.bats +++ b/test/integration/connect/envoy/case-http/verify.bats @@ -23,7 +23,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2 with http/1.1" { diff --git a/test/integration/connect/envoy/case-http2/verify.bats b/test/integration/connect/envoy/case-http2/verify.bats index 9dfcc09a5..7062c487b 100644 --- a/test/integration/connect/envoy/case-http2/verify.bats +++ b/test/integration/connect/envoy/case-http2/verify.bats @@ -23,7 +23,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 49c19fe6:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2 via http2" { diff --git a/test/integration/connect/envoy/case-prometheus/verify.bats b/test/integration/connect/envoy/case-prometheus/verify.bats index 4cbb28a2c..ffa4fcf7e 100644 --- a/test/integration/connect/envoy/case-prometheus/verify.bats +++ b/test/integration/connect/envoy/case-prometheus/verify.bats @@ -23,7 +23,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2 with http/1.1" { diff --git a/test/integration/connect/envoy/case-statsd-udp/verify.bats b/test/integration/connect/envoy/case-statsd-udp/verify.bats index 8bf959007..093eb3819 100644 --- a/test/integration/connect/envoy/case-statsd-udp/verify.bats +++ b/test/integration/connect/envoy/case-statsd-udp/verify.bats @@ -15,7 +15,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2" { diff --git a/test/integration/connect/envoy/case-zipkin/verify.bats b/test/integration/connect/envoy/case-zipkin/verify.bats index 0489fed1a..d56c3f190 100644 --- a/test/integration/connect/envoy/case-zipkin/verify.bats +++ b/test/integration/connect/envoy/case-zipkin/verify.bats @@ -23,7 +23,8 @@ load helpers } @test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 + # protocol is configured in an upstream override so the cluster name is customized here + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1:s2.default.primary HEALTHY 1 } @test "s1 upstream should be able to connect to s2" { diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh index f4afae448..571651509 100755 --- a/test/integration/connect/envoy/run-tests.sh +++ b/test/integration/connect/envoy/run-tests.sh @@ -90,10 +90,10 @@ function init_workdir { cp consul-base-cfg/* workdir/${DC}/consul/ # Add any overrides if there are any (no op if not) - find ${CASE_DIR} -name '*.hcl' -maxdepth 1 -type f -exec cp -f {} workdir/${DC}/consul \; + find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${DC}/consul \; # Copy all the test files - find ${CASE_DIR} -name '*.bats' -maxdepth 1 -type f -exec cp -f {} workdir/${DC}/bats \; + find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${DC}/bats \; # Copy DC specific bats cp helpers.bash workdir/${DC}/bats