From f759a48726436af9eda308bfd3bda019b13a46e4 Mon Sep 17 00:00:00 2001 From: Freddy Date: Wed, 3 Jun 2020 15:28:45 -0600 Subject: [PATCH] Enable gateways to resolve hostnames to IPv4 addresses (#7999) The DNS resolution will be handled by Envoy and defaults to LOGICAL_DNS. This discovery type can be overridden on a per-gateway basis with the envoy_dns_discovery_type Gateway Option. If a service contains an instance with a hostname as an address we set the Envoy cluster to use DNS as the discovery type rather than EDS. Since both mesh gateways and terminating gateways route to clusters using SNI, whenever there is a mix of hostnames and IP addresses associated with a service we use the hostname + CDS rather than the IPs + EDS. Note that we detect hostnames by attempting to parse the service instance's address as an IP. If it is not a valid IP we assume it is a hostname. --- agent/proxycfg/snapshot.go | 14 +- agent/proxycfg/state.go | 63 +++++- agent/proxycfg/state_test.go | 207 +++++++++++++++++- agent/proxycfg/testing.go | 172 ++++++++++++++- agent/xds/clusters.go | 177 +++++++++++---- agent/xds/clusters_test.go | 17 ++ agent/xds/config.go | 7 + agent/xds/config_test.go | 2 + agent/xds/endpoints.go | 17 +- agent/xds/listeners_test.go | 6 +- ...mesh-gateway-ignore-extra-resolvers.golden | 44 ++++ .../mesh-gateway-service-subsets.golden | 44 ++++ .../mesh-gateway-service-timeouts.golden | 44 ++++ ...esh-gateway-using-federation-states.golden | 44 ++++ .../xds/testdata/clusters/mesh-gateway.golden | 44 ++++ ...ng-gateway-hostname-service-subsets.golden | 155 +++++++++++++ ...ting-gateway-ignore-extra-resolvers.golden | 44 +++- ...terminating-gateway-service-subsets.golden | 44 +++- .../clusters/terminating-gateway.golden | 44 +++- ...th-default-chain-and-custom-cluster.golden | 2 +- ...ting-gateway-default-service-subset.golden | 34 --- ...terminating-gateway-service-subsets.golden | 34 --- .../endpoints/terminating-gateway.golden | 34 --- .../mesh-gateway-custom-addresses.golden | 64 ++++++ .../mesh-gateway-tagged-addresses.golden | 32 +++ ...esh-gateway-using-federation-states.golden | 16 ++ .../testdata/listeners/mesh-gateway.golden | 16 ++ .../capture.sh | 4 + .../config_entries.hcl | 14 ++ .../gateway.hcl | 5 + .../case-terminating-gateway-hostnames/s1.hcl | 16 ++ .../case-terminating-gateway-hostnames/s4.hcl | 7 + .../setup.sh | 9 + .../vars.sh | 4 + .../verify.bats | 33 +++ .../connect/envoy/docker-compose.yml | 16 ++ test/integration/connect/envoy/helpers.bash | 2 +- test/integration/connect/envoy/main_test.go | 1 + website/pages/docs/connect/proxies/envoy.mdx | 7 + .../docs/connect/terminating-gateway.mdx | 3 +- 40 files changed, 1342 insertions(+), 200 deletions(-) create mode 100644 agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh create mode 100644 test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index b5ec6d691..929e75a97 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -100,6 +100,10 @@ type configSnapshotTerminatingGateway struct { // between the gateway and a service. TLS configuration stored here is // used for TLS origination from the gateway to the linked service. GatewayServices map[structs.ServiceID]structs.GatewayService + + // HostnameServices is a map of service id to service instances with a hostname as the address. + // If hostnames are configured they must be provided to Envoy via CDS not EDS. + HostnameServices map[structs.ServiceID]structs.CheckServiceNodes } func (c *configSnapshotTerminatingGateway) IsEmpty() bool { @@ -113,7 +117,8 @@ func (c *configSnapshotTerminatingGateway) IsEmpty() bool { len(c.WatchedServices) == 0 && len(c.ServiceResolvers) == 0 && len(c.WatchedResolvers) == 0 && - len(c.GatewayServices) == 0 + len(c.GatewayServices) == 0 && + len(c.HostnameServices) == 0 } type configSnapshotMeshGateway struct { @@ -155,6 +160,10 @@ type configSnapshotMeshGateway struct { // ConsulServers is the list of consul servers in this datacenter. ConsulServers structs.CheckServiceNodes + + // HostnameDatacenters is a map of datacenters to mesh gateway instances with a hostname as the address. + // If hostnames are configured they must be provided to Envoy via CDS not EDS. + HostnameDatacenters map[string]structs.CheckServiceNodes } func (c *configSnapshotMeshGateway) Datacenters() []string { @@ -188,7 +197,8 @@ func (c *configSnapshotMeshGateway) IsEmpty() bool { len(c.ServiceResolvers) == 0 && len(c.GatewayGroups) == 0 && len(c.FedStateGateways) == 0 && - len(c.ConsulServers) == 0 + len(c.ConsulServers) == 0 && + len(c.HostnameDatacenters) == 0 } type configSnapshotIngressGateway struct { diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 1cd703eb5..97801a84e 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "reflect" "strings" "time" @@ -552,12 +553,14 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot { snap.TerminatingGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes) snap.TerminatingGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry) snap.TerminatingGateway.GatewayServices = make(map[structs.ServiceID]structs.GatewayService) + snap.TerminatingGateway.HostnameServices = make(map[structs.ServiceID]structs.CheckServiceNodes) case structs.ServiceKindMeshGateway: snap.MeshGateway.WatchedServices = make(map[structs.ServiceID]context.CancelFunc) snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc) snap.MeshGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes) snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes) snap.MeshGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry) + snap.MeshGateway.HostnameDatacenters = make(map[string]structs.CheckServiceNodes) // there is no need to initialize the map of service resolvers as we // fully rebuild it every time we get updates case structs.ServiceKindIngressGateway: @@ -1032,6 +1035,13 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config } } + // Clean up services with hostname mapping for services that were not in the update + for sid, _ := range snap.TerminatingGateway.HostnameServices { + if _, ok := svcMap[sid]; !ok { + delete(snap.TerminatingGateway.HostnameServices, sid) + } + } + // Cancel service instance watches for services that were not in the update for sid, cancelFn := range snap.TerminatingGateway.WatchedServices { if _, ok := svcMap[sid]; !ok { @@ -1081,11 +1091,12 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config } sid := structs.ServiceIDFromString(strings.TrimPrefix(u.CorrelationID, externalServiceIDPrefix)) + delete(snap.TerminatingGateway.ServiceGroups, sid) + delete(snap.TerminatingGateway.HostnameServices, sid) if len(resp.Nodes) > 0 { snap.TerminatingGateway.ServiceGroups[sid] = resp.Nodes - } else if _, ok := snap.TerminatingGateway.ServiceGroups[sid]; ok { - delete(snap.TerminatingGateway.ServiceGroups, sid) + snap.TerminatingGateway.HostnameServices[sid] = s.hostnameEndpoints(logging.TerminatingGateway, snap.Datacenter, resp.Nodes) } // Store leaf cert for watched service @@ -1141,6 +1152,17 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho return fmt.Errorf("invalid type for response: %T", u.Result) } snap.MeshGateway.FedStateGateways = dcIndexedNodes.DatacenterNodes + + for dc, nodes := range dcIndexedNodes.DatacenterNodes { + snap.MeshGateway.HostnameDatacenters[dc] = s.hostnameEndpoints(logging.MeshGateway, snap.Datacenter, nodes) + } + + for dc, _ := range snap.MeshGateway.HostnameDatacenters { + if _, ok := dcIndexedNodes.DatacenterNodes[dc]; !ok { + delete(snap.MeshGateway.HostnameDatacenters, dc) + } + } + case serviceListWatchID: services, ok := u.Result.(*structs.IndexedServiceList) if !ok { @@ -1297,11 +1319,12 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho } dc := strings.TrimPrefix(u.CorrelationID, "mesh-gateway:") + delete(snap.MeshGateway.GatewayGroups, dc) + delete(snap.MeshGateway.HostnameDatacenters, dc) if len(resp.Nodes) > 0 { snap.MeshGateway.GatewayGroups[dc] = resp.Nodes - } else if _, ok := snap.MeshGateway.GatewayGroups[dc]; ok { - delete(snap.MeshGateway.GatewayGroups, dc) + snap.MeshGateway.HostnameDatacenters[dc] = s.hostnameEndpoints(logging.MeshGateway, snap.Datacenter, resp.Nodes) } default: // do nothing for now @@ -1518,3 +1541,35 @@ func (s *state) Changed(ns *structs.NodeService, token string) bool { !reflect.DeepEqual(s.proxyCfg, proxyCfg) || s.token != token } + +// hostnameEndpoints returns all CheckServiceNodes that have hostnames instead of IPs as the address. +// Envoy cannot resolve hostnames provided through EDS, so we exclusively use CDS for these clusters. +// If there is a mix of hostnames and addresses we exclusively use the hostnames, since clusters cannot discover +// services with both EDS and DNS. +func (s *state) hostnameEndpoints(loggerName string, localDC string, nodes structs.CheckServiceNodes) structs.CheckServiceNodes { + var ( + hasIP bool + hasHostname bool + resp structs.CheckServiceNodes + ) + + for _, n := range nodes { + addr, _ := n.BestAddress(localDC != n.Node.Datacenter) + if net.ParseIP(addr) != nil { + hasIP = true + continue + } + hasHostname = true + resp = append(resp, n) + } + + if hasHostname && hasIP { + dc := nodes[0].Node.Datacenter + sid := nodes[0].Service.CompoundServiceName() + + s.logger.Named(loggerName). + Warn("service contains instances with mix of hostnames and IP addresses; only hostnames will be passed to Envoy.", + "dc", dc, "service", sid.String()) + } + return resp +} diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 1d741b6e5..9067a6f69 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -599,6 +599,9 @@ func TestState_WatchesAndUpdates(t *testing.T) { db := structs.NewServiceID("db", nil) dbStr := db.String() + api := structs.NewServiceID("api", nil) + apiStr := api.String() + cases := map[string]testCase{ "initial-gateway": testCase{ ns: structs.NodeService{ @@ -714,6 +717,96 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.True(t, snap.MeshGateway.WatchedServicesSet) }, }, + verificationStage{ + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: "mesh-gateway:dc4", + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestGatewayNodesDC4Hostname(t), + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "gateway with service list is valid") + require.Len(t, snap.MeshGateway.WatchedServices, 2) + require.True(t, snap.MeshGateway.WatchedServicesSet) + + expect := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + } + require.Equal(t, snap.MeshGateway.HostnameDatacenters["dc4"], expect) + }, + }, + verificationStage{ + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: federationStateListGatewaysWatchID, + Result: &structs.DatacenterIndexedCheckServiceNodes{ + DatacenterNodes: map[string]structs.CheckServiceNodes{ + "dc5": TestGatewayNodesDC5Hostname(t), + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "gateway with service list is valid") + require.Len(t, snap.MeshGateway.WatchedServices, 2) + require.True(t, snap.MeshGateway.WatchedServicesSet) + + expect := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + } + require.Equal(t, snap.MeshGateway.HostnameDatacenters["dc5"], expect) + }, + }, }, }, "ingress-gateway": testCase{ @@ -1039,6 +1132,10 @@ func TestState_WatchesAndUpdates(t *testing.T) { Service: structs.NewServiceID("billing", nil), Gateway: structs.NewServiceID("terminating-gateway", nil), }, + { + Service: structs.NewServiceID("api", nil), + Gateway: structs.NewServiceID("terminating-gateway", nil), + }, }, }, Err: nil, @@ -1047,27 +1144,33 @@ func TestState_WatchesAndUpdates(t *testing.T) { verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { db := structs.NewServiceID("db", nil) billing := structs.NewServiceID("billing", nil) + api := structs.NewServiceID("api", nil) require.True(t, snap.Valid(), "gateway with service list is valid") - require.Len(t, snap.TerminatingGateway.WatchedServices, 2) + require.Len(t, snap.TerminatingGateway.WatchedServices, 3) require.Contains(t, snap.TerminatingGateway.WatchedServices, db) require.Contains(t, snap.TerminatingGateway.WatchedServices, billing) + require.Contains(t, snap.TerminatingGateway.WatchedServices, api) - require.Len(t, snap.TerminatingGateway.WatchedIntentions, 2) + require.Len(t, snap.TerminatingGateway.WatchedIntentions, 3) require.Contains(t, snap.TerminatingGateway.WatchedIntentions, db) require.Contains(t, snap.TerminatingGateway.WatchedIntentions, billing) + require.Contains(t, snap.TerminatingGateway.WatchedIntentions, api) - require.Len(t, snap.TerminatingGateway.WatchedLeaves, 2) + require.Len(t, snap.TerminatingGateway.WatchedLeaves, 3) require.Contains(t, snap.TerminatingGateway.WatchedLeaves, db) require.Contains(t, snap.TerminatingGateway.WatchedLeaves, billing) + require.Contains(t, snap.TerminatingGateway.WatchedLeaves, api) - require.Len(t, snap.TerminatingGateway.WatchedResolvers, 2) + require.Len(t, snap.TerminatingGateway.WatchedResolvers, 3) require.Contains(t, snap.TerminatingGateway.WatchedResolvers, db) require.Contains(t, snap.TerminatingGateway.WatchedResolvers, billing) + require.Contains(t, snap.TerminatingGateway.WatchedResolvers, api) - require.Len(t, snap.TerminatingGateway.GatewayServices, 2) + require.Len(t, snap.TerminatingGateway.GatewayServices, 3) require.Contains(t, snap.TerminatingGateway.GatewayServices, db) require.Contains(t, snap.TerminatingGateway.GatewayServices, billing) + require.Contains(t, snap.TerminatingGateway.GatewayServices, api) }, }, verificationStage{ @@ -1112,6 +1215,97 @@ func TestState_WatchesAndUpdates(t *testing.T) { ) }, }, + verificationStage{ + requiredWatches: map[string]verifyWatchRequest{ + "external-service:" + apiStr: genVerifyServiceWatch("api", "", "dc1", false), + }, + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: "external-service:" + apiStr, + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "10.0.1.1", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.mydomain", + }, + }, + { + Node: &structs.Node{ + Node: "node2", + Address: "10.0.1.2", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.altdomain", + }, + }, + { + Node: &structs.Node{ + Node: "node3", + Address: "10.0.1.3", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "10.0.1.3", + }, + }, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.Len(t, snap.TerminatingGateway.ServiceGroups, 2) + expect := structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "10.0.1.1", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.mydomain", + }, + }, + { + Node: &structs.Node{ + Node: "node2", + Address: "10.0.1.2", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.altdomain", + }, + }, + { + Node: &structs.Node{ + Node: "node3", + Address: "10.0.1.3", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "10.0.1.3", + }, + }, + } + sid := structs.NewServiceID("api", nil) + require.Equal(t, snap.TerminatingGateway.ServiceGroups[sid], expect) + + // The instance in node3 should not be present in HostnameDatacenters because it has a valid IP + require.ElementsMatch(t, snap.TerminatingGateway.HostnameServices[sid], expect[:2]) + }, + }, verificationStage{ requiredWatches: map[string]verifyWatchRequest{ "service-leaf:" + dbStr: genVerifyLeafWatch("db", "dc1"), @@ -1198,10 +1392,11 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Len(t, snap.TerminatingGateway.GatewayServices, 1) require.Contains(t, snap.TerminatingGateway.GatewayServices, billing) - // There was no update event for billing's leaf/endpoints, so length is 0 + // There was no update event for billing's leaf/endpoints/resolvers, so length is 0 require.Len(t, snap.TerminatingGateway.ServiceGroups, 0) require.Len(t, snap.TerminatingGateway.ServiceLeaves, 0) require.Len(t, snap.TerminatingGateway.ServiceResolvers, 0) + require.Len(t, snap.TerminatingGateway.HostnameServices, 0) }, }, }, diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 00c530f56..8dfc6a746 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -374,6 +374,88 @@ func TestGatewayNodesDC3(t testing.T) structs.CheckServiceNodes { } } +func TestGatewayNodesDC4Hostname(t testing.T) structs.CheckServiceNodes { + return structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-3", + Node: "mesh-gateway", + Address: "10.30.1.3", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.3", 8443, + structs.ServiceAddress{Address: "10.30.1.3", Port: 8443}, + structs.ServiceAddress{Address: "198.38.1.1", Port: 443}), + }, + } +} + +func TestGatewayNodesDC5Hostname(t testing.T) structs.CheckServiceNodes { + return structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-3", + Node: "mesh-gateway", + Address: "10.30.1.3", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.3", 8443, + structs.ServiceAddress{Address: "10.30.1.3", Port: 8443}, + structs.ServiceAddress{Address: "198.38.1.1", Port: 443}), + }, + } +} + func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes { return structs.CheckServiceNodes{ structs.CheckServiceNode{ @@ -1305,11 +1387,41 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera }, GatewayGroups: map[string]structs.CheckServiceNodes{ "dc2": TestGatewayNodesDC2(t), + "dc4": TestGatewayNodesDC4Hostname(t), + }, + HostnameDatacenters: map[string]structs.CheckServiceNodes{ + "dc4": { + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + }, }, } if useFederationStates { snap.MeshGateway.FedStateGateways = map[string]structs.CheckServiceNodes{ "dc2": TestGatewayNodesDC2(t), + "dc4": TestGatewayNodesDC4Hostname(t), } delete(snap.MeshGateway.GatewayGroups, "dc2") @@ -1505,10 +1617,59 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C } api := structs.NewServiceID("api", nil) - apiNodes := TestUpstreamNodes(t) - for i := 0; i < len(apiNodes); i++ { - apiNodes[i].Service.Service = "api" - apiNodes[i].Service.Port = 8081 + apiNodes := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "api", + Node: "test1", + Address: "10.10.1.1", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "api", + Address: "api.mydomain", + Port: 8081, + }, + Checks: structs.HealthChecks{ + { + Status: "critical", + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "test2", + Node: "test2", + Address: "10.10.1.2", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "api", + Address: "api.altdomain", + Port: 8081, + Meta: map[string]string{ + "domain": "alt", + }, + }, + Checks: structs.HealthChecks{ + { + Status: "passing", + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "test3", + Node: "test3", + Address: "10.10.1.3", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "api", + Address: "10.10.1.3", + Port: 8081, + }, + }, } snap.TerminatingGateway = configSnapshotTerminatingGateway{ @@ -1528,6 +1689,9 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C KeyFile: "api.key.pem", }, }, + HostnameServices: map[structs.ServiceID]structs.CheckServiceNodes{ + api: {apiNodes[0], apiNodes[1]}, + }, } snap.TerminatingGateway.ServiceLeaves = map[structs.ServiceID]*structs.IssuedCert{ structs.NewServiceID("web", nil): { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index f74b6e5dd..99c15a34a 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -133,35 +133,41 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho if dc == cfgSnap.Datacenter { continue // skip local } - clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain) - cluster, err := s.makeGatewayCluster(cfgSnap, clusterName) - if err != nil { - return nil, err + opts := gatewayClusterOpts{ + name: connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain), + hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[dc], + isRemote: dc != cfgSnap.Datacenter, } + cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) } if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { // Add all of the remote wildcard datacenter mappings for servers. for _, dc := range datacenters { - clusterName := cfgSnap.ServerSNIFn(dc, "") + hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[dc] - cluster, err := s.makeGatewayCluster(cfgSnap, clusterName) - if err != nil { - return nil, err + // If the DC is our current DC then this cluster is for traffic from a remote DC to a local server. + // HostnameDatacenters is populated with gateway addresses, so it does not apply here. + if dc == cfgSnap.Datacenter { + hostnameEndpoints = nil } + opts := gatewayClusterOpts{ + name: cfgSnap.ServerSNIFn(dc, ""), + hostnameEndpoints: hostnameEndpoints, + isRemote: dc != cfgSnap.Datacenter, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) } // And for the current datacenter, send all flavors appropriately. for _, srv := range cfgSnap.MeshGateway.ConsulServers { - clusterName := cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node) - - cluster, err := s.makeGatewayCluster(cfgSnap, clusterName) - if err != nil { - return nil, err + opts := gatewayClusterOpts{ + name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node), } + cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) } } @@ -179,6 +185,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var services map[structs.ServiceID]structs.CheckServiceNodes var resolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry + var hostnameEndpoints structs.CheckServiceNodes switch cfgSnap.Kind { case structs.ServiceKindTerminatingGateway: @@ -197,33 +204,44 @@ func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([ clusterName := connect.ServiceSNI(svc.ID, "", svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) resolver, hasResolver := resolvers[svc] - // Create the cluster for default/unnamed services - var cluster *envoy.Cluster - var err error - if !hasResolver { // Use a zero value resolver with no timeout and no subsets resolver = &structs.ServiceResolverConfigEntry{} } - cluster, err = s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout) - if err != nil { - return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err) + + // When making service clusters we only pass endpoints with hostnames if the kind is a terminating gateway + // This is because the services a mesh gateway will route to are not external services and are not addressed by a hostname. + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway { + hostnameEndpoints = cfgSnap.TerminatingGateway.HostnameServices[svc] } + opts := gatewayClusterOpts{ + name: clusterName, + hostnameEndpoints: hostnameEndpoints, + connectTimeout: resolver.ConnectTimeout, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway { injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc) } clusters = append(clusters, cluster) // If there is a service-resolver for this service then also setup a cluster for each subset - for subsetName := range resolver.Subsets { - clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - - cluster, err := s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout) + for name, subset := range resolver.Subsets { + subsetHostnameEndpoints, err := s.filterSubsetEndpoints(&subset, hostnameEndpoints) if err != nil { - return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err) + return nil, err } + opts := gatewayClusterOpts{ + name: connect.ServiceSNI(svc.ID, name, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain), + hostnameEndpoints: subsetHostnameEndpoints, + onlyPassing: subset.OnlyPassing, + connectTimeout: resolver.ConnectTimeout, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway { injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc) } @@ -547,43 +565,86 @@ func makeClusterFromUserConfig(configJSON string) (*envoy.Cluster, error) { return &c, err } -func (s *Server) makeGatewayCluster(cfgSnap *proxycfg.ConfigSnapshot, clusterName string) (*envoy.Cluster, error) { - return s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, 0) +type gatewayClusterOpts struct { + // name for the cluster + name string + + // isRemote determines whether the cluster is in a remote DC and we should prefer a WAN address + isRemote bool + + // onlyPassing determines whether endpoints that do not have a passing status should be considered unhealthy + onlyPassing bool + + // connectTimeout is the timeout for new network connections to hosts in the cluster + connectTimeout time.Duration + + // hostnameEndpoints is a list of endpoints with a hostname as their address + hostnameEndpoints structs.CheckServiceNodes } -// makeGatewayClusterWithConnectTimeout initializes a gateway cluster -// with the specified connect timeout. If the timeout is 0, the connect timeout -// defaults to use the configured gateway timeout. -func (s *Server) makeGatewayClusterWithConnectTimeout(cfgSnap *proxycfg.ConfigSnapshot, - clusterName string, connectTimeout time.Duration) (*envoy.Cluster, error) { - - cfg, err := ParseGatewayConfig(cfgSnap.Proxy.Config) +func (s *Server) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts gatewayClusterOpts) *envoy.Cluster { + cfg, err := ParseGatewayConfig(snap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. s.Logger.Warn("failed to parse gateway config", "error", err) } - - if connectTimeout <= 0 { - connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond + if opts.connectTimeout <= 0 { + opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond } - cluster := envoy.Cluster{ - Name: clusterName, - ConnectTimeout: connectTimeout, - ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_EDS}, - EdsClusterConfig: &envoy.Cluster_EdsClusterConfig{ + cluster := &envoy.Cluster{ + Name: opts.name, + ConnectTimeout: opts.connectTimeout, + + // Having an empty config enables outlier detection with default config. + OutlierDetection: &envoycluster.OutlierDetection{}, + } + + useEDS := true + if len(opts.hostnameEndpoints) > 0 { + useEDS = false + } + + // If none of the service instances are addressed by a hostname we provide the endpoint IP addresses via EDS + if useEDS { + cluster.ClusterDiscoveryType = &envoy.Cluster_Type{Type: envoy.Cluster_EDS} + cluster.EdsClusterConfig = &envoy.Cluster_EdsClusterConfig{ EdsConfig: &envoycore.ConfigSource{ ConfigSourceSpecifier: &envoycore.ConfigSource_Ads{ Ads: &envoycore.AggregatedConfigSource{}, }, }, - }, - // Having an empty config enables outlier detection with default config. - OutlierDetection: &envoycluster.OutlierDetection{}, + } + return cluster } - return &cluster, nil + // When a service instance is addressed by a hostname we have Envoy do the DNS resolution + // by setting a DNS cluster type and passing the hostname endpoints via CDS. + rate := 10 * time.Second + cluster.DnsRefreshRate = &rate + cluster.DnsLookupFamily = envoy.Cluster_V4_ONLY + + discoveryType := envoy.Cluster_Type{Type: envoy.Cluster_LOGICAL_DNS} + if cfg.DNSDiscoveryType == "strict_dns" { + discoveryType.Type = envoy.Cluster_STRICT_DNS + } + cluster.ClusterDiscoveryType = &discoveryType + + endpoints := make([]envoyendpoint.LbEndpoint, 0, len(opts.hostnameEndpoints)) + + for _, e := range opts.hostnameEndpoints { + endpoints = append(endpoints, makeLbEndpoint(e, opts.isRemote, opts.onlyPassing)) + } + cluster.LoadAssignment = &envoy.ClusterLoadAssignment{ + ClusterName: cluster.Name, + Endpoints: []envoyendpoint.LocalityLbEndpoints{ + { + LbEndpoints: endpoints, + }, + }, + } + return cluster } // injectTerminatingGatewayTLSContext adds an UpstreamTlsContext to a cluster for TLS origination @@ -621,3 +682,27 @@ func makeThresholdsIfNeeded(limits UpstreamLimits) []*envoycluster.CircuitBreake return []*envoycluster.CircuitBreakers_Thresholds{threshold} } + +func makeLbEndpoint(csn structs.CheckServiceNode, isRemote, onlyPassing bool) envoyendpoint.LbEndpoint { + health, weight := calculateEndpointHealthAndWeight(csn, onlyPassing) + addr, port := csn.BestAddress(isRemote) + + return envoyendpoint.LbEndpoint{ + HostIdentifier: &envoyendpoint.LbEndpoint_Endpoint{ + Endpoint: &envoyendpoint.Endpoint{ + Address: &envoycore.Address{ + Address: &envoycore.Address_SocketAddress{ + SocketAddress: &envoycore.SocketAddress{ + Address: addr, + PortSpecifier: &envoycore.SocketAddress_PortValue{ + PortValue: uint32(port), + }, + }, + }, + }, + }, + }, + HealthStatus: health, + LoadBalancingWeight: makeUint32Value(weight), + } +} diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 09b6f3762..53c5b4bcc 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -449,6 +449,23 @@ func TestClustersFromSnapshot(t *testing.T) { } }, }, + { + name: "terminating-gateway-hostname-service-subsets", + create: proxycfg.TestConfigSnapshotTerminatingGateway, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceID]*structs.ServiceResolverConfigEntry{ + structs.NewServiceID("api", nil): { + Kind: structs.ServiceResolver, + Name: "api", + Subsets: map[string]structs.ServiceResolverSubset{ + "alt": { + Filter: "Service.Meta.domain == alt", + }, + }, + }, + } + }, + }, { name: "terminating-gateway-ignore-extra-resolvers", create: proxycfg.TestConfigSnapshotTerminatingGateway, diff --git a/agent/xds/config.go b/agent/xds/config.go index d2a54e443..514ebd787 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -88,6 +88,10 @@ type GatewayConfig struct { // gateway service NoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind" alias:"envoy_mesh_gateway_no_default_bind"` + // DNSDiscoveryType indicates the DNS service discovery type. + // See: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/service_discovery#arch-overview-service-discovery-types + DNSDiscoveryType string `mapstructure:"envoy_dns_discovery_type"` + // ConnectTimeoutMs is the number of milliseconds to timeout making a new // connection to this upstream. Defaults to 5000 (5 seconds) if not set. ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"` @@ -113,6 +117,9 @@ func ParseGatewayConfig(m map[string]interface{}) (GatewayConfig, error) { if cfg.ConnectTimeoutMs < 1 { cfg.ConnectTimeoutMs = 5000 } + + cfg.DNSDiscoveryType = strings.ToLower(cfg.DNSDiscoveryType) + return cfg, err } diff --git a/agent/xds/config_test.go b/agent/xds/config_test.go index 23629089e..cc007d6da 100644 --- a/agent/xds/config_test.go +++ b/agent/xds/config_test.go @@ -308,6 +308,7 @@ func TestParseGatewayConfig(t *testing.T) { "envoy_gateway_bind_tagged_addresses": true, "envoy_gateway_bind_addresses": map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, "envoy_gateway_no_default_bind": true, + "envoy_dns_discovery_type": "StRiCt_DnS", "connect_timeout_ms": 10, }, want: GatewayConfig{ @@ -315,6 +316,7 @@ func TestParseGatewayConfig(t *testing.T) { BindTaggedAddresses: true, NoDefaultBind: true, BindAddresses: map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + DNSDiscoveryType: "strict_dns", }, }, { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index b492fde49..ea39f83a8 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -3,7 +3,6 @@ package xds import ( "errors" "fmt" - envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoycore "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" @@ -119,9 +118,12 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh // generate the endpoints for the gateways in the remote datacenters for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { - continue // skip local + // Skip creating endpoints for mesh gateways in local DC and gateways in remote DCs with a hostname as their address + // EDS cannot resolve hostnames so we provide them through CDS instead + if dc == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[dc]) > 0 { + continue } + endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[dc] if !ok { endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[dc] @@ -158,9 +160,8 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh } } + // generate endpoints for our servers if WAN federation is enabled if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - // generate endpoints for our servers - var allServersLbEndpoints []envoyendpoint.LbEndpoint for _, srv := range cfgSnap.MeshGateway.ConsulServers { @@ -217,6 +218,12 @@ func (s *Server) endpointsFromServicesAndResolvers( // generate the endpoints for the linked service groups for svc, endpoints := range services { + // Skip creating endpoints for services that have hostnames as addresses + // EDS cannot resolve hostnames so we provide them through CDS instead + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway && len(cfgSnap.TerminatingGateway.HostnameServices[svc]) > 0 { + continue + } + clusterEndpoints := make(map[string][]loadAssignmentEndpointGroup) clusterEndpoints[UnnamedSubset] = []loadAssignmentEndpointGroup{{Endpoints: endpoints, OnlyPassing: false}} diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 9b262c4be..f8ed331a7 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -461,10 +461,10 @@ func TestListenersFromSnapshot(t *testing.T) { }) // For terminating gateways we create filter chain matches for services/subsets from the ServiceGroups map - if snap.Kind == structs.ServiceKindTerminatingGateway { - for i := 0; i < len(listeners); i++ { - l := listeners[i].(*envoy.Listener) + for i := 0; i < len(listeners); i++ { + l := listeners[i].(*envoy.Listener) + if l.FilterChains != nil { // Sort chains by the matched name with the exception of the last one // The last chain is a fallback and does not have a FilterChainMatch sort.Slice(l.FilterChains[:len(l.FilterChains)-1], func(i, j int) bool { diff --git a/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden b/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden index aa8d42611..95e66aad5 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden b/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden index aa8d42611..95e66aad5 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden b/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden index 7419ce28b..7941d1510 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden b/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden index f6d5f1f08..a2b1ade8e 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway.golden b/agent/xds/testdata/clusters/mesh-gateway.golden index f6d5f1f08..a2b1ade8e 100644 --- a/agent/xds/testdata/clusters/mesh-gateway.golden +++ b/agent/xds/testdata/clusters/mesh-gateway.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden b/agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden new file mode 100644 index 000000000..1ff54b0be --- /dev/null +++ b/agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden @@ -0,0 +1,155 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "alt.api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "alt.api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "filename": "api.cert.pem" + }, + "privateKey": { + "filename": "api.key.pem" + } + } + ], + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "filename": "api.cert.pem" + }, + "privateKey": { + "filename": "api.key.pem" + } + } + ], + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + }, + "outlierDetection": { + + } + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden b/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden index 7271b0a06..e9261c21c 100644 --- a/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden +++ b/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden @@ -4,15 +4,41 @@ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - } - } - }, + "type": "LOGICAL_DNS", "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, "tlsContext": { "commonTlsContext": { "tlsParams": { @@ -35,6 +61,8 @@ } } }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", "outlierDetection": { } diff --git a/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden b/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden index 7271b0a06..e9261c21c 100644 --- a/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden +++ b/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden @@ -4,15 +4,41 @@ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - } - } - }, + "type": "LOGICAL_DNS", "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, "tlsContext": { "commonTlsContext": { "tlsParams": { @@ -35,6 +61,8 @@ } } }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", "outlierDetection": { } diff --git a/agent/xds/testdata/clusters/terminating-gateway.golden b/agent/xds/testdata/clusters/terminating-gateway.golden index 1e911e485..a4c473305 100644 --- a/agent/xds/testdata/clusters/terminating-gateway.golden +++ b/agent/xds/testdata/clusters/terminating-gateway.golden @@ -4,15 +4,41 @@ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - } - } - }, + "type": "LOGICAL_DNS", "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, "tlsContext": { "commonTlsContext": { "tlsParams": { @@ -35,6 +61,8 @@ } } }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", "outlierDetection": { } diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden b/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden index 20066655e..9e0033c9b 100644 --- a/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden +++ b/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden @@ -38,4 +38,4 @@ ], "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "nonce": "00000001" -} +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden b/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden index 85b000afb..e24d69d09 100644 --- a/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden +++ b/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden @@ -1,40 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden b/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden index da0624796..fa696684e 100644 --- a/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden +++ b/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden @@ -1,40 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/endpoints/terminating-gateway.golden b/agent/xds/testdata/endpoints/terminating-gateway.golden index 187a772f7..fb5379117 100644 --- a/agent/xds/testdata/endpoints/terminating-gateway.golden +++ b/agent/xds/testdata/endpoints/terminating-gateway.golden @@ -1,40 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "clusterName": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden b/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden index 0ef43158b..9ed414831 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_bar_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -74,6 +90,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_baz_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -121,6 +153,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_default_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -168,6 +216,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_foo_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden b/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden index 7ee019966..21ec907d5 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_lan_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -74,6 +90,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_wan_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden b/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden index 85b6ad1f4..157f996a5 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_default_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/agent/xds/testdata/listeners/mesh-gateway.golden b/agent/xds/testdata/listeners/mesh-gateway.golden index 85b6ad1f4..157f996a5 100644 --- a/agent/xds/testdata/listeners/mesh-gateway.golden +++ b/agent/xds/testdata/listeners/mesh-gateway.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_default_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh b/test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh new file mode 100644 index 000000000..2ef0c41a2 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 terminating-gateway primary || true +snapshot_envoy_admin localhost:19000 s1 primary || true diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl new file mode 100644 index 000000000..103049da1 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl @@ -0,0 +1,14 @@ +enable_central_service_config = true + +config_entries { + bootstrap { + kind = "terminating-gateway" + name = "terminating-gateway" + + services = [ + { + name = "s4" + } + ] + } +} diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl new file mode 100644 index 000000000..0958221ed --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl @@ -0,0 +1,5 @@ +services { + name = "terminating-gateway" + kind = "terminating-gateway" + port = 8443 +} diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl new file mode 100644 index 000000000..f95092cd0 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl @@ -0,0 +1,16 @@ +services { + name = "s1" + port = 8080 + connect { + sidecar_service { + proxy { + upstreams = [ + { + destination_name = "s4" + local_bind_port = 5000 + } + ] + } + } + } +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl new file mode 100644 index 000000000..0cabbc930 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl @@ -0,0 +1,7 @@ +services { + name = "s4" + + // EDS cannot resolve localhost to an IP address + address = "localhost" + port = 8382 +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh b/test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh new file mode 100644 index 000000000..94c79d2fb --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +# wait for bootstrap to apply config entries +wait_for_config_entry terminating-gateway terminating-gateway + +gen_envoy_bootstrap terminating-gateway 20000 primary true +gen_envoy_bootstrap s1 19000 diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh b/test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh new file mode 100644 index 000000000..ca739c9f5 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# There is no sidecar proxy for s2, since the terminating gateway acts as the proxy +export REQUIRED_SERVICES="s1 s1-sidecar-proxy s4 terminating-gateway-primary" diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats b/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats new file mode 100644 index 000000000..6b4fba66b --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats + +load helpers + +@test "terminating proxy admin is up on :20000" { + retry_default curl -f -s localhost:20000/stats -o /dev/null +} + +@test "s1 proxy admin is up on :19000" { + retry_default curl -f -s localhost:19000/stats -o /dev/null +} + +@test "terminating-gateway-primary listener is up on :8443" { + retry_default nc -z localhost:8443 +} + +@test "terminating-gateway should have healthy endpoints for s4" { + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1 +} + +@test "s1 upstream should have healthy endpoints for s4" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s4.default.primary HEALTHY 1 +} + +@test "s1 upstream should be able to connect to s4" { + run retry_default curl -s -f -d hello localhost:5000 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "terminating-gateway is used for the upstream connection" { + assert_envoy_metric_at_least 127.0.0.1:20000 "s4.default.primary.*cx_total" 1 +} diff --git a/test/integration/connect/envoy/docker-compose.yml b/test/integration/connect/envoy/docker-compose.yml index 80dfccaff..c214eea41 100644 --- a/test/integration/connect/envoy/docker-compose.yml +++ b/test/integration/connect/envoy/docker-compose.yml @@ -172,6 +172,22 @@ services: - "disabled" network_mode: service:consul-primary + s4: + depends_on: + - consul-primary + image: "fortio/fortio" + environment: + - "FORTIO_NAME=s4" + command: + - "server" + - "-http-port" + - ":8382" + - "-grpc-port" + - ":8281" + - "-redirect-port" + - "disabled" + network_mode: service:consul-primary + s1-sidecar-proxy: depends_on: - consul-primary diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index a163d261f..7d03f3fb7 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -425,7 +425,7 @@ function docker_consul { function docker_wget { local DC=$1 shift 1 - docker run -ti --rm --network container:envoy_consul-${DC}_1 alpine:3.9 wget "$@" + docker run --rm --network container:envoy_consul-${DC}_1 alpine:3.9 wget "$@" } function docker_curl { diff --git a/test/integration/connect/envoy/main_test.go b/test/integration/connect/envoy/main_test.go index 1dbd84d14..e694b20f6 100644 --- a/test/integration/connect/envoy/main_test.go +++ b/test/integration/connect/envoy/main_test.go @@ -38,6 +38,7 @@ func TestEnvoy(t *testing.T) { "case-prometheus", "case-statsd-udp", "case-stats-proxy", + "case-terminating-gateway-hostnames", "case-terminating-gateway-simple", "case-terminating-gateway-subsets", "case-terminating-gateway-without-services", diff --git a/website/pages/docs/connect/proxies/envoy.mdx b/website/pages/docs/connect/proxies/envoy.mdx index 8209ea7ea..f1d7a2153 100644 --- a/website/pages/docs/connect/proxies/envoy.mdx +++ b/website/pages/docs/connect/proxies/envoy.mdx @@ -319,6 +319,13 @@ will continue to be supported. - `envoy_gateway_no_default_bind` - Prevents binding to the default address of the gateway service. This should be used with one of the other options to configure the gateway's bind addresses. + +- `envoy_dns_discovery_type` - Determines how Envoy will resolve hostnames. Defaults to `LOGICAL_DNS`. + Must be one of `STRICT_DNS` or `LOGICAL_DNS`. Details for each type are available in + the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/v1.14.1/intro/arch_overview/upstream/service_discovery). + This option applies to terminating gateways that route to services + addressed by a hostname, such as a managed databased. It also applies to mesh gateways, + such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname. ## Advanced Configuration diff --git a/website/pages/docs/connect/terminating-gateway.mdx b/website/pages/docs/connect/terminating-gateway.mdx index 3b097caa3..bb3b1ee90 100644 --- a/website/pages/docs/connect/terminating-gateway.mdx +++ b/website/pages/docs/connect/terminating-gateway.mdx @@ -24,8 +24,7 @@ services outside the mesh, review the [terminating gateway guide](https://learn. ~> **Beta limitations:** Terminating Gateways currently do not support targeting service subsets with [L7 configuration](/docs/connect/l7-traffic-management). They route to all instances of a service with no capabilities -for filtering by instance. Terminating Gateways also currently do not support routing to services with a hostname -defined as a their address. The service address registered with Consul, that the gateway will route traffic to, **must** be a resolved IP address. +for filtering by instance. ## Security Considerations