diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index b5ec6d691..929e75a97 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -100,6 +100,10 @@ type configSnapshotTerminatingGateway struct { // between the gateway and a service. TLS configuration stored here is // used for TLS origination from the gateway to the linked service. GatewayServices map[structs.ServiceID]structs.GatewayService + + // HostnameServices is a map of service id to service instances with a hostname as the address. + // If hostnames are configured they must be provided to Envoy via CDS not EDS. + HostnameServices map[structs.ServiceID]structs.CheckServiceNodes } func (c *configSnapshotTerminatingGateway) IsEmpty() bool { @@ -113,7 +117,8 @@ func (c *configSnapshotTerminatingGateway) IsEmpty() bool { len(c.WatchedServices) == 0 && len(c.ServiceResolvers) == 0 && len(c.WatchedResolvers) == 0 && - len(c.GatewayServices) == 0 + len(c.GatewayServices) == 0 && + len(c.HostnameServices) == 0 } type configSnapshotMeshGateway struct { @@ -155,6 +160,10 @@ type configSnapshotMeshGateway struct { // ConsulServers is the list of consul servers in this datacenter. ConsulServers structs.CheckServiceNodes + + // HostnameDatacenters is a map of datacenters to mesh gateway instances with a hostname as the address. + // If hostnames are configured they must be provided to Envoy via CDS not EDS. + HostnameDatacenters map[string]structs.CheckServiceNodes } func (c *configSnapshotMeshGateway) Datacenters() []string { @@ -188,7 +197,8 @@ func (c *configSnapshotMeshGateway) IsEmpty() bool { len(c.ServiceResolvers) == 0 && len(c.GatewayGroups) == 0 && len(c.FedStateGateways) == 0 && - len(c.ConsulServers) == 0 + len(c.ConsulServers) == 0 && + len(c.HostnameDatacenters) == 0 } type configSnapshotIngressGateway struct { diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 1cd703eb5..97801a84e 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "reflect" "strings" "time" @@ -552,12 +553,14 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot { snap.TerminatingGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes) snap.TerminatingGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry) snap.TerminatingGateway.GatewayServices = make(map[structs.ServiceID]structs.GatewayService) + snap.TerminatingGateway.HostnameServices = make(map[structs.ServiceID]structs.CheckServiceNodes) case structs.ServiceKindMeshGateway: snap.MeshGateway.WatchedServices = make(map[structs.ServiceID]context.CancelFunc) snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc) snap.MeshGateway.ServiceGroups = make(map[structs.ServiceID]structs.CheckServiceNodes) snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes) snap.MeshGateway.ServiceResolvers = make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry) + snap.MeshGateway.HostnameDatacenters = make(map[string]structs.CheckServiceNodes) // there is no need to initialize the map of service resolvers as we // fully rebuild it every time we get updates case structs.ServiceKindIngressGateway: @@ -1032,6 +1035,13 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config } } + // Clean up services with hostname mapping for services that were not in the update + for sid, _ := range snap.TerminatingGateway.HostnameServices { + if _, ok := svcMap[sid]; !ok { + delete(snap.TerminatingGateway.HostnameServices, sid) + } + } + // Cancel service instance watches for services that were not in the update for sid, cancelFn := range snap.TerminatingGateway.WatchedServices { if _, ok := svcMap[sid]; !ok { @@ -1081,11 +1091,12 @@ func (s *state) handleUpdateTerminatingGateway(u cache.UpdateEvent, snap *Config } sid := structs.ServiceIDFromString(strings.TrimPrefix(u.CorrelationID, externalServiceIDPrefix)) + delete(snap.TerminatingGateway.ServiceGroups, sid) + delete(snap.TerminatingGateway.HostnameServices, sid) if len(resp.Nodes) > 0 { snap.TerminatingGateway.ServiceGroups[sid] = resp.Nodes - } else if _, ok := snap.TerminatingGateway.ServiceGroups[sid]; ok { - delete(snap.TerminatingGateway.ServiceGroups, sid) + snap.TerminatingGateway.HostnameServices[sid] = s.hostnameEndpoints(logging.TerminatingGateway, snap.Datacenter, resp.Nodes) } // Store leaf cert for watched service @@ -1141,6 +1152,17 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho return fmt.Errorf("invalid type for response: %T", u.Result) } snap.MeshGateway.FedStateGateways = dcIndexedNodes.DatacenterNodes + + for dc, nodes := range dcIndexedNodes.DatacenterNodes { + snap.MeshGateway.HostnameDatacenters[dc] = s.hostnameEndpoints(logging.MeshGateway, snap.Datacenter, nodes) + } + + for dc, _ := range snap.MeshGateway.HostnameDatacenters { + if _, ok := dcIndexedNodes.DatacenterNodes[dc]; !ok { + delete(snap.MeshGateway.HostnameDatacenters, dc) + } + } + case serviceListWatchID: services, ok := u.Result.(*structs.IndexedServiceList) if !ok { @@ -1297,11 +1319,12 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho } dc := strings.TrimPrefix(u.CorrelationID, "mesh-gateway:") + delete(snap.MeshGateway.GatewayGroups, dc) + delete(snap.MeshGateway.HostnameDatacenters, dc) if len(resp.Nodes) > 0 { snap.MeshGateway.GatewayGroups[dc] = resp.Nodes - } else if _, ok := snap.MeshGateway.GatewayGroups[dc]; ok { - delete(snap.MeshGateway.GatewayGroups, dc) + snap.MeshGateway.HostnameDatacenters[dc] = s.hostnameEndpoints(logging.MeshGateway, snap.Datacenter, resp.Nodes) } default: // do nothing for now @@ -1518,3 +1541,35 @@ func (s *state) Changed(ns *structs.NodeService, token string) bool { !reflect.DeepEqual(s.proxyCfg, proxyCfg) || s.token != token } + +// hostnameEndpoints returns all CheckServiceNodes that have hostnames instead of IPs as the address. +// Envoy cannot resolve hostnames provided through EDS, so we exclusively use CDS for these clusters. +// If there is a mix of hostnames and addresses we exclusively use the hostnames, since clusters cannot discover +// services with both EDS and DNS. +func (s *state) hostnameEndpoints(loggerName string, localDC string, nodes structs.CheckServiceNodes) structs.CheckServiceNodes { + var ( + hasIP bool + hasHostname bool + resp structs.CheckServiceNodes + ) + + for _, n := range nodes { + addr, _ := n.BestAddress(localDC != n.Node.Datacenter) + if net.ParseIP(addr) != nil { + hasIP = true + continue + } + hasHostname = true + resp = append(resp, n) + } + + if hasHostname && hasIP { + dc := nodes[0].Node.Datacenter + sid := nodes[0].Service.CompoundServiceName() + + s.logger.Named(loggerName). + Warn("service contains instances with mix of hostnames and IP addresses; only hostnames will be passed to Envoy.", + "dc", dc, "service", sid.String()) + } + return resp +} diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 1d741b6e5..9067a6f69 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -599,6 +599,9 @@ func TestState_WatchesAndUpdates(t *testing.T) { db := structs.NewServiceID("db", nil) dbStr := db.String() + api := structs.NewServiceID("api", nil) + apiStr := api.String() + cases := map[string]testCase{ "initial-gateway": testCase{ ns: structs.NodeService{ @@ -714,6 +717,96 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.True(t, snap.MeshGateway.WatchedServicesSet) }, }, + verificationStage{ + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: "mesh-gateway:dc4", + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestGatewayNodesDC4Hostname(t), + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "gateway with service list is valid") + require.Len(t, snap.MeshGateway.WatchedServices, 2) + require.True(t, snap.MeshGateway.WatchedServicesSet) + + expect := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + } + require.Equal(t, snap.MeshGateway.HostnameDatacenters["dc4"], expect) + }, + }, + verificationStage{ + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: federationStateListGatewaysWatchID, + Result: &structs.DatacenterIndexedCheckServiceNodes{ + DatacenterNodes: map[string]structs.CheckServiceNodes{ + "dc5": TestGatewayNodesDC5Hostname(t), + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "gateway with service list is valid") + require.Len(t, snap.MeshGateway.WatchedServices, 2) + require.True(t, snap.MeshGateway.WatchedServicesSet) + + expect := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + } + require.Equal(t, snap.MeshGateway.HostnameDatacenters["dc5"], expect) + }, + }, }, }, "ingress-gateway": testCase{ @@ -1039,6 +1132,10 @@ func TestState_WatchesAndUpdates(t *testing.T) { Service: structs.NewServiceID("billing", nil), Gateway: structs.NewServiceID("terminating-gateway", nil), }, + { + Service: structs.NewServiceID("api", nil), + Gateway: structs.NewServiceID("terminating-gateway", nil), + }, }, }, Err: nil, @@ -1047,27 +1144,33 @@ func TestState_WatchesAndUpdates(t *testing.T) { verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { db := structs.NewServiceID("db", nil) billing := structs.NewServiceID("billing", nil) + api := structs.NewServiceID("api", nil) require.True(t, snap.Valid(), "gateway with service list is valid") - require.Len(t, snap.TerminatingGateway.WatchedServices, 2) + require.Len(t, snap.TerminatingGateway.WatchedServices, 3) require.Contains(t, snap.TerminatingGateway.WatchedServices, db) require.Contains(t, snap.TerminatingGateway.WatchedServices, billing) + require.Contains(t, snap.TerminatingGateway.WatchedServices, api) - require.Len(t, snap.TerminatingGateway.WatchedIntentions, 2) + require.Len(t, snap.TerminatingGateway.WatchedIntentions, 3) require.Contains(t, snap.TerminatingGateway.WatchedIntentions, db) require.Contains(t, snap.TerminatingGateway.WatchedIntentions, billing) + require.Contains(t, snap.TerminatingGateway.WatchedIntentions, api) - require.Len(t, snap.TerminatingGateway.WatchedLeaves, 2) + require.Len(t, snap.TerminatingGateway.WatchedLeaves, 3) require.Contains(t, snap.TerminatingGateway.WatchedLeaves, db) require.Contains(t, snap.TerminatingGateway.WatchedLeaves, billing) + require.Contains(t, snap.TerminatingGateway.WatchedLeaves, api) - require.Len(t, snap.TerminatingGateway.WatchedResolvers, 2) + require.Len(t, snap.TerminatingGateway.WatchedResolvers, 3) require.Contains(t, snap.TerminatingGateway.WatchedResolvers, db) require.Contains(t, snap.TerminatingGateway.WatchedResolvers, billing) + require.Contains(t, snap.TerminatingGateway.WatchedResolvers, api) - require.Len(t, snap.TerminatingGateway.GatewayServices, 2) + require.Len(t, snap.TerminatingGateway.GatewayServices, 3) require.Contains(t, snap.TerminatingGateway.GatewayServices, db) require.Contains(t, snap.TerminatingGateway.GatewayServices, billing) + require.Contains(t, snap.TerminatingGateway.GatewayServices, api) }, }, verificationStage{ @@ -1112,6 +1215,97 @@ func TestState_WatchesAndUpdates(t *testing.T) { ) }, }, + verificationStage{ + requiredWatches: map[string]verifyWatchRequest{ + "external-service:" + apiStr: genVerifyServiceWatch("api", "", "dc1", false), + }, + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: "external-service:" + apiStr, + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "10.0.1.1", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.mydomain", + }, + }, + { + Node: &structs.Node{ + Node: "node2", + Address: "10.0.1.2", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.altdomain", + }, + }, + { + Node: &structs.Node{ + Node: "node3", + Address: "10.0.1.3", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "10.0.1.3", + }, + }, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.Len(t, snap.TerminatingGateway.ServiceGroups, 2) + expect := structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "10.0.1.1", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.mydomain", + }, + }, + { + Node: &structs.Node{ + Node: "node2", + Address: "10.0.1.2", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "api.altdomain", + }, + }, + { + Node: &structs.Node{ + Node: "node3", + Address: "10.0.1.3", + }, + Service: &structs.NodeService{ + ID: "api", + Service: "api", + Address: "10.0.1.3", + }, + }, + } + sid := structs.NewServiceID("api", nil) + require.Equal(t, snap.TerminatingGateway.ServiceGroups[sid], expect) + + // The instance in node3 should not be present in HostnameDatacenters because it has a valid IP + require.ElementsMatch(t, snap.TerminatingGateway.HostnameServices[sid], expect[:2]) + }, + }, verificationStage{ requiredWatches: map[string]verifyWatchRequest{ "service-leaf:" + dbStr: genVerifyLeafWatch("db", "dc1"), @@ -1198,10 +1392,11 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Len(t, snap.TerminatingGateway.GatewayServices, 1) require.Contains(t, snap.TerminatingGateway.GatewayServices, billing) - // There was no update event for billing's leaf/endpoints, so length is 0 + // There was no update event for billing's leaf/endpoints/resolvers, so length is 0 require.Len(t, snap.TerminatingGateway.ServiceGroups, 0) require.Len(t, snap.TerminatingGateway.ServiceLeaves, 0) require.Len(t, snap.TerminatingGateway.ServiceResolvers, 0) + require.Len(t, snap.TerminatingGateway.HostnameServices, 0) }, }, }, diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 00c530f56..8dfc6a746 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -374,6 +374,88 @@ func TestGatewayNodesDC3(t testing.T) structs.CheckServiceNodes { } } +func TestGatewayNodesDC4Hostname(t testing.T) structs.CheckServiceNodes { + return structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-3", + Node: "mesh-gateway", + Address: "10.30.1.3", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.3", 8443, + structs.ServiceAddress{Address: "10.30.1.3", Port: 8443}, + structs.ServiceAddress{Address: "198.38.1.1", Port: 443}), + }, + } +} + +func TestGatewayNodesDC5Hostname(t testing.T) structs.CheckServiceNodes { + return structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-3", + Node: "mesh-gateway", + Address: "10.30.1.3", + Datacenter: "dc5", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.3", 8443, + structs.ServiceAddress{Address: "10.30.1.3", Port: 8443}, + structs.ServiceAddress{Address: "198.38.1.1", Port: 443}), + }, + } +} + func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes { return structs.CheckServiceNodes{ structs.CheckServiceNode{ @@ -1305,11 +1387,41 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera }, GatewayGroups: map[string]structs.CheckServiceNodes{ "dc2": TestGatewayNodesDC2(t), + "dc4": TestGatewayNodesDC4Hostname(t), + }, + HostnameDatacenters: map[string]structs.CheckServiceNodes{ + "dc4": { + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-1", + Node: "mesh-gateway", + Address: "10.30.1.1", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.1", 8443, + structs.ServiceAddress{Address: "10.0.1.1", Port: 8443}, + structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}), + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "mesh-gateway-2", + Node: "mesh-gateway", + Address: "10.30.1.2", + Datacenter: "dc4", + }, + Service: structs.TestNodeServiceMeshGatewayWithAddrs(t, + "10.30.1.2", 8443, + structs.ServiceAddress{Address: "10.30.1.2", Port: 8443}, + structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}), + }, + }, }, } if useFederationStates { snap.MeshGateway.FedStateGateways = map[string]structs.CheckServiceNodes{ "dc2": TestGatewayNodesDC2(t), + "dc4": TestGatewayNodesDC4Hostname(t), } delete(snap.MeshGateway.GatewayGroups, "dc2") @@ -1505,10 +1617,59 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C } api := structs.NewServiceID("api", nil) - apiNodes := TestUpstreamNodes(t) - for i := 0; i < len(apiNodes); i++ { - apiNodes[i].Service.Service = "api" - apiNodes[i].Service.Port = 8081 + apiNodes := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "api", + Node: "test1", + Address: "10.10.1.1", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "api", + Address: "api.mydomain", + Port: 8081, + }, + Checks: structs.HealthChecks{ + { + Status: "critical", + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "test2", + Node: "test2", + Address: "10.10.1.2", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "api", + Address: "api.altdomain", + Port: 8081, + Meta: map[string]string{ + "domain": "alt", + }, + }, + Checks: structs.HealthChecks{ + { + Status: "passing", + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "test3", + Node: "test3", + Address: "10.10.1.3", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "api", + Address: "10.10.1.3", + Port: 8081, + }, + }, } snap.TerminatingGateway = configSnapshotTerminatingGateway{ @@ -1528,6 +1689,9 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C KeyFile: "api.key.pem", }, }, + HostnameServices: map[structs.ServiceID]structs.CheckServiceNodes{ + api: {apiNodes[0], apiNodes[1]}, + }, } snap.TerminatingGateway.ServiceLeaves = map[structs.ServiceID]*structs.IssuedCert{ structs.NewServiceID("web", nil): { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index f74b6e5dd..99c15a34a 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -133,35 +133,41 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho if dc == cfgSnap.Datacenter { continue // skip local } - clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain) - cluster, err := s.makeGatewayCluster(cfgSnap, clusterName) - if err != nil { - return nil, err + opts := gatewayClusterOpts{ + name: connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain), + hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[dc], + isRemote: dc != cfgSnap.Datacenter, } + cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) } if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { // Add all of the remote wildcard datacenter mappings for servers. for _, dc := range datacenters { - clusterName := cfgSnap.ServerSNIFn(dc, "") + hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[dc] - cluster, err := s.makeGatewayCluster(cfgSnap, clusterName) - if err != nil { - return nil, err + // If the DC is our current DC then this cluster is for traffic from a remote DC to a local server. + // HostnameDatacenters is populated with gateway addresses, so it does not apply here. + if dc == cfgSnap.Datacenter { + hostnameEndpoints = nil } + opts := gatewayClusterOpts{ + name: cfgSnap.ServerSNIFn(dc, ""), + hostnameEndpoints: hostnameEndpoints, + isRemote: dc != cfgSnap.Datacenter, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) } // And for the current datacenter, send all flavors appropriately. for _, srv := range cfgSnap.MeshGateway.ConsulServers { - clusterName := cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node) - - cluster, err := s.makeGatewayCluster(cfgSnap, clusterName) - if err != nil { - return nil, err + opts := gatewayClusterOpts{ + name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node), } + cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) } } @@ -179,6 +185,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var services map[structs.ServiceID]structs.CheckServiceNodes var resolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry + var hostnameEndpoints structs.CheckServiceNodes switch cfgSnap.Kind { case structs.ServiceKindTerminatingGateway: @@ -197,33 +204,44 @@ func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([ clusterName := connect.ServiceSNI(svc.ID, "", svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) resolver, hasResolver := resolvers[svc] - // Create the cluster for default/unnamed services - var cluster *envoy.Cluster - var err error - if !hasResolver { // Use a zero value resolver with no timeout and no subsets resolver = &structs.ServiceResolverConfigEntry{} } - cluster, err = s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout) - if err != nil { - return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err) + + // When making service clusters we only pass endpoints with hostnames if the kind is a terminating gateway + // This is because the services a mesh gateway will route to are not external services and are not addressed by a hostname. + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway { + hostnameEndpoints = cfgSnap.TerminatingGateway.HostnameServices[svc] } + opts := gatewayClusterOpts{ + name: clusterName, + hostnameEndpoints: hostnameEndpoints, + connectTimeout: resolver.ConnectTimeout, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway { injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc) } clusters = append(clusters, cluster) // If there is a service-resolver for this service then also setup a cluster for each subset - for subsetName := range resolver.Subsets { - clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - - cluster, err := s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, resolver.ConnectTimeout) + for name, subset := range resolver.Subsets { + subsetHostnameEndpoints, err := s.filterSubsetEndpoints(&subset, hostnameEndpoints) if err != nil { - return nil, fmt.Errorf("failed to make %s cluster: %v", cfgSnap.Kind, err) + return nil, err } + opts := gatewayClusterOpts{ + name: connect.ServiceSNI(svc.ID, name, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain), + hostnameEndpoints: subsetHostnameEndpoints, + onlyPassing: subset.OnlyPassing, + connectTimeout: resolver.ConnectTimeout, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway { injectTerminatingGatewayTLSContext(cfgSnap, cluster, svc) } @@ -547,43 +565,86 @@ func makeClusterFromUserConfig(configJSON string) (*envoy.Cluster, error) { return &c, err } -func (s *Server) makeGatewayCluster(cfgSnap *proxycfg.ConfigSnapshot, clusterName string) (*envoy.Cluster, error) { - return s.makeGatewayClusterWithConnectTimeout(cfgSnap, clusterName, 0) +type gatewayClusterOpts struct { + // name for the cluster + name string + + // isRemote determines whether the cluster is in a remote DC and we should prefer a WAN address + isRemote bool + + // onlyPassing determines whether endpoints that do not have a passing status should be considered unhealthy + onlyPassing bool + + // connectTimeout is the timeout for new network connections to hosts in the cluster + connectTimeout time.Duration + + // hostnameEndpoints is a list of endpoints with a hostname as their address + hostnameEndpoints structs.CheckServiceNodes } -// makeGatewayClusterWithConnectTimeout initializes a gateway cluster -// with the specified connect timeout. If the timeout is 0, the connect timeout -// defaults to use the configured gateway timeout. -func (s *Server) makeGatewayClusterWithConnectTimeout(cfgSnap *proxycfg.ConfigSnapshot, - clusterName string, connectTimeout time.Duration) (*envoy.Cluster, error) { - - cfg, err := ParseGatewayConfig(cfgSnap.Proxy.Config) +func (s *Server) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts gatewayClusterOpts) *envoy.Cluster { + cfg, err := ParseGatewayConfig(snap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. s.Logger.Warn("failed to parse gateway config", "error", err) } - - if connectTimeout <= 0 { - connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond + if opts.connectTimeout <= 0 { + opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond } - cluster := envoy.Cluster{ - Name: clusterName, - ConnectTimeout: connectTimeout, - ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_EDS}, - EdsClusterConfig: &envoy.Cluster_EdsClusterConfig{ + cluster := &envoy.Cluster{ + Name: opts.name, + ConnectTimeout: opts.connectTimeout, + + // Having an empty config enables outlier detection with default config. + OutlierDetection: &envoycluster.OutlierDetection{}, + } + + useEDS := true + if len(opts.hostnameEndpoints) > 0 { + useEDS = false + } + + // If none of the service instances are addressed by a hostname we provide the endpoint IP addresses via EDS + if useEDS { + cluster.ClusterDiscoveryType = &envoy.Cluster_Type{Type: envoy.Cluster_EDS} + cluster.EdsClusterConfig = &envoy.Cluster_EdsClusterConfig{ EdsConfig: &envoycore.ConfigSource{ ConfigSourceSpecifier: &envoycore.ConfigSource_Ads{ Ads: &envoycore.AggregatedConfigSource{}, }, }, - }, - // Having an empty config enables outlier detection with default config. - OutlierDetection: &envoycluster.OutlierDetection{}, + } + return cluster } - return &cluster, nil + // When a service instance is addressed by a hostname we have Envoy do the DNS resolution + // by setting a DNS cluster type and passing the hostname endpoints via CDS. + rate := 10 * time.Second + cluster.DnsRefreshRate = &rate + cluster.DnsLookupFamily = envoy.Cluster_V4_ONLY + + discoveryType := envoy.Cluster_Type{Type: envoy.Cluster_LOGICAL_DNS} + if cfg.DNSDiscoveryType == "strict_dns" { + discoveryType.Type = envoy.Cluster_STRICT_DNS + } + cluster.ClusterDiscoveryType = &discoveryType + + endpoints := make([]envoyendpoint.LbEndpoint, 0, len(opts.hostnameEndpoints)) + + for _, e := range opts.hostnameEndpoints { + endpoints = append(endpoints, makeLbEndpoint(e, opts.isRemote, opts.onlyPassing)) + } + cluster.LoadAssignment = &envoy.ClusterLoadAssignment{ + ClusterName: cluster.Name, + Endpoints: []envoyendpoint.LocalityLbEndpoints{ + { + LbEndpoints: endpoints, + }, + }, + } + return cluster } // injectTerminatingGatewayTLSContext adds an UpstreamTlsContext to a cluster for TLS origination @@ -621,3 +682,27 @@ func makeThresholdsIfNeeded(limits UpstreamLimits) []*envoycluster.CircuitBreake return []*envoycluster.CircuitBreakers_Thresholds{threshold} } + +func makeLbEndpoint(csn structs.CheckServiceNode, isRemote, onlyPassing bool) envoyendpoint.LbEndpoint { + health, weight := calculateEndpointHealthAndWeight(csn, onlyPassing) + addr, port := csn.BestAddress(isRemote) + + return envoyendpoint.LbEndpoint{ + HostIdentifier: &envoyendpoint.LbEndpoint_Endpoint{ + Endpoint: &envoyendpoint.Endpoint{ + Address: &envoycore.Address{ + Address: &envoycore.Address_SocketAddress{ + SocketAddress: &envoycore.SocketAddress{ + Address: addr, + PortSpecifier: &envoycore.SocketAddress_PortValue{ + PortValue: uint32(port), + }, + }, + }, + }, + }, + }, + HealthStatus: health, + LoadBalancingWeight: makeUint32Value(weight), + } +} diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 09b6f3762..53c5b4bcc 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -449,6 +449,23 @@ func TestClustersFromSnapshot(t *testing.T) { } }, }, + { + name: "terminating-gateway-hostname-service-subsets", + create: proxycfg.TestConfigSnapshotTerminatingGateway, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceID]*structs.ServiceResolverConfigEntry{ + structs.NewServiceID("api", nil): { + Kind: structs.ServiceResolver, + Name: "api", + Subsets: map[string]structs.ServiceResolverSubset{ + "alt": { + Filter: "Service.Meta.domain == alt", + }, + }, + }, + } + }, + }, { name: "terminating-gateway-ignore-extra-resolvers", create: proxycfg.TestConfigSnapshotTerminatingGateway, diff --git a/agent/xds/config.go b/agent/xds/config.go index d2a54e443..514ebd787 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -88,6 +88,10 @@ type GatewayConfig struct { // gateway service NoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind" alias:"envoy_mesh_gateway_no_default_bind"` + // DNSDiscoveryType indicates the DNS service discovery type. + // See: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/service_discovery#arch-overview-service-discovery-types + DNSDiscoveryType string `mapstructure:"envoy_dns_discovery_type"` + // ConnectTimeoutMs is the number of milliseconds to timeout making a new // connection to this upstream. Defaults to 5000 (5 seconds) if not set. ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"` @@ -113,6 +117,9 @@ func ParseGatewayConfig(m map[string]interface{}) (GatewayConfig, error) { if cfg.ConnectTimeoutMs < 1 { cfg.ConnectTimeoutMs = 5000 } + + cfg.DNSDiscoveryType = strings.ToLower(cfg.DNSDiscoveryType) + return cfg, err } diff --git a/agent/xds/config_test.go b/agent/xds/config_test.go index 23629089e..cc007d6da 100644 --- a/agent/xds/config_test.go +++ b/agent/xds/config_test.go @@ -308,6 +308,7 @@ func TestParseGatewayConfig(t *testing.T) { "envoy_gateway_bind_tagged_addresses": true, "envoy_gateway_bind_addresses": map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, "envoy_gateway_no_default_bind": true, + "envoy_dns_discovery_type": "StRiCt_DnS", "connect_timeout_ms": 10, }, want: GatewayConfig{ @@ -315,6 +316,7 @@ func TestParseGatewayConfig(t *testing.T) { BindTaggedAddresses: true, NoDefaultBind: true, BindAddresses: map[string]structs.ServiceAddress{"foo": {Address: "127.0.0.1", Port: 80}}, + DNSDiscoveryType: "strict_dns", }, }, { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index b492fde49..ea39f83a8 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -3,7 +3,6 @@ package xds import ( "errors" "fmt" - envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoycore "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" @@ -119,9 +118,12 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh // generate the endpoints for the gateways in the remote datacenters for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { - continue // skip local + // Skip creating endpoints for mesh gateways in local DC and gateways in remote DCs with a hostname as their address + // EDS cannot resolve hostnames so we provide them through CDS instead + if dc == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[dc]) > 0 { + continue } + endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[dc] if !ok { endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[dc] @@ -158,9 +160,8 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh } } + // generate endpoints for our servers if WAN federation is enabled if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - // generate endpoints for our servers - var allServersLbEndpoints []envoyendpoint.LbEndpoint for _, srv := range cfgSnap.MeshGateway.ConsulServers { @@ -217,6 +218,12 @@ func (s *Server) endpointsFromServicesAndResolvers( // generate the endpoints for the linked service groups for svc, endpoints := range services { + // Skip creating endpoints for services that have hostnames as addresses + // EDS cannot resolve hostnames so we provide them through CDS instead + if cfgSnap.Kind == structs.ServiceKindTerminatingGateway && len(cfgSnap.TerminatingGateway.HostnameServices[svc]) > 0 { + continue + } + clusterEndpoints := make(map[string][]loadAssignmentEndpointGroup) clusterEndpoints[UnnamedSubset] = []loadAssignmentEndpointGroup{{Endpoints: endpoints, OnlyPassing: false}} diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 9b262c4be..f8ed331a7 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -461,10 +461,10 @@ func TestListenersFromSnapshot(t *testing.T) { }) // For terminating gateways we create filter chain matches for services/subsets from the ServiceGroups map - if snap.Kind == structs.ServiceKindTerminatingGateway { - for i := 0; i < len(listeners); i++ { - l := listeners[i].(*envoy.Listener) + for i := 0; i < len(listeners); i++ { + l := listeners[i].(*envoy.Listener) + if l.FilterChains != nil { // Sort chains by the matched name with the exception of the last one // The last chain is a fallback and does not have a FilterChainMatch sort.Slice(l.FilterChains[:len(l.FilterChains)-1], func(i, j int) bool { diff --git a/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden b/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden index aa8d42611..95e66aad5 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden b/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden index aa8d42611..95e66aad5 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden b/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden index 7419ce28b..7941d1510 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden b/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden index f6d5f1f08..a2b1ade8e 100644 --- a/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden +++ b/agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/mesh-gateway.golden b/agent/xds/testdata/clusters/mesh-gateway.golden index f6d5f1f08..a2b1ade8e 100644 --- a/agent/xds/testdata/clusters/mesh-gateway.golden +++ b/agent/xds/testdata/clusters/mesh-gateway.golden @@ -33,6 +33,50 @@ } }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "456.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden b/agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden new file mode 100644 index 000000000..1ff54b0be --- /dev/null +++ b/agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden @@ -0,0 +1,155 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "alt.api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "alt.api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "filename": "api.cert.pem" + }, + "privateKey": { + "filename": "api.key.pem" + } + } + ], + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "filename": "api.cert.pem" + }, + "privateKey": { + "filename": "api.key.pem" + } + } + ], + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "tlsContext": { + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + }, + "outlierDetection": { + + } + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden b/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden index 7271b0a06..e9261c21c 100644 --- a/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden +++ b/agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden @@ -4,15 +4,41 @@ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - } - } - }, + "type": "LOGICAL_DNS", "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, "tlsContext": { "commonTlsContext": { "tlsParams": { @@ -35,6 +61,8 @@ } } }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", "outlierDetection": { } diff --git a/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden b/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden index 7271b0a06..e9261c21c 100644 --- a/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden +++ b/agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden @@ -4,15 +4,41 @@ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - } - } - }, + "type": "LOGICAL_DNS", "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, "tlsContext": { "commonTlsContext": { "tlsParams": { @@ -35,6 +61,8 @@ } } }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", "outlierDetection": { } diff --git a/agent/xds/testdata/clusters/terminating-gateway.golden b/agent/xds/testdata/clusters/terminating-gateway.golden index 1e911e485..a4c473305 100644 --- a/agent/xds/testdata/clusters/terminating-gateway.golden +++ b/agent/xds/testdata/clusters/terminating-gateway.golden @@ -4,15 +4,41 @@ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", "name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - } - } - }, + "type": "LOGICAL_DNS", "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.mydomain", + "portValue": 8081 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.altdomain", + "portValue": 8081 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, "tlsContext": { "commonTlsContext": { "tlsParams": { @@ -35,6 +61,8 @@ } } }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", "outlierDetection": { } diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden b/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden index 20066655e..9e0033c9b 100644 --- a/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden +++ b/agent/xds/testdata/endpoints/connect-proxy-with-default-chain-and-custom-cluster.golden @@ -38,4 +38,4 @@ ], "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "nonce": "00000001" -} +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden b/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden index 85b000afb..e24d69d09 100644 --- a/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden +++ b/agent/xds/testdata/endpoints/terminating-gateway-default-service-subset.golden @@ -1,40 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden b/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden index da0624796..fa696684e 100644 --- a/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden +++ b/agent/xds/testdata/endpoints/terminating-gateway-service-subsets.golden @@ -1,40 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/endpoints/terminating-gateway.golden b/agent/xds/testdata/endpoints/terminating-gateway.golden index 187a772f7..fb5379117 100644 --- a/agent/xds/testdata/endpoints/terminating-gateway.golden +++ b/agent/xds/testdata/endpoints/terminating-gateway.golden @@ -1,40 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8081 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "clusterName": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", diff --git a/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden b/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden index 0ef43158b..9ed414831 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_bar_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -74,6 +90,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_baz_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -121,6 +153,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_default_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -168,6 +216,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_foo_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden b/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden index 7ee019966..21ec907d5 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_lan_dc4_tcp" + } + } + ] + }, { "filters": [ { @@ -74,6 +90,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_wan_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden b/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden index 85b6ad1f4..157f996a5 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-using-federation-states.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_default_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/agent/xds/testdata/listeners/mesh-gateway.golden b/agent/xds/testdata/listeners/mesh-gateway.golden index 85b6ad1f4..157f996a5 100644 --- a/agent/xds/testdata/listeners/mesh-gateway.golden +++ b/agent/xds/testdata/listeners/mesh-gateway.golden @@ -27,6 +27,22 @@ } ] }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "stat_prefix": "mesh_gateway_remote_default_dc4_tcp" + } + } + ] + }, { "filters": [ { diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh b/test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh new file mode 100644 index 000000000..2ef0c41a2 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/capture.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 terminating-gateway primary || true +snapshot_envoy_admin localhost:19000 s1 primary || true diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl new file mode 100644 index 000000000..103049da1 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/config_entries.hcl @@ -0,0 +1,14 @@ +enable_central_service_config = true + +config_entries { + bootstrap { + kind = "terminating-gateway" + name = "terminating-gateway" + + services = [ + { + name = "s4" + } + ] + } +} diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl new file mode 100644 index 000000000..0958221ed --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/gateway.hcl @@ -0,0 +1,5 @@ +services { + name = "terminating-gateway" + kind = "terminating-gateway" + port = 8443 +} diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl new file mode 100644 index 000000000..f95092cd0 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s1.hcl @@ -0,0 +1,16 @@ +services { + name = "s1" + port = 8080 + connect { + sidecar_service { + proxy { + upstreams = [ + { + destination_name = "s4" + local_bind_port = 5000 + } + ] + } + } + } +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl new file mode 100644 index 000000000..0cabbc930 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/s4.hcl @@ -0,0 +1,7 @@ +services { + name = "s4" + + // EDS cannot resolve localhost to an IP address + address = "localhost" + port = 8382 +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh b/test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh new file mode 100644 index 000000000..94c79d2fb --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +# wait for bootstrap to apply config entries +wait_for_config_entry terminating-gateway terminating-gateway + +gen_envoy_bootstrap terminating-gateway 20000 primary true +gen_envoy_bootstrap s1 19000 diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh b/test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh new file mode 100644 index 000000000..ca739c9f5 --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/vars.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# There is no sidecar proxy for s2, since the terminating gateway acts as the proxy +export REQUIRED_SERVICES="s1 s1-sidecar-proxy s4 terminating-gateway-primary" diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats b/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats new file mode 100644 index 000000000..6b4fba66b --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats + +load helpers + +@test "terminating proxy admin is up on :20000" { + retry_default curl -f -s localhost:20000/stats -o /dev/null +} + +@test "s1 proxy admin is up on :19000" { + retry_default curl -f -s localhost:19000/stats -o /dev/null +} + +@test "terminating-gateway-primary listener is up on :8443" { + retry_default nc -z localhost:8443 +} + +@test "terminating-gateway should have healthy endpoints for s4" { + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1 +} + +@test "s1 upstream should have healthy endpoints for s4" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s4.default.primary HEALTHY 1 +} + +@test "s1 upstream should be able to connect to s4" { + run retry_default curl -s -f -d hello localhost:5000 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "terminating-gateway is used for the upstream connection" { + assert_envoy_metric_at_least 127.0.0.1:20000 "s4.default.primary.*cx_total" 1 +} diff --git a/test/integration/connect/envoy/docker-compose.yml b/test/integration/connect/envoy/docker-compose.yml index 80dfccaff..c214eea41 100644 --- a/test/integration/connect/envoy/docker-compose.yml +++ b/test/integration/connect/envoy/docker-compose.yml @@ -172,6 +172,22 @@ services: - "disabled" network_mode: service:consul-primary + s4: + depends_on: + - consul-primary + image: "fortio/fortio" + environment: + - "FORTIO_NAME=s4" + command: + - "server" + - "-http-port" + - ":8382" + - "-grpc-port" + - ":8281" + - "-redirect-port" + - "disabled" + network_mode: service:consul-primary + s1-sidecar-proxy: depends_on: - consul-primary diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index a163d261f..7d03f3fb7 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -425,7 +425,7 @@ function docker_consul { function docker_wget { local DC=$1 shift 1 - docker run -ti --rm --network container:envoy_consul-${DC}_1 alpine:3.9 wget "$@" + docker run --rm --network container:envoy_consul-${DC}_1 alpine:3.9 wget "$@" } function docker_curl { diff --git a/test/integration/connect/envoy/main_test.go b/test/integration/connect/envoy/main_test.go index 1dbd84d14..e694b20f6 100644 --- a/test/integration/connect/envoy/main_test.go +++ b/test/integration/connect/envoy/main_test.go @@ -38,6 +38,7 @@ func TestEnvoy(t *testing.T) { "case-prometheus", "case-statsd-udp", "case-stats-proxy", + "case-terminating-gateway-hostnames", "case-terminating-gateway-simple", "case-terminating-gateway-subsets", "case-terminating-gateway-without-services", diff --git a/website/pages/docs/connect/proxies/envoy.mdx b/website/pages/docs/connect/proxies/envoy.mdx index 8209ea7ea..f1d7a2153 100644 --- a/website/pages/docs/connect/proxies/envoy.mdx +++ b/website/pages/docs/connect/proxies/envoy.mdx @@ -319,6 +319,13 @@ will continue to be supported. - `envoy_gateway_no_default_bind` - Prevents binding to the default address of the gateway service. This should be used with one of the other options to configure the gateway's bind addresses. + +- `envoy_dns_discovery_type` - Determines how Envoy will resolve hostnames. Defaults to `LOGICAL_DNS`. + Must be one of `STRICT_DNS` or `LOGICAL_DNS`. Details for each type are available in + the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/v1.14.1/intro/arch_overview/upstream/service_discovery). + This option applies to terminating gateways that route to services + addressed by a hostname, such as a managed databased. It also applies to mesh gateways, + such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname. ## Advanced Configuration diff --git a/website/pages/docs/connect/terminating-gateway.mdx b/website/pages/docs/connect/terminating-gateway.mdx index 3b097caa3..bb3b1ee90 100644 --- a/website/pages/docs/connect/terminating-gateway.mdx +++ b/website/pages/docs/connect/terminating-gateway.mdx @@ -24,8 +24,7 @@ services outside the mesh, review the [terminating gateway guide](https://learn. ~> **Beta limitations:** Terminating Gateways currently do not support targeting service subsets with [L7 configuration](/docs/connect/l7-traffic-management). They route to all instances of a service with no capabilities -for filtering by instance. Terminating Gateways also currently do not support routing to services with a hostname -defined as a their address. The service address registered with Consul, that the gateway will route traffic to, **must** be a resolved IP address. +for filtering by instance. ## Security Considerations