Allow ingress gateways to route traffic based on Host header

This commit adds the necessary changes to allow an ingress gateway to
route traffic from a single defined port to multiple different upstream
services in the Consul mesh.

To do this, we now require all HTTP requests coming into the ingress
gateway to specify a Host header that matches "<service-name>.*" in
order to correctly route traffic to the correct service.

- Differentiate multiple listener's route names by port
- Adds a case in xds for allowing default discovery chains to create a
  route configuration when on an ingress gateway. This allows default
  services to easily use host header routing
- ingress-gateways have a single route config for each listener
  that utilizes domain matching to route to different services.

agent/consul/internal_endpoint_test.go

@@ -935,6 +935,7 @@ func TestInternal_GatewayServices_BothGateways(t *testing.T) {
 				Service:     structs.NewServiceID("db", nil),
 				Gateway:     structs.NewServiceID("ingress", nil),
 				GatewayKind: structs.ServiceKindIngressGateway,
+				Protocol:    "tcp",
 				Port:        8888,
 			},
 		}

agent/consul/state/catalog.go

@@ -2540,6 +2540,7 @@ func (s *Store) ingressConfigGatewayServices(tx *memdb.Txn, gateway structs.Serv
 				Service:     service.ToServiceID(),
 				GatewayKind: structs.ServiceKindIngressGateway,
 				Port:        listener.Port,
+				Protocol:    listener.Protocol,
 			}
 
 			gatewayServices = append(gatewayServices, mapping)

agent/proxycfg/snapshot.go

@@ -2,6 +2,8 @@ package proxycfg
 
 import (
 	"context"
+	"fmt"
+
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/mitchellh/copystructure"
 )
@@ -194,7 +196,7 @@ type configSnapshotIngressGateway struct {
 	// Upstreams is a list of upstreams this ingress gateway should serve traffic
 	// to. This is constructed from the ingress-gateway config entry, and uses
 	// the GatewayServices RPC to retrieve them.
-	Upstreams []structs.Upstream
+	Upstreams map[IngressListenerKey]structs.Upstreams
 
 	// WatchedDiscoveryChains is a map of upstream.Identifier() -> CancelFunc's
 	// in order to cancel any watches when the ingress gateway configuration is
@@ -214,6 +216,15 @@ func (c *configSnapshotIngressGateway) IsEmpty() bool {
 		len(c.WatchedUpstreamEndpoints) == 0
 }
 
+type IngressListenerKey struct {
+	Protocol string
+	Port     int
+}
+
+func (k *IngressListenerKey) RouteName() string {
+	return fmt.Sprintf("%s_%d", k.Protocol, k.Port)
+}
+
 // ConfigSnapshot captures all the resulting config needed for a proxy instance.
 // It is meant to be point-in-time coherent and is used to deliver the current
 // config state to observers who need it to be pushed in (e.g. XDS server).

agent/proxycfg/state.go

@@ -1320,8 +1320,8 @@ func (s *state) handleUpdateIngressGateway(u cache.UpdateEvent, snap *ConfigSnap
 			return fmt.Errorf("invalid type for response: %T", u.Result)
 		}
 
-		var upstreams structs.Upstreams
 		watchedSvcs := make(map[string]struct{})
+		upstreamsMap := make(map[IngressListenerKey]structs.Upstreams)
 		for _, service := range services.Services {
 			u := makeUpstream(service, s.address)
 
@@ -1330,9 +1330,11 @@ func (s *state) handleUpdateIngressGateway(u cache.UpdateEvent, snap *ConfigSnap
 				return err
 			}
 			watchedSvcs[u.Identifier()] = struct{}{}
-			upstreams = append(upstreams, u)
+
+			id := IngressListenerKey{Protocol: service.Protocol, Port: service.Port}
+			upstreamsMap[id] = append(upstreamsMap[id], u)
 		}
-		snap.IngressGateway.Upstreams = upstreams
+		snap.IngressGateway.Upstreams = upstreamsMap
 
 		for id, cancelFn := range snap.IngressGateway.WatchedDiscoveryChains {
 			if _, ok := watchedSvcs[id]; !ok {

agent/proxycfg/state_test.go

@@ -810,6 +810,66 @@ func TestState_WatchesAndUpdates(t *testing.T) {
 				},
 			},
 		},
+		"ingress-gateway-update-upstreams": testCase{
+			ns: structs.NodeService{
+				Kind:    structs.ServiceKindIngressGateway,
+				ID:      "ingress-gateway",
+				Service: "ingress-gateway",
+				Address: "10.0.1.1",
+			},
+			sourceDC: "dc1",
+			stages: []verificationStage{
+				verificationStage{
+					requiredWatches: map[string]verifyWatchRequest{
+						rootsWatchID: genVerifyRootsWatch("dc1"),
+						leafWatchID:  genVerifyLeafWatch("ingress-gateway", "dc1"),
+					},
+					events: []cache.UpdateEvent{
+						rootWatchEvent(),
+						cache.UpdateEvent{
+							CorrelationID: leafWatchID,
+							Result:        issuedCert,
+							Err:           nil,
+						},
+						cache.UpdateEvent{
+							CorrelationID: gatewayServicesWatchID,
+							Result: &structs.IndexedGatewayServices{
+								Services: structs.GatewayServices{
+									{
+										Gateway: structs.NewServiceID("ingress-gateway", nil),
+										Service: structs.NewServiceID("api", nil),
+										Port:    9999,
+									},
+								},
+							},
+							Err: nil,
+						},
+					},
+					verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
+						require.True(t, snap.Valid())
+						require.Len(t, snap.IngressGateway.Upstreams, 1)
+						require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 1)
+						require.Contains(t, snap.IngressGateway.WatchedDiscoveryChains, "api")
+					},
+				},
+				verificationStage{
+					requiredWatches: map[string]verifyWatchRequest{},
+					events: []cache.UpdateEvent{
+						cache.UpdateEvent{
+							CorrelationID: gatewayServicesWatchID,
+							Result:        &structs.IndexedGatewayServices{},
+							Err:           nil,
+						},
+					},
+					verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
+						require.True(t, snap.Valid())
+						require.Len(t, snap.IngressGateway.Upstreams, 0)
+						require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 0)
+						require.NotContains(t, snap.IngressGateway.WatchedDiscoveryChains, "api")
+					},
+				},
+			},
+		},
 		"terminating-gateway-initial": testCase{
 			ns: structs.NodeService{
 				Kind:    structs.ServiceKindTerminatingGateway,

agent/proxycfg/testing.go

@@ -1143,6 +1143,7 @@ func setupTestVariationConfigEntriesAndSnapshot(
 				},
 			},
 		)
+	case "http-multiple-services":
 	default:
 		t.Fatalf("unexpected variation: %q", variation)
 		return ConfigSnapshotUpstreams{}
@@ -1233,6 +1234,13 @@ func setupTestVariationConfigEntriesAndSnapshot(
 	case "chain-and-splitter":
 	case "grpc-router":
 	case "chain-and-router":
+	case "http-multiple-services":
+		snap.WatchedUpstreamEndpoints["foo"] = map[string]structs.CheckServiceNodes{
+			"foo.default.dc1": TestUpstreamNodes(t),
+		}
+		snap.WatchedUpstreamEndpoints["bar"] = map[string]structs.CheckServiceNodes{
+			"bar.default.dc1": TestUpstreamNodesAlternate(t),
+		}
 	default:
 		t.Fatalf("unexpected variation: %q", variation)
 		return ConfigSnapshotUpstreams{}
@@ -1312,82 +1320,86 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera
 }
 
 func TestConfigSnapshotIngress(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "simple")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "simple")
 }
 
 func TestConfigSnapshotIngressWithOverrides(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "simple-with-overrides")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "simple-with-overrides")
 }
 func TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "splitter-with-resolver-redirect-multidc")
+	return testConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc")
+}
+
+func TestConfigSnapshotIngress_HTTPMultipleServices(t testing.T) *ConfigSnapshot {
+	return testConfigSnapshotIngressGateway(t, true, "http", "http-multiple-services")
 }
 
 func TestConfigSnapshotIngressExternalSNI(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "external-sni")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "external-sni")
 }
 
 func TestConfigSnapshotIngressWithFailover(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover")
 }
 
 func TestConfigSnapshotIngressWithFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-remote-gateway")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway")
 }
 
 func TestConfigSnapshotIngressWithFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-remote-gateway-triggered")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered")
 }
 
 func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-double-remote-gateway")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway")
 }
 
 func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-double-remote-gateway-triggered")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered")
 }
 
 func TestConfigSnapshotIngressWithFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-local-gateway")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway")
 }
 
 func TestConfigSnapshotIngressWithFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-local-gateway-triggered")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered")
 }
 
 func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-double-local-gateway")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway")
 }
 
 func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "failover-through-double-local-gateway-triggered")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered")
 }
 
 func TestConfigSnapshotIngressWithSplitter(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "chain-and-splitter")
+	return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-splitter")
 }
 
 func TestConfigSnapshotIngressWithGRPCRouter(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "grpc-router")
+	return testConfigSnapshotIngressGateway(t, true, "http", "grpc-router")
 }
 
 func TestConfigSnapshotIngressWithRouter(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "chain-and-router")
+	return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-router")
 }
 
 func TestConfigSnapshotIngressGateway(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "default")
+	return testConfigSnapshotIngressGateway(t, true, "tcp", "default")
 }
 
 func TestConfigSnapshotIngressGatewayNoServices(t testing.T) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, false, "default")
+	return testConfigSnapshotIngressGateway(t, false, "tcp", "default")
 }
 
 func TestConfigSnapshotIngressDiscoveryChainWithEntries(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
-	return testConfigSnapshotIngressGateway(t, true, "simple", additionalEntries...)
+	return testConfigSnapshotIngressGateway(t, true, "http", "simple", additionalEntries...)
 }
 
 func testConfigSnapshotIngressGateway(
-	t testing.T, populateServices bool, variation string,
+	t testing.T, populateServices bool, protocol, variation string,
 	additionalEntries ...structs.ConfigEntry,
 ) *ConfigSnapshot {
 	roots, leaf := TestCerts(t)
@@ -1404,7 +1416,8 @@ func testConfigSnapshotIngressGateway(
 			ConfigSnapshotUpstreams: setupTestVariationConfigEntriesAndSnapshot(
 				t, variation, leaf, additionalEntries...,
 			),
-			Upstreams: structs.Upstreams{
+			Upstreams: map[IngressListenerKey]structs.Upstreams{
+				IngressListenerKey{protocol, 9191}: structs.Upstreams{
 					{
 						// We rely on this one having default type in a few tests...
 						DestinationName:  "db",
@@ -1412,6 +1425,7 @@ func testConfigSnapshotIngressGateway(
 						LocalBindAddress: "2.3.4.5",
 					},
 				},
+			},
 		}
 	}
 	return snap

agent/structs/config_entry_gateways.go

@@ -300,6 +300,7 @@ type GatewayService struct {
 	Service      ServiceID
 	GatewayKind  ServiceKind
 	Port         int
+	Protocol     string
 	CAFile       string
 	CertFile     string
 	KeyFile      string
@@ -315,6 +316,7 @@ func (g *GatewayService) IsSame(o *GatewayService) bool {
 		g.Service.Matches(&o.Service) &&
 		g.GatewayKind == o.GatewayKind &&
 		g.Port == o.Port &&
+		g.Protocol == o.Protocol &&
 		g.CAFile == o.CAFile &&
 		g.CertFile == o.CertFile &&
 		g.KeyFile == o.KeyFile &&
@@ -328,6 +330,7 @@ func (g *GatewayService) Clone() *GatewayService {
 		Service:      g.Service,
 		GatewayKind:  g.GatewayKind,
 		Port:         g.Port,
+		Protocol:     g.Protocol,
 		CAFile:       g.CAFile,
 		CertFile:     g.CertFile,
 		KeyFile:      g.KeyFile,

agent/xds/clusters.go

@@ -236,7 +236,8 @@ func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([
 
 func (s *Server) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
 	var clusters []proto.Message
-	for _, u := range cfgSnap.IngressGateway.Upstreams {
+	for _, upstreams := range cfgSnap.IngressGateway.Upstreams {
+		for _, u := range upstreams {
 			id := u.Identifier()
 			chain, ok := cfgSnap.IngressGateway.DiscoveryChain[id]
 			if !ok {
@@ -259,6 +260,7 @@ func (s *Server) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnap
 				clusters = append(clusters, c)
 			}
 		}
+	}
 	return clusters, nil
 }
 

agent/xds/endpoints.go

@@ -255,7 +255,8 @@ func (s *Server) endpointsFromServicesAndResolvers(
 
 func (s *Server) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
 	var resources []proto.Message
-	for _, u := range cfgSnap.IngressGateway.Upstreams {
+	for _, upstreams := range cfgSnap.IngressGateway.Upstreams {
+		for _, u := range upstreams {
 			id := u.Identifier()
 
 			es := s.endpointsFromDiscoveryChain(
@@ -266,6 +267,7 @@ func (s *Server) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSna
 			)
 			resources = append(resources, es...)
 		}
+	}
 	return resources, nil
 }
 

agent/xds/listeners.go

@@ -270,10 +270,9 @@ func (s *Server) listenersFromSnapshotGateway(cfgSnap *proxycfg.ConfigSnapshot,
 // See: https://www.consul.io/docs/connect/proxies/envoy.html#mesh-gateway-options
 func (s *Server) listenersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
 	var resources []proto.Message
-	// TODO(ingress): We give each upstream a distinct listener at the moment,
+	for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams {
-	// for http listeners we will need to multiplex upstreams on a single
+		if listenerKey.Protocol == "tcp" {
-	// listener.
+			u := upstreams[0]
-	for _, u := range cfgSnap.IngressGateway.Upstreams {
 			id := u.Identifier()
 
 			chain := cfgSnap.IngressGateway.DiscoveryChain[id]
@@ -289,6 +288,29 @@ func (s *Server) listenersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSna
 				return nil, err
 			}
 			resources = append(resources, upstreamListener)
+		} else {
+			// If multiple upstreams share this port, make a special listener for the protocol.
+			addr := cfgSnap.Address
+			if addr == "" {
+				addr = "0.0.0.0"
+			}
+
+			listener := makeListener(listenerKey.Protocol, addr, listenerKey.Port)
+			filter, err := makeListenerFilter(
+				true, listenerKey.Protocol, listenerKey.RouteName(), "", "ingress_upstream_", "", false)
+			if err != nil {
+				return nil, err
+			}
+
+			listener.FilterChains = []envoylistener.FilterChain{
+				{
+					Filters: []envoylistener.Filter{
+						filter,
+					},
+				},
+			}
+			resources = append(resources, listener)
+		}
 	}
 
 	return resources, nil

agent/xds/listeners_test.go

@@ -363,6 +363,34 @@ func TestListenersFromSnapshot(t *testing.T) {
 				}
 			},
 		},
+		{
+			name:   "ingress-http-multiple-services",
+			create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices,
+			setup: func(snap *proxycfg.ConfigSnapshot) {
+				snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
+					proxycfg.IngressListenerKey{Protocol: "http", Port: 8080}: structs.Upstreams{
+						{
+							DestinationName: "foo",
+							LocalBindPort:   8080,
+						},
+						{
+							DestinationName: "bar",
+							LocalBindPort:   8080,
+						},
+					},
+					proxycfg.IngressListenerKey{Protocol: "http", Port: 443}: structs.Upstreams{
+						{
+							DestinationName: "baz",
+							LocalBindPort:   443,
+						},
+						{
+							DestinationName: "qux",
+							LocalBindPort:   443,
+						},
+					},
+				}
+			},
+		},
 		{
 			name:   "terminating-gateway-no-api-cert",
 			create: proxycfg.TestConfigSnapshotTerminatingGateway,

agent/xds/routes.go

@@ -37,7 +37,34 @@ func routesFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.M
 		return nil, errors.New("nil config given")
 	}
 
-	return routesFromUpstreams(cfgSnap.ConnectProxy.ConfigSnapshotUpstreams, cfgSnap.Proxy.Upstreams)
+	var resources []proto.Message
+	for _, u := range cfgSnap.Proxy.Upstreams {
+		upstreamID := u.Identifier()
+
+		var chain *structs.CompiledDiscoveryChain
+		if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
+			chain = cfgSnap.ConnectProxy.DiscoveryChain[upstreamID]
+		}
+
+		if chain == nil || chain.IsDefault() {
+			// TODO(rb): make this do the old school stuff too
+		} else {
+			virtualHost, err := makeUpstreamRouteForDiscoveryChain(upstreamID, chain, "*")
+			if err != nil {
+				return nil, err
+			}
+
+			route := &envoy.RouteConfiguration{
+				Name:             upstreamID,
+				VirtualHosts:     []envoyroute.VirtualHost{*virtualHost},
+				ValidateClusters: makeBoolValue(true),
+			}
+			resources = append(resources, route)
+		}
+	}
+
+	// TODO(rb): make sure we don't generate an empty result
+	return resources, nil
 }
 
 // routesFromSnapshotIngressGateway returns the xDS API representation of the
@@ -47,45 +74,51 @@ func routesFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto
 		return nil, errors.New("nil config given")
 	}
 
-	return routesFromUpstreams(cfgSnap.IngressGateway.ConfigSnapshotUpstreams, cfgSnap.IngressGateway.Upstreams)
+	var result []proto.Message
+	for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams {
+		// Do not create any route configuration for TCP listeners
+		if listenerKey.Protocol == "tcp" {
+			continue
 		}
 
-func routesFromUpstreams(snap proxycfg.ConfigSnapshotUpstreams, upstreams structs.Upstreams) ([]proto.Message, error) {
+		upstreamRoute := &envoy.RouteConfiguration{
-	var resources []proto.Message
+			Name: listenerKey.RouteName(),
-
+			// ValidateClusters defaults to true when defined statically and false
+			// when done via RDS. Re-set the sane value of true to prevent
+			// null-routing traffic.
+			ValidateClusters: makeBoolValue(true),
+		}
 		for _, u := range upstreams {
 			upstreamID := u.Identifier()
-
+			chain := cfgSnap.IngressGateway.DiscoveryChain[upstreamID]
-		var chain *structs.CompiledDiscoveryChain
+			if chain != nil {
-		if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
+				domain := fmt.Sprintf("%s.*", chain.ServiceName)
-			chain = snap.DiscoveryChain[upstreamID]
+				// Don't require a service prefix on the domain if there is only 1
+				// upstream. This makes it a smoother experience when only having a
+				// single service associated to a listener, which is probably a common
+				// case when demoing/testing
+				if len(upstreams) == 1 {
+					domain = "*"
 				}
-
+				virtualHost, err := makeUpstreamRouteForDiscoveryChain(upstreamID, chain, domain)
-		if chain == nil || chain.IsDefault() {
-			// TODO(rb): make this do the old school stuff too
-		} else {
-			upstreamRoute, err := makeUpstreamRouteForDiscoveryChain(&u, chain)
 				if err != nil {
 					return nil, err
 				}
-			if upstreamRoute != nil {
+				upstreamRoute.VirtualHosts = append(upstreamRoute.VirtualHosts, *virtualHost)
-				resources = append(resources, upstreamRoute)
-			}
 			}
 		}
 
-	// TODO(rb): make sure we don't generate an empty result
+		result = append(result, upstreamRoute)
+	}
 
-	return resources, nil
+	return result, nil
 }
 
 func makeUpstreamRouteForDiscoveryChain(
-	u *structs.Upstream,
+	routeName string,
 	chain *structs.CompiledDiscoveryChain,
-) (*envoy.RouteConfiguration, error) {
+	serviceDomain string,
-	upstreamID := u.Identifier()
+) (*envoyroute.VirtualHost, error) {
-	routeName := upstreamID
-
 	var routes []envoyroute.Route
 
 	startNode := chain.Nodes[chain.StartNode]
@@ -188,20 +221,13 @@ func makeUpstreamRouteForDiscoveryChain(
 		panic("unknown first node in discovery chain of type: " + startNode.Type)
 	}
 
-	return &envoy.RouteConfiguration{
+	host := &envoyroute.VirtualHost{
 		Name:    routeName,
-		VirtualHosts: []envoyroute.VirtualHost{
+		Domains: []string{serviceDomain},
-			envoyroute.VirtualHost{
-				Name:    routeName,
-				Domains: []string{"*"},
 		Routes:  routes,
-			},
+	}
-		},
+
-		// ValidateClusters defaults to true when defined statically and false
+	return host, nil
-		// when done via RDS. Re-set the sane value of true to prevent
-		// null-routing traffic.
-		ValidateClusters: makeBoolValue(true),
-	}, nil
 }
 
 func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute, protocol string) envoyroute.RouteMatch {

agent/xds/routes_test.go

@@ -4,9 +4,13 @@ import (
 	"path"
 	"sort"
 	"testing"
+	"time"
 
 	envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	"github.com/hashicorp/consul/agent/connect"
+	"github.com/hashicorp/consul/agent/consul/discoverychain"
 	"github.com/hashicorp/consul/agent/proxycfg"
+	"github.com/hashicorp/consul/agent/structs"
 	testinf "github.com/mitchellh/go-testing-interface"
 	"github.com/stretchr/testify/require"
 )
@@ -104,6 +108,66 @@ func TestRoutesFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotIngressWithRouter,
 			setup:  nil,
 		},
+		{
+			name:   "ingress-http-multiple-services",
+			create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices,
+			setup: func(snap *proxycfg.ConfigSnapshot) {
+				snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
+					proxycfg.IngressListenerKey{Protocol: "http", Port: 8080}: structs.Upstreams{
+						{
+							DestinationName: "foo",
+							LocalBindPort:   8080,
+						},
+						{
+							DestinationName: "bar",
+							LocalBindPort:   8080,
+						},
+					},
+					proxycfg.IngressListenerKey{Protocol: "http", Port: 443}: structs.Upstreams{
+						{
+							DestinationName: "baz",
+							LocalBindPort:   443,
+						},
+						{
+							DestinationName: "qux",
+							LocalBindPort:   443,
+						},
+					},
+				}
+
+				// We do not add baz/qux here so that we test the chain.IsDefault() case
+				entries := []structs.ConfigEntry{
+					&structs.ProxyConfigEntry{
+						Kind: structs.ProxyDefaults,
+						Name: structs.ProxyConfigGlobal,
+						Config: map[string]interface{}{
+							"protocol": "http",
+						},
+					},
+					&structs.ServiceResolverConfigEntry{
+						Kind:           structs.ServiceResolver,
+						Name:           "foo",
+						ConnectTimeout: 22 * time.Second,
+					},
+					&structs.ServiceResolverConfigEntry{
+						Kind:           structs.ServiceResolver,
+						Name:           "bar",
+						ConnectTimeout: 22 * time.Second,
+					},
+				}
+				fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
+				barChain := discoverychain.TestCompileConfigEntries(t, "bar", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
+				bazChain := discoverychain.TestCompileConfigEntries(t, "baz", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
+				quxChain := discoverychain.TestCompileConfigEntries(t, "qux", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
+
+				snap.IngressGateway.DiscoveryChain = map[string]*structs.CompiledDiscoveryChain{
+					"foo": fooChain,
+					"bar": barChain,
+					"baz": bazChain,
+					"qux": quxChain,
+				}
+			},
+		},
 	}
 
 	for _, tt := range tests {

agent/xds/testdata/listeners/ingress-http-multiple-services.golden

@@ -0,0 +1,85 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Listener",
+      "name": "http:1.2.3.4:443",
+      "address": {
+        "socketAddress": {
+          "address": "1.2.3.4",
+          "portValue": 443
+        }
+      },
+      "filterChains": [
+        {
+          "filters": [
+            {
+              "name": "envoy.http_connection_manager",
+              "config": {
+                  "http_filters": [
+                        {
+                              "name": "envoy.router"
+                            }
+                      ],
+                  "rds": {
+                        "config_source": {
+                              "ads": {
+                                  }
+                            },
+                        "route_config_name": "http_443"
+                      },
+                  "stat_prefix": "ingress_upstream_http_443_http",
+                  "tracing": {
+                        "operation_name": "EGRESS",
+                        "random_sampling": {
+                            }
+                      }
+                }
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Listener",
+      "name": "http:1.2.3.4:8080",
+      "address": {
+        "socketAddress": {
+          "address": "1.2.3.4",
+          "portValue": 8080
+        }
+      },
+      "filterChains": [
+        {
+          "filters": [
+            {
+              "name": "envoy.http_connection_manager",
+              "config": {
+                  "http_filters": [
+                        {
+                              "name": "envoy.router"
+                            }
+                      ],
+                  "rds": {
+                        "config_source": {
+                              "ads": {
+                                  }
+                            },
+                        "route_config_name": "http_8080"
+                      },
+                  "stat_prefix": "ingress_upstream_http_8080_http",
+                  "tracing": {
+                        "operation_name": "EGRESS",
+                        "random_sampling": {
+                            }
+                      }
+                }
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
+  "nonce": "00000001"
+}

agent/xds/testdata/listeners/ingress-splitter-with-resolver-redirect.golden

@@ -3,10 +3,10 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.Listener",
-      "name": "db:2.3.4.5:9191",
+      "name": "http:1.2.3.4:9191",
       "address": {
         "socketAddress": {
-          "address": "2.3.4.5",
+          "address": "1.2.3.4",
           "portValue": 9191
         }
       },
@@ -26,9 +26,9 @@
                               "ads": {
                                   }
                             },
-                        "route_config_name": "db"
+                        "route_config_name": "http_9191"
                       },
-                  "stat_prefix": "upstream_db_http",
+                  "stat_prefix": "ingress_upstream_http_9191_http",
                   "tracing": {
                         "operation_name": "EGRESS",
                         "random_sampling": {

agent/xds/testdata/routes/ingress-http-multiple-services.golden

@@ -0,0 +1,85 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
+      "name": "http_443",
+      "virtualHosts": [
+        {
+          "name": "baz",
+          "domains": [
+            "baz.*"
+          ],
+          "routes": [
+            {
+              "match": {
+                "prefix": "/"
+              },
+              "route": {
+                "cluster": "baz.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+              }
+            }
+          ]
+        },
+        {
+          "name": "qux",
+          "domains": [
+            "qux.*"
+          ],
+          "routes": [
+            {
+              "match": {
+                "prefix": "/"
+              },
+              "route": {
+                "cluster": "qux.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+              }
+            }
+          ]
+        }
+      ],
+      "validateClusters": true
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
+      "name": "http_8080",
+      "virtualHosts": [
+        {
+          "name": "foo",
+          "domains": [
+            "foo.*"
+          ],
+          "routes": [
+            {
+              "match": {
+                "prefix": "/"
+              },
+              "route": {
+                "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+              }
+            }
+          ]
+        },
+        {
+          "name": "bar",
+          "domains": [
+            "bar.*"
+          ],
+          "routes": [
+            {
+              "match": {
+                "prefix": "/"
+              },
+              "route": {
+                "cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+              }
+            }
+          ]
+        }
+      ],
+      "validateClusters": true
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
+  "nonce": "00000001"
+}

agent/xds/testdata/routes/ingress-splitter-with-resolver-redirect.golden

@@ -3,7 +3,7 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
+      "name": "http_9191",
       "virtualHosts": [
         {
           "name": "db",

agent/xds/testdata/routes/ingress-with-chain-and-overrides.golden

@@ -1,29 +1,6 @@
 {
   "versionInfo": "00000001",
   "resources": [
-    {
-      "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
-      "virtualHosts": [
-        {
-          "name": "db",
-          "domains": [
-            "*"
-          ],
-          "routes": [
-            {
-              "match": {
-                "prefix": "/"
-              },
-              "route": {
-                "cluster": "a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
-              }
-            }
-          ]
-        }
-      ],
-      "validateClusters": true
-    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
   "nonce": "00000001"

agent/xds/testdata/routes/ingress-with-chain-and-router.golden

@@ -3,7 +3,7 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
+      "name": "http_9191",
       "virtualHosts": [
         {
           "name": "db",

agent/xds/testdata/routes/ingress-with-chain-and-splitter.golden

@@ -3,7 +3,7 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
+      "name": "http_9191",
       "virtualHosts": [
         {
           "name": "db",

agent/xds/testdata/routes/ingress-with-chain-external-sni.golden

@@ -1,29 +1,6 @@
 {
   "versionInfo": "00000001",
   "resources": [
-    {
-      "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
-      "virtualHosts": [
-        {
-          "name": "db",
-          "domains": [
-            "*"
-          ],
-          "routes": [
-            {
-              "match": {
-                "prefix": "/"
-              },
-              "route": {
-                "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
-              }
-            }
-          ]
-        }
-      ],
-      "validateClusters": true
-    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
   "nonce": "00000001"

agent/xds/testdata/routes/ingress-with-chain.golden

@@ -1,29 +1,6 @@
 {
   "versionInfo": "00000001",
   "resources": [
-    {
-      "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
-      "virtualHosts": [
-        {
-          "name": "db",
-          "domains": [
-            "*"
-          ],
-          "routes": [
-            {
-              "match": {
-                "prefix": "/"
-              },
-              "route": {
-                "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
-              }
-            }
-          ]
-        }
-      ],
-      "validateClusters": true
-    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
   "nonce": "00000001"

agent/xds/testdata/routes/ingress-with-grpc-router.golden

@@ -3,7 +3,7 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-      "name": "db",
+      "name": "http_9191",
       "virtualHosts": [
         {
           "name": "db",

test/integration/connect/envoy/case-ingress-gateway-multiple-services/capture.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+snapshot_envoy_admin localhost:20000 ingress-gateway primary || true

test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl

@@ -0,0 +1,29 @@
+enable_central_service_config = true
+
+config_entries {
+  bootstrap = [
+    {
+      kind = "ingress-gateway"
+      name = "ingress-gateway"
+
+      listeners = [
+        {
+          port = 9999
+          protocol = "http"
+          services = [
+            {
+              name = "*"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      kind = "proxy-defaults"
+      name = "global"
+      config {
+        protocol = "http"
+      }
+    }
+  ]
+}

test/integration/connect/envoy/case-ingress-gateway-multiple-services/gateway.hcl

@@ -0,0 +1,4 @@
+services {
+  name = "ingress-gateway"
+  kind = "ingress-gateway"
+}

test/integration/connect/envoy/case-ingress-gateway-multiple-services/setup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# wait for bootstrap to apply config entries
+wait_for_config_entry ingress-gateway ingress-gateway
+wait_for_config_entry proxy-defaults global
+
+gen_envoy_bootstrap ingress-gateway 20000 primary true
+gen_envoy_bootstrap s1 19000
+gen_envoy_bootstrap s2 19001

test/integration/connect/envoy/case-ingress-gateway-multiple-services/vars.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+export REQUIRED_SERVICES="$DEFAULT_REQUIRED_SERVICES ingress-gateway-primary"

test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats

@@ -0,0 +1,58 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "ingress proxy admin is up on :20000" {
+  retry_default curl -f -s localhost:20000/stats -o /dev/null
+}
+
+@test "s1 proxy admin is up on :19000" {
+  retry_default curl -f -s localhost:19000/stats -o /dev/null
+}
+
+@test "s2 proxy admin is up on :19001" {
+  retry_default curl -f -s localhost:19001/stats -o /dev/null
+}
+
+@test "s1 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21000 s1
+}
+
+@test "s2 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21001 s2
+}
+
+@test "ingress-gateway should have healthy endpoints for s1" {
+   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1
+}
+
+@test "ingress-gateway should have healthy endpoints for s2" {
+   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1
+}
+
+@test "ingress should be able to connect to s1 using Host header" {
+  run retry_default curl -H"Host: s1.example.consul" -s -f localhost:9999/debug?env=dump
+  [ "$status" -eq 0 ]
+
+  GOT=$(echo "$output" | grep -E "^FORTIO_NAME=")
+  EXPECT_NAME="s1"
+
+  if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then
+    echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2
+    return 1
+  fi
+}
+
+@test "ingress should be able to connect to s2 using Host header" {
+  run retry_default curl -H"Host: s2.example.consul" -s -f localhost:9999/debug?env=dump
+  [ "$status" -eq 0 ]
+
+  GOT=$(echo "$output" | grep -E "^FORTIO_NAME=")
+  EXPECT_NAME="s2"
+
+  if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then
+    echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2
+    return 1
+  fi
+}
+