Merge pull request #6053 from hashicorp/gateways_and_resolvers

Integrate Mesh Gateways with ServiceResolverSubsets

agent/agent.go

@@ -3954,6 +3954,15 @@ func (a *Agent) registerCache() {
 		RefreshTimer:   0 * time.Second,
 		RefreshTimeout: 10 * time.Minute,
 	})
+
+	a.cache.RegisterType(cachetype.ConfigEntriesName, &cachetype.ConfigEntries{
+		RPC: a,
+	}, &cache.RegisterOptions{
+		// Maintain a blocking query, retry dropped connections quickly
+		Refresh:        true,
+		RefreshTimer:   0 * time.Second,
+		RefreshTimeout: 10 * time.Minute,
+	})
 }
 
 // defaultProxyCommand returns the default Connect managed proxy command.

agent/cache-types/config_entry.go

@@ -0,0 +1,51 @@
+package cachetype
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+)
+
+// Recommended name for registration.
+const ConfigEntriesName = "config-entries"
+
+// ConfigEntries supports fetching discovering configuration entries
+type ConfigEntries struct {
+	RPC RPC
+}
+
+func (c *ConfigEntries) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) {
+	var result cache.FetchResult
+
+	// The request should be a ConfigEntryQuery.
+	reqReal, ok := req.(*structs.ConfigEntryQuery)
+	if !ok {
+		return result, fmt.Errorf(
+			"Internal cache failure: request wrong type: %T", req)
+	}
+
+	// Set the minimum query index to our current index so we block
+	reqReal.QueryOptions.MinQueryIndex = opts.MinIndex
+	reqReal.QueryOptions.MaxQueryTime = opts.Timeout
+
+	// Always allow stale - there's no point in hitting leader if the request is
+	// going to be served from cache and endup arbitrarily stale anyway. This
+	// allows cached service-discover to automatically read scale across all
+	// servers too.
+	reqReal.AllowStale = true
+
+	// Fetch
+	var reply structs.IndexedConfigEntries
+	if err := c.RPC.RPC("ConfigEntry.List", reqReal, &reply); err != nil {
+		return result, err
+	}
+
+	result.Value = &reply
+	result.Index = reply.QueryMeta.Index
+	return result, nil
+}
+
+func (c *ConfigEntries) SupportsBlocking() bool {
+	return true
+}

agent/cache-types/config_entry_test.go

@@ -0,0 +1,66 @@
+package cachetype
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigEntries(t *testing.T) {
+	rpc := TestRPC(t)
+	typ := &ConfigEntries{RPC: rpc}
+
+	// Expect the proper RPC call. This also sets the expected value
+	// since that is return-by-pointer in the arguments.
+	var resp *structs.IndexedConfigEntries
+	rpc.On("RPC", "ConfigEntry.List", mock.Anything, mock.Anything).Return(nil).
+		Run(func(args mock.Arguments) {
+			req := args.Get(1).(*structs.ConfigEntryQuery)
+			require.Equal(t, uint64(24), req.QueryOptions.MinQueryIndex)
+			require.Equal(t, 1*time.Second, req.QueryOptions.MaxQueryTime)
+			require.True(t, req.AllowStale)
+			require.Equal(t, structs.ServiceResolver, req.Kind)
+			require.Equal(t, "", req.Name)
+
+			reply := args.Get(2).(*structs.IndexedConfigEntries)
+			reply.Kind = structs.ServiceResolver
+			reply.Entries = []structs.ConfigEntry{
+				&structs.ServiceResolverConfigEntry{Kind: structs.ServiceResolver, Name: "foo"},
+				&structs.ServiceResolverConfigEntry{Kind: structs.ServiceResolver, Name: "bar"},
+			}
+			reply.QueryMeta.Index = 48
+			resp = reply
+		})
+
+	// Fetch
+	resultA, err := typ.Fetch(cache.FetchOptions{
+		MinIndex: 24,
+		Timeout:  1 * time.Second,
+	}, &structs.ConfigEntryQuery{
+		Datacenter: "dc1",
+		Kind:       structs.ServiceResolver,
+	})
+	require.NoError(t, err)
+	require.Equal(t, cache.FetchResult{
+		Value: resp,
+		Index: 48,
+	}, resultA)
+
+	rpc.AssertExpectations(t)
+}
+
+func TestConfigEntries_badReqType(t *testing.T) {
+	rpc := TestRPC(t)
+	typ := &ConfigEntries{RPC: rpc}
+
+	// Fetch
+	_, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest(
+		t, cache.RequestInfo{Key: "foo", MinIndex: 64}))
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "wrong type")
+	rpc.AssertExpectations(t)
+}

agent/proxycfg/snapshot.go

@@ -19,6 +19,7 @@ type configSnapshotMeshGateway struct {
 	WatchedServices    map[string]context.CancelFunc
 	WatchedDatacenters map[string]context.CancelFunc
 	ServiceGroups      map[string]structs.CheckServiceNodes
+	ServiceResolvers   map[string]*structs.ServiceResolverConfigEntry
 	GatewayGroups      map[string]structs.CheckServiceNodes
 }
 

agent/proxycfg/state.go

@@ -22,6 +22,7 @@ const (
 	intentionsWatchID                = "intentions"
 	serviceListWatchID               = "service-list"
 	datacentersWatchID               = "datacenters"
+	serviceResolversWatchID          = "service-resolvers"
 	serviceIDPrefix                  = string(structs.UpstreamDestTypeService) + ":"
 	preparedQueryIDPrefix            = string(structs.UpstreamDestTypePreparedQuery) + ":"
 	defaultPreparedQueryPollInterval = 30 * time.Second
@@ -347,9 +348,10 @@ func (s *state) run() {
 	case structs.ServiceKindMeshGateway:
 		snap.MeshGateway.WatchedServices = make(map[string]context.CancelFunc)
 		snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc)
-		// TODO (mesh-gateway) - maybe reuse UpstreamEndpoints?
 		snap.MeshGateway.ServiceGroups = make(map[string]structs.CheckServiceNodes)
 		snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes)
+		// there is no need to initialize the map of service resolvers as we
+		// fully rebuild it every time we get updates
 	}
 
 	// This turns out to be really fiddly/painful by just using time.Timer.C
@@ -675,7 +677,17 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
 					return err
 				}
 
-				// TODO (mesh-gateway) also should watch the resolver config for the service here
+				err = s.cache.Notify(ctx, cachetype.ConfigEntriesName, &structs.ConfigEntryQuery{
+					Datacenter:   s.source.Datacenter,
+					QueryOptions: structs.QueryOptions{Token: s.token},
+					Kind:         structs.ServiceResolver,
+				}, serviceResolversWatchID, s.ch)
+
+				if err != nil {
+					s.logger.Printf("[ERR] mesh-gateway: failed to register watch for service-resolver config entries")
+					cancel()
+					return err
+				}
 				snap.MeshGateway.WatchedServices[svcName] = cancel
 			}
 		}
@@ -735,6 +747,19 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
 				cancelFn()
 			}
 		}
+	case serviceResolversWatchID:
+		configEntries, ok := u.Result.(*structs.IndexedConfigEntries)
+		if !ok {
+			return fmt.Errorf("invalid type for services response: %T", u.Result)
+		}
+
+		resolvers := make(map[string]*structs.ServiceResolverConfigEntry)
+		for _, entry := range configEntries.Entries {
+			if resolver, ok := entry.(*structs.ServiceResolverConfigEntry); ok {
+				resolvers[resolver.Name] = resolver
+			}
+		}
+		snap.MeshGateway.ServiceResolvers = resolvers
 	default:
 		switch {
 		case strings.HasPrefix(u.CorrelationID, "connect-service:"):

agent/proxycfg/testing.go

@@ -2,6 +2,7 @@ package proxycfg
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -85,7 +86,7 @@ func TestCerts(t testing.T) (*structs.IndexedCARoots, *structs.IssuedCert) {
 	ca := connect.TestCA(t, nil)
 	roots := &structs.IndexedCARoots{
 		ActiveRootID: ca.ID,
-		TrustDomain:  connect.TestClusterID,
+		TrustDomain:  fmt.Sprintf("%s.consul", connect.TestClusterID),
 		Roots:        []*structs.CARoot{ca},
 	}
 	return roots, TestLeafForCA(t, ca)
@@ -183,7 +184,75 @@ func TestGatewayNodesDC2(t testing.T) structs.CheckServiceNodes {
 	}
 }
 
-func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes {
+func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes {
+	return structs.CheckServiceNodes{
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "bar-node-1",
+				Node:       "bar-node-1",
+				Address:    "10.1.1.4",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "bar-sidecar-proxy",
+				Address: "172.16.1.6",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "bar",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "bar-node-2",
+				Node:       "bar-node-2",
+				Address:    "10.1.1.5",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "bar-sidecar-proxy",
+				Address: "172.16.1.7",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "bar",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "bar-node-3",
+				Node:       "bar-node-3",
+				Address:    "10.1.1.6",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "bar-sidecar-proxy",
+				Address: "172.16.1.8",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "2",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "bar",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+	}
+}
+
+func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes {
 	return structs.CheckServiceNodes{
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -192,7 +261,19 @@ func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.1.1.1",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeServiceProxy(t),
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.3",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
 		},
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -201,7 +282,69 @@ func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.1.1.2",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeServiceProxy(t),
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.4",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "foo-node-3",
+				Node:       "foo-node-3",
+				Address:    "10.1.1.3",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.5",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "2",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "foo-node-4",
+				Node:       "foo-node-4",
+				Address:    "10.1.1.7",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.9",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "2",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+			Checks: structs.HealthChecks{
+				&structs.HealthCheck{
+					Node:        "foo-node-4",
+					ServiceName: "foo-sidecar-proxy",
+					Name:        "proxy-alive",
+					Status:      "warning",
+				},
+			},
 		},
 	}
 }
@@ -268,7 +411,8 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
 				"dc2": nil,
 			},
 			ServiceGroups: map[string]structs.CheckServiceNodes{
-				"foo": TestGatewayServicesDC1(t),
+				"foo": TestGatewayServiceGroupFooDC1(t),
+				"bar": TestGatewayServiceGroupBarDC1(t),
 			},
 			GatewayGroups: map[string]structs.CheckServiceNodes{
 				"dc2": TestGatewayNodesDC2(t),

agent/structs/config_entry.go

@@ -387,6 +387,31 @@ func (c *ConfigEntryQuery) RequestDatacenter() string {
 	return c.Datacenter
 }
 
+func (r *ConfigEntryQuery) CacheInfo() cache.RequestInfo {
+	info := cache.RequestInfo{
+		Token:          r.Token,
+		Datacenter:     r.Datacenter,
+		MinIndex:       r.MinQueryIndex,
+		Timeout:        r.MaxQueryTime,
+		MaxAge:         r.MaxAge,
+		MustRevalidate: r.MustRevalidate,
+	}
+
+	v, err := hashstructure.Hash([]interface{}{
+		r.Kind,
+		r.Name,
+		r.Filter,
+	}, nil)
+	if err == nil {
+		// If there is an error, we don't set the key. A blank key forces
+		// no cache for this request so the request is forwarded directly
+		// to the server.
+		info.Key = strconv.FormatUint(v, 10)
+	}
+
+	return info
+}
+
 // ServiceConfigRequest is used when requesting the resolved configuration
 // for a service.
 type ServiceConfigRequest struct {

agent/xds/clusters.go

@@ -84,33 +84,42 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh
 // for a mesh gateway. This will include 1 cluster per remote datacenter as well as
 // 1 cluster for each service subset.
 func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) {
-	// TODO (mesh-gateway) will need to generate 1 cluster for each service subset as well.
+	// 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended)
+	clusters := make([]proto.Message, 0, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups))
 
-	// 1 cluster per remote dc + 1 cluster per local service
-	clusters := make([]proto.Message, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups))
-
-	var err error
-	idx := 0
 	// generate the remote dc clusters
 	for dc, _ := range cfgSnap.MeshGateway.GatewayGroups {
 		clusterName := DatacenterSNI(dc, cfgSnap)
 
-		clusters[idx], err = s.makeMeshGatewayCluster(clusterName, cfgSnap)
+		cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
 		if err != nil {
 			return nil, err
 		}
-		idx += 1
+		clusters = append(clusters, cluster)
 	}
 
 	// generate the per-service clusters
 	for svc, _ := range cfgSnap.MeshGateway.ServiceGroups {
-		clusterName := ServiceSNI(svc, "default", cfgSnap.Datacenter, cfgSnap)
+		clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap)
 
-		clusters[idx], err = s.makeMeshGatewayCluster(clusterName, cfgSnap)
+		cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
 		if err != nil {
 			return nil, err
 		}
-		idx += 1
+		clusters = append(clusters, cluster)
+	}
+
+	// generate the service subset clusters
+	for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
+		for subsetName, _ := range resolver.Subsets {
+			clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap)
+
+			cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
+			if err != nil {
+				return nil, err
+			}
+			clusters = append(clusters, cluster)
+		}
 	}
 
 	return clusters, nil
@@ -174,7 +183,7 @@ func (s *Server) makeUpstreamCluster(upstream structs.Upstream, cfgSnap *proxycf
 	if upstream.Datacenter != "" {
 		dc = upstream.Datacenter
 	}
-	sni := ServiceSNI(upstream.DestinationName, ns, dc, cfgSnap)
+	sni := ServiceSNI(upstream.DestinationName, "", ns, dc, cfgSnap)
 
 	cfg, err := ParseUpstreamConfig(upstream.Config)
 	if err != nil {

agent/xds/clusters_test.go

@@ -9,6 +9,7 @@ import (
 	"text/template"
 
 	"github.com/hashicorp/consul/agent/proxycfg"
+	"github.com/hashicorp/consul/agent/structs"
 	testinf "github.com/mitchellh/go-testing-interface"
 	"github.com/stretchr/testify/require"
 )
@@ -102,6 +103,27 @@ func TestClustersFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotMeshGateway,
 			setup:  nil,
 		},
+		{
+			name:   "mesh-gateway-service-subsets",
+			create: proxycfg.TestConfigSnapshotMeshGateway,
+			setup: func(snap *proxycfg.ConfigSnapshot) {
+				snap.MeshGateway.ServiceResolvers = map[string]*structs.ServiceResolverConfigEntry{
+					"bar": &structs.ServiceResolverConfigEntry{
+						Kind: structs.ServiceResolver,
+						Name: "bar",
+						Subsets: map[string]structs.ServiceResolverSubset{
+							"v1": structs.ServiceResolverSubset{
+								Filter: "Service.Meta.Version == 1",
+							},
+							"v2": structs.ServiceResolverSubset{
+								Filter:      "Service.Meta.Version == 2",
+								OnlyPassing: true,
+							},
+						},
+					},
+				}
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -190,7 +212,7 @@ func expectClustersJSONResources(t *testing.T, snap *proxycfg.ConfigSnapshot, to
 
 				},
 				"connectTimeout": "1s",
-				"tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "db.default.dc1.internal.11111111-2222-3333-4444-555555555555") + `
+				"tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul") + `
 			}`,
 		"prepared_query:geo-cache": `
 			{
@@ -208,7 +230,7 @@ func expectClustersJSONResources(t *testing.T, snap *proxycfg.ConfigSnapshot, to
 
 				},
 				"connectTimeout": "5s",
-				"tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555") + `
+				"tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul") + `
 			}`,
 	}
 }

agent/xds/endpoints.go

@@ -12,6 +12,8 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
+
+	bexpr "github.com/hashicorp/go-bexpr"
 )
 
 // endpointsFromSnapshot returns the xDS API representation of the "endpoints"
@@ -143,7 +145,7 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
 
 	// generate the endpoints for the local service groups
 	for svc, endpoints := range cfgSnap.MeshGateway.ServiceGroups {
-		clusterName := ServiceSNI(svc, "default", cfgSnap.Datacenter, cfgSnap)
+		clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap)
 		la := makeLoadAssignment(
 			clusterName,
 			0,
@@ -155,6 +157,51 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
 		resources = append(resources, la)
 	}
 
+	// generate the endpoints for the service subsets
+	for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
+		for subsetName, subset := range resolver.Subsets {
+			clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap)
+
+			endpoints := cfgSnap.MeshGateway.ServiceGroups[svc]
+
+			// locally execute the subsets filter
+			filterExp := subset.Filter
+			if subset.OnlyPassing {
+				// we could do another filter pass without bexpr but this simplifies things a bit
+				if filterExp != "" {
+					// TODO (filtering) - Update to "and all Checks as chk { chk.Status == passing }"
+					//                    once the syntax is supported
+					filterExp = fmt.Sprintf("(%s) and not Checks.Status != passing", filterExp)
+				} else {
+					filterExp = "not Checks.Status != passing"
+				}
+			}
+
+			if filterExp != "" {
+				filter, err := bexpr.CreateFilter(filterExp, nil, endpoints)
+				if err != nil {
+					return nil, err
+				}
+
+				raw, err := filter.Execute(endpoints)
+				if err != nil {
+					return nil, err
+				}
+				endpoints = raw.(structs.CheckServiceNodes)
+			}
+
+			la := makeLoadAssignment(
+				clusterName,
+				0,
+				[]structs.CheckServiceNodes{
+					endpoints,
+				},
+				cfgSnap.Datacenter,
+			)
+			resources = append(resources, la)
+		}
+	}
+
 	return resources, nil
 }
 

agent/xds/endpoints_test.go

@@ -233,6 +233,40 @@ func Test_endpointsFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotMeshGateway,
 			setup:  nil,
 		},
+		{
+			name:   "mesh-gateway-service-subsets",
+			create: proxycfg.TestConfigSnapshotMeshGateway,
+			setup: func(snap *proxycfg.ConfigSnapshot) {
+				snap.MeshGateway.ServiceResolvers = map[string]*structs.ServiceResolverConfigEntry{
+					"bar": &structs.ServiceResolverConfigEntry{
+						Kind: structs.ServiceResolver,
+						Name: "bar",
+						Subsets: map[string]structs.ServiceResolverSubset{
+							"v1": structs.ServiceResolverSubset{
+								Filter: "Service.Meta.version == 1",
+							},
+							"v2": structs.ServiceResolverSubset{
+								Filter:      "Service.Meta.version == 2",
+								OnlyPassing: true,
+							},
+						},
+					},
+					"foo": &structs.ServiceResolverConfigEntry{
+						Kind: structs.ServiceResolver,
+						Name: "foo",
+						Subsets: map[string]structs.ServiceResolverSubset{
+							"v1": structs.ServiceResolverSubset{
+								Filter: "Service.Meta.version == 1",
+							},
+							"v2": structs.ServiceResolverSubset{
+								Filter:      "Service.Meta.version == 2",
+								OnlyPassing: true,
+							},
+						},
+					},
+				}
+			},
+		},
 	}
 
 	for _, tt := range tests {

agent/xds/sni.go

@@ -10,7 +10,10 @@ func DatacenterSNI(dc string, cfgSnap *proxycfg.ConfigSnapshot) string {
 	return fmt.Sprintf("%s.internal.%s", dc, cfgSnap.Roots.TrustDomain)
 }
 
-func ServiceSNI(service string, namespace string, datacenter string, cfgSnap *proxycfg.ConfigSnapshot) string {
+func ServiceSNI(service string, subset string, namespace string, datacenter string, cfgSnap *proxycfg.ConfigSnapshot) string {
-	// TODO (mesh-gateway) - support service subsets here too
+	if subset == "" {
-	return fmt.Sprintf("%s.%s.%s.internal.%s", service, namespace, datacenter, cfgSnap.Roots.TrustDomain)
+		return fmt.Sprintf("%s.%s.%s.internal.%s", service, namespace, datacenter, cfgSnap.Roots.TrustDomain)
+	} else {
+		return fmt.Sprintf("%s.%s.%s.%s.internal.%s", subset, service, namespace, datacenter, cfgSnap.Roots.TrustDomain)
+	}
 }

agent/xds/testdata/clusters/custom-local-app.golden

@@ -47,7 +47,7 @@
             }
           }
         },
-        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 
@@ -86,7 +86,7 @@
             }
           }
         },
-        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 

agent/xds/testdata/clusters/custom-timeouts.golden

@@ -59,7 +59,7 @@
             }
           }
         },
-        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 
@@ -98,7 +98,7 @@
             }
           }
         },
-        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 

agent/xds/testdata/clusters/custom-upstream.golden

@@ -59,7 +59,7 @@
             }
           }
         },
-        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       }
     },
     {
@@ -95,7 +95,7 @@
             }
           }
         },
-        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 

agent/xds/testdata/clusters/defaults.golden

@@ -59,7 +59,7 @@
             }
           }
         },
-        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 
@@ -98,7 +98,7 @@
             }
           }
         },
-        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555"
+        "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
       },
       "outlierDetection": {
 

agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden

@@ -0,0 +1,87 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster",
+  "nonce": "00000001"
+}

agent/xds/testdata/clusters/mesh-gateway.golden

@@ -3,7 +3,7 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.Cluster",
-      "name": "dc2.internal.11111111-2222-3333-4444-555555555555",
+      "name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
       "type": "EDS",
       "edsClusterConfig": {
         "edsConfig": {
@@ -19,7 +19,23 @@
     },
     {
       "@type": "type.googleapis.com/envoy.api.v2.Cluster",
-      "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
       "type": "EDS",
       "edsClusterConfig": {
         "edsConfig": {

agent/xds/testdata/endpoints/mesh-gateway-service-subsets.golden

@@ -0,0 +1,257 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.1",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.2",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.3",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.9",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v1.foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.3",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v2.foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+  "nonce": "00000001"
+}

agent/xds/testdata/endpoints/mesh-gateway.golden

@@ -3,7 +3,7 @@
   "resources": [
     {
       "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
-      "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555",
+      "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
       "endpoints": [
         {
           "lbEndpoints": [
@@ -37,7 +37,7 @@
     },
     {
       "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
-      "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
       "endpoints": [
         {
           "lbEndpoints": [
@@ -45,7 +45,7 @@
               "endpoint": {
                 "address": {
                   "socketAddress": {
-                    "address": "127.0.0.2",
+                    "address": "172.16.1.3",
                     "portValue": 2222
                   }
                 }
@@ -57,7 +57,77 @@
               "endpoint": {
                 "address": {
                   "socketAddress": {
-                    "address": "127.0.0.2",
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.9",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
                     "portValue": 2222
                   }
                 }

agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden

@@ -14,14 +14,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_bar_dc2_tcp"
                 }
             }
@@ -61,14 +61,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_baz_dc2_tcp"
                 }
             }
@@ -108,14 +108,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_default_dc2_tcp"
                 }
             }
@@ -155,14 +155,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_foo_dc2_tcp"
                 }
             }

agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden

@@ -14,14 +14,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_lan_dc2_tcp"
                 }
             }
@@ -61,14 +61,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_wan_dc2_tcp"
                 }
             }

agent/xds/testdata/listeners/mesh-gateway.golden

@@ -14,14 +14,14 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "*.dc2.internal.11111111-2222-3333-4444-555555555555"
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
             {
               "name": "envoy.tcp_proxy",
               "config": {
-                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555",
+                  "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
                   "stat_prefix": "mesh_gateway_remote_default_dc2_tcp"
                 }
             }