From 2f08fab31785553567639ea313510eb32ef51daa Mon Sep 17 00:00:00 2001 From: Eric Haberkorn Date: Thu, 6 Oct 2022 09:54:14 -0400 Subject: [PATCH] Make the mesh gateway changes to allow `local` mode for cluster peering data plane traffic (#14817) Make the mesh gateway changes to allow `local` mode for cluster peering data plane traffic --- .changelog/14817.txt | 3 + agent/proxycfg/mesh_gateway.go | 132 ++++++++++++++++++ agent/proxycfg/snapshot.go | 18 +++ agent/proxycfg/state.go | 1 + agent/proxycfg/state_test.go | 47 ++++++- agent/proxycfg/testing.go | 25 ---- agent/proxycfg/testing_mesh_gateway.go | 42 ++++++ agent/proxycfg/testing_upstreams.go | 4 +- agent/structs/testing_catalog.go | 39 ++++-- agent/xds/clusters.go | 63 +++++++++ agent/xds/endpoints.go | 42 ++++++ agent/xds/resources_test.go | 6 + ...and-failover-to-cluster-peer.latest.golden | 4 +- ...and-redirect-to-cluster-peer.latest.golden | 4 +- ...and-failover-to-cluster-peer.latest.golden | 4 +- ...ults-service-max-connections.latest.golden | 2 +- ...ults-service-max-connections.latest.golden | 2 +- ...with-service-max-connections.latest.golden | 2 +- ...ith-imported-peered-services.latest.golden | 64 +++++++++ ...and-failover-to-cluster-peer.latest.golden | 12 -- ...and-redirect-to-cluster-peer.latest.golden | 12 -- ...and-failover-to-cluster-peer.latest.golden | 12 -- ...ith-imported-peered-services.latest.golden | 41 ++++++ ...-balance-inbound-connections.latest.golden | 4 +- ...tbound-connections-bind-port.latest.golden | 4 +- ...ith-imported-peered-services.latest.golden | 45 ++++++ ...ith-imported-peered-services.latest.golden | 5 + .../case-cross-peers/primary/service_s1.hcl | 3 + .../case-cross-peers/primary/verify.bats | 4 + 29 files changed, 562 insertions(+), 84 deletions(-) create mode 100644 .changelog/14817.txt create mode 100644 agent/xds/testdata/clusters/mesh-gateway-with-imported-peered-services.latest.golden create mode 100644 agent/xds/testdata/endpoints/mesh-gateway-with-imported-peered-services.latest.golden create mode 100644 agent/xds/testdata/listeners/mesh-gateway-with-imported-peered-services.latest.golden create mode 100644 agent/xds/testdata/routes/mesh-gateway-with-imported-peered-services.latest.golden diff --git a/.changelog/14817.txt b/.changelog/14817.txt new file mode 100644 index 000000000..7a695c164 --- /dev/null +++ b/.changelog/14817.txt @@ -0,0 +1,3 @@ +```release-note:feature +peering: Add mesh gateway local mode support for cluster peering. +``` diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index bd9f14042..9b134c3e6 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -120,6 +120,9 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er snap.MeshGateway.ExportedServicesWithPeers = make(map[structs.ServiceName][]string) snap.MeshGateway.DiscoveryChain = make(map[structs.ServiceName]*structs.CompiledDiscoveryChain) snap.MeshGateway.WatchedDiscoveryChains = make(map[structs.ServiceName]context.CancelFunc) + snap.MeshGateway.WatchedPeeringServices = make(map[string]map[structs.ServiceName]context.CancelFunc) + snap.MeshGateway.WatchedPeers = make(map[string]context.CancelFunc) + snap.MeshGateway.PeeringServices = make(map[string]map[structs.ServiceName]PeeringServiceValue) // there is no need to initialize the map of service resolvers as we // fully rebuild it every time we get updates @@ -460,6 +463,52 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u UpdateEvent, sn } snap.MeshGateway.PeeringTrustBundlesSet = true + wildcardEntMeta := s.proxyID.WithWildcardNamespace() + + // For each peer, fetch the imported services to support mesh gateway local + // mode. + for _, tb := range resp.Bundles { + entMeta := structs.DefaultEnterpriseMetaInDefaultPartition() + + if _, ok := snap.MeshGateway.WatchedPeers[tb.PeerName]; !ok { + ctx, cancel := context.WithCancel(ctx) + + err := s.dataSources.ServiceList.Notify(ctx, &structs.DCSpecificRequest{ + PeerName: tb.PeerName, + QueryOptions: structs.QueryOptions{Token: s.token}, + Source: *s.source, + EnterpriseMeta: *wildcardEntMeta, + }, peeringServiceListWatchID+tb.PeerName, s.ch) + + if err != nil { + meshLogger.Error("failed to register watch for mesh-gateway", + "peer", tb.PeerName, + "partition", entMeta.PartitionOrDefault(), + "error", err, + ) + cancel() + return err + } + snap.MeshGateway.WatchedPeers[tb.PeerName] = cancel + } + } + + for peerName, cancelFn := range snap.MeshGateway.WatchedPeers { + found := false + for _, bundle := range resp.Bundles { + if peerName == bundle.PeerName { + found = true + break + } + } + if !found { + delete(snap.MeshGateway.PeeringServices, peerName) + delete(snap.MeshGateway.WatchedPeers, peerName) + delete(snap.MeshGateway.WatchedPeeringServices, peerName) + cancelFn() + } + } + case meshConfigEntryID: resp, ok := u.Result.(*structs.ConfigEntryResponse) if !ok { @@ -517,6 +566,57 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u UpdateEvent, sn default: switch { + case strings.HasPrefix(u.CorrelationID, peeringServiceListWatchID): + services, ok := u.Result.(*structs.IndexedServiceList) + if !ok { + return fmt.Errorf("invalid type for response: %T", u.Result) + } + + peerName := strings.TrimPrefix(u.CorrelationID, peeringServiceListWatchID) + + svcMap := make(map[structs.ServiceName]struct{}) + + if _, ok := snap.MeshGateway.WatchedPeeringServices[peerName]; !ok { + snap.MeshGateway.WatchedPeeringServices[peerName] = make(map[structs.ServiceName]context.CancelFunc) + } + + for _, svc := range services.Services { + // Make sure to add every service to this map, we use it to cancel + // watches below. + svcMap[svc] = struct{}{} + + if _, ok := snap.MeshGateway.WatchedPeeringServices[peerName][svc]; !ok { + ctx, cancel := context.WithCancel(ctx) + err := s.dataSources.Health.Notify(ctx, &structs.ServiceSpecificRequest{ + PeerName: peerName, + QueryOptions: structs.QueryOptions{Token: s.token}, + ServiceName: svc.Name, + Connect: true, + EnterpriseMeta: svc.EnterpriseMeta, + }, fmt.Sprintf("peering-connect-service:%s:%s", peerName, svc.String()), s.ch) + + if err != nil { + meshLogger.Error("failed to register watch for connect-service", + "service", svc.String(), + "error", err, + ) + cancel() + return err + } + snap.MeshGateway.WatchedPeeringServices[peerName][svc] = cancel + } + } + + watchedServices := snap.MeshGateway.WatchedPeeringServices[peerName] + for sn, cancelFn := range watchedServices { + if _, ok := svcMap[sn]; !ok { + meshLogger.Debug("canceling watch for service", "service", sn.String()) + delete(snap.MeshGateway.WatchedPeeringServices[peerName], sn) + delete(snap.MeshGateway.PeeringServices[peerName], sn) + cancelFn() + } + } + case strings.HasPrefix(u.CorrelationID, "connect-service:"): resp, ok := u.Result.(*structs.IndexedCheckServiceNodes) if !ok { @@ -530,6 +630,38 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u UpdateEvent, sn } else if _, ok := snap.MeshGateway.ServiceGroups[sn]; ok { delete(snap.MeshGateway.ServiceGroups, sn) } + case strings.HasPrefix(u.CorrelationID, "peering-connect-service:"): + resp, ok := u.Result.(*structs.IndexedCheckServiceNodes) + + if !ok { + return fmt.Errorf("invalid type for response: %T", u.Result) + } + + key := strings.TrimPrefix(u.CorrelationID, "peering-connect-service:") + peer, snString, ok := strings.Cut(key, ":") + + if ok { + sn := structs.ServiceNameFromString(snString) + + if len(resp.Nodes) > 0 { + if _, ok := snap.MeshGateway.PeeringServices[peer]; !ok { + snap.MeshGateway.PeeringServices[peer] = make(map[structs.ServiceName]PeeringServiceValue) + } + + if eps := hostnameEndpoints(s.logger, GatewayKey{}, resp.Nodes); len(eps) > 0 { + snap.MeshGateway.PeeringServices[peer][sn] = PeeringServiceValue{ + Nodes: eps, + UseCDS: true, + } + } else { + snap.MeshGateway.PeeringServices[peer][sn] = PeeringServiceValue{ + Nodes: resp.Nodes, + } + } + } else if _, ok := snap.MeshGateway.PeeringServices[peer]; ok { + delete(snap.MeshGateway.PeeringServices[peer], sn) + } + } case strings.HasPrefix(u.CorrelationID, "mesh-gateway:"): resp, ok := u.Result.(*structs.IndexedCheckServiceNodes) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 4a0bbc1b7..a52d568cc 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -348,6 +348,11 @@ func (c *configSnapshotTerminatingGateway) isEmpty() bool { !c.MeshConfigSet } +type PeeringServiceValue struct { + Nodes structs.CheckServiceNodes + UseCDS bool +} + type configSnapshotMeshGateway struct { // WatchedServices is a map of service name to a cancel function. This cancel // function is tied to the watch of connect enabled services for the given @@ -373,6 +378,19 @@ type configSnapshotMeshGateway struct { // service in the local datacenter. ServiceGroups map[structs.ServiceName]structs.CheckServiceNodes + // PeeringServices is a map of peer name -> (map of + // service name -> CheckServiceNodes) and is used to determine the backing + // endpoints of a service on a peer. + PeeringServices map[string]map[structs.ServiceName]PeeringServiceValue + + // WatchedPeeringServices is a map of peer name -> (map of service name -> + // cancel function) and is used to track watches on services within a peer. + WatchedPeeringServices map[string]map[structs.ServiceName]context.CancelFunc + + // WatchedPeers is a map of peer name -> cancel functions. It is used to + // track watches on peers. + WatchedPeers map[string]context.CancelFunc + // ServiceResolvers is a map of service name to an associated // service-resolver config entry for that service. ServiceResolvers map[structs.ServiceName]*structs.ServiceResolverConfigEntry diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index e069e80ee..52df57428 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -25,6 +25,7 @@ const ( peerTrustBundleIDPrefix = "peer-trust-bundle:" intentionsWatchID = "intentions" serviceListWatchID = "service-list" + peeringServiceListWatchID = "peering-service-list:" federationStateListGatewaysWatchID = "federation-state-list-mesh-gateways" consulServerListWatchID = "consul-server-list" datacentersWatchID = "datacenters" diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 3add369a8..7e5675e97 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -830,6 +830,9 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Empty(t, snap.MeshGateway.ServiceGroups) require.Empty(t, snap.MeshGateway.ServiceResolvers) require.Empty(t, snap.MeshGateway.GatewayGroups) + require.Empty(t, snap.MeshGateway.WatchedPeeringServices) + require.Empty(t, snap.MeshGateway.WatchedPeers) + require.Empty(t, snap.MeshGateway.PeeringServices) }, }, { @@ -897,8 +900,22 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, { CorrelationID: peeringTrustBundlesWatchID, - Result: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: nil, + Result: peerTrustBundles, + }, + { + CorrelationID: peeringServiceListWatchID + "peer-a", + Result: &structs.IndexedServiceList{ + Services: structs.ServiceList{ + {Name: "service-1"}, + }, + }, + }, + { + CorrelationID: peeringServiceListWatchID + "peer-b", + Result: &structs.IndexedServiceList{ + Services: structs.ServiceList{ + {Name: "service-10"}, + }, }, }, }, @@ -906,6 +923,10 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.True(t, snap.Valid(), "gateway with service list is valid") require.Len(t, snap.MeshGateway.WatchedServices, 1) require.True(t, snap.MeshGateway.WatchedServicesSet) + require.Len(t, snap.MeshGateway.WatchedPeers, 2) + require.Len(t, snap.MeshGateway.WatchedPeeringServices, 2) + require.Len(t, snap.MeshGateway.WatchedPeeringServices["peer-a"], 1) + require.Len(t, snap.MeshGateway.WatchedPeeringServices["peer-b"], 1) }, }, { @@ -920,11 +941,33 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, Err: nil, }, + { + CorrelationID: peeringServiceListWatchID + "peer-a", + Result: &structs.IndexedServiceList{ + Services: structs.ServiceList{ + {Name: "service-1"}, + {Name: "service-2"}, + {Name: "service-3"}, + }, + }, + Err: nil, + }, + { + CorrelationID: peeringServiceListWatchID + "peer-b", + Result: &structs.IndexedServiceList{ + Services: structs.ServiceList{}, + }, + Err: nil, + }, }, verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { require.True(t, snap.Valid(), "gateway with service list is valid") require.Len(t, snap.MeshGateway.WatchedServices, 2) require.True(t, snap.MeshGateway.WatchedServicesSet) + require.Len(t, snap.MeshGateway.WatchedPeers, 2) + require.Len(t, snap.MeshGateway.WatchedPeeringServices, 2) + require.Len(t, snap.MeshGateway.WatchedPeeringServices["peer-a"], 3) + require.Len(t, snap.MeshGateway.WatchedPeeringServices["peer-b"], 0) }, }, { diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index d5a3d8224..6382deb66 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -280,31 +280,6 @@ func TestUpstreamNodesDC2(t testing.T) structs.CheckServiceNodes { } } -func TestUpstreamNodesPeerCluster01(t testing.T) structs.CheckServiceNodes { - peer := "cluster-01" - service := structs.TestNodeServiceWithNameInPeer(t, "web", peer) - return structs.CheckServiceNodes{ - structs.CheckServiceNode{ - Node: &structs.Node{ - ID: "test1", - Node: "test1", - Address: "10.40.1.1", - PeerName: peer, - }, - Service: service, - }, - structs.CheckServiceNode{ - Node: &structs.Node{ - ID: "test2", - Node: "test2", - Address: "10.40.1.2", - PeerName: peer, - }, - Service: service, - }, - } -} - func TestUpstreamNodesInStatusDC2(t testing.T, status string) structs.CheckServiceNodes { return structs.CheckServiceNodes{ structs.CheckServiceNode{ diff --git a/agent/proxycfg/testing_mesh_gateway.go b/agent/proxycfg/testing_mesh_gateway.go index d5868d58e..49205fe14 100644 --- a/agent/proxycfg/testing_mesh_gateway.go +++ b/agent/proxycfg/testing_mesh_gateway.go @@ -632,6 +632,48 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( }, }, ) + case "imported-services": + peerTrustBundles := TestPeerTrustBundles(t).Bundles + dbSN := structs.NewServiceName("db", nil) + altSN := structs.NewServiceName("alt", nil) + extraUpdates = append(extraUpdates, + UpdateEvent{ + CorrelationID: peeringTrustBundlesWatchID, + Result: &pbpeering.TrustBundleListByServiceResponse{ + Bundles: peerTrustBundles, + }, + }, + UpdateEvent{ + CorrelationID: peeringServiceListWatchID + "peer-a", + Result: &structs.IndexedServiceList{ + Services: []structs.ServiceName{altSN}, + }, + }, + UpdateEvent{ + CorrelationID: peeringServiceListWatchID + "peer-b", + Result: &structs.IndexedServiceList{ + Services: []structs.ServiceName{dbSN}, + }, + }, + UpdateEvent{ + CorrelationID: "peering-connect-service:peer-a:db", + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + structs.TestCheckNodeServiceWithNameInPeer(t, "db", "peer-a", "10.40.1.1", false), + structs.TestCheckNodeServiceWithNameInPeer(t, "db", "peer-a", "10.40.1.2", false), + }, + }, + }, + UpdateEvent{ + CorrelationID: "peering-connect-service:peer-b:alt", + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "peer-b", "10.40.2.1", false), + structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "peer-b", "10.40.2.2", true), + }, + }, + }, + ) case "chain-and-l7-stuff": entries = []structs.ConfigEntry{ &structs.ProxyConfigEntry{ diff --git a/agent/proxycfg/testing_upstreams.go b/agent/proxycfg/testing_upstreams.go index 25679c692..bbf541f40 100644 --- a/agent/proxycfg/testing_upstreams.go +++ b/agent/proxycfg/testing_upstreams.go @@ -88,7 +88,7 @@ func setupTestVariationConfigEntriesAndSnapshot( events = append(events, UpdateEvent{ CorrelationID: "upstream-peer:db?peer=cluster-01", Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestUpstreamNodesPeerCluster01(t), + Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "cluster-01", "10.40.1.1", false)}, }, }) case "redirect-to-cluster-peer": @@ -106,7 +106,7 @@ func setupTestVariationConfigEntriesAndSnapshot( events = append(events, UpdateEvent{ CorrelationID: "upstream-peer:db?peer=cluster-01", Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestUpstreamNodesPeerCluster01(t), + Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "cluster-01", "10.40.1.1", false)}, }, }) case "failover-through-double-remote-gateway-triggered": diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index f026f6091..113169051 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -55,24 +55,47 @@ func TestNodeServiceWithName(t testing.T, name string) *NodeService { const peerTrustDomain = "1c053652-8512-4373-90cf-5a7f6263a994.consul" -func TestNodeServiceWithNameInPeer(t testing.T, name string, peer string) *NodeService { - service := "payments" - return &NodeService{ - Kind: ServiceKindTypical, - Service: name, - Port: 8080, +func TestCheckNodeServiceWithNameInPeer(t testing.T, name, peer, ip string, useHostname bool) CheckServiceNode { + service := &NodeService{ + Kind: ServiceKindTypical, + Service: name, + Port: 8080, + PeerName: peer, Connect: ServiceConnect{ PeerMeta: &PeeringServiceMeta{ SNI: []string{ - service + ".default.default." + peer + ".external." + peerTrustDomain, + name + ".default.default." + peer + ".external." + peerTrustDomain, }, SpiffeID: []string{ - "spiffe://" + peerTrustDomain + "/ns/default/dc/" + peer + "-dc/svc/" + service, + "spiffe://" + peerTrustDomain + "/ns/default/dc/" + peer + "-dc/svc/" + name, }, Protocol: "tcp", }, }, } + + if useHostname { + service.TaggedAddresses = map[string]ServiceAddress{ + TaggedAddressLAN: { + Address: ip, + Port: 443, + }, + TaggedAddressWAN: { + Address: name + ".us-east-1.elb.notaws.com", + Port: 8443, + }, + } + } + + return CheckServiceNode{ + Node: &Node{ + ID: "test1", + Node: "test1", + Address: ip, + Datacenter: "cloud-dc", + }, + Service: service, + } } // TestNodeServiceProxy returns a *NodeService representing a valid diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 6e34ce2e1..4604c69e8 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -418,6 +418,13 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co } clusters = append(clusters, c...) + // generate the outgoing clusters for imported peer services. + c, err = s.makeGatewayOutgoingClusterPeeringServiceClusters(cfgSnap) + if err != nil { + return nil, err + } + clusters = append(clusters, c...) + // Generate per-target clusters for all exported discovery chains. c, err = s.makeExportedUpstreamClustersForMeshGateway(cfgSnap) if err != nil { @@ -559,6 +566,62 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( return clusters, nil } +func (s *ResourceGenerator) makeGatewayOutgoingClusterPeeringServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { + if cfgSnap.Kind != structs.ServiceKindMeshGateway { + return nil, fmt.Errorf("unsupported gateway kind %q", cfgSnap.Kind) + } + + var clusters []proto.Message + + for _, serviceGroups := range cfgSnap.MeshGateway.PeeringServices { + for sn, serviceGroup := range serviceGroups { + if len(serviceGroup.Nodes) == 0 { + continue + } + + node := serviceGroup.Nodes[0] + if node.Service == nil { + return nil, fmt.Errorf("couldn't get SNI for peered service %s", sn.String()) + } + // This uses the SNI in the accepting cluster peer so the remote mesh + // gateway can distinguish between an exported service as opposed to the + // usual mesh gateway route for a service. + clusterName := node.Service.Connect.PeerMeta.PrimarySNI() + + opts := clusterOpts{ + name: clusterName, + isRemote: true, + } + cluster := s.makeGatewayCluster(cfgSnap, opts) + + if serviceGroup.UseCDS { + configureClusterWithHostnames( + s.Logger, + cluster, + "", /*TODO:make configurable?*/ + serviceGroup.Nodes, + true, /*isRemote*/ + false, /*onlyPassing*/ + ) + } else { + cluster.ClusterDiscoveryType = &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS} + cluster.EdsClusterConfig = &envoy_cluster_v3.Cluster_EdsClusterConfig{ + EdsConfig: &envoy_core_v3.ConfigSource{ + ResourceApiVersion: envoy_core_v3.ApiVersion_V3, + ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{ + Ads: &envoy_core_v3.AggregatedConfigSource{}, + }, + }, + } + } + + clusters = append(clusters, cluster) + } + } + + return clusters, nil +} + func (s *ResourceGenerator) makeDestinationClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { serviceConfigs := cfgSnap.TerminatingGateway.ServiceConfigs diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index c92a78526..6a6f8c844 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -342,6 +342,13 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C } resources = append(resources, e...) + // generate the outgoing endpoints for imported peer services. + e, err = s.makeEndpointsForOutgoingPeeredServices(cfgSnap) + if err != nil { + return nil, err + } + resources = append(resources, e...) + return resources, nil } @@ -398,6 +405,41 @@ func (s *ResourceGenerator) endpointsFromServicesAndResolvers( return resources, nil } +func (s *ResourceGenerator) makeEndpointsForOutgoingPeeredServices( + cfgSnap *proxycfg.ConfigSnapshot, +) ([]proto.Message, error) { + var resources []proto.Message + + // generate the endpoints for the linked service groups + for _, serviceGroups := range cfgSnap.MeshGateway.PeeringServices { + for sn, serviceGroup := range serviceGroups { + if serviceGroup.UseCDS || len(serviceGroup.Nodes) == 0 { + continue + } + + node := serviceGroup.Nodes[0] + if node.Service == nil { + return nil, fmt.Errorf("couldn't get SNI for peered service %s", sn.String()) + } + // This uses the SNI in the accepting cluster peer so the remote mesh + // gateway can distinguish between an exported service as opposed to the + // usual mesh gateway route for a service. + clusterName := node.Service.Connect.PeerMeta.PrimarySNI() + + groups := []loadAssignmentEndpointGroup{{Endpoints: serviceGroup.Nodes, OnlyPassing: false}} + + la := makeLoadAssignment( + clusterName, + groups, + cfgSnap.Locality, + ) + resources = append(resources, la) + } + } + + return resources, nil +} + func (s *ResourceGenerator) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var resources []proto.Message createdClusters := make(map[proxycfg.UpstreamID]bool) diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 6666b6454..f7c6e9d75 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -225,6 +225,12 @@ func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase { return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "control-plane", nil, nil) }, }, + { + name: "mesh-gateway-with-imported-peered-services", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "imported-services", nil, nil) + }, + }, } } diff --git a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden index 61de6b2e2..78d23803d 100644 --- a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -66,12 +66,12 @@ }, "matchSubjectAltNames": [ { - "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/payments" + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/db" } ] } }, - "sni": "payments.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" } } }, diff --git a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden index 0bd578a1c..a797d7e89 100644 --- a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -49,12 +49,12 @@ }, "matchSubjectAltNames": [ { - "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/payments" + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/db" } ] } }, - "sni": "payments.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" } } }, diff --git a/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden index 94521dc8f..5f75c93a3 100644 --- a/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden @@ -66,12 +66,12 @@ }, "matchSubjectAltNames": [ { - "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/payments" + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/db" } ] } }, - "sni": "payments.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" } } }, diff --git a/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden b/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden index 08d2c471e..24bb5835d 100644 --- a/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden +++ b/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden @@ -16,7 +16,7 @@ }, "connectTimeout": "33s", "circuitBreakers": { - "thresholds":[ + "thresholds": [ { "maxConnections": 2048, "maxPendingRequests": 512, diff --git a/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden b/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden index 61101551c..2ea775950 100644 --- a/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden +++ b/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden @@ -16,7 +16,7 @@ }, "connectTimeout": "33s", "circuitBreakers": { - "thresholds":[ + "thresholds": [ { "maxConnections": 4096, "maxPendingRequests": 2048 diff --git a/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden b/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden index 6ca8d60c6..3ac392d40 100644 --- a/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden +++ b/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden @@ -16,7 +16,7 @@ }, "connectTimeout": "33s", "circuitBreakers": { - "thresholds":[ + "thresholds": [ { "maxConnections": 4096 } diff --git a/agent/xds/testdata/clusters/mesh-gateway-with-imported-peered-services.latest.golden b/agent/xds/testdata/clusters/mesh-gateway-with-imported-peered-services.latest.golden new file mode 100644 index 000000000..27761e811 --- /dev/null +++ b/agent/xds/testdata/clusters/mesh-gateway-with-imported-peered-services.latest.golden @@ -0,0 +1,64 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "alt.default.default.peer-b.external.1c053652-8512-4373-90cf-5a7f6263a994.consul", + "type": "LOGICAL_DNS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "alt.default.default.peer-b.external.1c053652-8512-4373-90cf-5a7f6263a994.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "alt.us-east-1.elb.notaws.com", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.default.peer-a.external.1c053652-8512-4373-90cf-5a7f6263a994.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden index feaea9055..5b767ac86 100644 --- a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -18,18 +18,6 @@ }, "healthStatus": "HEALTHY", "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.40.1.2", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 } ] } diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden index 830d3941e..8cb6ce20a 100644 --- a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -18,18 +18,6 @@ }, "healthStatus": "HEALTHY", "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.40.1.2", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 } ] } diff --git a/agent/xds/testdata/endpoints/ingress-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/endpoints/ingress-with-chain-and-failover-to-cluster-peer.latest.golden index c799a5a0c..d919f8195 100644 --- a/agent/xds/testdata/endpoints/ingress-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/endpoints/ingress-with-chain-and-failover-to-cluster-peer.latest.golden @@ -18,18 +18,6 @@ }, "healthStatus": "HEALTHY", "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.40.1.2", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 } ] } diff --git a/agent/xds/testdata/endpoints/mesh-gateway-with-imported-peered-services.latest.golden b/agent/xds/testdata/endpoints/mesh-gateway-with-imported-peered-services.latest.golden new file mode 100644 index 000000000..efb3c588b --- /dev/null +++ b/agent/xds/testdata/endpoints/mesh-gateway-with-imported-peered-services.latest.golden @@ -0,0 +1,41 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.default.peer-a.external.1c053652-8512-4373-90cf-5a7f6263a994.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.40.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.40.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden b/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden index 9c8b0a581..c04122640 100644 --- a/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden +++ b/agent/xds/testdata/listeners/listener-balance-inbound-connections.latest.golden @@ -113,7 +113,9 @@ ], "trafficDirection": "INBOUND", "connectionBalanceConfig": { - "exactBalance": {} + "exactBalance": { + + } } } ], diff --git a/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden b/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden index 1181ff019..889873f55 100644 --- a/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden +++ b/agent/xds/testdata/listeners/listener-balance-outbound-connections-bind-port.latest.golden @@ -26,7 +26,9 @@ ], "trafficDirection": "OUTBOUND", "connectionBalanceConfig": { - "exactBalance": {} + "exactBalance": { + + } } }, { diff --git a/agent/xds/testdata/listeners/mesh-gateway-with-imported-peered-services.latest.golden b/agent/xds/testdata/listeners/mesh-gateway-with-imported-peered-services.latest.golden new file mode 100644 index 000000000..40d5b919b --- /dev/null +++ b/agent/xds/testdata/listeners/mesh-gateway-with-imported-peered-services.latest.golden @@ -0,0 +1,45 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "default:1.2.3.4:8443", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8443 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.sni_cluster", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.sni_cluster.v3.SniCluster" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "mesh_gateway_local.default", + "cluster": "" + } + } + ] + } + ], + "listenerFilters": [ + { + "name": "envoy.filters.listener.tls_inspector", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector" + } + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/mesh-gateway-with-imported-peered-services.latest.golden b/agent/xds/testdata/routes/mesh-gateway-with-imported-peered-services.latest.golden new file mode 100644 index 000000000..9c050cbe6 --- /dev/null +++ b/agent/xds/testdata/routes/mesh-gateway-with-imported-peered-services.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-cross-peers/primary/service_s1.hcl b/test/integration/connect/envoy/case-cross-peers/primary/service_s1.hcl index 0e3dcbc3e..af0773c29 100644 --- a/test/integration/connect/envoy/case-cross-peers/primary/service_s1.hcl +++ b/test/integration/connect/envoy/case-cross-peers/primary/service_s1.hcl @@ -9,6 +9,9 @@ services { destination_name = "s2" destination_peer = "primary-to-alpha" local_bind_port = 5000 + mesh_gateway { + mode = "local" + } } ] } diff --git a/test/integration/connect/envoy/case-cross-peers/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers/primary/verify.bats index 93837be61..af45d2ad3 100644 --- a/test/integration/connect/envoy/case-cross-peers/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers/primary/verify.bats @@ -55,3 +55,7 @@ load helpers @test "s1 upstream made 1 connection to s2" { assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary-to-alpha.external.*cx_total" 1 } + +@test "s1 upstream made 1 connection to s2 through the primary mesh gateway" { + assert_envoy_metric_at_least 127.0.0.1:19001 "cluster.s2.default.default.alpha-to-primary.external.*cx_total" 1 +}