From 1490eedfbc3743935c96dd20c02646bf2c29f7dc Mon Sep 17 00:00:00 2001 From: Eric Haberkorn Date: Fri, 9 Sep 2022 13:58:28 -0400 Subject: [PATCH] Implement Cluster Peering Redirects (#14445) implement cluster peering redirects --- .changelog/14445.txt | 3 + agent/proxycfg/testing_upstreams.go | 29 ++++ agent/xds/clusters.go | 107 +++++++------ agent/xds/clusters_test.go | 6 - agent/xds/endpoints.go | 63 +++----- agent/xds/endpoints_test.go | 6 - agent/xds/listeners.go | 38 +++-- agent/xds/resources_test.go | 18 +++ agent/xds/routes.go | 69 ++++++--- ...and-redirect-to-cluster-peer.latest.golden | 144 ++++++++++++++++++ ...and-redirect-to-cluster-peer.latest.golden | 75 +++++++++ ...and-failover-to-cluster-peer.latest.golden | 119 +++++++++++++++ ...and-redirect-to-cluster-peer.latest.golden | 119 +++++++++++++++ ...and-failover-to-cluster-peer.latest.golden | 30 ++++ ...and-redirect-to-cluster-peer.latest.golden | 30 ++++ .../primary/config_entries.hcl | 10 ++ .../primary/service_s1.hcl | 4 + .../primary/verify.bats | 28 +++- 18 files changed, 753 insertions(+), 145 deletions(-) create mode 100644 .changelog/14445.txt create mode 100644 agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden diff --git a/.changelog/14445.txt b/.changelog/14445.txt new file mode 100644 index 000000000..ec2d1e764 --- /dev/null +++ b/.changelog/14445.txt @@ -0,0 +1,3 @@ +```release-note:feature +peering: Add support to redirect to services running on cluster peers with service resolvers. +``` diff --git a/agent/proxycfg/testing_upstreams.go b/agent/proxycfg/testing_upstreams.go index f851ea059..afb310c75 100644 --- a/agent/proxycfg/testing_upstreams.go +++ b/agent/proxycfg/testing_upstreams.go @@ -91,6 +91,24 @@ func setupTestVariationConfigEntriesAndSnapshot( Nodes: TestUpstreamNodesPeerCluster01(t), }, }) + case "redirect-to-cluster-peer": + events = append(events, UpdateEvent{ + CorrelationID: "peer-trust-bundle:cluster-01", + Result: &pbpeering.TrustBundleReadResponse{ + Bundle: &pbpeering.PeeringTrustBundle{ + PeerName: "peer1", + TrustDomain: "peer1.domain", + ExportedPartition: "peer1ap", + RootPEMs: []string{"peer1-root-1"}, + }, + }, + }) + events = append(events, UpdateEvent{ + CorrelationID: "upstream-peer:db?peer=cluster-01", + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestUpstreamNodesPeerCluster01(t), + }, + }) case "failover-through-double-remote-gateway-triggered": events = append(events, UpdateEvent{ CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(), @@ -289,6 +307,17 @@ func setupTestVariationDiscoveryChain( }, }, ) + case "redirect-to-cluster-peer": + entries = append(entries, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "db", + ConnectTimeout: 33 * time.Second, + Redirect: &structs.ServiceResolverRedirect{ + Peer: "cluster-01", + }, + }, + ) case "failover-through-double-remote-gateway-triggered": fallthrough case "failover-through-double-remote-gateway": diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 6b171a27f..a425f829e 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -980,7 +980,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( // Mesh gateways are exempt because upstreamsSnapshot is only used for // cluster peering targets and transative failover/redirects are unsupported. if err != nil && !forMeshGateway { - return nil, fmt.Errorf("No upstream snapshot for gateway mode %q", cfgSnap.Kind) + return nil, err } rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) @@ -1038,11 +1038,11 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( // These variables are prefixed with primary to avoid shaddowing bugs. primaryTargetID := node.Resolver.Target primaryTarget := chain.Targets[primaryTargetID] - primaryClusterName := CustomizeClusterName(primaryTarget.Name, chain) - upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, node.Resolver.ConnectTimeout) - if forMeshGateway { - primaryClusterName = meshGatewayExportedClusterNamePrefix + primaryClusterName + primaryTargetClusterData, ok := s.getTargetClusterData(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway, false) + if !ok { + continue } + upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, node.Resolver.ConnectTimeout) if forMeshGateway && !cfgSnap.Locality.Matches(primaryTarget.Datacenter, primaryTarget.Partition) { s.Logger.Warn("ignoring discovery chain target that crosses a datacenter or partition boundary in a mesh gateway", @@ -1052,54 +1052,28 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( continue } - type targetClusterOption struct { - targetID string - clusterName string - } - // Construct the information required to make target clusters. When // failover is configured, create the aggregate cluster. - var targetClustersOptions []targetClusterOption + var targetClustersData []targetClusterData if failover != nil && !forMeshGateway { var failoverClusterNames []string for _, tid := range append([]string{primaryTargetID}, failover.Targets...) { - target := chain.Targets[tid] - clusterName := target.Name - targetUID := proxycfg.NewUpstreamIDFromTargetID(tid) - if targetUID.Peer != "" { - tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer) - // We can't generate cluster on peers without the trust bundle. The - // trust bundle should be ready soon. - if !ok { - s.Logger.Debug("peer trust bundle not ready for discovery chain target", - "peer", targetUID.Peer, - "target", tid, - ) - continue - } - - clusterName = generatePeeredClusterName(targetUID, tbs) + if td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, tid, forMeshGateway, true); ok { + targetClustersData = append(targetClustersData, td) + failoverClusterNames = append(failoverClusterNames, td.clusterName) } - clusterName = CustomizeClusterName(clusterName, chain) - clusterName = failoverClusterNamePrefix + clusterName - - targetClustersOptions = append(targetClustersOptions, targetClusterOption{ - targetID: tid, - clusterName: clusterName, - }) - failoverClusterNames = append(failoverClusterNames, clusterName) } aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{ Clusters: failoverClusterNames, }) if err != nil { - return nil, fmt.Errorf("failed to construct the aggregate cluster %q: %v", primaryClusterName, err) + return nil, fmt.Errorf("failed to construct the aggregate cluster %q: %v", primaryTargetClusterData.clusterName, err) } c := &envoy_cluster_v3.Cluster{ - Name: primaryClusterName, - AltStatName: primaryClusterName, + Name: primaryTargetClusterData.clusterName, + AltStatName: primaryTargetClusterData.clusterName, ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{ @@ -1112,15 +1086,12 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( out = append(out, c) } else { - targetClustersOptions = append(targetClustersOptions, targetClusterOption{ - targetID: primaryTargetID, - clusterName: primaryClusterName, - }) + targetClustersData = append(targetClustersData, primaryTargetClusterData) } // Construct the target clusters. - for _, targetInfo := range targetClustersOptions { - target := chain.Targets[targetInfo.targetID] + for _, targetData := range targetClustersData { + target := chain.Targets[targetData.targetID] sni := target.SNI var additionalSpiffeIDs []string @@ -1131,8 +1102,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( Datacenter: target.Datacenter, Service: target.Service, }.URI().String() - targetUID := proxycfg.NewUpstreamIDFromTargetID(targetInfo.targetID) - s.Logger.Debug("generating cluster for", "cluster", targetInfo.clusterName) + targetUID := proxycfg.NewUpstreamIDFromTargetID(targetData.targetID) if targetUID.Peer != "" { peerMeta := upstreamsSnapshot.UpstreamPeerMeta(targetUID) upstreamCluster, err := s.makeUpstreamClusterForPeerService(targetUID, upstreamConfig, peerMeta, cfgSnap) @@ -1140,14 +1110,15 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( continue } // Override the cluster name to include the failover-target~ prefix. - upstreamCluster.Name = targetInfo.clusterName + upstreamCluster.Name = targetData.clusterName out = append(out, upstreamCluster) continue } + s.Logger.Debug("generating cluster for", "cluster", targetData.clusterName) c := &envoy_cluster_v3.Cluster{ - Name: targetInfo.clusterName, - AltStatName: targetInfo.clusterName, + Name: targetData.clusterName, + AltStatName: targetData.clusterName, ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}, CommonLbConfig: &envoy_cluster_v3.Cluster_CommonLbConfig{ @@ -1175,7 +1146,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( lb = node.LoadBalancer } if err := injectLBToCluster(lb, c); err != nil { - return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", targetInfo.clusterName, err) + return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", targetData.clusterName, err) } if upstreamConfig.Protocol == "http2" || upstreamConfig.Protocol == "grpc" { @@ -1675,3 +1646,39 @@ func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTru tb.TrustDomain, }, ".") } + +type targetClusterData struct { + targetID string + clusterName string +} + +func (s *ResourceGenerator) getTargetClusterData(upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams, chain *structs.CompiledDiscoveryChain, tid string, forMeshGateway bool, failover bool) (targetClusterData, bool) { + target := chain.Targets[tid] + clusterName := target.Name + targetUID := proxycfg.NewUpstreamIDFromTargetID(tid) + if targetUID.Peer != "" { + tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer) + // We can't generate cluster on peers without the trust bundle. The + // trust bundle should be ready soon. + if !ok { + s.Logger.Debug("peer trust bundle not ready for discovery chain target", + "peer", targetUID.Peer, + "target", tid, + ) + return targetClusterData{}, false + } + + clusterName = generatePeeredClusterName(targetUID, tbs) + } + clusterName = CustomizeClusterName(clusterName, chain) + if failover { + clusterName = failoverClusterNamePrefix + clusterName + } + if forMeshGateway { + clusterName = meshGatewayExportedClusterNamePrefix + clusterName + } + return targetClusterData{ + targetID: tid, + clusterName: clusterName, + }, true +} diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 5efd5029c..de0fefcd7 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -269,12 +269,6 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) }, }, - { - name: "connect-proxy-with-chain-and-failover-to-cluster-peer", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil) - }, - }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index c1501f0f7..b5588ce64 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -462,7 +462,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain( // Mesh gateways are exempt because upstreamsSnapshot is only used for // cluster peering targets and transative failover/redirects are unsupported. if err != nil && !forMeshGateway { - return nil, fmt.Errorf("No upstream snapshot for gateway mode %q", cfgSnap.Kind) + return nil, err } var resources []proto.Message @@ -505,65 +505,38 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain( targetID string clusterName string } - var targetLoadAssignmentOptions []targetLoadAssignmentOption + var targetsClustersData []targetClusterData var numFailoverTargets int if failover != nil { numFailoverTargets = len(failover.Targets) } - clusterNamePrefix := "" if numFailoverTargets > 0 && !forMeshGateway { - clusterNamePrefix = failoverClusterNamePrefix for _, targetID := range append([]string{primaryTargetID}, failover.Targets...) { - target := chain.Targets[targetID] - clusterName := target.Name - targetUID := proxycfg.NewUpstreamIDFromTargetID(targetID) - if targetUID.Peer != "" { - tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer) - // We can't generate cluster on peers without the trust bundle. The - // trust bundle should be ready soon. - if !ok { - s.Logger.Debug("peer trust bundle not ready for discovery chain target", - "peer", targetUID.Peer, - "target", targetID, - ) - continue - } - - clusterName = generatePeeredClusterName(targetUID, tbs) + targetData, ok := s.getTargetClusterData(upstreamsSnapshot, chain, targetID, forMeshGateway, true) + if !ok { + continue } - clusterName = CustomizeClusterName(clusterName, chain) - clusterName = failoverClusterNamePrefix + clusterName if escapeHatchCluster != nil { - clusterName = escapeHatchCluster.Name + targetData.clusterName = escapeHatchCluster.Name } - targetLoadAssignmentOptions = append(targetLoadAssignmentOptions, targetLoadAssignmentOption{ - targetID: targetID, - clusterName: clusterName, - }) + targetsClustersData = append(targetsClustersData, targetData) } } else { - target := chain.Targets[primaryTargetID] - clusterName := CustomizeClusterName(target.Name, chain) - clusterName = clusterNamePrefix + clusterName - if escapeHatchCluster != nil { - clusterName = escapeHatchCluster.Name + if td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway, false); ok { + if escapeHatchCluster != nil { + td.clusterName = escapeHatchCluster.Name + } + targetsClustersData = append(targetsClustersData, td) } - if forMeshGateway { - clusterName = meshGatewayExportedClusterNamePrefix + clusterName - } - targetLoadAssignmentOptions = append(targetLoadAssignmentOptions, targetLoadAssignmentOption{ - targetID: primaryTargetID, - clusterName: clusterName, - }) } - for _, targetInfo := range targetLoadAssignmentOptions { - s.Logger.Debug("generating endpoints for", "cluster", targetInfo.clusterName) - targetUID := proxycfg.NewUpstreamIDFromTargetID(targetInfo.targetID) + for _, targetOpt := range targetsClustersData { + s.Logger.Debug("generating endpoints for", "cluster", targetOpt.clusterName) + targetUID := proxycfg.NewUpstreamIDFromTargetID(targetOpt.targetID) if targetUID.Peer != "" { - loadAssignment, err := s.makeUpstreamLoadAssignmentForPeerService(cfgSnap, targetInfo.clusterName, targetUID) + loadAssignment, err := s.makeUpstreamLoadAssignmentForPeerService(cfgSnap, targetOpt.clusterName, targetUID) if err != nil { return nil, err } @@ -577,7 +550,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain( chain.Targets, upstreamEndpoints, gatewayEndpoints, - targetInfo.targetID, + targetOpt.targetID, gatewayKey, forMeshGateway, ) @@ -586,7 +559,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain( } la := makeLoadAssignment( - targetInfo.clusterName, + targetOpt.clusterName, []loadAssignmentEndpointGroup{endpointGroup}, gatewayKey, ) diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index 90fad78e2..432ecfa2c 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -284,12 +284,6 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) }, }, - { - name: "connect-proxy-with-chain-and-failover-to-cluster-peer", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil) - }, - }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 488cc6eb8..cfea25cbc 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -121,12 +121,23 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. } } - for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { - upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid] + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil { + return nil, err + } - explicit := upstreamCfg.HasLocalPortOrSocket() + getUpstream := func(uid proxycfg.UpstreamID) (*structs.Upstream, bool) { + upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid] + + explicit := upstream.HasLocalPortOrSocket() implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid) - if !implicit && !explicit { + return upstream, !implicit && !explicit + } + + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + upstreamCfg, skip := getUpstream(uid) + + if skip { // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. continue } @@ -146,7 +157,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. // RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain. useRDS := chain.Protocol != "tcp" && !chain.Default - var clusterName string + var targetClusterData targetClusterData if !useRDS { // When not using RDS we must generate a cluster name to attach to the filter chain. // With RDS, cluster names get attached to the dynamic routes instead. @@ -154,7 +165,12 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. if err != nil { return nil, err } - clusterName = CustomizeClusterName(target.Name, chain) + + td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, target.ID, false, false) + if !ok { + continue + } + targetClusterData = td } filterName := fmt.Sprintf("%s.%s.%s.%s", chain.ServiceName, chain.Namespace, chain.Partition, chain.Datacenter) @@ -163,7 +179,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. if upstreamCfg != nil && upstreamCfg.HasLocalPortOrSocket() { filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ routeName: uid.EnvoyID(), - clusterName: clusterName, + clusterName: targetClusterData.clusterName, filterName: filterName, protocol: cfg.Protocol, useRDS: useRDS, @@ -189,7 +205,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ routeName: uid.EnvoyID(), - clusterName: clusterName, + clusterName: targetClusterData.clusterName, filterName: filterName, protocol: cfg.Protocol, useRDS: useRDS, @@ -319,11 +335,9 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. // Looping over explicit and implicit upstreams is only needed for cross-peer // because they do not have discovery chains. for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { - upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid] + upstreamCfg, skip := getUpstream(uid) - explicit := upstreamCfg.HasLocalPortOrSocket() - implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid) - if !implicit && !explicit { + if skip { // Not associated with a known explicit or implicit upstream so it is skipped. continue } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index fbb98b121..53274d719 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -159,6 +159,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { } tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...) tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) + tests = append(tests, getTrafficControlPeeringGoldenTestCases()...) tests = append(tests, getEnterpriseGoldenTestCases()...) latestEnvoyVersion := proxysupport.EnvoyVersions[0] @@ -216,3 +217,20 @@ func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase { }, } } + +func getTrafficControlPeeringGoldenTestCases() []goldenTestCase { + return []goldenTestCase{ + { + name: "connect-proxy-with-chain-and-failover-to-cluster-peer", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil) + }, + }, + { + name: "connect-proxy-with-chain-and-redirect-to-cluster-peer", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", nil, nil) + }, + }, + } +} diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 321faa5cd..9bbf3ff71 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -57,7 +57,7 @@ func (s *ResourceGenerator) routesForConnectProxy(cfgSnap *proxycfg.ConfigSnapsh continue } - virtualHost, err := makeUpstreamRouteForDiscoveryChain(uid.EnvoyID(), chain, []string{"*"}, "") + virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(cfgSnap, uid, chain, []string{"*"}, false) if err != nil { return nil, err } @@ -249,11 +249,12 @@ func (s *ResourceGenerator) routesForMeshGateway(cfgSnap *proxycfg.ConfigSnapsho uid := proxycfg.NewUpstreamIDFromServiceName(svc) - virtualHost, err := makeUpstreamRouteForDiscoveryChain( - uid.EnvoyID(), + virtualHost, err := s.makeUpstreamRouteForDiscoveryChain( + cfgSnap, + uid, chain, []string{"*"}, - meshGatewayExportedClusterNamePrefix, + true, ) if err != nil { return nil, err @@ -367,7 +368,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap } domains := generateUpstreamIngressDomains(listenerKey, u) - virtualHost, err := makeUpstreamRouteForDiscoveryChain(uid.EnvoyID(), chain, domains, "") + virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(cfgSnap, uid, chain, domains, false) if err != nil { return nil, err } @@ -500,12 +501,14 @@ func generateUpstreamIngressDomains(listenerKey proxycfg.IngressListenerKey, u s return domains } -func makeUpstreamRouteForDiscoveryChain( - routeName string, +func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain( + cfgSnap *proxycfg.ConfigSnapshot, + uid proxycfg.UpstreamID, chain *structs.CompiledDiscoveryChain, serviceDomains []string, - clusterNamePrefix string, + forMeshGateway bool, ) (*envoy_route_v3.VirtualHost, error) { + routeName := uid.EnvoyID() var routes []*envoy_route_v3.Route startNode := chain.Nodes[chain.StartNode] @@ -513,6 +516,11 @@ func makeUpstreamRouteForDiscoveryChain( return nil, fmt.Errorf("missing first node in compiled discovery chain for: %s", chain.ServiceName) } + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil && !forMeshGateway { + return nil, err + } + switch startNode.Type { case structs.DiscoveryGraphNodeTypeRouter: routes = make([]*envoy_route_v3.Route, 0, len(startNode.Routes)) @@ -534,13 +542,17 @@ func makeUpstreamRouteForDiscoveryChain( switch nextNode.Type { case structs.DiscoveryGraphNodeTypeSplitter: - routeAction, err = makeRouteActionForSplitter(nextNode.Splits, chain, clusterNamePrefix) + routeAction, err = s.makeRouteActionForSplitter(upstreamsSnapshot, nextNode.Splits, chain, forMeshGateway) if err != nil { return nil, err } case structs.DiscoveryGraphNodeTypeResolver: - routeAction = makeRouteActionForChainCluster(nextNode.Resolver.Target, chain, clusterNamePrefix) + ra, ok := s.makeRouteActionForChainCluster(upstreamsSnapshot, nextNode.Resolver.Target, chain, forMeshGateway) + if !ok { + continue + } + routeAction = ra default: return nil, fmt.Errorf("unexpected graph node after route %q", nextNode.Type) @@ -599,11 +611,10 @@ func makeUpstreamRouteForDiscoveryChain( } case structs.DiscoveryGraphNodeTypeSplitter: - routeAction, err := makeRouteActionForSplitter(startNode.Splits, chain, clusterNamePrefix) + routeAction, err := s.makeRouteActionForSplitter(upstreamsSnapshot, startNode.Splits, chain, forMeshGateway) if err != nil { return nil, err } - var lb *structs.LoadBalancer if startNode.LoadBalancer != nil { lb = startNode.LoadBalancer @@ -620,8 +631,10 @@ func makeUpstreamRouteForDiscoveryChain( routes = []*envoy_route_v3.Route{defaultRoute} case structs.DiscoveryGraphNodeTypeResolver: - routeAction := makeRouteActionForChainCluster(startNode.Resolver.Target, chain, clusterNamePrefix) - + routeAction, ok := s.makeRouteActionForChainCluster(upstreamsSnapshot, startNode.Resolver.Target, chain, forMeshGateway) + if !ok { + break + } var lb *structs.LoadBalancer if startNode.LoadBalancer != nil { lb = startNode.LoadBalancer @@ -782,13 +795,17 @@ func makeDefaultRouteMatch() *envoy_route_v3.RouteMatch { } } -func makeRouteActionForChainCluster( +func (s *ResourceGenerator) makeRouteActionForChainCluster( + upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams, targetID string, chain *structs.CompiledDiscoveryChain, - clusterNamePrefix string, -) *envoy_route_v3.Route_Route { - target := chain.Targets[targetID] - return makeRouteActionFromName(clusterNamePrefix + CustomizeClusterName(target.Name, chain)) + forMeshGateway bool, +) (*envoy_route_v3.Route_Route, bool) { + td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, targetID, forMeshGateway, false) + if !ok { + return nil, false + } + return makeRouteActionFromName(td.clusterName), true } func makeRouteActionFromName(clusterName string) *envoy_route_v3.Route_Route { @@ -801,10 +818,11 @@ func makeRouteActionFromName(clusterName string) *envoy_route_v3.Route_Route { } } -func makeRouteActionForSplitter( +func (s *ResourceGenerator) makeRouteActionForSplitter( + upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams, splits []*structs.DiscoverySplit, chain *structs.CompiledDiscoveryChain, - clusterNamePrefix string, + forMeshGateway bool, ) (*envoy_route_v3.Route_Route, error) { clusters := make([]*envoy_route_v3.WeightedCluster_ClusterWeight, 0, len(splits)) for _, split := range splits { @@ -815,15 +833,16 @@ func makeRouteActionForSplitter( } targetID := nextNode.Resolver.Target - target := chain.Targets[targetID] - - clusterName := clusterNamePrefix + CustomizeClusterName(target.Name, chain) + targetOptions, ok := s.getTargetClusterData(upstreamsSnapshot, chain, targetID, forMeshGateway, false) + if !ok { + continue + } // The smallest representable weight is 1/10000 or .01% but envoy // deals with integers so scale everything up by 100x. cw := &envoy_route_v3.WeightedCluster_ClusterWeight{ Weight: makeUint32Value(int(split.Weight * 100)), - Name: clusterName, + Name: targetOptions.clusterName, } if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil { return nil, err diff --git a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden new file mode 100644 index 000000000..0bd578a1c --- /dev/null +++ b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -0,0 +1,144 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.cluster-01.external.peer1.domain", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "1s", + "circuitBreakers": { + + }, + "outlierDetection": { + "maxEjectionPercent": 100 + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "peer1-root-1\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/payments" + } + ] + } + }, + "sni": "payments.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" + }, + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" + } + ] + } + }, + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden new file mode 100644 index 000000000..830d3941e --- /dev/null +++ b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -0,0 +1,75 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.cluster-01.external.peer1.domain", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.40.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.40.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.20.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden new file mode 100644 index 000000000..57d50f71c --- /dev/null +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -0,0 +1,119 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden new file mode 100644 index 000000000..e061148c0 --- /dev/null +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -0,0 +1,119 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.cluster-01.external.peer1.domain" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden new file mode 100644 index 000000000..547b923b0 --- /dev/null +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -0,0 +1,30 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "db", + "virtualHosts": [ + { + "name": "db", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden new file mode 100644 index 000000000..e63a643ef --- /dev/null +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -0,0 +1,30 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "db", + "virtualHosts": [ + { + "name": "db", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "db.default.cluster-01.external.peer1.domain" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/config_entries.hcl b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/config_entries.hcl index d9b4ba03b..65163b111 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/config_entries.hcl +++ b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/config_entries.hcl @@ -18,4 +18,14 @@ config_entries { } } } + + bootstrap { + kind = "service-resolver" + name = "virtual-s2" + + redirect = { + service = "s2" + peer = "primary-to-alpha" + } + } } diff --git a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/service_s1.hcl b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/service_s1.hcl index 842490e63..d10535324 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/service_s1.hcl +++ b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/service_s1.hcl @@ -8,6 +8,10 @@ services { { destination_name = "s2" local_bind_port = 5000 + }, + { + destination_name = "virtual-s2" + local_bind_port = 5001 } ] } diff --git a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats index 543459333..5b059bece 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats @@ -42,6 +42,8 @@ load helpers assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha } +# Failover + @test "s1 upstream should have healthy endpoints for s2 in both primary and failover" { assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal HEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 @@ -66,15 +68,19 @@ load helpers assert_service_has_healthy_instances s2 0 primary } + @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" { assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 } -@test "reset envoy statistics" { +@test "reset envoy statistics for failover" { reset_envoy_metrics 127.0.0.1:19000 } +@test "gateway-alpha should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status consul-alpha-client:19003 exported~s2.default.alpha HEALTHY 1 +} @test "s1 upstream should be able to connect to s2 in the failover cluster peer" { run retry_default curl -s -f -d hello localhost:5000 @@ -85,3 +91,23 @@ load helpers @test "s1 upstream made 1 connection to s2 through the cluster peer" { assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.failover-target~s2.default.primary-to-alpha.external.*cx_total" 1 } + +# Redirect + +@test "reset envoy statistics for redirect" { + reset_envoy_metrics 127.0.0.1:19000 +} + +@test "s1 upstream should have healthy endpoints for s2 (virtual-s2) in the cluster peer" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary-to-alpha.external HEALTHY 1 +} + +@test "s1 upstream should be able to connect to s2 via virtual-s2" { + run retry_default curl -s -f -d hello localhost:5001 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" { + assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary-to-alpha.external.*cx_total" 1 +}