Implement Cluster Peering Redirects (#14445)

implement cluster peering redirects
This commit is contained in:
Eric Haberkorn 2022-09-09 13:58:28 -04:00 committed by GitHub
parent 29772eac2a
commit 1490eedfbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 753 additions and 145 deletions

3
.changelog/14445.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:feature
peering: Add support to redirect to services running on cluster peers with service resolvers.
```

View File

@ -91,6 +91,24 @@ func setupTestVariationConfigEntriesAndSnapshot(
Nodes: TestUpstreamNodesPeerCluster01(t), Nodes: TestUpstreamNodesPeerCluster01(t),
}, },
}) })
case "redirect-to-cluster-peer":
events = append(events, UpdateEvent{
CorrelationID: "peer-trust-bundle:cluster-01",
Result: &pbpeering.TrustBundleReadResponse{
Bundle: &pbpeering.PeeringTrustBundle{
PeerName: "peer1",
TrustDomain: "peer1.domain",
ExportedPartition: "peer1ap",
RootPEMs: []string{"peer1-root-1"},
},
},
})
events = append(events, UpdateEvent{
CorrelationID: "upstream-peer:db?peer=cluster-01",
Result: &structs.IndexedCheckServiceNodes{
Nodes: TestUpstreamNodesPeerCluster01(t),
},
})
case "failover-through-double-remote-gateway-triggered": case "failover-through-double-remote-gateway-triggered":
events = append(events, UpdateEvent{ events = append(events, UpdateEvent{
CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(), CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(),
@ -289,6 +307,17 @@ func setupTestVariationDiscoveryChain(
}, },
}, },
) )
case "redirect-to-cluster-peer":
entries = append(entries,
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
Redirect: &structs.ServiceResolverRedirect{
Peer: "cluster-01",
},
},
)
case "failover-through-double-remote-gateway-triggered": case "failover-through-double-remote-gateway-triggered":
fallthrough fallthrough
case "failover-through-double-remote-gateway": case "failover-through-double-remote-gateway":

View File

@ -980,7 +980,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
// Mesh gateways are exempt because upstreamsSnapshot is only used for // Mesh gateways are exempt because upstreamsSnapshot is only used for
// cluster peering targets and transative failover/redirects are unsupported. // cluster peering targets and transative failover/redirects are unsupported.
if err != nil && !forMeshGateway { if err != nil && !forMeshGateway {
return nil, fmt.Errorf("No upstream snapshot for gateway mode %q", cfgSnap.Kind) return nil, err
} }
rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap)
@ -1038,11 +1038,11 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
// These variables are prefixed with primary to avoid shaddowing bugs. // These variables are prefixed with primary to avoid shaddowing bugs.
primaryTargetID := node.Resolver.Target primaryTargetID := node.Resolver.Target
primaryTarget := chain.Targets[primaryTargetID] primaryTarget := chain.Targets[primaryTargetID]
primaryClusterName := CustomizeClusterName(primaryTarget.Name, chain) primaryTargetClusterData, ok := s.getTargetClusterData(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway, false)
upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, node.Resolver.ConnectTimeout) if !ok {
if forMeshGateway { continue
primaryClusterName = meshGatewayExportedClusterNamePrefix + primaryClusterName
} }
upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, node.Resolver.ConnectTimeout)
if forMeshGateway && !cfgSnap.Locality.Matches(primaryTarget.Datacenter, primaryTarget.Partition) { if forMeshGateway && !cfgSnap.Locality.Matches(primaryTarget.Datacenter, primaryTarget.Partition) {
s.Logger.Warn("ignoring discovery chain target that crosses a datacenter or partition boundary in a mesh gateway", s.Logger.Warn("ignoring discovery chain target that crosses a datacenter or partition boundary in a mesh gateway",
@ -1052,54 +1052,28 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
continue continue
} }
type targetClusterOption struct {
targetID string
clusterName string
}
// Construct the information required to make target clusters. When // Construct the information required to make target clusters. When
// failover is configured, create the aggregate cluster. // failover is configured, create the aggregate cluster.
var targetClustersOptions []targetClusterOption var targetClustersData []targetClusterData
if failover != nil && !forMeshGateway { if failover != nil && !forMeshGateway {
var failoverClusterNames []string var failoverClusterNames []string
for _, tid := range append([]string{primaryTargetID}, failover.Targets...) { for _, tid := range append([]string{primaryTargetID}, failover.Targets...) {
target := chain.Targets[tid] if td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, tid, forMeshGateway, true); ok {
clusterName := target.Name targetClustersData = append(targetClustersData, td)
targetUID := proxycfg.NewUpstreamIDFromTargetID(tid) failoverClusterNames = append(failoverClusterNames, td.clusterName)
if targetUID.Peer != "" {
tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer)
// We can't generate cluster on peers without the trust bundle. The
// trust bundle should be ready soon.
if !ok {
s.Logger.Debug("peer trust bundle not ready for discovery chain target",
"peer", targetUID.Peer,
"target", tid,
)
continue
}
clusterName = generatePeeredClusterName(targetUID, tbs)
} }
clusterName = CustomizeClusterName(clusterName, chain)
clusterName = failoverClusterNamePrefix + clusterName
targetClustersOptions = append(targetClustersOptions, targetClusterOption{
targetID: tid,
clusterName: clusterName,
})
failoverClusterNames = append(failoverClusterNames, clusterName)
} }
aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{ aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{
Clusters: failoverClusterNames, Clusters: failoverClusterNames,
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to construct the aggregate cluster %q: %v", primaryClusterName, err) return nil, fmt.Errorf("failed to construct the aggregate cluster %q: %v", primaryTargetClusterData.clusterName, err)
} }
c := &envoy_cluster_v3.Cluster{ c := &envoy_cluster_v3.Cluster{
Name: primaryClusterName, Name: primaryTargetClusterData.clusterName,
AltStatName: primaryClusterName, AltStatName: primaryTargetClusterData.clusterName,
ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout),
LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED,
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{ ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{
@ -1112,15 +1086,12 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
out = append(out, c) out = append(out, c)
} else { } else {
targetClustersOptions = append(targetClustersOptions, targetClusterOption{ targetClustersData = append(targetClustersData, primaryTargetClusterData)
targetID: primaryTargetID,
clusterName: primaryClusterName,
})
} }
// Construct the target clusters. // Construct the target clusters.
for _, targetInfo := range targetClustersOptions { for _, targetData := range targetClustersData {
target := chain.Targets[targetInfo.targetID] target := chain.Targets[targetData.targetID]
sni := target.SNI sni := target.SNI
var additionalSpiffeIDs []string var additionalSpiffeIDs []string
@ -1131,8 +1102,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
Datacenter: target.Datacenter, Datacenter: target.Datacenter,
Service: target.Service, Service: target.Service,
}.URI().String() }.URI().String()
targetUID := proxycfg.NewUpstreamIDFromTargetID(targetInfo.targetID) targetUID := proxycfg.NewUpstreamIDFromTargetID(targetData.targetID)
s.Logger.Debug("generating cluster for", "cluster", targetInfo.clusterName)
if targetUID.Peer != "" { if targetUID.Peer != "" {
peerMeta := upstreamsSnapshot.UpstreamPeerMeta(targetUID) peerMeta := upstreamsSnapshot.UpstreamPeerMeta(targetUID)
upstreamCluster, err := s.makeUpstreamClusterForPeerService(targetUID, upstreamConfig, peerMeta, cfgSnap) upstreamCluster, err := s.makeUpstreamClusterForPeerService(targetUID, upstreamConfig, peerMeta, cfgSnap)
@ -1140,14 +1110,15 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
continue continue
} }
// Override the cluster name to include the failover-target~ prefix. // Override the cluster name to include the failover-target~ prefix.
upstreamCluster.Name = targetInfo.clusterName upstreamCluster.Name = targetData.clusterName
out = append(out, upstreamCluster) out = append(out, upstreamCluster)
continue continue
} }
s.Logger.Debug("generating cluster for", "cluster", targetData.clusterName)
c := &envoy_cluster_v3.Cluster{ c := &envoy_cluster_v3.Cluster{
Name: targetInfo.clusterName, Name: targetData.clusterName,
AltStatName: targetInfo.clusterName, AltStatName: targetData.clusterName,
ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout),
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}, ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS},
CommonLbConfig: &envoy_cluster_v3.Cluster_CommonLbConfig{ CommonLbConfig: &envoy_cluster_v3.Cluster_CommonLbConfig{
@ -1175,7 +1146,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
lb = node.LoadBalancer lb = node.LoadBalancer
} }
if err := injectLBToCluster(lb, c); err != nil { if err := injectLBToCluster(lb, c); err != nil {
return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", targetInfo.clusterName, err) return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", targetData.clusterName, err)
} }
if upstreamConfig.Protocol == "http2" || upstreamConfig.Protocol == "grpc" { if upstreamConfig.Protocol == "http2" || upstreamConfig.Protocol == "grpc" {
@ -1675,3 +1646,39 @@ func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTru
tb.TrustDomain, tb.TrustDomain,
}, ".") }, ".")
} }
type targetClusterData struct {
targetID string
clusterName string
}
func (s *ResourceGenerator) getTargetClusterData(upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams, chain *structs.CompiledDiscoveryChain, tid string, forMeshGateway bool, failover bool) (targetClusterData, bool) {
target := chain.Targets[tid]
clusterName := target.Name
targetUID := proxycfg.NewUpstreamIDFromTargetID(tid)
if targetUID.Peer != "" {
tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer)
// We can't generate cluster on peers without the trust bundle. The
// trust bundle should be ready soon.
if !ok {
s.Logger.Debug("peer trust bundle not ready for discovery chain target",
"peer", targetUID.Peer,
"target", tid,
)
return targetClusterData{}, false
}
clusterName = generatePeeredClusterName(targetUID, tbs)
}
clusterName = CustomizeClusterName(clusterName, chain)
if failover {
clusterName = failoverClusterNamePrefix + clusterName
}
if forMeshGateway {
clusterName = meshGatewayExportedClusterNamePrefix + clusterName
}
return targetClusterData{
targetID: tid,
clusterName: clusterName,
}, true
}

View File

@ -269,12 +269,6 @@ func TestClustersFromSnapshot(t *testing.T) {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil)
}, },
}, },
{
name: "connect-proxy-with-chain-and-failover-to-cluster-peer",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil)
},
},
{ {
name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway",
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {

View File

@ -462,7 +462,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain(
// Mesh gateways are exempt because upstreamsSnapshot is only used for // Mesh gateways are exempt because upstreamsSnapshot is only used for
// cluster peering targets and transative failover/redirects are unsupported. // cluster peering targets and transative failover/redirects are unsupported.
if err != nil && !forMeshGateway { if err != nil && !forMeshGateway {
return nil, fmt.Errorf("No upstream snapshot for gateway mode %q", cfgSnap.Kind) return nil, err
} }
var resources []proto.Message var resources []proto.Message
@ -505,65 +505,38 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain(
targetID string targetID string
clusterName string clusterName string
} }
var targetLoadAssignmentOptions []targetLoadAssignmentOption var targetsClustersData []targetClusterData
var numFailoverTargets int var numFailoverTargets int
if failover != nil { if failover != nil {
numFailoverTargets = len(failover.Targets) numFailoverTargets = len(failover.Targets)
} }
clusterNamePrefix := ""
if numFailoverTargets > 0 && !forMeshGateway { if numFailoverTargets > 0 && !forMeshGateway {
clusterNamePrefix = failoverClusterNamePrefix
for _, targetID := range append([]string{primaryTargetID}, failover.Targets...) { for _, targetID := range append([]string{primaryTargetID}, failover.Targets...) {
target := chain.Targets[targetID] targetData, ok := s.getTargetClusterData(upstreamsSnapshot, chain, targetID, forMeshGateway, true)
clusterName := target.Name if !ok {
targetUID := proxycfg.NewUpstreamIDFromTargetID(targetID) continue
if targetUID.Peer != "" {
tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer)
// We can't generate cluster on peers without the trust bundle. The
// trust bundle should be ready soon.
if !ok {
s.Logger.Debug("peer trust bundle not ready for discovery chain target",
"peer", targetUID.Peer,
"target", targetID,
)
continue
}
clusterName = generatePeeredClusterName(targetUID, tbs)
} }
clusterName = CustomizeClusterName(clusterName, chain)
clusterName = failoverClusterNamePrefix + clusterName
if escapeHatchCluster != nil { if escapeHatchCluster != nil {
clusterName = escapeHatchCluster.Name targetData.clusterName = escapeHatchCluster.Name
} }
targetLoadAssignmentOptions = append(targetLoadAssignmentOptions, targetLoadAssignmentOption{ targetsClustersData = append(targetsClustersData, targetData)
targetID: targetID,
clusterName: clusterName,
})
} }
} else { } else {
target := chain.Targets[primaryTargetID] if td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway, false); ok {
clusterName := CustomizeClusterName(target.Name, chain) if escapeHatchCluster != nil {
clusterName = clusterNamePrefix + clusterName td.clusterName = escapeHatchCluster.Name
if escapeHatchCluster != nil { }
clusterName = escapeHatchCluster.Name targetsClustersData = append(targetsClustersData, td)
} }
if forMeshGateway {
clusterName = meshGatewayExportedClusterNamePrefix + clusterName
}
targetLoadAssignmentOptions = append(targetLoadAssignmentOptions, targetLoadAssignmentOption{
targetID: primaryTargetID,
clusterName: clusterName,
})
} }
for _, targetInfo := range targetLoadAssignmentOptions { for _, targetOpt := range targetsClustersData {
s.Logger.Debug("generating endpoints for", "cluster", targetInfo.clusterName) s.Logger.Debug("generating endpoints for", "cluster", targetOpt.clusterName)
targetUID := proxycfg.NewUpstreamIDFromTargetID(targetInfo.targetID) targetUID := proxycfg.NewUpstreamIDFromTargetID(targetOpt.targetID)
if targetUID.Peer != "" { if targetUID.Peer != "" {
loadAssignment, err := s.makeUpstreamLoadAssignmentForPeerService(cfgSnap, targetInfo.clusterName, targetUID) loadAssignment, err := s.makeUpstreamLoadAssignmentForPeerService(cfgSnap, targetOpt.clusterName, targetUID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -577,7 +550,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain(
chain.Targets, chain.Targets,
upstreamEndpoints, upstreamEndpoints,
gatewayEndpoints, gatewayEndpoints,
targetInfo.targetID, targetOpt.targetID,
gatewayKey, gatewayKey,
forMeshGateway, forMeshGateway,
) )
@ -586,7 +559,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain(
} }
la := makeLoadAssignment( la := makeLoadAssignment(
targetInfo.clusterName, targetOpt.clusterName,
[]loadAssignmentEndpointGroup{endpointGroup}, []loadAssignmentEndpointGroup{endpointGroup},
gatewayKey, gatewayKey,
) )

View File

@ -284,12 +284,6 @@ func TestEndpointsFromSnapshot(t *testing.T) {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil)
}, },
}, },
{
name: "connect-proxy-with-chain-and-failover-to-cluster-peer",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil)
},
},
{ {
name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway",
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {

View File

@ -121,12 +121,23 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
} }
} }
for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams()
upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid] if err != nil {
return nil, err
}
explicit := upstreamCfg.HasLocalPortOrSocket() getUpstream := func(uid proxycfg.UpstreamID) (*structs.Upstream, bool) {
upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid]
explicit := upstream.HasLocalPortOrSocket()
implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid) implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid)
if !implicit && !explicit { return upstream, !implicit && !explicit
}
for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain {
upstreamCfg, skip := getUpstream(uid)
if skip {
// Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped.
continue continue
} }
@ -146,7 +157,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
// RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain. // RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain.
useRDS := chain.Protocol != "tcp" && !chain.Default useRDS := chain.Protocol != "tcp" && !chain.Default
var clusterName string var targetClusterData targetClusterData
if !useRDS { if !useRDS {
// When not using RDS we must generate a cluster name to attach to the filter chain. // When not using RDS we must generate a cluster name to attach to the filter chain.
// With RDS, cluster names get attached to the dynamic routes instead. // With RDS, cluster names get attached to the dynamic routes instead.
@ -154,7 +165,12 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
if err != nil { if err != nil {
return nil, err return nil, err
} }
clusterName = CustomizeClusterName(target.Name, chain)
td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, target.ID, false, false)
if !ok {
continue
}
targetClusterData = td
} }
filterName := fmt.Sprintf("%s.%s.%s.%s", chain.ServiceName, chain.Namespace, chain.Partition, chain.Datacenter) filterName := fmt.Sprintf("%s.%s.%s.%s", chain.ServiceName, chain.Namespace, chain.Partition, chain.Datacenter)
@ -163,7 +179,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
if upstreamCfg != nil && upstreamCfg.HasLocalPortOrSocket() { if upstreamCfg != nil && upstreamCfg.HasLocalPortOrSocket() {
filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{
routeName: uid.EnvoyID(), routeName: uid.EnvoyID(),
clusterName: clusterName, clusterName: targetClusterData.clusterName,
filterName: filterName, filterName: filterName,
protocol: cfg.Protocol, protocol: cfg.Protocol,
useRDS: useRDS, useRDS: useRDS,
@ -189,7 +205,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{
routeName: uid.EnvoyID(), routeName: uid.EnvoyID(),
clusterName: clusterName, clusterName: targetClusterData.clusterName,
filterName: filterName, filterName: filterName,
protocol: cfg.Protocol, protocol: cfg.Protocol,
useRDS: useRDS, useRDS: useRDS,
@ -319,11 +335,9 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
// Looping over explicit and implicit upstreams is only needed for cross-peer // Looping over explicit and implicit upstreams is only needed for cross-peer
// because they do not have discovery chains. // because they do not have discovery chains.
for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() {
upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid] upstreamCfg, skip := getUpstream(uid)
explicit := upstreamCfg.HasLocalPortOrSocket() if skip {
implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid)
if !implicit && !explicit {
// Not associated with a known explicit or implicit upstream so it is skipped. // Not associated with a known explicit or implicit upstream so it is skipped.
continue continue
} }

View File

@ -159,6 +159,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) {
} }
tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...) tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...)
tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...)
tests = append(tests, getTrafficControlPeeringGoldenTestCases()...)
tests = append(tests, getEnterpriseGoldenTestCases()...) tests = append(tests, getEnterpriseGoldenTestCases()...)
latestEnvoyVersion := proxysupport.EnvoyVersions[0] latestEnvoyVersion := proxysupport.EnvoyVersions[0]
@ -216,3 +217,20 @@ func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase {
}, },
} }
} }
func getTrafficControlPeeringGoldenTestCases() []goldenTestCase {
return []goldenTestCase{
{
name: "connect-proxy-with-chain-and-failover-to-cluster-peer",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil)
},
},
{
name: "connect-proxy-with-chain-and-redirect-to-cluster-peer",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", nil, nil)
},
},
}
}

View File

@ -57,7 +57,7 @@ func (s *ResourceGenerator) routesForConnectProxy(cfgSnap *proxycfg.ConfigSnapsh
continue continue
} }
virtualHost, err := makeUpstreamRouteForDiscoveryChain(uid.EnvoyID(), chain, []string{"*"}, "") virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(cfgSnap, uid, chain, []string{"*"}, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -249,11 +249,12 @@ func (s *ResourceGenerator) routesForMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
uid := proxycfg.NewUpstreamIDFromServiceName(svc) uid := proxycfg.NewUpstreamIDFromServiceName(svc)
virtualHost, err := makeUpstreamRouteForDiscoveryChain( virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(
uid.EnvoyID(), cfgSnap,
uid,
chain, chain,
[]string{"*"}, []string{"*"},
meshGatewayExportedClusterNamePrefix, true,
) )
if err != nil { if err != nil {
return nil, err return nil, err
@ -367,7 +368,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap
} }
domains := generateUpstreamIngressDomains(listenerKey, u) domains := generateUpstreamIngressDomains(listenerKey, u)
virtualHost, err := makeUpstreamRouteForDiscoveryChain(uid.EnvoyID(), chain, domains, "") virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(cfgSnap, uid, chain, domains, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -500,12 +501,14 @@ func generateUpstreamIngressDomains(listenerKey proxycfg.IngressListenerKey, u s
return domains return domains
} }
func makeUpstreamRouteForDiscoveryChain( func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain(
routeName string, cfgSnap *proxycfg.ConfigSnapshot,
uid proxycfg.UpstreamID,
chain *structs.CompiledDiscoveryChain, chain *structs.CompiledDiscoveryChain,
serviceDomains []string, serviceDomains []string,
clusterNamePrefix string, forMeshGateway bool,
) (*envoy_route_v3.VirtualHost, error) { ) (*envoy_route_v3.VirtualHost, error) {
routeName := uid.EnvoyID()
var routes []*envoy_route_v3.Route var routes []*envoy_route_v3.Route
startNode := chain.Nodes[chain.StartNode] startNode := chain.Nodes[chain.StartNode]
@ -513,6 +516,11 @@ func makeUpstreamRouteForDiscoveryChain(
return nil, fmt.Errorf("missing first node in compiled discovery chain for: %s", chain.ServiceName) return nil, fmt.Errorf("missing first node in compiled discovery chain for: %s", chain.ServiceName)
} }
upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams()
if err != nil && !forMeshGateway {
return nil, err
}
switch startNode.Type { switch startNode.Type {
case structs.DiscoveryGraphNodeTypeRouter: case structs.DiscoveryGraphNodeTypeRouter:
routes = make([]*envoy_route_v3.Route, 0, len(startNode.Routes)) routes = make([]*envoy_route_v3.Route, 0, len(startNode.Routes))
@ -534,13 +542,17 @@ func makeUpstreamRouteForDiscoveryChain(
switch nextNode.Type { switch nextNode.Type {
case structs.DiscoveryGraphNodeTypeSplitter: case structs.DiscoveryGraphNodeTypeSplitter:
routeAction, err = makeRouteActionForSplitter(nextNode.Splits, chain, clusterNamePrefix) routeAction, err = s.makeRouteActionForSplitter(upstreamsSnapshot, nextNode.Splits, chain, forMeshGateway)
if err != nil { if err != nil {
return nil, err return nil, err
} }
case structs.DiscoveryGraphNodeTypeResolver: case structs.DiscoveryGraphNodeTypeResolver:
routeAction = makeRouteActionForChainCluster(nextNode.Resolver.Target, chain, clusterNamePrefix) ra, ok := s.makeRouteActionForChainCluster(upstreamsSnapshot, nextNode.Resolver.Target, chain, forMeshGateway)
if !ok {
continue
}
routeAction = ra
default: default:
return nil, fmt.Errorf("unexpected graph node after route %q", nextNode.Type) return nil, fmt.Errorf("unexpected graph node after route %q", nextNode.Type)
@ -599,11 +611,10 @@ func makeUpstreamRouteForDiscoveryChain(
} }
case structs.DiscoveryGraphNodeTypeSplitter: case structs.DiscoveryGraphNodeTypeSplitter:
routeAction, err := makeRouteActionForSplitter(startNode.Splits, chain, clusterNamePrefix) routeAction, err := s.makeRouteActionForSplitter(upstreamsSnapshot, startNode.Splits, chain, forMeshGateway)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var lb *structs.LoadBalancer var lb *structs.LoadBalancer
if startNode.LoadBalancer != nil { if startNode.LoadBalancer != nil {
lb = startNode.LoadBalancer lb = startNode.LoadBalancer
@ -620,8 +631,10 @@ func makeUpstreamRouteForDiscoveryChain(
routes = []*envoy_route_v3.Route{defaultRoute} routes = []*envoy_route_v3.Route{defaultRoute}
case structs.DiscoveryGraphNodeTypeResolver: case structs.DiscoveryGraphNodeTypeResolver:
routeAction := makeRouteActionForChainCluster(startNode.Resolver.Target, chain, clusterNamePrefix) routeAction, ok := s.makeRouteActionForChainCluster(upstreamsSnapshot, startNode.Resolver.Target, chain, forMeshGateway)
if !ok {
break
}
var lb *structs.LoadBalancer var lb *structs.LoadBalancer
if startNode.LoadBalancer != nil { if startNode.LoadBalancer != nil {
lb = startNode.LoadBalancer lb = startNode.LoadBalancer
@ -782,13 +795,17 @@ func makeDefaultRouteMatch() *envoy_route_v3.RouteMatch {
} }
} }
func makeRouteActionForChainCluster( func (s *ResourceGenerator) makeRouteActionForChainCluster(
upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams,
targetID string, targetID string,
chain *structs.CompiledDiscoveryChain, chain *structs.CompiledDiscoveryChain,
clusterNamePrefix string, forMeshGateway bool,
) *envoy_route_v3.Route_Route { ) (*envoy_route_v3.Route_Route, bool) {
target := chain.Targets[targetID] td, ok := s.getTargetClusterData(upstreamsSnapshot, chain, targetID, forMeshGateway, false)
return makeRouteActionFromName(clusterNamePrefix + CustomizeClusterName(target.Name, chain)) if !ok {
return nil, false
}
return makeRouteActionFromName(td.clusterName), true
} }
func makeRouteActionFromName(clusterName string) *envoy_route_v3.Route_Route { func makeRouteActionFromName(clusterName string) *envoy_route_v3.Route_Route {
@ -801,10 +818,11 @@ func makeRouteActionFromName(clusterName string) *envoy_route_v3.Route_Route {
} }
} }
func makeRouteActionForSplitter( func (s *ResourceGenerator) makeRouteActionForSplitter(
upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams,
splits []*structs.DiscoverySplit, splits []*structs.DiscoverySplit,
chain *structs.CompiledDiscoveryChain, chain *structs.CompiledDiscoveryChain,
clusterNamePrefix string, forMeshGateway bool,
) (*envoy_route_v3.Route_Route, error) { ) (*envoy_route_v3.Route_Route, error) {
clusters := make([]*envoy_route_v3.WeightedCluster_ClusterWeight, 0, len(splits)) clusters := make([]*envoy_route_v3.WeightedCluster_ClusterWeight, 0, len(splits))
for _, split := range splits { for _, split := range splits {
@ -815,15 +833,16 @@ func makeRouteActionForSplitter(
} }
targetID := nextNode.Resolver.Target targetID := nextNode.Resolver.Target
target := chain.Targets[targetID] targetOptions, ok := s.getTargetClusterData(upstreamsSnapshot, chain, targetID, forMeshGateway, false)
if !ok {
clusterName := clusterNamePrefix + CustomizeClusterName(target.Name, chain) continue
}
// The smallest representable weight is 1/10000 or .01% but envoy // The smallest representable weight is 1/10000 or .01% but envoy
// deals with integers so scale everything up by 100x. // deals with integers so scale everything up by 100x.
cw := &envoy_route_v3.WeightedCluster_ClusterWeight{ cw := &envoy_route_v3.WeightedCluster_ClusterWeight{
Weight: makeUint32Value(int(split.Weight * 100)), Weight: makeUint32Value(int(split.Weight * 100)),
Name: clusterName, Name: targetOptions.clusterName,
} }
if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil { if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil {
return nil, err return nil, err

View File

@ -0,0 +1,144 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "db.default.cluster-01.external.peer1.domain",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "1s",
"circuitBreakers": {
},
"outlierDetection": {
"maxEjectionPercent": 100
},
"commonLbConfig": {
"healthyPanicThreshold": {
}
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "peer1-root-1\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/payments"
}
]
}
},
"sni": "payments.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {
},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {
},
"outlierDetection": {
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target"
},
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target"
}
]
}
},
"sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "local_app",
"type": "STATIC",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "local_app",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 8080
}
}
}
}
]
}
]
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

View File

@ -0,0 +1,75 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "db.default.cluster-01.external.peer1.domain",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.40.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.40.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.20.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"nonce": "00000001"
}

View File

@ -0,0 +1,119 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "db:127.0.0.1:9191",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9191
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.db.default.default.dc1",
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "prepared_query:geo-cache:127.10.10.10:8181",
"address": {
"socketAddress": {
"address": "127.10.10.10",
"portValue": 8181
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.prepared_query_geo-cache",
"cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:0.0.0.0:9999",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 9999
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "public_listener",
"cluster": "local_app"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
}
],
"trafficDirection": "INBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,119 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "db:127.0.0.1:9191",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9191
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.db.default.default.dc1",
"cluster": "db.default.cluster-01.external.peer1.domain"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "prepared_query:geo-cache:127.10.10.10:8181",
"address": {
"socketAddress": {
"address": "127.10.10.10",
"portValue": 8181
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.prepared_query_geo-cache",
"cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:0.0.0.0:9999",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 9999
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "public_listener",
"cluster": "local_app"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
}
],
"trafficDirection": "INBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,30 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "db",
"virtualHosts": [
{
"name": "db",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,30 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "db",
"virtualHosts": [
{
"name": "db",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "db.default.cluster-01.external.peer1.domain"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -18,4 +18,14 @@ config_entries {
} }
} }
} }
bootstrap {
kind = "service-resolver"
name = "virtual-s2"
redirect = {
service = "s2"
peer = "primary-to-alpha"
}
}
} }

View File

@ -8,6 +8,10 @@ services {
{ {
destination_name = "s2" destination_name = "s2"
local_bind_port = 5000 local_bind_port = 5000
},
{
destination_name = "virtual-s2"
local_bind_port = 5001
} }
] ]
} }

View File

@ -42,6 +42,8 @@ load helpers
assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha assert_service_has_healthy_instances s2 1 primary "" "" primary-to-alpha
} }
# Failover
@test "s1 upstream should have healthy endpoints for s2 in both primary and failover" { @test "s1 upstream should have healthy endpoints for s2 in both primary and failover" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal HEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal HEALTHY 1
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
@ -66,15 +68,19 @@ load helpers
assert_service_has_healthy_instances s2 0 primary assert_service_has_healthy_instances s2 0 primary
} }
@test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" { @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
} }
@test "reset envoy statistics" { @test "reset envoy statistics for failover" {
reset_envoy_metrics 127.0.0.1:19000 reset_envoy_metrics 127.0.0.1:19000
} }
@test "gateway-alpha should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status consul-alpha-client:19003 exported~s2.default.alpha HEALTHY 1
}
@test "s1 upstream should be able to connect to s2 in the failover cluster peer" { @test "s1 upstream should be able to connect to s2 in the failover cluster peer" {
run retry_default curl -s -f -d hello localhost:5000 run retry_default curl -s -f -d hello localhost:5000
@ -85,3 +91,23 @@ load helpers
@test "s1 upstream made 1 connection to s2 through the cluster peer" { @test "s1 upstream made 1 connection to s2 through the cluster peer" {
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.failover-target~s2.default.primary-to-alpha.external.*cx_total" 1 assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.failover-target~s2.default.primary-to-alpha.external.*cx_total" 1
} }
# Redirect
@test "reset envoy statistics for redirect" {
reset_envoy_metrics 127.0.0.1:19000
}
@test "s1 upstream should have healthy endpoints for s2 (virtual-s2) in the cluster peer" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary-to-alpha.external HEALTHY 1
}
@test "s1 upstream should be able to connect to s2 via virtual-s2" {
run retry_default curl -s -f -d hello localhost:5001
[ "$status" -eq 0 ]
[ "$output" = "hello" ]
}
@test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" {
assert_envoy_metric_at_least 127.0.0.1:19000 "cluster.s2.default.primary-to-alpha.external.*cx_total" 1
}