open-consul/agent/xds/routes_test.go
R.B. Boyer baf886c6f3
proxycfg: introduce explicit UpstreamID in lieu of bare string (#12125)
The gist here is that now we use a value-type struct proxycfg.UpstreamID
as the map key in ConfigSnapshot maps where we used to use "upstream
id-ish" strings. These are internal only and used just for bidirectional
trips through the agent cache keyspace (like the discovery chain target
struct).

For the few places where the upstream id needs to be projected into xDS,
that's what (proxycfg.UpstreamID).EnvoyID() is for. This lets us ALWAYS
inject the partition and namespace into these things without making
stuff like the golden testdata diverge.
2022-01-20 10:12:04 -06:00

785 lines
22 KiB
Go

package xds
import (
"path/filepath"
"sort"
"testing"
"time"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/golang/protobuf/ptypes"
testinf "github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/consul/discoverychain"
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/xds/proxysupport"
"github.com/hashicorp/consul/sdk/testutil"
)
func TestRoutesFromSnapshot(t *testing.T) {
tests := []struct {
name string
create func(t testinf.T) *proxycfg.ConfigSnapshot
// Setup is called before the test starts. It is passed the snapshot from
// create func and is allowed to modify it in any way to setup the
// test input.
setup func(snap *proxycfg.ConfigSnapshot)
overrideGoldenName string
}{
{
name: "defaults-no-chain",
create: proxycfg.TestConfigSnapshot,
setup: nil, // Default snapshot
},
{
name: "connect-proxy-with-chain",
create: proxycfg.TestConfigSnapshotDiscoveryChain,
setup: nil,
},
{
name: "connect-proxy-with-chain-external-sni",
create: proxycfg.TestConfigSnapshotDiscoveryChainExternalSNI,
setup: nil,
},
{
name: "connect-proxy-with-chain-and-overrides",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides,
setup: nil,
},
{
name: "splitter-with-resolver-redirect",
create: proxycfg.TestConfigSnapshotDiscoveryChain_SplitterWithResolverRedirectMultiDC,
setup: nil,
},
{
name: "connect-proxy-with-chain-and-splitter",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithSplitter,
setup: nil,
},
{
name: "connect-proxy-with-grpc-router",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithGRPCRouter,
setup: nil,
},
{
name: "connect-proxy-with-chain-and-router",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithRouter,
setup: nil,
},
{
name: "connect-proxy-lb-in-resolver",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithLB,
setup: nil,
},
// TODO(rb): test match stanza skipped for grpc
// Start ingress gateway test cases
{
name: "ingress-defaults-no-chain",
create: proxycfg.TestConfigSnapshotIngressGateway,
setup: nil, // Default snapshot
},
{
name: "ingress-with-chain",
create: proxycfg.TestConfigSnapshotIngress,
setup: nil,
},
{
name: "ingress-with-chain-external-sni",
create: proxycfg.TestConfigSnapshotIngressExternalSNI,
setup: nil,
},
{
name: "ingress-with-chain-and-overrides",
create: proxycfg.TestConfigSnapshotIngressWithOverrides,
setup: nil,
},
{
name: "ingress-splitter-with-resolver-redirect",
create: proxycfg.TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC,
setup: nil,
},
{
name: "ingress-with-chain-and-splitter",
create: proxycfg.TestConfigSnapshotIngressWithSplitter,
setup: nil,
},
{
name: "ingress-with-grpc-router",
create: proxycfg.TestConfigSnapshotIngressWithGRPCRouter,
setup: nil,
},
{
name: "ingress-with-chain-and-router",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: nil,
},
{
name: "ingress-lb-in-resolver",
create: proxycfg.TestConfigSnapshotIngressWithLB,
setup: nil,
},
{
name: "ingress-http-multiple-services",
create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
{Protocol: "http", Port: 8080}: {
{
DestinationName: "foo",
LocalBindPort: 8080,
IngressHosts: []string{
"test1.example.com",
"test2.example.com",
"test2.example.com:8080",
},
},
{
DestinationName: "bar",
LocalBindPort: 8080,
},
},
{Protocol: "http", Port: 443}: {
{
DestinationName: "baz",
LocalBindPort: 443,
},
{
DestinationName: "qux",
LocalBindPort: 443,
},
},
}
snap.IngressGateway.Listeners = map[proxycfg.IngressListenerKey]structs.IngressListener{
{Protocol: "http", Port: 8080}: {
Port: 8080,
Services: []structs.IngressService{
{
Name: "foo",
},
{
Name: "bar",
},
},
},
{Protocol: "http", Port: 443}: {
Port: 443,
Services: []structs.IngressService{
{
Name: "baz",
},
{
Name: "qux",
},
},
},
}
// We do not add baz/qux here so that we test the chain.IsDefault() case
entries := []structs.ConfigEntry{
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
ConnectTimeout: 22 * time.Second,
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "bar",
ConnectTimeout: 22 * time.Second,
},
}
fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "default", "dc1", connect.TestClusterID+".consul", nil, entries...)
barChain := discoverychain.TestCompileConfigEntries(t, "bar", "default", "default", "dc1", connect.TestClusterID+".consul", nil, entries...)
bazChain := discoverychain.TestCompileConfigEntries(t, "baz", "default", "default", "dc1", connect.TestClusterID+".consul", nil, entries...)
quxChain := discoverychain.TestCompileConfigEntries(t, "qux", "default", "default", "dc1", connect.TestClusterID+".consul", nil, entries...)
snap.IngressGateway.DiscoveryChain = map[proxycfg.UpstreamID]*structs.CompiledDiscoveryChain{
UID("foo"): fooChain,
UID("bar"): barChain,
UID("baz"): bazChain,
UID("qux"): quxChain,
}
},
},
{
name: "ingress-with-chain-and-router-header-manip",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: func(snap *proxycfg.ConfigSnapshot) {
k := proxycfg.IngressListenerKey{Port: 9191, Protocol: "http"}
l := snap.IngressGateway.Listeners[k]
l.Services[0].RequestHeaders = &structs.HTTPHeaderModifiers{
Add: map[string]string{
"foo": "bar",
},
Set: map[string]string{
"bar": "baz",
},
Remove: []string{"qux"},
}
l.Services[0].ResponseHeaders = &structs.HTTPHeaderModifiers{
Add: map[string]string{
"foo": "bar",
},
Set: map[string]string{
"bar": "baz",
},
Remove: []string{"qux"},
}
snap.IngressGateway.Listeners[k] = l
},
},
{
name: "ingress-with-sds-listener-level",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
// Listener-level SDS means all services share the default route.
listenerSDS: true,
}),
},
{
name: "ingress-with-sds-listener-level-wildcard",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
// Listener-level SDS means all services share the default route.
listenerSDS: true,
wildcard: true,
}),
},
{
name: "ingress-with-sds-service-level",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
listenerSDS: false,
// Services should get separate routes and no default since they all
// have custom certs.
webSDS: true,
fooSDS: true,
}),
},
{
name: "ingress-with-sds-service-level-mixed-tls",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
listenerSDS: false,
// Web needs a separate route as it has custom filter chain but foo
// should use default route for listener.
webSDS: true,
fooSDS: false,
}),
},
{
name: "terminating-gateway-lb-config",
create: proxycfg.TestConfigSnapshotTerminatingGateway,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{
structs.NewServiceName("web", nil): {
Kind: structs.ServiceResolver,
Name: "web",
DefaultSubset: "v2",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": {
Filter: "Service.Meta.Version == 1",
},
"v2": {
Filter: "Service.Meta.Version == 2",
OnlyPassing: true,
},
},
LoadBalancer: &structs.LoadBalancer{
Policy: "ring_hash",
RingHashConfig: &structs.RingHashConfig{
MinimumRingSize: 20,
MaximumRingSize: 50,
},
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "chocolate-chip",
Terminal: true,
},
{
Field: structs.HashPolicyHeader,
FieldValue: "x-user-id",
},
{
SourceIP: true,
Terminal: true,
},
},
},
},
}
snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("web", nil)] = &structs.ServiceConfigResponse{
ProxyConfig: map[string]interface{}{"protocol": "http"},
}
},
},
}
latestEnvoyVersion := proxysupport.EnvoyVersions[0]
for _, envoyVersion := range proxysupport.EnvoyVersions {
sf, err := determineSupportedProxyFeaturesFromString(envoyVersion)
require.NoError(t, err)
t.Run("envoy-"+envoyVersion, func(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Sanity check default with no overrides first
snap := tt.create(t)
// We need to replace the TLS certs with deterministic ones to make golden
// files workable. Note we don't update these otherwise they'd change
// golden files for every test case and so not be any use!
setupTLSRootsAndLeaf(t, snap)
if tt.setup != nil {
tt.setup(snap)
}
g := newResourceGenerator(testutil.Logger(t), nil, nil, false)
g.ProxyFeatures = sf
routes, err := g.routesFromSnapshot(snap)
require.NoError(t, err)
sort.Slice(routes, func(i, j int) bool {
return routes[i].(*envoy_route_v3.RouteConfiguration).Name < routes[j].(*envoy_route_v3.RouteConfiguration).Name
})
r, err := createResponse(RouteType, "00000001", "00000001", routes)
require.NoError(t, err)
t.Run("current", func(t *testing.T) {
gotJSON := protoToJSON(t, r)
gName := tt.name
if tt.overrideGoldenName != "" {
gName = tt.overrideGoldenName
}
require.JSONEq(t, goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON)
})
})
}
})
}
}
func TestEnvoyLBConfig_InjectToRouteAction(t *testing.T) {
var tests = []struct {
name string
lb *structs.LoadBalancer
expected *envoy_route_v3.RouteAction
}{
{
name: "empty",
lb: &structs.LoadBalancer{
Policy: "",
},
// we only modify route actions for hash-based LB policies
expected: &envoy_route_v3.RouteAction{},
},
{
name: "least request",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyLeastRequest,
LeastRequestConfig: &structs.LeastRequestConfig{
ChoiceCount: 3,
},
},
// we only modify route actions for hash-based LB policies
expected: &envoy_route_v3.RouteAction{},
},
{
name: "headers",
lb: &structs.LoadBalancer{
Policy: "ring_hash",
RingHashConfig: &structs.RingHashConfig{
MinimumRingSize: 3,
MaximumRingSize: 7,
},
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyHeader,
FieldValue: "x-route-key",
Terminal: true,
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Header_{
Header: &envoy_route_v3.RouteAction_HashPolicy_Header{
HeaderName: "x-route-key",
},
},
Terminal: true,
},
},
},
},
{
name: "cookies",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "red-velvet",
Terminal: true,
},
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "red-velvet",
},
},
Terminal: true,
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
},
},
},
},
},
},
{
name: "non-zero session ttl gets zeroed out",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
CookieConfig: &structs.CookieConfig{
TTL: 10 * time.Second,
Session: true,
},
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
Ttl: ptypes.DurationProto(0 * time.Second),
},
},
},
},
},
},
{
name: "zero value ttl omitted if not session cookie",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
CookieConfig: &structs.CookieConfig{
Path: "/oven",
},
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
Path: "/oven",
Ttl: nil,
},
},
},
},
},
},
{
name: "source addr",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
SourceIP: true,
Terminal: true,
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties_{
ConnectionProperties: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties{
SourceIp: true,
},
},
Terminal: true,
},
},
},
},
{
name: "kitchen sink",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
SourceIP: true,
Terminal: true,
},
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
CookieConfig: &structs.CookieConfig{
TTL: 10 * time.Second,
Path: "/oven",
},
},
{
Field: structs.HashPolicyCookie,
FieldValue: "chocolate-chip",
CookieConfig: &structs.CookieConfig{
Session: true,
Path: "/oven",
},
},
{
Field: structs.HashPolicyHeader,
FieldValue: "special-header",
Terminal: true,
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties_{
ConnectionProperties: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties{
SourceIp: true,
},
},
Terminal: true,
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
Ttl: ptypes.DurationProto(10 * time.Second),
Path: "/oven",
},
},
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "chocolate-chip",
Ttl: ptypes.DurationProto(0 * time.Second),
Path: "/oven",
},
},
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Header_{
Header: &envoy_route_v3.RouteAction_HashPolicy_Header{
HeaderName: "special-header",
},
},
Terminal: true,
},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var ra envoy_route_v3.RouteAction
err := injectLBToRouteAction(tc.lb, &ra)
require.NoError(t, err)
require.Equal(t, tc.expected, &ra)
})
}
}
type ingressSDSOpts struct {
listenerSDS, webSDS, fooSDS, wildcard bool
entMetas map[string]*structs.EnterpriseMeta
}
// setupIngressWithTwoHTTPServices can be used with
// proxycfg.TestConfigSnapshotIngressWithRouter to generate a setup func for an
// ingress listener with multiple HTTP services and varying SDS configurations
// since those affect how we generate routes.
func setupIngressWithTwoHTTPServices(t *testing.T, o ingressSDSOpts) func(snap *proxycfg.ConfigSnapshot) {
return func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.TLSConfig.SDS = nil
webUpstream := structs.Upstream{
DestinationName: "web",
// We use empty not default here because of the way upstream identifiers
// vary between OSS and Enterprise currently causing test conflicts. In
// real life `proxycfg` always sets ingress upstream namespaces to
// `NamespaceOrDefault` which shouldn't matter because we should be
// consistent within a single binary it's just inconvenient if OSS and
// enterprise tests generate different output.
DestinationNamespace: o.entMetas["web"].NamespaceOrEmpty(),
DestinationPartition: o.entMetas["web"].PartitionOrEmpty(),
LocalBindPort: 9191,
IngressHosts: []string{
"www.example.com",
},
}
fooUpstream := structs.Upstream{
DestinationName: "foo",
DestinationNamespace: o.entMetas["foo"].NamespaceOrEmpty(),
DestinationPartition: o.entMetas["foo"].PartitionOrEmpty(),
LocalBindPort: 9191,
IngressHosts: []string{
"foo.example.com",
},
}
webUID := proxycfg.NewUpstreamID(&webUpstream)
fooUID := proxycfg.NewUpstreamID(&fooUpstream)
// Setup additional HTTP service on same listener with default router
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
{Protocol: "http", Port: 9191}: {webUpstream, fooUpstream},
}
il := structs.IngressListener{
Port: 9191,
Services: []structs.IngressService{
{
Name: "web",
Hosts: []string{"www.example.com"},
},
{
Name: "foo",
Hosts: []string{"foo.example.com"},
},
},
}
for i, svc := range il.Services {
if em, ok := o.entMetas[svc.Name]; ok && em != nil {
il.Services[i].EnterpriseMeta = *em
}
}
// Now set the appropriate SDS configs
if o.listenerSDS {
il.TLS = &structs.GatewayTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "listener-cluster",
CertResource: "listener-cert",
},
}
}
if o.webSDS {
il.Services[0].TLS = &structs.GatewayServiceTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "web-cluster",
CertResource: "www-cert",
},
}
}
if o.fooSDS {
il.Services[1].TLS = &structs.GatewayServiceTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "foo-cluster",
CertResource: "foo-cert",
},
}
}
if o.wildcard {
// undo all that and set just a single wildcard config with no TLS to test
// the lookup path where we have to compare an actual resolved upstream to
// a wildcard config.
il.Services = []structs.IngressService{
{
Name: "*",
},
}
// We also don't support user-specified hosts with wildcard so remove
// those from the upstreams.
ups := snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}]
for i := range ups {
ups[i].IngressHosts = nil
}
snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = ups
}
snap.IngressGateway.Listeners[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = il
entries := []structs.ConfigEntry{
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "web",
ConnectTimeout: 22 * time.Second,
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
ConnectTimeout: 22 * time.Second,
},
}
for i, e := range entries {
switch v := e.(type) {
// Add other Service types here if we ever need them above
case *structs.ServiceResolverConfigEntry:
if em, ok := o.entMetas[v.Name]; ok && em != nil {
v.EnterpriseMeta = *em
entries[i] = v
}
}
}
webChain := discoverychain.TestCompileConfigEntries(t, "web",
o.entMetas["web"].NamespaceOrDefault(),
o.entMetas["web"].PartitionOrDefault(), "dc1",
connect.TestClusterID+".consul", nil, entries...)
fooChain := discoverychain.TestCompileConfigEntries(t, "foo",
o.entMetas["foo"].NamespaceOrDefault(),
o.entMetas["web"].PartitionOrDefault(), "dc1",
connect.TestClusterID+".consul", nil, entries...)
snap.IngressGateway.DiscoveryChain[webUID] = webChain
snap.IngressGateway.DiscoveryChain[fooUID] = fooChain
}
}