open-consul/agent/xds/routes_test.go
2021-09-23 10:12:37 +01:00

814 lines
23 KiB
Go

package xds
import (
"path/filepath"
"sort"
"testing"
"time"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/golang/protobuf/ptypes"
testinf "github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/consul/discoverychain"
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/xds/proxysupport"
"github.com/hashicorp/consul/lib/stringslice"
"github.com/hashicorp/consul/sdk/testutil"
)
func TestRoutesFromSnapshot(t *testing.T) {
tests := []struct {
name string
create func(t testinf.T) *proxycfg.ConfigSnapshot
// Setup is called before the test starts. It is passed the snapshot from
// create func and is allowed to modify it in any way to setup the
// test input.
setup func(snap *proxycfg.ConfigSnapshot)
overrideGoldenName string
}{
{
name: "defaults-no-chain",
create: proxycfg.TestConfigSnapshot,
setup: nil, // Default snapshot
},
{
name: "connect-proxy-with-chain",
create: proxycfg.TestConfigSnapshotDiscoveryChain,
setup: nil,
},
{
name: "connect-proxy-with-chain-external-sni",
create: proxycfg.TestConfigSnapshotDiscoveryChainExternalSNI,
setup: nil,
},
{
name: "connect-proxy-with-chain-and-overrides",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides,
setup: nil,
},
{
name: "splitter-with-resolver-redirect",
create: proxycfg.TestConfigSnapshotDiscoveryChain_SplitterWithResolverRedirectMultiDC,
setup: nil,
},
{
name: "connect-proxy-with-chain-and-splitter",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithSplitter,
setup: nil,
},
{
name: "connect-proxy-with-grpc-router",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithGRPCRouter,
setup: nil,
},
{
name: "connect-proxy-with-chain-and-router",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithRouter,
setup: nil,
},
{
name: "connect-proxy-lb-in-resolver",
create: proxycfg.TestConfigSnapshotDiscoveryChainWithLB,
setup: nil,
},
// TODO(rb): test match stanza skipped for grpc
// Start ingress gateway test cases
{
name: "ingress-defaults-no-chain",
create: proxycfg.TestConfigSnapshotIngressGateway,
setup: nil, // Default snapshot
},
{
name: "ingress-with-chain",
create: proxycfg.TestConfigSnapshotIngress,
setup: nil,
},
{
name: "ingress-with-chain-external-sni",
create: proxycfg.TestConfigSnapshotIngressExternalSNI,
setup: nil,
},
{
name: "ingress-with-chain-and-overrides",
create: proxycfg.TestConfigSnapshotIngressWithOverrides,
setup: nil,
},
{
name: "ingress-splitter-with-resolver-redirect",
create: proxycfg.TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC,
setup: nil,
},
{
name: "ingress-with-chain-and-splitter",
create: proxycfg.TestConfigSnapshotIngressWithSplitter,
setup: nil,
},
{
name: "ingress-with-grpc-router",
create: proxycfg.TestConfigSnapshotIngressWithGRPCRouter,
setup: nil,
},
{
name: "ingress-with-chain-and-router",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: nil,
},
{
name: "ingress-lb-in-resolver",
create: proxycfg.TestConfigSnapshotIngressWithLB,
setup: nil,
},
{
name: "ingress-http-multiple-services",
create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
{Protocol: "http", Port: 8080}: {
{
DestinationName: "foo",
LocalBindPort: 8080,
IngressHosts: []string{
"test1.example.com",
"test2.example.com",
"test2.example.com:8080",
},
},
{
DestinationName: "bar",
LocalBindPort: 8080,
},
},
{Protocol: "http", Port: 443}: {
{
DestinationName: "baz",
LocalBindPort: 443,
},
{
DestinationName: "qux",
LocalBindPort: 443,
},
},
}
snap.IngressGateway.Listeners = map[proxycfg.IngressListenerKey]structs.IngressListener{
{Protocol: "http", Port: 8080}: {
Port: 8080,
Services: []structs.IngressService{
{
Name: "foo",
},
{
Name: "bar",
},
},
},
{Protocol: "http", Port: 443}: {
Port: 443,
Services: []structs.IngressService{
{
Name: "baz",
},
{
Name: "qux",
},
},
},
}
// We do not add baz/qux here so that we test the chain.IsDefault() case
entries := []structs.ConfigEntry{
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
ConnectTimeout: 22 * time.Second,
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "bar",
ConnectTimeout: 22 * time.Second,
},
}
fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
barChain := discoverychain.TestCompileConfigEntries(t, "bar", "default", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
bazChain := discoverychain.TestCompileConfigEntries(t, "baz", "default", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
quxChain := discoverychain.TestCompileConfigEntries(t, "qux", "default", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
snap.IngressGateway.DiscoveryChain = map[string]*structs.CompiledDiscoveryChain{
"foo": fooChain,
"bar": barChain,
"baz": bazChain,
"qux": quxChain,
}
},
},
{
name: "ingress-with-chain-and-router-header-manip",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: func(snap *proxycfg.ConfigSnapshot) {
k := proxycfg.IngressListenerKey{Port: 9191, Protocol: "http"}
l := snap.IngressGateway.Listeners[k]
l.Services[0].RequestHeaders = &structs.HTTPHeaderModifiers{
Add: map[string]string{
"foo": "bar",
},
Set: map[string]string{
"bar": "baz",
},
Remove: []string{"qux"},
}
l.Services[0].ResponseHeaders = &structs.HTTPHeaderModifiers{
Add: map[string]string{
"foo": "bar",
},
Set: map[string]string{
"bar": "baz",
},
Remove: []string{"qux"},
}
snap.IngressGateway.Listeners[k] = l
},
},
{
name: "ingress-with-sds-listener-level",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
// Listener-level SDS means all services share the default route.
listenerSDS: true,
}),
},
{
name: "ingress-with-sds-listener-level-wildcard",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
// Listener-level SDS means all services share the default route.
listenerSDS: true,
wildcard: true,
}),
},
{
name: "ingress-with-sds-service-level",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
listenerSDS: false,
// Services should get separate routes and no default since they all
// have custom certs.
webSDS: true,
fooSDS: true,
}),
},
{
name: "ingress-with-sds-service-level-mixed-tls",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
listenerSDS: false,
// Web needs a separate route as it has custom filter chain but foo
// should use default route for listener.
webSDS: true,
fooSDS: false,
}),
},
{
name: "terminating-gateway-lb-config",
create: proxycfg.TestConfigSnapshotTerminatingGateway,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{
structs.NewServiceName("web", nil): {
Kind: structs.ServiceResolver,
Name: "web",
DefaultSubset: "v2",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": {
Filter: "Service.Meta.Version == 1",
},
"v2": {
Filter: "Service.Meta.Version == 2",
OnlyPassing: true,
},
},
LoadBalancer: &structs.LoadBalancer{
Policy: "ring_hash",
RingHashConfig: &structs.RingHashConfig{
MinimumRingSize: 20,
MaximumRingSize: 50,
},
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "chocolate-chip",
Terminal: true,
},
{
Field: structs.HashPolicyHeader,
FieldValue: "x-user-id",
},
{
SourceIP: true,
Terminal: true,
},
},
},
},
}
snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("web", nil)] = &structs.ServiceConfigResponse{
ProxyConfig: map[string]interface{}{"protocol": "http"},
}
},
},
}
latestEnvoyVersion := proxysupport.EnvoyVersions[0]
latestEnvoyVersion_v2 := proxysupport.EnvoyVersionsV2[0]
for _, envoyVersion := range proxysupport.EnvoyVersions {
sf, err := determineSupportedProxyFeaturesFromString(envoyVersion)
require.NoError(t, err)
t.Run("envoy-"+envoyVersion, func(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Sanity check default with no overrides first
snap := tt.create(t)
// We need to replace the TLS certs with deterministic ones to make golden
// files workable. Note we don't update these otherwise they'd change
// golden files for every test case and so not be any use!
setupTLSRootsAndLeaf(t, snap)
if tt.setup != nil {
tt.setup(snap)
}
g := newResourceGenerator(testutil.Logger(t), nil, nil, false)
g.ProxyFeatures = sf
routes, err := g.routesFromSnapshot(snap)
require.NoError(t, err)
sort.Slice(routes, func(i, j int) bool {
return routes[i].(*envoy_route_v3.RouteConfiguration).Name < routes[j].(*envoy_route_v3.RouteConfiguration).Name
})
r, err := createResponse(RouteType, "00000001", "00000001", routes)
require.NoError(t, err)
t.Run("current", func(t *testing.T) {
gotJSON := protoToJSON(t, r)
gName := tt.name
if tt.overrideGoldenName != "" {
gName = tt.overrideGoldenName
}
require.JSONEq(t, goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON)
})
t.Run("v2-compat", func(t *testing.T) {
if !stringslice.Contains(proxysupport.EnvoyVersionsV2, envoyVersion) {
t.Skip()
}
respV2, err := convertDiscoveryResponseToV2(r)
require.NoError(t, err)
gotJSON := protoToJSON(t, respV2)
gName := tt.name
if tt.overrideGoldenName != "" {
gName = tt.overrideGoldenName
}
gName += ".v2compat"
// It's easy to miss a new type that encodes a version from just
// looking at the golden files so lets make it an error here. If
// there are ever false positives we can maybe include an allow list
// here as it seems safer to assume something was missed than to
// assume we'll notice the golden file being wrong. Note the first
// one matches both resourceApiVersion and transportApiVersion. I
// left it as a suffix in case there are other field names that
// follow that convention now or in the future.
require.NotContains(t, gotJSON, `ApiVersion": "V3"`)
require.NotContains(t, gotJSON, `type.googleapis.com/envoy.api.v3`)
require.JSONEq(t, goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion_v2, gotJSON), gotJSON)
})
})
}
})
}
}
func TestEnvoyLBConfig_InjectToRouteAction(t *testing.T) {
var tests = []struct {
name string
lb *structs.LoadBalancer
expected *envoy_route_v3.RouteAction
}{
{
name: "empty",
lb: &structs.LoadBalancer{
Policy: "",
},
// we only modify route actions for hash-based LB policies
expected: &envoy_route_v3.RouteAction{},
},
{
name: "least request",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyLeastRequest,
LeastRequestConfig: &structs.LeastRequestConfig{
ChoiceCount: 3,
},
},
// we only modify route actions for hash-based LB policies
expected: &envoy_route_v3.RouteAction{},
},
{
name: "headers",
lb: &structs.LoadBalancer{
Policy: "ring_hash",
RingHashConfig: &structs.RingHashConfig{
MinimumRingSize: 3,
MaximumRingSize: 7,
},
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyHeader,
FieldValue: "x-route-key",
Terminal: true,
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Header_{
Header: &envoy_route_v3.RouteAction_HashPolicy_Header{
HeaderName: "x-route-key",
},
},
Terminal: true,
},
},
},
},
{
name: "cookies",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "red-velvet",
Terminal: true,
},
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "red-velvet",
},
},
Terminal: true,
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
},
},
},
},
},
},
{
name: "non-zero session ttl gets zeroed out",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
CookieConfig: &structs.CookieConfig{
TTL: 10 * time.Second,
Session: true,
},
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
Ttl: ptypes.DurationProto(0 * time.Second),
},
},
},
},
},
},
{
name: "zero value ttl omitted if not session cookie",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
CookieConfig: &structs.CookieConfig{
Path: "/oven",
},
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
Path: "/oven",
Ttl: nil,
},
},
},
},
},
},
{
name: "source addr",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
SourceIP: true,
Terminal: true,
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties_{
ConnectionProperties: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties{
SourceIp: true,
},
},
Terminal: true,
},
},
},
},
{
name: "kitchen sink",
lb: &structs.LoadBalancer{
Policy: structs.LBPolicyMaglev,
HashPolicies: []structs.HashPolicy{
{
SourceIP: true,
Terminal: true,
},
{
Field: structs.HashPolicyCookie,
FieldValue: "oatmeal",
CookieConfig: &structs.CookieConfig{
TTL: 10 * time.Second,
Path: "/oven",
},
},
{
Field: structs.HashPolicyCookie,
FieldValue: "chocolate-chip",
CookieConfig: &structs.CookieConfig{
Session: true,
Path: "/oven",
},
},
{
Field: structs.HashPolicyHeader,
FieldValue: "special-header",
Terminal: true,
},
},
},
expected: &envoy_route_v3.RouteAction{
HashPolicy: []*envoy_route_v3.RouteAction_HashPolicy{
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties_{
ConnectionProperties: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties{
SourceIp: true,
},
},
Terminal: true,
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "oatmeal",
Ttl: ptypes.DurationProto(10 * time.Second),
Path: "/oven",
},
},
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{
Cookie: &envoy_route_v3.RouteAction_HashPolicy_Cookie{
Name: "chocolate-chip",
Ttl: ptypes.DurationProto(0 * time.Second),
Path: "/oven",
},
},
},
{
PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Header_{
Header: &envoy_route_v3.RouteAction_HashPolicy_Header{
HeaderName: "special-header",
},
},
Terminal: true,
},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var ra envoy_route_v3.RouteAction
err := injectLBToRouteAction(tc.lb, &ra)
require.NoError(t, err)
require.Equal(t, tc.expected, &ra)
})
}
}
type ingressSDSOpts struct {
listenerSDS, webSDS, fooSDS, wildcard bool
entMetas map[string]*structs.EnterpriseMeta
}
// setupIngressWithTwoHTTPServices can be used with
// proxycfg.TestConfigSnapshotIngressWithRouter to generate a setup func for an
// ingress listener with multiple HTTP services and varying SDS configurations
// since those affect how we generate routes.
func setupIngressWithTwoHTTPServices(t *testing.T, o ingressSDSOpts) func(snap *proxycfg.ConfigSnapshot) {
return func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.TLSConfig.SDS = nil
webUpstream := structs.Upstream{
DestinationName: "web",
// We use empty not default here because of the way upstream identifiers
// vary between OSS and Enterprise currently causing test conflicts. In
// real life `proxycfg` always sets ingress upstream namespaces to
// `NamespaceOrDefault` which shouldn't matter because we should be
// consistent within a single binary it's just inconvenient if OSS and
// enterprise tests generate different output.
DestinationNamespace: o.entMetas["web"].NamespaceOrEmpty(),
DestinationPartition: o.entMetas["web"].PartitionOrEmpty(),
LocalBindPort: 9191,
IngressHosts: []string{
"www.example.com",
},
}
fooUpstream := structs.Upstream{
DestinationName: "foo",
DestinationNamespace: o.entMetas["foo"].NamespaceOrEmpty(),
DestinationPartition: o.entMetas["foo"].PartitionOrEmpty(),
LocalBindPort: 9191,
IngressHosts: []string{
"foo.example.com",
},
}
// Setup additional HTTP service on same listener with default router
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
{Protocol: "http", Port: 9191}: {webUpstream, fooUpstream},
}
il := structs.IngressListener{
Port: 9191,
Services: []structs.IngressService{
{
Name: "web",
Hosts: []string{"www.example.com"},
},
{
Name: "foo",
Hosts: []string{"foo.example.com"},
},
},
}
for i, svc := range il.Services {
if em, ok := o.entMetas[svc.Name]; ok && em != nil {
il.Services[i].EnterpriseMeta = *em
}
}
// Now set the appropriate SDS configs
if o.listenerSDS {
il.TLS = &structs.GatewayTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "listener-cluster",
CertResource: "listener-cert",
},
}
}
if o.webSDS {
il.Services[0].TLS = &structs.GatewayServiceTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "web-cluster",
CertResource: "www-cert",
},
}
}
if o.fooSDS {
il.Services[1].TLS = &structs.GatewayServiceTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "foo-cluster",
CertResource: "foo-cert",
},
}
}
if o.wildcard {
// undo all that and set just a single wildcard config with no TLS to test
// the lookup path where we have to compare an actual resolved upstream to
// a wildcard config.
il.Services = []structs.IngressService{
{
Name: "*",
},
}
// We also don't support user-specified hosts with wildcard so remove
// those from the upstreams.
ups := snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}]
for i := range ups {
ups[i].IngressHosts = nil
}
snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = ups
}
snap.IngressGateway.Listeners[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = il
entries := []structs.ConfigEntry{
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "web",
ConnectTimeout: 22 * time.Second,
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
ConnectTimeout: 22 * time.Second,
},
}
for i, e := range entries {
switch v := e.(type) {
// Add other Service types here if we ever need them above
case *structs.ServiceResolverConfigEntry:
if em, ok := o.entMetas[v.Name]; ok && em != nil {
v.EnterpriseMeta = *em
entries[i] = v
}
}
}
webChain := discoverychain.TestCompileConfigEntries(t, "web",
o.entMetas["web"].NamespaceOrDefault(),
o.entMetas["web"].PartitionOrDefault(), "dc1",
connect.TestClusterID+".consul", "dc1", nil, entries...)
fooChain := discoverychain.TestCompileConfigEntries(t, "foo",
o.entMetas["foo"].NamespaceOrDefault(),
o.entMetas["web"].PartitionOrDefault(), "dc1",
connect.TestClusterID+".consul", "dc1", nil, entries...)
snap.IngressGateway.DiscoveryChain[webUpstream.Identifier()] = webChain
snap.IngressGateway.DiscoveryChain[fooUpstream.Identifier()] = fooChain
}
}