open-consul/agent/proxycfg/testing.go

2267 lines
65 KiB
Go

package proxycfg
import (
"context"
"fmt"
"io/ioutil"
"math"
"path"
"path/filepath"
"sync"
"sync/atomic"
"time"
"github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/cache"
cachetype "github.com/hashicorp/consul/agent/cache-types"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/consul/discoverychain"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/api"
)
// TestCacheTypes encapsulates all the different cache types proxycfg.State will
// watch/request for controlling one during testing.
type TestCacheTypes struct {
roots *ControllableCacheType
leaf *ControllableCacheType
intentions *ControllableCacheType
health *ControllableCacheType
query *ControllableCacheType
compiledChain *ControllableCacheType
serviceHTTPChecks *ControllableCacheType
}
// NewTestCacheTypes creates a set of ControllableCacheTypes for all types that
// proxycfg will watch suitable for testing a proxycfg.State or Manager.
func NewTestCacheTypes(t testing.T) *TestCacheTypes {
t.Helper()
ct := &TestCacheTypes{
roots: NewControllableCacheType(t),
leaf: NewControllableCacheType(t),
intentions: NewControllableCacheType(t),
health: NewControllableCacheType(t),
query: NewControllableCacheType(t),
compiledChain: NewControllableCacheType(t),
serviceHTTPChecks: NewControllableCacheType(t),
}
ct.query.blocking = false
return ct
}
// TestCacheWithTypes registers ControllableCacheTypes for all types that
// proxycfg will watch suitable for testing a proxycfg.State or Manager.
func TestCacheWithTypes(t testing.T, types *TestCacheTypes) *cache.Cache {
c := cache.New(cache.Options{})
c.RegisterType(cachetype.ConnectCARootName, types.roots)
c.RegisterType(cachetype.ConnectCALeafName, types.leaf)
c.RegisterType(cachetype.IntentionMatchName, types.intentions)
c.RegisterType(cachetype.HealthServicesName, types.health)
c.RegisterType(cachetype.PreparedQueryName, types.query)
c.RegisterType(cachetype.CompiledDiscoveryChainName, types.compiledChain)
c.RegisterType(cachetype.ServiceHTTPChecksName, types.serviceHTTPChecks)
return c
}
// TestCerts generates a CA and Leaf suitable for returning as mock CA
// root/leaf cache requests.
func TestCerts(t testing.T) (*structs.IndexedCARoots, *structs.IssuedCert) {
t.Helper()
ca := connect.TestCA(t, nil)
roots := &structs.IndexedCARoots{
ActiveRootID: ca.ID,
TrustDomain: fmt.Sprintf("%s.consul", connect.TestClusterID),
Roots: []*structs.CARoot{ca},
}
return roots, TestLeafForCA(t, ca)
}
// TestLeafForCA generates new Leaf suitable for returning as mock CA
// leaf cache response, signed by an existing CA.
func TestLeafForCA(t testing.T, ca *structs.CARoot) *structs.IssuedCert {
leafPEM, pkPEM := connect.TestLeaf(t, "web", ca)
leafCert, err := connect.ParseCert(leafPEM)
require.NoError(t, err)
return &structs.IssuedCert{
SerialNumber: connect.EncodeSerialNumber(leafCert.SerialNumber),
CertPEM: leafPEM,
PrivateKeyPEM: pkPEM,
Service: "web",
ServiceURI: leafCert.URIs[0].String(),
ValidAfter: leafCert.NotBefore,
ValidBefore: leafCert.NotAfter,
}
}
// TestIntentions returns a sample intentions match result useful to
// mocking service discovery cache results.
func TestIntentions() *structs.IndexedIntentionMatches {
return &structs.IndexedIntentionMatches{
Matches: []structs.Intentions{
[]*structs.Intention{
{
ID: "foo",
SourceNS: "default",
SourceName: "billing",
DestinationNS: "default",
DestinationName: "web",
Action: structs.IntentionActionAllow,
},
},
},
}
}
// TestUpstreamNodes returns a sample service discovery result useful to
// mocking service discovery cache results.
func TestUpstreamNodes(t testing.T, service string) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test1",
Node: "test1",
Address: "10.10.1.1",
Datacenter: "dc1",
Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(),
},
Service: structs.TestNodeServiceWithName(t, service),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.10.1.2",
Datacenter: "dc1",
Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(),
},
Service: structs.TestNodeServiceWithName(t, service),
},
}
}
// TestPreparedQueryNodes returns instances of a service spread across two datacenters.
// The service instance names use a "-target" suffix to ensure we don't use the
// prepared query's name for SAN validation.
// The name of prepared queries won't always match the name of the service they target.
func TestPreparedQueryNodes(t testing.T, query string) structs.CheckServiceNodes {
nodes := structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test1",
Node: "test1",
Address: "10.10.1.1",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: query + "-sidecar-proxy",
Port: 8080,
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: query + "-target",
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.20.1.2",
Datacenter: "dc2",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindTypical,
Service: query + "-target",
Port: 8080,
Connect: structs.ServiceConnect{Native: true},
},
},
}
return nodes
}
func TestUpstreamNodesInStatus(t testing.T, status string) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test1",
Node: "test1",
Address: "10.10.1.1",
Datacenter: "dc1",
},
Service: structs.TestNodeService(t),
Checks: structs.HealthChecks{
&structs.HealthCheck{
Node: "test1",
ServiceName: "web",
Name: "force",
Status: status,
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.10.1.2",
Datacenter: "dc1",
},
Service: structs.TestNodeService(t),
Checks: structs.HealthChecks{
&structs.HealthCheck{
Node: "test2",
ServiceName: "web",
Name: "force",
Status: status,
},
},
},
}
}
func TestUpstreamNodesDC2(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test1",
Node: "test1",
Address: "10.20.1.1",
Datacenter: "dc2",
},
Service: structs.TestNodeService(t),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.20.1.2",
Datacenter: "dc2",
},
Service: structs.TestNodeService(t),
},
}
}
func TestUpstreamNodesInStatusDC2(t testing.T, status string) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test1",
Node: "test1",
Address: "10.20.1.1",
Datacenter: "dc2",
},
Service: structs.TestNodeService(t),
Checks: structs.HealthChecks{
&structs.HealthCheck{
Node: "test1",
ServiceName: "web",
Name: "force",
Status: status,
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.20.1.2",
Datacenter: "dc2",
},
Service: structs.TestNodeService(t),
Checks: structs.HealthChecks{
&structs.HealthCheck{
Node: "test2",
ServiceName: "web",
Name: "force",
Status: status,
},
},
},
}
}
func TestUpstreamNodesAlternate(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "alt-test1",
Node: "alt-test1",
Address: "10.20.1.1",
Datacenter: "dc1",
},
Service: structs.TestNodeService(t),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "alt-test2",
Node: "alt-test2",
Address: "10.20.1.2",
Datacenter: "dc1",
},
Service: structs.TestNodeService(t),
},
}
}
func TestGatewayNodesDC1(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.10.1.1",
Datacenter: "dc1",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.10.1.1", 8443,
structs.ServiceAddress{Address: "10.10.1.1", Port: 8443},
structs.ServiceAddress{Address: "198.118.1.1", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.10.1.2",
Datacenter: "dc1",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.10.1.2", 8443,
structs.ServiceAddress{Address: "10.0.1.2", Port: 8443},
structs.ServiceAddress{Address: "198.118.1.2", Port: 443}),
},
}
}
func TestGatewayNodesDC2(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.0.1.1",
Datacenter: "dc2",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.0.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "198.18.1.1", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.0.1.2",
Datacenter: "dc2",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.0.1.2", 8443,
structs.ServiceAddress{Address: "10.0.1.2", Port: 8443},
structs.ServiceAddress{Address: "198.18.1.2", Port: 443}),
},
}
}
func TestGatewayNodesDC3(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc3",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "198.38.1.1", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc3",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "198.38.1.2", Port: 443}),
},
}
}
func TestGatewayNodesDC4Hostname(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-3",
Node: "mesh-gateway",
Address: "10.30.1.3",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.3", 8443,
structs.ServiceAddress{Address: "10.30.1.3", Port: 8443},
structs.ServiceAddress{Address: "198.38.1.1", Port: 443}),
},
}
}
func TestGatewayNodesDC5Hostname(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-3",
Node: "mesh-gateway",
Address: "10.30.1.3",
Datacenter: "dc5",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.3", 8443,
structs.ServiceAddress{Address: "10.30.1.3", Port: 8443},
structs.ServiceAddress{Address: "198.38.1.1", Port: 443}),
},
}
}
func TestGatewayNodesDC6Hostname(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc6",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-east-1.elb.notaws.com", Port: 443}),
Checks: structs.HealthChecks{
{
Status: api.HealthCritical,
},
},
},
}
}
func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "bar-node-1",
Node: "bar-node-1",
Address: "10.1.1.4",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "bar-sidecar-proxy",
Address: "172.16.1.6",
Port: 2222,
Meta: map[string]string{
"version": "1",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "bar",
Upstreams: structs.TestUpstreams(t),
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "bar-node-2",
Node: "bar-node-2",
Address: "10.1.1.5",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "bar-sidecar-proxy",
Address: "172.16.1.7",
Port: 2222,
Meta: map[string]string{
"version": "1",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "bar",
Upstreams: structs.TestUpstreams(t),
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "bar-node-3",
Node: "bar-node-3",
Address: "10.1.1.6",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "bar-sidecar-proxy",
Address: "172.16.1.8",
Port: 2222,
Meta: map[string]string{
"version": "2",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "bar",
Upstreams: structs.TestUpstreams(t),
},
},
},
}
}
func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes {
return structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "foo-node-1",
Node: "foo-node-1",
Address: "10.1.1.1",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "foo-sidecar-proxy",
Address: "172.16.1.3",
Port: 2222,
Meta: map[string]string{
"version": "1",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "foo",
Upstreams: structs.TestUpstreams(t),
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "foo-node-2",
Node: "foo-node-2",
Address: "10.1.1.2",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "foo-sidecar-proxy",
Address: "172.16.1.4",
Port: 2222,
Meta: map[string]string{
"version": "1",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "foo",
Upstreams: structs.TestUpstreams(t),
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "foo-node-3",
Node: "foo-node-3",
Address: "10.1.1.3",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "foo-sidecar-proxy",
Address: "172.16.1.5",
Port: 2222,
Meta: map[string]string{
"version": "2",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "foo",
Upstreams: structs.TestUpstreams(t),
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "foo-node-4",
Node: "foo-node-4",
Address: "10.1.1.7",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Kind: structs.ServiceKindConnectProxy,
Service: "foo-sidecar-proxy",
Address: "172.16.1.9",
Port: 2222,
Meta: map[string]string{
"version": "2",
},
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "foo",
Upstreams: structs.TestUpstreams(t),
},
},
Checks: structs.HealthChecks{
&structs.HealthCheck{
Node: "foo-node-4",
ServiceName: "foo-sidecar-proxy",
Name: "proxy-alive",
Status: "warning",
},
},
},
}
}
// TestConfigSnapshot returns a fully populated snapshot
func TestConfigSnapshot(t testing.T) *ConfigSnapshot {
roots, leaf := TestCerts(t)
// no entries implies we'll get a default chain
dbChain := discoverychain.TestCompileConfigEntries(t, "db", "default", "default", "dc1", connect.TestClusterID+".consul", nil)
upstreams := structs.TestUpstreams(t)
return &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindConnectProxy,
Service: "web-sidecar-proxy",
ProxyID: structs.NewServiceID("web-sidecar-proxy", nil),
Address: "0.0.0.0",
Port: 9999,
Proxy: structs.ConnectProxyConfig{
DestinationServiceID: "web",
DestinationServiceName: "web",
LocalServiceAddress: "127.0.0.1",
LocalServicePort: 8080,
Config: map[string]interface{}{
"foo": "bar",
},
Upstreams: upstreams,
},
Roots: roots,
ConnectProxy: configSnapshotConnectProxy{
ConfigSnapshotUpstreams: ConfigSnapshotUpstreams{
Leaf: leaf,
UpstreamConfig: UpstreamsToMap(upstreams),
DiscoveryChain: map[UpstreamID]*structs.CompiledDiscoveryChain{
UpstreamIDFromString("db"): dbChain,
},
WatchedUpstreamEndpoints: map[UpstreamID]map[string]structs.CheckServiceNodes{
UpstreamIDFromString("db"): {
"db.default.default.dc1": TestUpstreamNodes(t, "db"),
},
},
},
PreparedQueryEndpoints: map[UpstreamID]structs.CheckServiceNodes{
UpstreamIDFromString("prepared_query:geo-cache"): TestPreparedQueryNodes(t, "geo-cache"),
},
Intentions: nil, // no intentions defined
IntentionsSet: true,
},
Datacenter: "dc1",
}
}
// TestConfigSnapshotDiscoveryChain returns a fully populated snapshot using a discovery chain
func TestConfigSnapshotDiscoveryChain(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "simple")
}
func TestConfigSnapshotDiscoveryChainExternalSNI(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "external-sni")
}
func TestConfigSnapshotDiscoveryChainWithOverrides(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "simple-with-overrides")
}
func TestConfigSnapshotDiscoveryChainWithFailover(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover")
}
func TestConfigSnapshotDiscoveryChainWithFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway")
}
func TestConfigSnapshotDiscoveryChainWithFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered")
}
func TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway")
}
func TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered")
}
func TestConfigSnapshotDiscoveryChainWithFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway")
}
func TestConfigSnapshotDiscoveryChainWithFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered")
}
func TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway")
}
func TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered")
}
func TestConfigSnapshotDiscoveryChain_SplitterWithResolverRedirectMultiDC(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc")
}
func TestConfigSnapshotDiscoveryChainWithEntries(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "simple", additionalEntries...)
}
func TestConfigSnapshotDiscoveryChainDefaultWithEntries(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "default", additionalEntries...)
}
func TestConfigSnapshotDiscoveryChainDefault(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "default")
}
func TestConfigSnapshotDiscoveryChainWithSplitter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "chain-and-splitter")
}
func TestConfigSnapshotDiscoveryChainWithGRPCRouter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "grpc-router")
}
func TestConfigSnapshotDiscoveryChainWithRouter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "chain-and-router")
}
func TestConfigSnapshotDiscoveryChainWithLB(t testing.T) *ConfigSnapshot {
return testConfigSnapshotDiscoveryChain(t, "lb-resolver")
}
func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
roots, leaf := TestCerts(t)
snap := &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindConnectProxy,
Service: "web-sidecar-proxy",
ProxyID: structs.NewServiceID("web-sidecar-proxy", nil),
Address: "0.0.0.0",
Port: 9999,
Proxy: structs.ConnectProxyConfig{
DestinationServiceID: "web",
DestinationServiceName: "web",
LocalServiceAddress: "127.0.0.1",
LocalServicePort: 8080,
Config: map[string]interface{}{
"foo": "bar",
},
Upstreams: structs.TestUpstreams(t),
},
Roots: roots,
ConnectProxy: configSnapshotConnectProxy{
ConfigSnapshotUpstreams: setupTestVariationConfigEntriesAndSnapshot(
t, variation, leaf, additionalEntries...,
),
PreparedQueryEndpoints: map[UpstreamID]structs.CheckServiceNodes{
UpstreamIDFromString("prepared_query:geo-cache"): TestPreparedQueryNodes(t, "geo-cache"),
},
Intentions: nil, // no intentions defined
IntentionsSet: true,
},
Datacenter: "dc1",
}
return snap
}
func setupTestVariationConfigEntriesAndSnapshot(
t testing.T,
variation string,
leaf *structs.IssuedCert,
additionalEntries ...structs.ConfigEntry,
) ConfigSnapshotUpstreams {
// Compile a chain.
var (
entries []structs.ConfigEntry
compileSetup func(req *discoverychain.CompileRequest)
)
switch variation {
case "default":
// no config entries
case "simple-with-overrides":
compileSetup = func(req *discoverychain.CompileRequest) {
req.OverrideMeshGateway.Mode = structs.MeshGatewayModeLocal
req.OverrideProtocol = "grpc"
req.OverrideConnectTimeout = 66 * time.Second
}
fallthrough
case "simple":
entries = append(entries,
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
},
)
case "external-sni":
entries = append(entries,
&structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
ExternalSNI: "db.some.other.service.mesh",
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
},
)
case "failover":
entries = append(entries,
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
Failover: map[string]structs.ServiceResolverFailover{
"*": {
Service: "fail",
},
},
},
)
case "failover-through-remote-gateway-triggered":
fallthrough
case "failover-through-remote-gateway":
entries = append(entries,
&structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
MeshGateway: structs.MeshGatewayConfig{
Mode: structs.MeshGatewayModeRemote,
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
Failover: map[string]structs.ServiceResolverFailover{
"*": {
Datacenters: []string{"dc2"},
},
},
},
)
case "failover-through-double-remote-gateway-triggered":
fallthrough
case "failover-through-double-remote-gateway":
entries = append(entries,
&structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
MeshGateway: structs.MeshGatewayConfig{
Mode: structs.MeshGatewayModeRemote,
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
Failover: map[string]structs.ServiceResolverFailover{
"*": {
Datacenters: []string{"dc2", "dc3"},
},
},
},
)
case "failover-through-local-gateway-triggered":
fallthrough
case "failover-through-local-gateway":
entries = append(entries,
&structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
MeshGateway: structs.MeshGatewayConfig{
Mode: structs.MeshGatewayModeLocal,
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
Failover: map[string]structs.ServiceResolverFailover{
"*": {
Datacenters: []string{"dc2"},
},
},
},
)
case "failover-through-double-local-gateway-triggered":
fallthrough
case "failover-through-double-local-gateway":
entries = append(entries,
&structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "db",
MeshGateway: structs.MeshGatewayConfig{
Mode: structs.MeshGatewayModeLocal,
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
Failover: map[string]structs.ServiceResolverFailover{
"*": {
Datacenters: []string{"dc2", "dc3"},
},
},
},
)
case "splitter-with-resolver-redirect-multidc":
entries = append(entries,
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
Splits: []structs.ServiceSplit{
{Weight: 50, Service: "db-dc1"},
{Weight: 50, Service: "db-dc2"},
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db-dc1",
Redirect: &structs.ServiceResolverRedirect{
Service: "db",
ServiceSubset: "v1",
Datacenter: "dc1",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db-dc2",
Redirect: &structs.ServiceResolverRedirect{
Service: "db",
ServiceSubset: "v2",
Datacenter: "dc2",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": {
Filter: "Service.Meta.version == v1",
},
"v2": {
Filter: "Service.Meta.version == v2",
},
},
},
)
case "chain-and-splitter":
entries = append(entries,
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
},
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceSplitter,
Name: "db",
Splits: []structs.ServiceSplit{
{
Weight: 95.5,
Service: "big-side",
RequestHeaders: &structs.HTTPHeaderModifiers{
Set: map[string]string{"x-split-leg": "big"},
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Set: map[string]string{"x-split-leg": "big"},
},
},
{
Weight: 4,
Service: "goldilocks-side",
RequestHeaders: &structs.HTTPHeaderModifiers{
Set: map[string]string{"x-split-leg": "goldilocks"},
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Set: map[string]string{"x-split-leg": "goldilocks"},
},
},
{
Weight: 0.5,
Service: "lil-bit-side",
RequestHeaders: &structs.HTTPHeaderModifiers{
Set: map[string]string{"x-split-leg": "small"},
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Set: map[string]string{"x-split-leg": "small"},
},
},
},
},
)
case "grpc-router":
entries = append(entries,
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
},
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "grpc",
},
},
&structs.ServiceRouterConfigEntry{
Kind: structs.ServiceRouter,
Name: "db",
Routes: []structs.ServiceRoute{
{
Match: &structs.ServiceRouteMatch{
HTTP: &structs.ServiceRouteHTTPMatch{
PathExact: "/fgrpc.PingServer/Ping",
},
},
Destination: &structs.ServiceRouteDestination{
Service: "prefix",
},
},
},
},
)
case "chain-and-router":
entries = append(entries,
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
ConnectTimeout: 33 * time.Second,
},
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceSplitter,
Name: "split-3-ways",
Splits: []structs.ServiceSplit{
{Weight: 95.5, Service: "big-side"},
{Weight: 4, Service: "goldilocks-side"},
{Weight: 0.5, Service: "lil-bit-side"},
},
},
&structs.ServiceRouterConfigEntry{
Kind: structs.ServiceRouter,
Name: "db",
Routes: []structs.ServiceRoute{
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/prefix",
}),
Destination: toService("prefix"),
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathExact: "/exact",
}),
Destination: toService("exact"),
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathRegex: "/regex",
}),
Destination: toService("regex"),
},
{
Match: httpMatchHeader(structs.ServiceRouteHTTPMatchHeader{
Name: "x-debug",
Present: true,
}),
Destination: toService("hdr-present"),
},
{
Match: httpMatchHeader(structs.ServiceRouteHTTPMatchHeader{
Name: "x-debug",
Present: true,
Invert: true,
}),
Destination: toService("hdr-not-present"),
},
{
Match: httpMatchHeader(structs.ServiceRouteHTTPMatchHeader{
Name: "x-debug",
Exact: "exact",
}),
Destination: toService("hdr-exact"),
},
{
Match: httpMatchHeader(structs.ServiceRouteHTTPMatchHeader{
Name: "x-debug",
Prefix: "prefix",
}),
Destination: toService("hdr-prefix"),
},
{
Match: httpMatchHeader(structs.ServiceRouteHTTPMatchHeader{
Name: "x-debug",
Suffix: "suffix",
}),
Destination: toService("hdr-suffix"),
},
{
Match: httpMatchHeader(structs.ServiceRouteHTTPMatchHeader{
Name: "x-debug",
Regex: "regex",
}),
Destination: toService("hdr-regex"),
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
Methods: []string{"GET", "PUT"},
}),
Destination: toService("just-methods"),
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
Header: []structs.ServiceRouteHTTPMatchHeader{
{
Name: "x-debug",
Exact: "exact",
},
},
Methods: []string{"GET", "PUT"},
}),
Destination: toService("hdr-exact-with-method"),
},
{
Match: httpMatchParam(structs.ServiceRouteHTTPMatchQueryParam{
Name: "secretparam1",
Exact: "exact",
}),
Destination: toService("prm-exact"),
},
{
Match: httpMatchParam(structs.ServiceRouteHTTPMatchQueryParam{
Name: "secretparam2",
Regex: "regex",
}),
Destination: toService("prm-regex"),
},
{
Match: httpMatchParam(structs.ServiceRouteHTTPMatchQueryParam{
Name: "secretparam3",
Present: true,
}),
Destination: toService("prm-present"),
},
{
Match: nil,
Destination: toService("nil-match"),
},
{
Match: &structs.ServiceRouteMatch{},
Destination: toService("empty-match-1"),
},
{
Match: &structs.ServiceRouteMatch{
HTTP: &structs.ServiceRouteHTTPMatch{},
},
Destination: toService("empty-match-2"),
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/prefix",
}),
Destination: &structs.ServiceRouteDestination{
Service: "prefix-rewrite-1",
PrefixRewrite: "/",
},
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/prefix",
}),
Destination: &structs.ServiceRouteDestination{
Service: "prefix-rewrite-2",
PrefixRewrite: "/nested/newlocation",
},
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/timeout",
}),
Destination: &structs.ServiceRouteDestination{
Service: "req-timeout",
RequestTimeout: 33 * time.Second,
},
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/retry-connect",
}),
Destination: &structs.ServiceRouteDestination{
Service: "retry-connect",
NumRetries: 15,
RetryOnConnectFailure: true,
},
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/retry-codes",
}),
Destination: &structs.ServiceRouteDestination{
Service: "retry-codes",
NumRetries: 15,
RetryOnStatusCodes: []uint32{401, 409, 451},
},
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/retry-both",
}),
Destination: &structs.ServiceRouteDestination{
Service: "retry-both",
RetryOnConnectFailure: true,
RetryOnStatusCodes: []uint32{401, 409, 451},
},
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathPrefix: "/split-3-ways",
}),
Destination: toService("split-3-ways"),
},
{
Match: httpMatch(&structs.ServiceRouteHTTPMatch{
PathExact: "/header-manip",
}),
Destination: &structs.ServiceRouteDestination{
Service: "header-manip",
RequestHeaders: &structs.HTTPHeaderModifiers{
Add: map[string]string{
"request": "bar",
},
Set: map[string]string{
"bar": "baz",
},
Remove: []string{"qux"},
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Add: map[string]string{
"response": "bar",
},
Set: map[string]string{
"bar": "baz",
},
Remove: []string{"qux"},
},
},
},
},
},
)
case "lb-resolver":
entries = append(entries,
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceSplitter,
Name: "db",
Splits: []structs.ServiceSplit{
{Weight: 95.5, Service: "something-else"},
{Weight: 4.5, Service: "db"},
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "db",
LoadBalancer: &structs.LoadBalancer{
Policy: "ring_hash",
RingHashConfig: &structs.RingHashConfig{
MinimumRingSize: 20,
MaximumRingSize: 30,
},
HashPolicies: []structs.HashPolicy{
{
Field: "cookie",
FieldValue: "chocolate-chip",
Terminal: true,
},
{
Field: "cookie",
FieldValue: "chocolate-chip",
CookieConfig: &structs.CookieConfig{Session: true},
},
{
Field: "header",
FieldValue: "x-user-id",
},
{
SourceIP: true,
Terminal: true,
},
},
},
},
)
case "http-multiple-services":
default:
t.Fatalf("unexpected variation: %q", variation)
return ConfigSnapshotUpstreams{}
}
if len(additionalEntries) > 0 {
entries = append(entries, additionalEntries...)
}
dbChain := discoverychain.TestCompileConfigEntries(t, "db", "default", "default", "dc1", connect.TestClusterID+".consul", compileSetup, entries...)
dbUID := UpstreamIDFromString("db")
upstreams := structs.TestUpstreams(t)
snap := ConfigSnapshotUpstreams{
Leaf: leaf,
DiscoveryChain: map[UpstreamID]*structs.CompiledDiscoveryChain{
dbUID: dbChain,
},
WatchedUpstreamEndpoints: map[UpstreamID]map[string]structs.CheckServiceNodes{
dbUID: {
"db.default.default.dc1": TestUpstreamNodes(t, "db"),
},
},
UpstreamConfig: UpstreamsToMap(upstreams),
}
switch variation {
case "default":
case "simple-with-overrides":
case "simple":
case "external-sni":
case "failover":
snap.WatchedUpstreamEndpoints[dbUID]["fail.default.default.dc1"] =
TestUpstreamNodesAlternate(t)
case "failover-through-remote-gateway-triggered":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc1"] =
TestUpstreamNodesInStatus(t, "critical")
fallthrough
case "failover-through-remote-gateway":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc2"] =
TestUpstreamNodesDC2(t)
snap.WatchedGatewayEndpoints = map[UpstreamID]map[string]structs.CheckServiceNodes{
dbUID: {
"dc2": TestGatewayNodesDC2(t),
},
}
case "failover-through-double-remote-gateway-triggered":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc1"] =
TestUpstreamNodesInStatus(t, "critical")
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc2"] =
TestUpstreamNodesInStatusDC2(t, "critical")
fallthrough
case "failover-through-double-remote-gateway":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc3"] = TestUpstreamNodesDC2(t)
snap.WatchedGatewayEndpoints = map[UpstreamID]map[string]structs.CheckServiceNodes{
dbUID: {
"dc2": TestGatewayNodesDC2(t),
"dc3": TestGatewayNodesDC3(t),
},
}
case "failover-through-local-gateway-triggered":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc1"] =
TestUpstreamNodesInStatus(t, "critical")
fallthrough
case "failover-through-local-gateway":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc2"] =
TestUpstreamNodesDC2(t)
snap.WatchedGatewayEndpoints = map[UpstreamID]map[string]structs.CheckServiceNodes{
dbUID: {
"dc1": TestGatewayNodesDC1(t),
},
}
case "failover-through-double-local-gateway-triggered":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc1"] =
TestUpstreamNodesInStatus(t, "critical")
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc2"] =
TestUpstreamNodesInStatusDC2(t, "critical")
fallthrough
case "failover-through-double-local-gateway":
snap.WatchedUpstreamEndpoints[dbUID]["db.default.default.dc3"] = TestUpstreamNodesDC2(t)
snap.WatchedGatewayEndpoints = map[UpstreamID]map[string]structs.CheckServiceNodes{
dbUID: {
"dc1": TestGatewayNodesDC1(t),
},
}
case "splitter-with-resolver-redirect-multidc":
snap.WatchedUpstreamEndpoints[dbUID] = map[string]structs.CheckServiceNodes{
"v1.db.default.default.dc1": TestUpstreamNodes(t, "db"),
"v2.db.default.default.dc2": TestUpstreamNodesDC2(t),
}
case "chain-and-splitter":
case "grpc-router":
case "chain-and-router":
case "http-multiple-services":
snap.WatchedUpstreamEndpoints[UpstreamIDFromString("foo")] = map[string]structs.CheckServiceNodes{
"foo.default.default.dc1": TestUpstreamNodes(t, "foo"),
}
snap.WatchedUpstreamEndpoints[UpstreamIDFromString("bar")] = map[string]structs.CheckServiceNodes{
"bar.default.default.dc1": TestUpstreamNodesAlternate(t),
}
case "lb-resolver":
default:
t.Fatalf("unexpected variation: %q", variation)
return ConfigSnapshotUpstreams{}
}
return snap
}
func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, true, false)
}
func TestConfigSnapshotMeshGatewayUsingFederationStates(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, true, true)
}
func TestConfigSnapshotMeshGatewayNewerInformationInFederationStates(t testing.T) *ConfigSnapshot {
snap := TestConfigSnapshotMeshGateway(t)
// Create a duplicate entry in FedStateGateways, with a high ModifyIndex, to
// verify that fresh data in the federation state is preferred over stale data
// in GatewayGroups.
svc := structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.0.1.3", 8443,
structs.ServiceAddress{Address: "10.0.1.3", Port: 8443},
structs.ServiceAddress{Address: "198.18.1.3", Port: 443},
)
svc.RaftIndex.ModifyIndex = math.MaxUint64
snap.MeshGateway.FedStateGateways = map[string]structs.CheckServiceNodes{
"dc2": {
{
Node: snap.MeshGateway.GatewayGroups["dc2"][0].Node,
Service: svc,
},
},
}
return snap
}
func TestConfigSnapshotMeshGatewayOlderInformationInFederationStates(t testing.T) *ConfigSnapshot {
snap := TestConfigSnapshotMeshGateway(t)
// Create a duplicate entry in FedStateGateways, with a low ModifyIndex, to
// verify that stale data in the federation state is ignored in favor of the
// fresher data in GatewayGroups.
svc := structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.0.1.3", 8443,
structs.ServiceAddress{Address: "10.0.1.3", Port: 8443},
structs.ServiceAddress{Address: "198.18.1.3", Port: 443},
)
svc.RaftIndex.ModifyIndex = 0
snap.MeshGateway.FedStateGateways = map[string]structs.CheckServiceNodes{
"dc2": {
{
Node: snap.MeshGateway.GatewayGroups["dc2"][0].Node,
Service: svc,
},
},
}
return snap
}
func TestConfigSnapshotMeshGatewayNoServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotMeshGateway(t, false, false)
}
func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFederationStates bool) *ConfigSnapshot {
roots, _ := TestCerts(t)
snap := &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindMeshGateway,
Service: "mesh-gateway",
ProxyID: structs.NewServiceID("mesh-gateway", nil),
Address: "1.2.3.4",
Port: 8443,
Proxy: structs.ConnectProxyConfig{
Config: map[string]interface{}{},
},
TaggedAddresses: map[string]structs.ServiceAddress{
structs.TaggedAddressLAN: {
Address: "1.2.3.4",
Port: 8443,
},
structs.TaggedAddressWAN: {
Address: "198.18.0.1",
Port: 443,
},
},
Roots: roots,
Datacenter: "dc1",
MeshGateway: configSnapshotMeshGateway{
WatchedServicesSet: true,
},
}
if populateServices {
snap.MeshGateway = configSnapshotMeshGateway{
WatchedServices: map[structs.ServiceName]context.CancelFunc{
structs.NewServiceName("foo", nil): nil,
structs.NewServiceName("bar", nil): nil,
},
WatchedServicesSet: true,
WatchedGateways: map[string]context.CancelFunc{
"dc2": nil,
},
ServiceGroups: map[structs.ServiceName]structs.CheckServiceNodes{
structs.NewServiceName("foo", nil): TestGatewayServiceGroupFooDC1(t),
structs.NewServiceName("bar", nil): TestGatewayServiceGroupBarDC1(t),
},
GatewayGroups: map[string]structs.CheckServiceNodes{
"dc2": TestGatewayNodesDC2(t),
"dc4": TestGatewayNodesDC4Hostname(t),
"dc6": TestGatewayNodesDC6Hostname(t),
},
HostnameDatacenters: map[string]structs.CheckServiceNodes{
"dc4": {
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-west-2.elb.notaws.com", Port: 443}),
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-2",
Node: "mesh-gateway",
Address: "10.30.1.2",
Datacenter: "dc4",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.2", 8443,
structs.ServiceAddress{Address: "10.30.1.2", Port: 8443},
structs.ServiceAddress{Address: "456.us-west-2.elb.notaws.com", Port: 443}),
},
},
"dc6": {
structs.CheckServiceNode{
Node: &structs.Node{
ID: "mesh-gateway-1",
Node: "mesh-gateway",
Address: "10.30.1.1",
Datacenter: "dc6",
},
Service: structs.TestNodeServiceMeshGatewayWithAddrs(t,
"10.30.1.1", 8443,
structs.ServiceAddress{Address: "10.0.1.1", Port: 8443},
structs.ServiceAddress{Address: "123.us-east-1.elb.notaws.com", Port: 443}),
Checks: structs.HealthChecks{
{
Status: api.HealthCritical,
},
},
},
},
},
}
if useFederationStates {
snap.MeshGateway.FedStateGateways = map[string]structs.CheckServiceNodes{
"dc2": TestGatewayNodesDC2(t),
"dc4": TestGatewayNodesDC4Hostname(t),
"dc6": TestGatewayNodesDC6Hostname(t),
}
delete(snap.MeshGateway.GatewayGroups, "dc2")
}
}
return snap
}
func TestConfigSnapshotIngress(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "simple")
}
func TestConfigSnapshotIngressWithTLSListener(t testing.T) *ConfigSnapshot {
snap := testConfigSnapshotIngressGateway(t, true, "tcp", "default")
snap.IngressGateway.TLSConfig.Enabled = true
return snap
}
func TestConfigSnapshotIngressWithGatewaySDS(t testing.T) *ConfigSnapshot {
snap := testConfigSnapshotIngressGateway(t, true, "tcp", "default")
snap.IngressGateway.TLSConfig.SDS = &structs.GatewayTLSSDSConfig{
ClusterName: "sds-cluster",
CertResource: "cert-resource",
}
return snap
}
func TestConfigSnapshotIngressWithOverrides(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "simple-with-overrides")
}
func TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc")
}
func TestConfigSnapshotIngress_HTTPMultipleServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "http-multiple-services")
}
func TestConfigSnapshotIngressExternalSNI(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "external-sni")
}
func TestConfigSnapshotIngressWithFailover(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover")
}
func TestConfigSnapshotIngressWithFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway")
}
func TestConfigSnapshotIngressWithFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered")
}
func TestConfigSnapshotIngressWithFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway")
}
func TestConfigSnapshotIngressWithFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered")
}
func TestConfigSnapshotIngressWithSplitter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-splitter")
}
func TestConfigSnapshotIngressWithGRPCRouter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "grpc-router")
}
func TestConfigSnapshotIngressWithRouter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-router")
}
func TestConfigSnapshotIngressGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "tcp", "default")
}
func TestConfigSnapshotIngressGatewayNoServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, false, "tcp", "default")
}
func TestConfigSnapshotIngressWithLB(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "lb-resolver")
}
func TestConfigSnapshotIngressDiscoveryChainWithEntries(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "simple", additionalEntries...)
}
func testConfigSnapshotIngressGateway(
t testing.T, populateServices bool, protocol, variation string,
additionalEntries ...structs.ConfigEntry,
) *ConfigSnapshot {
roots, leaf := TestCerts(t)
snap := &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindIngressGateway,
Service: "ingress-gateway",
ProxyID: structs.NewServiceID("ingress-gateway", nil),
Address: "1.2.3.4",
Roots: roots,
Datacenter: "dc1",
}
if populateServices {
snap.IngressGateway = configSnapshotIngressGateway{
ConfigSnapshotUpstreams: setupTestVariationConfigEntriesAndSnapshot(
t, variation, leaf, additionalEntries...,
),
Upstreams: map[IngressListenerKey]structs.Upstreams{
{protocol, 9191}: {
{
// We rely on this one having default type in a few tests...
DestinationName: "db",
DestinationPartition: "default",
LocalBindPort: 9191,
LocalBindAddress: "2.3.4.5",
},
},
},
Listeners: map[IngressListenerKey]structs.IngressListener{
{protocol, 9191}: {
Port: 9191,
Protocol: protocol,
Services: []structs.IngressService{
{Name: "db"},
},
},
},
}
}
return snap
}
func TestConfigSnapshotExposeConfig(t testing.T) *ConfigSnapshot {
return &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindConnectProxy,
Service: "web-proxy",
ProxyID: structs.NewServiceID("web-proxy", nil),
Address: "1.2.3.4",
Port: 8080,
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "web",
DestinationServiceID: "web",
LocalServicePort: 8080,
Expose: structs.ExposeConfig{
Checks: false,
Paths: []structs.ExposePath{
{
LocalPathPort: 8080,
Path: "/health1",
ListenerPort: 21500,
},
{
LocalPathPort: 8080,
Path: "/health2",
ListenerPort: 21501,
},
},
},
},
Datacenter: "dc1",
}
}
func TestConfigSnapshotTerminatingGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotTerminatingGateway(t, true)
}
func TestConfigSnapshotTerminatingGatewayNoServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotTerminatingGateway(t, false)
}
func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *ConfigSnapshot {
roots, _ := TestCerts(t)
snap := &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindTerminatingGateway,
Service: "terminating-gateway",
ProxyID: structs.NewServiceID("terminating-gateway", nil),
Address: "1.2.3.4",
TaggedAddresses: map[string]structs.ServiceAddress{
structs.TaggedAddressWAN: {
Address: "198.18.0.1",
Port: 443,
},
},
Port: 8443,
Roots: roots,
Datacenter: "dc1",
}
if populateServices {
web := structs.NewServiceName("web", nil)
webNodes := TestUpstreamNodes(t, web.Name)
webNodes[0].Service.Meta = map[string]string{
"version": "1",
}
webNodes[1].Service.Meta = map[string]string{
"version": "2",
}
api := structs.NewServiceName("api", nil)
apiNodes := structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "api",
Node: "test1",
Address: "10.10.1.1",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "api.mydomain",
Port: 8081,
},
Checks: structs.HealthChecks{
{
Status: "critical",
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test2",
Node: "test2",
Address: "10.10.1.2",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "api.altdomain",
Port: 8081,
Meta: map[string]string{
"domain": "alt",
},
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test3",
Node: "test3",
Address: "10.10.1.3",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "10.10.1.3",
Port: 8081,
},
},
structs.CheckServiceNode{
Node: &structs.Node{
ID: "test4",
Node: "test4",
Address: "10.10.1.4",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "api",
Address: "api.thirddomain",
Port: 8081,
},
},
}
// Has failing instance
db := structs.NewServiceName("db", nil)
dbNodes := structs.CheckServiceNodes{
structs.CheckServiceNode{
Node: &structs.Node{
ID: "db",
Node: "test4",
Address: "10.10.1.4",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "db",
Address: "db.mydomain",
Port: 8081,
},
Checks: structs.HealthChecks{
{
Status: "critical",
},
},
},
}
// Has passing instance but failing subset
cache := structs.NewServiceName("cache", nil)
cacheNodes := structs.CheckServiceNodes{
{
Node: &structs.Node{
ID: "cache",
Node: "test5",
Address: "10.10.1.5",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "cache",
Address: "cache.mydomain",
Port: 8081,
},
},
{
Node: &structs.Node{
ID: "cache",
Node: "test5",
Address: "10.10.1.5",
Datacenter: "dc1",
},
Service: &structs.NodeService{
Service: "cache",
Address: "cache.mydomain",
Port: 8081,
Meta: map[string]string{
"Env": "prod",
},
},
Checks: structs.HealthChecks{
{
Status: "critical",
},
},
},
}
snap.TerminatingGateway = configSnapshotTerminatingGateway{
ServiceGroups: map[structs.ServiceName]structs.CheckServiceNodes{
web: webNodes,
api: apiNodes,
db: dbNodes,
cache: cacheNodes,
},
ServiceResolversSet: map[structs.ServiceName]bool{
web: true,
api: true,
db: true,
cache: true,
},
GatewayServices: map[structs.ServiceName]structs.GatewayService{
web: {
Service: web,
CAFile: "ca.cert.pem",
},
api: {
Service: api,
CAFile: "ca.cert.pem",
CertFile: "api.cert.pem",
KeyFile: "api.key.pem",
},
db: {
Service: db,
},
cache: {
Service: cache,
},
},
HostnameServices: map[structs.ServiceName]structs.CheckServiceNodes{
api: {apiNodes[0], apiNodes[1]},
db: {dbNodes[0]},
cache: {cacheNodes[0], cacheNodes[1]},
},
}
snap.TerminatingGateway.ServiceConfigs = map[structs.ServiceName]*structs.ServiceConfigResponse{
web: {
ProxyConfig: map[string]interface{}{"protocol": "tcp"},
},
api: {
ProxyConfig: map[string]interface{}{"protocol": "tcp"},
},
db: {
ProxyConfig: map[string]interface{}{"protocol": "tcp"},
},
cache: {
ProxyConfig: map[string]interface{}{"protocol": "tcp"},
},
}
snap.TerminatingGateway.Intentions = map[structs.ServiceName]structs.Intentions{
// no intentions defined for thse services
web: nil,
api: nil,
db: nil,
cache: nil,
}
snap.TerminatingGateway.ServiceLeaves = map[structs.ServiceName]*structs.IssuedCert{
web: {
CertPEM: golden(t, "test-leaf-cert"),
PrivateKeyPEM: golden(t, "test-leaf-key"),
},
api: {
CertPEM: golden(t, "alt-test-leaf-cert"),
PrivateKeyPEM: golden(t, "alt-test-leaf-key"),
},
db: {
CertPEM: golden(t, "db-test-leaf-cert"),
PrivateKeyPEM: golden(t, "db-test-leaf-key"),
},
cache: {
CertPEM: golden(t, "cache-test-leaf-cert"),
PrivateKeyPEM: golden(t, "cache-test-leaf-key"),
},
}
}
return snap
}
func TestConfigSnapshotGRPCExposeHTTP1(t testing.T) *ConfigSnapshot {
return &ConfigSnapshot{
Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName},
Kind: structs.ServiceKindConnectProxy,
Service: "grpc-proxy",
ProxyID: structs.NewServiceID("grpc-proxy", nil),
Address: "1.2.3.4",
Port: 8080,
Proxy: structs.ConnectProxyConfig{
DestinationServiceName: "grpc",
DestinationServiceID: "grpc",
LocalServicePort: 8080,
Config: map[string]interface{}{
"protocol": "grpc",
},
Expose: structs.ExposeConfig{
Checks: false,
Paths: []structs.ExposePath{
{
LocalPathPort: 8090,
Path: "/healthz",
ListenerPort: 21500,
Protocol: "http",
},
},
},
},
Datacenter: "dc1",
}
}
func TestConfigSnapshotIngress_MultipleListenersDuplicateService(t testing.T) *ConfigSnapshot {
snap := TestConfigSnapshotIngress_HTTPMultipleServices(t)
snap.IngressGateway.Upstreams = map[IngressListenerKey]structs.Upstreams{
{Protocol: "http", Port: 8080}: {
{
DestinationName: "foo",
LocalBindPort: 8080,
},
{
DestinationName: "bar",
LocalBindPort: 8080,
},
},
{Protocol: "http", Port: 443}: {
{
DestinationName: "foo",
LocalBindPort: 443,
},
},
}
fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "default", "dc1", connect.TestClusterID+".consul", nil)
barChain := discoverychain.TestCompileConfigEntries(t, "bar", "default", "default", "dc1", connect.TestClusterID+".consul", nil)
snap.IngressGateway.DiscoveryChain = map[UpstreamID]*structs.CompiledDiscoveryChain{
UpstreamIDFromString("foo"): fooChain,
UpstreamIDFromString("bar"): barChain,
}
return snap
}
func httpMatch(http *structs.ServiceRouteHTTPMatch) *structs.ServiceRouteMatch {
return &structs.ServiceRouteMatch{HTTP: http}
}
func httpMatchHeader(headers ...structs.ServiceRouteHTTPMatchHeader) *structs.ServiceRouteMatch {
return httpMatch(&structs.ServiceRouteHTTPMatch{
Header: headers,
})
}
func httpMatchParam(params ...structs.ServiceRouteHTTPMatchQueryParam) *structs.ServiceRouteMatch {
return httpMatch(&structs.ServiceRouteHTTPMatch{
QueryParam: params,
})
}
func toService(svc string) *structs.ServiceRouteDestination {
return &structs.ServiceRouteDestination{Service: svc}
}
// ControllableCacheType is a cache.Type that simulates a typical blocking RPC
// but lets us control the responses and when they are delivered easily.
type ControllableCacheType struct {
index uint64
value sync.Map
// Need a condvar to trigger all blocking requests (there might be multiple
// for same type due to background refresh and timing issues) when values
// change. Chans make it nondeterministic which one triggers or need extra
// locking to coordinate replacing after close etc.
triggerMu sync.Mutex
trigger *sync.Cond
blocking bool
lastReq atomic.Value
}
// NewControllableCacheType returns a cache.Type that can be controlled for
// testing.
func NewControllableCacheType(t testing.T) *ControllableCacheType {
c := &ControllableCacheType{
index: 5,
blocking: true,
}
c.trigger = sync.NewCond(&c.triggerMu)
return c
}
// Set sets the response value to be returned from subsequent cache gets for the
// type.
func (ct *ControllableCacheType) Set(key string, value interface{}) {
atomic.AddUint64(&ct.index, 1)
ct.value.Store(key, value)
ct.triggerMu.Lock()
ct.trigger.Broadcast()
ct.triggerMu.Unlock()
}
// Fetch implements cache.Type. It simulates blocking or non-blocking queries.
func (ct *ControllableCacheType) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) {
index := atomic.LoadUint64(&ct.index)
ct.lastReq.Store(req)
shouldBlock := ct.blocking && opts.MinIndex > 0 && opts.MinIndex == index
if shouldBlock {
// Wait for return to be triggered. We ignore timeouts based on opts.Timeout
// since in practice they will always be way longer than our tests run for
// and the caller can simulate timeout by triggering return without changing
// index or value.
ct.triggerMu.Lock()
ct.trigger.Wait()
ct.triggerMu.Unlock()
}
info := req.CacheInfo()
key := path.Join(info.Key, info.Datacenter) // omit token for testing purposes
// reload index as it probably got bumped
index = atomic.LoadUint64(&ct.index)
val, _ := ct.value.Load(key)
if err, ok := val.(error); ok {
return cache.FetchResult{
Value: nil,
Index: index,
}, err
}
return cache.FetchResult{
Value: val,
Index: index,
}, nil
}
func (ct *ControllableCacheType) RegisterOptions() cache.RegisterOptions {
return cache.RegisterOptions{
Refresh: ct.blocking,
SupportsBlocking: ct.blocking,
QueryTimeout: 10 * time.Minute,
}
}
// golden is used to read golden files stores in consul/agent/xds/testdata
func golden(t testing.T, name string) string {
t.Helper()
golden := filepath.Join("../xds/testdata", name+".golden")
expected, err := ioutil.ReadFile(golden)
require.NoError(t, err)
return string(expected)
}