[API Gateway] Fix invalid cluster causing gateway programming delay (#16661)

* Add test for http routes

* Add fix

* Fix tests

* Add changelog entry

* Refactor and fix flaky tests
This commit is contained in:
Andrew Stucki 2023-03-17 13:31:04 -04:00 committed by GitHub
parent 7ff42ea796
commit a597cb3d57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 544 additions and 304 deletions

3
.changelog/16661.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:bug
gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS.
```

View File

@ -128,6 +128,29 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover
return nil, nil, err
}
node := compiled.Nodes[compiled.StartNode]
if node.IsRouter() {
resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + node.Name
// clean out the clusters that will get added for the router
for name := range compiled.Nodes {
if strings.HasPrefix(name, resolverPrefix) {
delete(compiled.Nodes, name)
}
}
// clean out the route rules that'll get added for the router
filtered := []*structs.DiscoveryRoute{}
for _, route := range node.Routes {
if strings.HasPrefix(route.NextNode, resolverPrefix) {
continue
}
filtered = append(filtered, route)
}
node.Routes = filtered
}
compiled.Nodes[compiled.StartNode] = node
// fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present
for name, node := range compiled.Nodes {
switch node.Type {

View File

@ -47,7 +47,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) {
route structs.HTTPRouteConfigEntry
expectedMatchesByHostname map[string][]hostnameMatch
}{
"no hostanames": {
"no hostnames": {
route: structs.HTTPRouteConfigEntry{
Kind: structs.HTTPRoute,
Name: "route",
@ -539,15 +539,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
Protocol: "http",
StartNode: "router:gateway-suffix-9b9265b.default.default",
Nodes: map[string]*structs.DiscoveryGraphNode{
"resolver:gateway-suffix-9b9265b.default.default.dc1": {
Type: "resolver",
Name: "gateway-suffix-9b9265b.default.default.dc1",
Resolver: &structs.DiscoveryResolver{
Target: "gateway-suffix-9b9265b.default.default.dc1",
Default: true,
ConnectTimeout: 5000000000,
},
},
"router:gateway-suffix-9b9265b.default.default": {
Type: "router",
Name: "gateway-suffix-9b9265b.default.default",
@ -569,20 +560,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
},
},
NextNode: "resolver:foo.default.default.dc1",
}, {
Definition: &structs.ServiceRoute{
Match: &structs.ServiceRouteMatch{
HTTP: &structs.ServiceRouteHTTPMatch{
PathPrefix: "/",
},
},
Destination: &structs.ServiceRouteDestination{
Service: "gateway-suffix-9b9265b",
Partition: "default",
Namespace: "default",
},
},
NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1",
}},
},
"resolver:foo.default.default.dc1": {
@ -704,15 +681,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
Protocol: "http",
StartNode: "router:gateway-suffix-9b9265b.default.default",
Nodes: map[string]*structs.DiscoveryGraphNode{
"resolver:gateway-suffix-9b9265b.default.default.dc1": {
Type: "resolver",
Name: "gateway-suffix-9b9265b.default.default.dc1",
Resolver: &structs.DiscoveryResolver{
Target: "gateway-suffix-9b9265b.default.default.dc1",
Default: true,
ConnectTimeout: 5000000000,
},
},
"resolver:service-one.default.default.dc1": {
Type: "resolver",
Name: "service-one.default.default.dc1",
@ -770,20 +738,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
},
},
NextNode: "splitter:splitter-one.default.default",
}, {
Definition: &structs.ServiceRoute{
Match: &structs.ServiceRouteMatch{
HTTP: &structs.ServiceRouteHTTPMatch{
PathPrefix: "/",
},
},
Destination: &structs.ServiceRouteDestination{
Service: "gateway-suffix-9b9265b",
Partition: "default",
Namespace: "default",
},
},
NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1",
}},
},
"splitter:splitter-one.default.default": {

View File

@ -11,6 +11,8 @@ import (
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/consul/discoverychain"
"github.com/hashicorp/consul/agent/xds/testcommon"
"github.com/hashicorp/consul/envoyextensions/xdscommon"
@ -175,7 +177,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) {
tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...)
tests = append(tests, getTrafficControlPeeringGoldenTestCases()...)
tests = append(tests, getEnterpriseGoldenTestCases()...)
tests = append(tests, getAPIGatewayGoldenTestCases()...)
tests = append(tests, getAPIGatewayGoldenTestCases(t)...)
latestEnvoyVersion := xdscommon.EnvoyVersions[0]
for _, envoyVersion := range xdscommon.EnvoyVersions {
@ -314,7 +316,13 @@ AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r
-----END CERTIFICATE-----`
)
func getAPIGatewayGoldenTestCases() []goldenTestCase {
func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase {
t.Helper()
service := structs.NewServiceName("service", nil)
serviceUID := proxycfg.NewUpstreamIDFromServiceName(service)
serviceChain := discoverychain.TestCompileConfigEntries(t, "service", "default", "default", "dc1", connect.TestClusterID+".consul", nil)
return []goldenTestCase{
{
name: "api-gateway-with-tcp-route-and-inline-certificate",
@ -362,5 +370,48 @@ func getAPIGatewayGoldenTestCases() []goldenTestCase {
}}, nil)
},
},
{
name: "api-gateway-with-http-route-and-inline-certificate",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) {
entry.Listeners = []structs.APIGatewayListener{
{
Name: "listener",
Protocol: structs.ListenerProtocolHTTP,
Port: 8080,
},
}
bound.Listeners = []structs.BoundAPIGatewayListener{
{
Name: "listener",
Routes: []structs.ResourceReference{{
Kind: structs.HTTPRoute,
Name: "route",
}},
},
}
}, []structs.BoundRoute{
&structs.HTTPRouteConfigEntry{
Kind: structs.HTTPRoute,
Name: "route",
Rules: []structs.HTTPRouteRule{{
Services: []structs.HTTPService{{
Name: "service",
}},
}},
},
}, nil, []proxycfg.UpdateEvent{{
CorrelationID: "discovery-chain:" + serviceUID.String(),
Result: &structs.DiscoveryChainResponse{
Chain: serviceChain,
},
}, {
CorrelationID: "upstream-target:" + serviceChain.ID() + ":" + serviceUID.String(),
Result: &structs.IndexedCheckServiceNodes{
Nodes: proxycfg.TestUpstreamNodes(t, "service"),
},
}})
},
},
}
}

View File

@ -0,0 +1,55 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"altStatName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {},
"outlierDetection": {},
"commonLbConfig": {
"healthyPanicThreshold": {}
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/service"
}
]
}
},
"sni": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

View File

@ -0,0 +1,41 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"nonce": "00000001"
}

View File

@ -0,0 +1,49 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "http:1.2.3.4:8080",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8080
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"statPrefix": "ingress_upstream_8080",
"rds": {
"configSource": {
"ads": {},
"resourceApiVersion": "V3"
},
"routeConfigName": "8080"
},
"httpFilters": [
{
"name": "envoy.filters.http.router",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"tracing": {
"randomSampling": {}
}
}
}
]
}
],
"trafficDirection": "OUTBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,31 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "8080",
"virtualHosts": [
{
"name": "api-gateway-listener-9b9265b",
"domains": [
"*",
"*:8080"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,5 @@
{
"versionInfo": "00000001",
"typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
"nonce": "00000001"
}

View File

@ -9,28 +9,30 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/sdk/testutil/retry"
libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
"github.com/hashicorp/go-cleanhttp"
)
// Creates a gateway service and tests to see if it is routable
func TestAPIGatewayCreate(t *testing.T) {
t.Skip()
if testing.Short() {
t.Skip("too slow for testing.Short")
}
t.Parallel()
gatewayName := randomName("gateway", 16)
routeName := randomName("route", 16)
serviceName := randomName("service", 16)
listenerPortOne := 6000
serviceHTTPPort := 6001
serviceGRPCPort := 6002
clusterConfig := &libtopology.ClusterConfig{
NumServers: 1,
@ -41,17 +43,28 @@ func TestAPIGatewayCreate(t *testing.T) {
InjectGossipEncryption: true,
AllowHTTPAnyway: true,
},
Ports: []int{listenerPortOne},
ApplyDefaultProxySettings: true,
Ports: []int{
listenerPortOne,
serviceHTTPPort,
serviceGRPCPort,
},
}
cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
client := cluster.APIClient(0)
namespace := getNamespace()
if namespace != "" {
ns := &api.Namespace{Name: namespace}
_, _, err := client.Namespaces().Create(ns, nil)
require.NoError(t, err)
}
// add api gateway config
apiGateway := &api.APIGatewayConfigEntry{
Kind: api.APIGateway,
Name: "api-gateway",
Kind: api.APIGateway,
Namespace: namespace,
Name: gatewayName,
Listeners: []api.APIGatewayListener{
{
Name: "listener",
@ -63,32 +76,48 @@ func TestAPIGatewayCreate(t *testing.T) {
require.NoError(t, cluster.ConfigEntryWrite(apiGateway))
_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
ID: serviceName,
Name: serviceName,
Namespace: namespace,
HTTPPort: serviceHTTPPort,
GRPCPort: serviceGRPCPort,
})
require.NoError(t, err)
tcpRoute := &api.TCPRouteConfigEntry{
Kind: api.TCPRoute,
Name: "api-gateway-route",
Kind: api.TCPRoute,
Name: routeName,
Namespace: namespace,
Parents: []api.ResourceReference{
{
Kind: api.APIGateway,
Name: "api-gateway",
Kind: api.APIGateway,
Namespace: namespace,
Name: gatewayName,
},
},
Services: []api.TCPService{
{
Name: libservice.StaticServerServiceName,
Namespace: namespace,
Name: serviceName,
},
},
}
require.NoError(t, cluster.ConfigEntryWrite(tcpRoute))
// Create a client proxy instance with the server as an upstream
_, gatewayService := createServices(t, cluster, listenerPortOne)
// Create a gateway
gatewayService, err := libservice.NewGatewayService(context.Background(), libservice.GatewayConfig{
Kind: "api",
Namespace: namespace,
Name: gatewayName,
}, cluster.Agents[0], listenerPortOne)
require.NoError(t, err)
// make sure the gateway/route come online
// make sure config entries have been properly created
namespace := getNamespace()
checkGatewayConfigEntry(t, client, "api-gateway", namespace)
checkTCPRouteConfigEntry(t, client, "api-gateway-route", namespace)
checkGatewayConfigEntry(t, client, gatewayName, namespace)
checkTCPRouteConfigEntry(t, client, routeName, namespace)
port, err := gatewayService.GetPort(listenerPortOne)
require.NoError(t, err)
@ -112,72 +141,36 @@ func conditionStatusIsValue(typeName string, statusValue string, conditions []ap
return false
}
// TODO this code is just copy pasted from elsewhere, it is likely we will need to modify it some
func createCluster(t *testing.T, ports ...int) *libcluster.Cluster {
opts := libcluster.BuildOptions{
InjectAutoEncryption: true,
InjectGossipEncryption: true,
AllowHTTPAnyway: true,
}
ctx := libcluster.NewBuildContext(t, opts)
conf := libcluster.NewConfigBuilder(ctx).
ToAgentConfig(t)
t.Logf("Cluster config:\n%s", conf.JSON)
configs := []libcluster.Config{*conf}
cluster, err := libcluster.New(t, configs, ports...)
require.NoError(t, err)
node := cluster.Agents[0]
client := node.GetClient()
libcluster.WaitForLeader(t, cluster, client)
libcluster.WaitForMembers(t, client, 1)
// Default Proxy Settings
ok, err := utils.ApplyDefaultProxySettings(client)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, err)
return cluster
}
func createGatewayConfigEntry(gatewayName, protocol, namespace string, listenerPort int) *api.APIGatewayConfigEntry {
return &api.APIGatewayConfigEntry{
Kind: api.APIGateway,
Name: gatewayName,
Listeners: []api.APIGatewayListener{
{
Name: "listener",
Port: listenerPort,
Protocol: protocol,
},
},
Namespace: namespace,
}
}
func checkGatewayConfigEntry(t *testing.T, client *api.Client, gatewayName string, namespace string) {
t.Helper()
require.Eventually(t, func() bool {
entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace})
require.NoError(t, err)
if entry == nil {
if err != nil {
t.Log("error constructing request", err)
return false
}
if entry == nil {
t.Log("returned entry is nil")
return false
}
apiEntry := entry.(*api.APIGatewayConfigEntry)
return isAccepted(apiEntry.Status.Conditions)
}, time.Second*10, time.Second*1)
}
func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) {
t.Helper()
require.Eventually(t, func() bool {
entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace})
require.NoError(t, err)
if err != nil {
t.Log("error constructing request", err)
return false
}
if entry == nil {
t.Log("returned entry is nil")
return false
}
@ -187,10 +180,16 @@ func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName strin
}
func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) {
t.Helper()
require.Eventually(t, func() bool {
entry, _, err := client.ConfigEntries().Get(api.TCPRoute, routeName, &api.QueryOptions{Namespace: namespace})
require.NoError(t, err)
if err != nil {
t.Log("error constructing request", err)
return false
}
if entry == nil {
t.Log("returned entry is nil")
return false
}
@ -199,43 +198,6 @@ func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string
}, time.Second*10, time.Second*1)
}
func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libservice.ServiceOpts, containerArgs []string) libservice.Service {
node := cluster.Agents[0]
client := node.GetClient()
// Create a service and proxy instance
service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...)
require.NoError(t, err)
libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy", &api.QueryOptions{Namespace: serviceOpts.Namespace})
libassert.CatalogServiceExists(t, client, serviceOpts.Name, &api.QueryOptions{Namespace: serviceOpts.Namespace})
return service
}
func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (libservice.Service, libservice.Service) {
node := cluster.Agents[0]
client := node.GetClient()
// Create a service and proxy instance
serviceOpts := &libservice.ServiceOpts{
Name: libservice.StaticServerServiceName,
ID: "static-server",
HTTPPort: 8080,
GRPCPort: 8079,
}
clientConnectProxy := createService(t, cluster, serviceOpts, nil)
gwCfg := libservice.GatewayConfig{
Name: "api-gateway",
Kind: "api",
}
gatewayService, err := libservice.NewGatewayService(context.Background(), gwCfg, cluster.Agents[0], ports...)
require.NoError(t, err)
libassert.CatalogServiceExists(t, client, "api-gateway", nil)
return clientConnectProxy, gatewayService
}
type checkOptions struct {
debug bool
statusCode int
@ -244,26 +206,23 @@ type checkOptions struct {
// checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances
func checkRoute(t *testing.T, port int, path string, headers map[string]string, expected checkOptions) {
ip := "localhost"
t.Helper()
if expected.testName != "" {
t.Log("running " + expected.testName)
}
const phrase = "hello"
failer := func() *retry.Timer {
return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60}
}
client := cleanhttp.DefaultClient()
path = strings.TrimPrefix(path, "/")
url := fmt.Sprintf("http://%s:%d/%s", ip, port, path)
url := fmt.Sprintf("http://localhost:%d/%s", port, path)
retry.RunWith(failer(), t, func(r *retry.R) {
t.Logf("making call to %s", url)
reader := strings.NewReader(phrase)
require.Eventually(t, func() bool {
reader := strings.NewReader("hello")
req, err := http.NewRequest("POST", url, reader)
require.NoError(t, err)
if err != nil {
t.Log("error constructing request", err)
return false
}
headers["content-type"] = "text/plain"
for k, v := range headers {
@ -273,39 +232,41 @@ func checkRoute(t *testing.T, port int, path string, headers map[string]string,
req.Host = v
}
}
res, err := client.Do(req)
if err != nil {
t.Log(err)
r.Fatal("could not make call to service ", url)
t.Log("error sending request", err)
return false
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
r.Fatal("could not read response body ", url)
t.Log("error reading response body", err)
return false
}
assert.Equal(t, expected.statusCode, res.StatusCode)
if expected.statusCode != res.StatusCode {
r.Fatal("unexpected response code returned")
t.Logf("bad status code - expected: %d, actual: %d", expected.statusCode, res.StatusCode)
return false
}
if expected.debug {
if !strings.Contains(string(body), "debug") {
t.Log("body does not contain 'debug'")
return false
}
}
if !strings.Contains(string(body), "hello") {
t.Log("body does not contain 'hello'")
return false
}
// if debug is expected, debug should be in the response body
assert.Equal(t, expected.debug, strings.Contains(string(body), "debug"))
if expected.statusCode != res.StatusCode {
r.Fatal("unexpected response body returned")
}
if !strings.Contains(string(body), phrase) {
r.Fatal("received an incorrect response ", string(body))
}
})
return true
}, time.Second*30, time.Second*1)
}
func checkRouteError(t *testing.T, ip string, port int, path string, headers map[string]string, expected string) {
failer := func() *retry.Timer {
return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60}
}
t.Helper()
client := cleanhttp.DefaultClient()
url := fmt.Sprintf("http://%s:%d", ip, port)
@ -314,11 +275,12 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map
url += "/" + path
}
retry.RunWith(failer(), t, func(r *retry.R) {
t.Logf("making call to %s", url)
require.Eventually(t, func() bool {
req, err := http.NewRequest("GET", url, nil)
assert.NoError(t, err)
if err != nil {
t.Log("error constructing request", err)
return false
}
for k, v := range headers {
req.Header.Set(k, v)
@ -327,10 +289,16 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map
}
}
_, err = client.Do(req)
assert.Error(t, err)
if expected != "" {
assert.ErrorContains(t, err, expected)
if err == nil {
t.Log("client request should have errored, but didn't")
return false
}
})
if expected != "" {
if !strings.Contains(err.Error(), expected) {
t.Logf("expected %q to contain %q", err.Error(), expected)
return false
}
}
return true
}, time.Second*30, time.Second*1)
}

View File

@ -36,10 +36,25 @@ func TestHTTPRouteFlattening(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
t.Parallel()
// infrastructure set up
listenerPort := 6000
listenerPort := 6004
serviceOneHTTPPort := 6005
serviceOneGRPCPort := 6006
serviceTwoHTTPPort := 6007
serviceTwoGRPCPort := 6008
serviceOneName := randomName("service", 16)
serviceTwoName := randomName("service", 16)
serviceOneResponseCode := 200
serviceTwoResponseCode := 418
gatewayName := randomName("gw", 16)
routeOneName := randomName("route", 16)
routeTwoName := randomName("route", 16)
path1 := "/"
path2 := "/v2"
clusterConfig := &libtopology.ClusterConfig{
NumServers: 1,
@ -50,7 +65,13 @@ func TestHTTPRouteFlattening(t *testing.T) {
InjectGossipEncryption: true,
AllowHTTPAnyway: true,
},
Ports: []int{listenerPort},
Ports: []int{
listenerPort,
serviceOneHTTPPort,
serviceOneGRPCPort,
serviceTwoHTTPPort,
serviceTwoGRPCPort,
},
ApplyDefaultProxySettings: true,
}
@ -64,40 +85,34 @@ func TestHTTPRouteFlattening(t *testing.T) {
require.NoError(t, err)
}
service1ResponseCode := 200
service2ResponseCode := 418
serviceOne := createService(t, cluster, &libservice.ServiceOpts{
Name: "service1",
ID: "service1",
HTTPPort: 8080,
GRPCPort: 8079,
_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
ID: serviceOneName,
Name: serviceOneName,
Namespace: namespace,
}, []string{
// customizes response code so we can distinguish between which service is responding
"-echo-server-default-params", fmt.Sprintf("status=%d", service1ResponseCode),
})
serviceTwo := createService(t, cluster, &libservice.ServiceOpts{
Name: "service2",
ID: "service2",
HTTPPort: 8081,
GRPCPort: 8082,
Namespace: namespace,
}, []string{
"-echo-server-default-params", fmt.Sprintf("status=%d", service2ResponseCode),
HTTPPort: serviceOneHTTPPort,
GRPCPort: serviceOneGRPCPort,
},
// customizes response code so we can distinguish between which service is responding
"-echo-server-default-params", fmt.Sprintf("status=%d", serviceOneResponseCode),
)
require.NoError(t, err)
gatewayName := randomName("gw", 16)
routeOneName := randomName("route", 16)
routeTwoName := randomName("route", 16)
path1 := "/"
path2 := "/v2"
_, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
ID: serviceTwoName,
Name: serviceTwoName,
Namespace: namespace,
HTTPPort: serviceTwoHTTPPort,
GRPCPort: serviceTwoGRPCPort,
},
// customizes response code so we can distinguish between which service is responding
"-echo-server-default-params", fmt.Sprintf("status=%d", serviceTwoResponseCode),
)
require.NoError(t, err)
// write config entries
proxyDefaults := &api.ProxyConfigEntry{
Kind: api.ProxyDefaults,
Name: api.ProxyConfigGlobal,
Namespace: "", // proxy-defaults can only be set in the default namespace
Kind: api.ProxyDefaults,
Name: api.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
@ -119,8 +134,9 @@ func TestHTTPRouteFlattening(t *testing.T) {
}
routeOne := &api.HTTPRouteConfigEntry{
Kind: api.HTTPRoute,
Name: routeOneName,
Kind: api.HTTPRoute,
Name: routeOneName,
Namespace: namespace,
Parents: []api.ResourceReference{
{
Kind: api.APIGateway,
@ -132,12 +148,11 @@ func TestHTTPRouteFlattening(t *testing.T) {
"test.foo",
"test.example",
},
Namespace: namespace,
Rules: []api.HTTPRouteRule{
{
Services: []api.HTTPService{
{
Name: serviceOne.GetServiceName(),
Name: serviceOneName,
Namespace: namespace,
},
},
@ -154,8 +169,9 @@ func TestHTTPRouteFlattening(t *testing.T) {
}
routeTwo := &api.HTTPRouteConfigEntry{
Kind: api.HTTPRoute,
Name: routeTwoName,
Kind: api.HTTPRoute,
Name: routeTwoName,
Namespace: namespace,
Parents: []api.ResourceReference{
{
Kind: api.APIGateway,
@ -166,12 +182,11 @@ func TestHTTPRouteFlattening(t *testing.T) {
Hostnames: []string{
"test.foo",
},
Namespace: namespace,
Rules: []api.HTTPRouteRule{
{
Services: []api.HTTPService{
{
Name: serviceTwo.GetServiceName(),
Name: serviceTwoName,
Namespace: namespace,
},
},
@ -210,6 +225,7 @@ func TestHTTPRouteFlattening(t *testing.T) {
// make sure config entries have been properly created
checkGatewayConfigEntry(t, client, gatewayName, namespace)
t.Log("checking route one")
checkHTTPRouteConfigEntry(t, client, routeOneName, namespace)
checkHTTPRouteConfigEntry(t, client, routeTwoName, namespace)
@ -222,31 +238,31 @@ func TestHTTPRouteFlattening(t *testing.T) {
checkRoute(t, gatewayPort, "/v2", map[string]string{
"Host": "test.foo",
"x-v2": "v2",
}, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"})
}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 header and path"})
checkRoute(t, gatewayPort, "/v2", map[string]string{
"Host": "test.foo",
}, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"})
}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just path match"})
// //v1 path with the header
checkRoute(t, gatewayPort, "/check", map[string]string{
"Host": "test.foo",
"x-v2": "v2",
}, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"})
}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just header match"})
checkRoute(t, gatewayPort, "/v2/path/value", map[string]string{
"Host": "test.foo",
"x-v2": "v2",
}, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"})
}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 v2 with path"})
// hit service 1 by hitting root path
checkRoute(t, gatewayPort, "", map[string]string{
"Host": "test.foo",
}, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"})
}, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1 root prefix"})
// hit service 1 by hitting v2 path with v1 hostname
checkRoute(t, gatewayPort, "/v2", map[string]string{
"Host": "test.example",
}, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"})
}, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1, v2 path with v2 hostname"})
}
func TestHTTPRoutePathRewrite(t *testing.T) {
@ -257,14 +273,46 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
t.Parallel()
// infrastructure set up
listenerPort := 6001
listenerPort := 6009
fooHTTPPort := 6010
fooGRPCPort := 6011
barHTTPPort := 6012
barGRPCPort := 6013
fooName := randomName("foo", 16)
barName := randomName("bar", 16)
gatewayName := randomName("gw", 16)
invalidRouteName := randomName("route", 16)
validRouteName := randomName("route", 16)
// create cluster
cluster := createCluster(t, listenerPort)
client := cluster.Agents[0].GetClient()
clusterConfig := &libtopology.ClusterConfig{
NumServers: 1,
NumClients: 1,
BuildOpts: &libcluster.BuildOptions{
Datacenter: "dc1",
InjectAutoEncryption: true,
InjectGossipEncryption: true,
AllowHTTPAnyway: true,
},
Ports: []int{
listenerPort,
fooHTTPPort,
fooGRPCPort,
barHTTPPort,
barGRPCPort,
},
ApplyDefaultProxySettings: true,
}
cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
client := cluster.APIClient(0)
fooStatusCode := 400
barStatusCode := 201
fooPath := "/v1/foo"
barPath := "/v1/bar"
namespace := getNamespace()
if namespace != "" {
ns := &api.Namespace{Name: namespace}
@ -272,33 +320,32 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
require.NoError(t, err)
}
fooService := createService(t, cluster, &libservice.ServiceOpts{
Name: "foo",
ID: "foo",
HTTPPort: 8080,
GRPCPort: 8081,
_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
ID: fooName,
Name: fooName,
Namespace: namespace,
}, []string{
HTTPPort: fooHTTPPort,
GRPCPort: fooGRPCPort,
},
// customizes response code so we can distinguish between which service is responding
"-echo-debug-path", fooPath,
"-echo-server-default-params", fmt.Sprintf("status=%d", fooStatusCode),
})
barService := createService(t, cluster, &libservice.ServiceOpts{
Name: "bar",
ID: "bar",
// TODO we can potentially get conflicts if these ports are the same
HTTPPort: 8079,
GRPCPort: 8078,
)
require.NoError(t, err)
_, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
ID: barName,
Name: barName,
Namespace: namespace,
}, []string{
HTTPPort: barHTTPPort,
GRPCPort: barGRPCPort,
},
// customizes response code so we can distinguish between which service is responding
"-echo-debug-path", barPath,
"-echo-server-default-params", fmt.Sprintf("status=%d", barStatusCode),
},
)
require.NoError(t, err)
gatewayName := randomName("gw", 16)
invalidRouteName := randomName("route", 16)
validRouteName := randomName("route", 16)
fooUnrewritten := "/foo"
barUnrewritten := "/bar"
@ -314,7 +361,18 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults))
apiGateway := createGatewayConfigEntry(gatewayName, "http", namespace, listenerPort)
apiGateway := &api.APIGatewayConfigEntry{
Kind: api.APIGateway,
Name: gatewayName,
Listeners: []api.APIGatewayListener{
{
Name: "listener",
Port: listenerPort,
Protocol: "http",
},
},
Namespace: namespace,
}
fooRoute := &api.HTTPRouteConfigEntry{
Kind: api.HTTPRoute,
@ -339,7 +397,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
},
Services: []api.HTTPService{
{
Name: fooService.GetServiceName(),
Name: fooName,
Namespace: namespace,
},
},
@ -378,7 +436,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
},
Services: []api.HTTPService{
{
Name: barService.GetServiceName(),
Name: barName,
Namespace: namespace,
},
},
@ -445,17 +503,43 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
t.Parallel()
// infrastructure set up
address := "localhost"
listenerOnePort := 6000
listenerTwoPort := 6001
listenerOnePort := 6014
listenerTwoPort := 6015
serviceHTTPPort := 6016
serviceGRPCPort := 6017
// create cluster and service
cluster := createCluster(t, listenerOnePort, listenerTwoPort)
client := cluster.Agents[0].GetClient()
serviceName := randomName("service", 16)
gatewayOneName := randomName("gw1", 16)
gatewayTwoName := randomName("gw2", 16)
routeName := randomName("route", 16)
// create cluster
clusterConfig := &libtopology.ClusterConfig{
NumServers: 1,
NumClients: 1,
BuildOpts: &libcluster.BuildOptions{
Datacenter: "dc1",
InjectAutoEncryption: true,
InjectGossipEncryption: true,
AllowHTTPAnyway: true,
},
Ports: []int{
listenerOnePort,
listenerTwoPort,
serviceHTTPPort,
serviceGRPCPort,
},
ApplyDefaultProxySettings: true,
}
cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
client := cluster.APIClient(0)
// getNamespace() should always return an empty string in Consul OSS
namespace := getNamespace()
@ -465,23 +549,19 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
require.NoError(t, err)
}
service := createService(t, cluster, &libservice.ServiceOpts{
Name: "service",
ID: "service",
HTTPPort: 8080,
GRPCPort: 8079,
_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
ID: serviceName,
Name: serviceName,
Namespace: namespace,
}, []string{})
gatewayOneName := randomName("gw1", 16)
gatewayTwoName := randomName("gw2", 16)
routeName := randomName("route", 16)
HTTPPort: serviceHTTPPort,
GRPCPort: serviceGRPCPort,
})
require.NoError(t, err)
// write config entries
proxyDefaults := &api.ProxyConfigEntry{
Kind: api.ProxyDefaults,
Name: api.ProxyConfigGlobal,
Namespace: "", // proxy-defaults can only be set in the default namespace
Kind: api.ProxyDefaults,
Name: api.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
@ -504,16 +584,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
Namespace: namespace,
}
require.NoError(t, cluster.ConfigEntryWrite(gatewayOne))
require.Eventually(t, func() bool {
entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayOneName, &api.QueryOptions{Namespace: namespace})
assert.NoError(t, err)
if entry == nil {
return false
}
apiEntry := entry.(*api.APIGatewayConfigEntry)
t.Log(entry)
return isAccepted(apiEntry.Status.Conditions)
}, time.Second*10, time.Second*1)
checkGatewayConfigEntry(t, client, gatewayOneName, namespace)
// create gateway service
gwOneCfg := libservice.GatewayConfig{
@ -539,19 +610,8 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
},
Namespace: namespace,
}
require.NoError(t, cluster.ConfigEntryWrite(gatewayTwo))
require.Eventually(t, func() bool {
entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayTwoName, &api.QueryOptions{Namespace: namespace})
assert.NoError(t, err)
if entry == nil {
return false
}
apiEntry := entry.(*api.APIGatewayConfigEntry)
t.Log(entry)
return isAccepted(apiEntry.Status.Conditions)
}, time.Second*10, time.Second*1)
checkGatewayConfigEntry(t, client, gatewayTwoName, namespace)
// create gateway service
gwTwoCfg := libservice.GatewayConfig{
@ -583,7 +643,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
{
Services: []api.HTTPService{
{
Name: service.GetServiceName(),
Name: serviceName,
Namespace: namespace,
},
},