[API Gateway] Fix invalid cluster causing gateway programming delay (#16661)

* Add test for http routes

* Add fix

* Fix tests

* Add changelog entry

* Refactor and fix flaky tests

.changelog/16661.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS.
+```

agent/consul/discoverychain/gateway.go

@@ -128,6 +128,29 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover
 			return nil, nil, err
 		}
 
+		node := compiled.Nodes[compiled.StartNode]
+		if node.IsRouter() {
+			resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + node.Name
+
+			// clean out the clusters that will get added for the router
+			for name := range compiled.Nodes {
+				if strings.HasPrefix(name, resolverPrefix) {
+					delete(compiled.Nodes, name)
+				}
+			}
+
+			// clean out the route rules that'll get added for the router
+			filtered := []*structs.DiscoveryRoute{}
+			for _, route := range node.Routes {
+				if strings.HasPrefix(route.NextNode, resolverPrefix) {
+					continue
+				}
+				filtered = append(filtered, route)
+			}
+			node.Routes = filtered
+		}
+		compiled.Nodes[compiled.StartNode] = node
+
 		// fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present
 		for name, node := range compiled.Nodes {
 			switch node.Type {

agent/consul/discoverychain/gateway_test.go

@@ -47,7 +47,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) {
 		route                     structs.HTTPRouteConfigEntry
 		expectedMatchesByHostname map[string][]hostnameMatch
 	}{
-		"no hostanames": {
+		"no hostnames": {
 			route: structs.HTTPRouteConfigEntry{
 				Kind: structs.HTTPRoute,
 				Name: "route",
@@ -539,15 +539,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
 				Protocol:    "http",
 				StartNode:   "router:gateway-suffix-9b9265b.default.default",
 				Nodes: map[string]*structs.DiscoveryGraphNode{
-					"resolver:gateway-suffix-9b9265b.default.default.dc1": {
-						Type: "resolver",
-						Name: "gateway-suffix-9b9265b.default.default.dc1",
-						Resolver: &structs.DiscoveryResolver{
-							Target:         "gateway-suffix-9b9265b.default.default.dc1",
-							Default:        true,
-							ConnectTimeout: 5000000000,
-						},
-					},
 					"router:gateway-suffix-9b9265b.default.default": {
 						Type: "router",
 						Name: "gateway-suffix-9b9265b.default.default",
@@ -569,20 +560,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
 								},
 							},
 							NextNode: "resolver:foo.default.default.dc1",
-						}, {
-							Definition: &structs.ServiceRoute{
-								Match: &structs.ServiceRouteMatch{
-									HTTP: &structs.ServiceRouteHTTPMatch{
-										PathPrefix: "/",
-									},
-								},
-								Destination: &structs.ServiceRouteDestination{
-									Service:   "gateway-suffix-9b9265b",
-									Partition: "default",
-									Namespace: "default",
-								},
-							},
-							NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1",
 						}},
 					},
 					"resolver:foo.default.default.dc1": {
@@ -704,15 +681,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
 				Protocol:    "http",
 				StartNode:   "router:gateway-suffix-9b9265b.default.default",
 				Nodes: map[string]*structs.DiscoveryGraphNode{
-					"resolver:gateway-suffix-9b9265b.default.default.dc1": {
-						Type: "resolver",
-						Name: "gateway-suffix-9b9265b.default.default.dc1",
-						Resolver: &structs.DiscoveryResolver{
-							Target:         "gateway-suffix-9b9265b.default.default.dc1",
-							Default:        true,
-							ConnectTimeout: 5000000000,
-						},
-					},
 					"resolver:service-one.default.default.dc1": {
 						Type: "resolver",
 						Name: "service-one.default.default.dc1",
@@ -770,20 +738,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
 								},
 							},
 							NextNode: "splitter:splitter-one.default.default",
-						}, {
-							Definition: &structs.ServiceRoute{
-								Match: &structs.ServiceRouteMatch{
-									HTTP: &structs.ServiceRouteHTTPMatch{
-										PathPrefix: "/",
-									},
-								},
-								Destination: &structs.ServiceRouteDestination{
-									Service:   "gateway-suffix-9b9265b",
-									Partition: "default",
-									Namespace: "default",
-								},
-							},
-							NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1",
 						}},
 					},
 					"splitter:splitter-one.default.default": {

agent/xds/resources_test.go

@@ -11,6 +11,8 @@ import (
 	envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 
+	"github.com/hashicorp/consul/agent/connect"
+	"github.com/hashicorp/consul/agent/consul/discoverychain"
 	"github.com/hashicorp/consul/agent/xds/testcommon"
 	"github.com/hashicorp/consul/envoyextensions/xdscommon"
 
@@ -175,7 +177,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) {
 	tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...)
 	tests = append(tests, getTrafficControlPeeringGoldenTestCases()...)
 	tests = append(tests, getEnterpriseGoldenTestCases()...)
-	tests = append(tests, getAPIGatewayGoldenTestCases()...)
+	tests = append(tests, getAPIGatewayGoldenTestCases(t)...)
 
 	latestEnvoyVersion := xdscommon.EnvoyVersions[0]
 	for _, envoyVersion := range xdscommon.EnvoyVersions {
@@ -314,7 +316,13 @@ AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r
 -----END CERTIFICATE-----`
 )
 
-func getAPIGatewayGoldenTestCases() []goldenTestCase {
+func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase {
+	t.Helper()
+
+	service := structs.NewServiceName("service", nil)
+	serviceUID := proxycfg.NewUpstreamIDFromServiceName(service)
+	serviceChain := discoverychain.TestCompileConfigEntries(t, "service", "default", "default", "dc1", connect.TestClusterID+".consul", nil)
+
 	return []goldenTestCase{
 		{
 			name: "api-gateway-with-tcp-route-and-inline-certificate",
@@ -362,5 +370,48 @@ func getAPIGatewayGoldenTestCases() []goldenTestCase {
 				}}, nil)
 			},
 		},
+		{
+			name: "api-gateway-with-http-route-and-inline-certificate",
+			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
+				return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) {
+					entry.Listeners = []structs.APIGatewayListener{
+						{
+							Name:     "listener",
+							Protocol: structs.ListenerProtocolHTTP,
+							Port:     8080,
+						},
+					}
+					bound.Listeners = []structs.BoundAPIGatewayListener{
+						{
+							Name: "listener",
+							Routes: []structs.ResourceReference{{
+								Kind: structs.HTTPRoute,
+								Name: "route",
+							}},
+						},
+					}
+				}, []structs.BoundRoute{
+					&structs.HTTPRouteConfigEntry{
+						Kind: structs.HTTPRoute,
+						Name: "route",
+						Rules: []structs.HTTPRouteRule{{
+							Services: []structs.HTTPService{{
+								Name: "service",
+							}},
+						}},
+					},
+				}, nil, []proxycfg.UpdateEvent{{
+					CorrelationID: "discovery-chain:" + serviceUID.String(),
+					Result: &structs.DiscoveryChainResponse{
+						Chain: serviceChain,
+					},
+				}, {
+					CorrelationID: "upstream-target:" + serviceChain.ID() + ":" + serviceUID.String(),
+					Result: &structs.IndexedCheckServiceNodes{
+						Nodes: proxycfg.TestUpstreamNodes(t, "service"),
+					},
+				}})
+			},
+		},
 	}
 }

agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden

@@ -0,0 +1,55 @@
+{
+  "versionInfo":  "00000001",
+  "resources":  [
+    {
+      "@type":  "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "name":  "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "altStatName":  "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "type":  "EDS",
+      "edsClusterConfig":  {
+        "edsConfig":  {
+          "ads":  {},
+          "resourceApiVersion":  "V3"
+        }
+      },
+      "connectTimeout":  "5s",
+      "circuitBreakers":  {},
+      "outlierDetection":  {},
+      "commonLbConfig":  {
+        "healthyPanicThreshold":  {}
+      },
+      "transportSocket":  {
+        "name":  "tls",
+        "typedConfig":  {
+          "@type":  "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
+          "commonTlsContext":  {
+            "tlsParams":  {},
+            "tlsCertificates":  [
+              {
+                "certificateChain":  {
+                  "inlineString":  "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
+                },
+                "privateKey":  {
+                  "inlineString":  "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
+                }
+              }
+            ],
+            "validationContext":  {
+              "trustedCa":  {
+                "inlineString":  "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
+              },
+              "matchSubjectAltNames":  [
+                {
+                  "exact":  "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/service"
+                }
+              ]
+            }
+          },
+          "sni":  "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+        }
+      }
+    }
+  ],
+  "typeUrl":  "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+  "nonce":  "00000001"
+}

agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden

@@ -0,0 +1,41 @@
+{
+  "versionInfo":  "00000001",
+  "resources":  [
+    {
+      "@type":  "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+      "clusterName":  "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints":  [
+        {
+          "lbEndpoints":  [
+            {
+              "endpoint":  {
+                "address":  {
+                  "socketAddress":  {
+                    "address":  "10.10.1.1",
+                    "portValue":  8080
+                  }
+                }
+              },
+              "healthStatus":  "HEALTHY",
+              "loadBalancingWeight":  1
+            },
+            {
+              "endpoint":  {
+                "address":  {
+                  "socketAddress":  {
+                    "address":  "10.10.1.2",
+                    "portValue":  8080
+                  }
+                }
+              },
+              "healthStatus":  "HEALTHY",
+              "loadBalancingWeight":  1
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "typeUrl":  "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+  "nonce":  "00000001"
+}

agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden

@@ -0,0 +1,49 @@
+{
+  "versionInfo":  "00000001",
+  "resources":  [
+    {
+      "@type":  "type.googleapis.com/envoy.config.listener.v3.Listener",
+      "name":  "http:1.2.3.4:8080",
+      "address":  {
+        "socketAddress":  {
+          "address":  "1.2.3.4",
+          "portValue":  8080
+        }
+      },
+      "filterChains":  [
+        {
+          "filters":  [
+            {
+              "name":  "envoy.filters.network.http_connection_manager",
+              "typedConfig":  {
+                "@type":  "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+                "statPrefix":  "ingress_upstream_8080",
+                "rds":  {
+                  "configSource":  {
+                    "ads":  {},
+                    "resourceApiVersion":  "V3"
+                  },
+                  "routeConfigName":  "8080"
+                },
+                "httpFilters":  [
+                  {
+                    "name":  "envoy.filters.http.router",
+                    "typedConfig":  {
+                      "@type":  "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
+                    }
+                  }
+                ],
+                "tracing":  {
+                  "randomSampling":  {}
+                }
+              }
+            }
+          ]
+        }
+      ],
+      "trafficDirection":  "OUTBOUND"
+    }
+  ],
+  "typeUrl":  "type.googleapis.com/envoy.config.listener.v3.Listener",
+  "nonce":  "00000001"
+}

agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden

@@ -0,0 +1,31 @@
+{
+  "versionInfo":  "00000001",
+  "resources":  [
+    {
+      "@type":  "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+      "name":  "8080",
+      "virtualHosts":  [
+        {
+          "name":  "api-gateway-listener-9b9265b",
+          "domains":  [
+            "*",
+            "*:8080"
+          ],
+          "routes":  [
+            {
+              "match":  {
+                "prefix":  "/"
+              },
+              "route":  {
+                "cluster":  "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+              }
+            }
+          ]
+        }
+      ],
+      "validateClusters":  true
+    }
+  ],
+  "typeUrl":  "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+  "nonce":  "00000001"
+}

agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden

@@ -0,0 +1,5 @@
+{
+  "versionInfo": "00000001",
+  "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
+  "nonce": "00000001"
+}

test/integration/consul-container/test/gateways/gateway_endpoint_test.go

@@ -9,28 +9,30 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/sdk/testutil/retry"
 	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
 	libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
-	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 	"github.com/hashicorp/go-cleanhttp"
 )
 
 // Creates a gateway service and tests to see if it is routable
 func TestAPIGatewayCreate(t *testing.T) {
-	t.Skip()
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
 
 	t.Parallel()
+
+	gatewayName := randomName("gateway", 16)
+	routeName := randomName("route", 16)
+	serviceName := randomName("service", 16)
 	listenerPortOne := 6000
+	serviceHTTPPort := 6001
+	serviceGRPCPort := 6002
 
 	clusterConfig := &libtopology.ClusterConfig{
 		NumServers: 1,
@@ -41,17 +43,28 @@ func TestAPIGatewayCreate(t *testing.T) {
 			InjectGossipEncryption: true,
 			AllowHTTPAnyway:        true,
 		},
-		Ports:                     []int{listenerPortOne},
+		Ports: []int{
-		ApplyDefaultProxySettings: true,
+			listenerPortOne,
+			serviceHTTPPort,
+			serviceGRPCPort,
+		},
 	}
 
 	cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
 	client := cluster.APIClient(0)
 
+	namespace := getNamespace()
+	if namespace != "" {
+		ns := &api.Namespace{Name: namespace}
+		_, _, err := client.Namespaces().Create(ns, nil)
+		require.NoError(t, err)
+	}
+
 	// add api gateway config
 	apiGateway := &api.APIGatewayConfigEntry{
-		Kind: api.APIGateway,
+		Kind:      api.APIGateway,
-		Name: "api-gateway",
+		Namespace: namespace,
+		Name:      gatewayName,
 		Listeners: []api.APIGatewayListener{
 			{
 				Name:     "listener",
@@ -63,32 +76,48 @@ func TestAPIGatewayCreate(t *testing.T) {
 
 	require.NoError(t, cluster.ConfigEntryWrite(apiGateway))
 
+	_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
+		ID:        serviceName,
+		Name:      serviceName,
+		Namespace: namespace,
+		HTTPPort:  serviceHTTPPort,
+		GRPCPort:  serviceGRPCPort,
+	})
+	require.NoError(t, err)
+
 	tcpRoute := &api.TCPRouteConfigEntry{
-		Kind: api.TCPRoute,
+		Kind:      api.TCPRoute,
-		Name: "api-gateway-route",
+		Name:      routeName,
+		Namespace: namespace,
 		Parents: []api.ResourceReference{
 			{
-				Kind: api.APIGateway,
+				Kind:      api.APIGateway,
-				Name: "api-gateway",
+				Namespace: namespace,
+				Name:      gatewayName,
 			},
 		},
 		Services: []api.TCPService{
 			{
-				Name: libservice.StaticServerServiceName,
+				Namespace: namespace,
+				Name:      serviceName,
 			},
 		},
 	}
 
 	require.NoError(t, cluster.ConfigEntryWrite(tcpRoute))
 
-	// Create a client proxy instance with the server as an upstream
+	// Create a gateway
-	_, gatewayService := createServices(t, cluster, listenerPortOne)
+	gatewayService, err := libservice.NewGatewayService(context.Background(), libservice.GatewayConfig{
+		Kind:      "api",
+		Namespace: namespace,
+		Name:      gatewayName,
+	}, cluster.Agents[0], listenerPortOne)
+	require.NoError(t, err)
 
 	// make sure the gateway/route come online
 	// make sure config entries have been properly created
-	namespace := getNamespace()
+	checkGatewayConfigEntry(t, client, gatewayName, namespace)
-	checkGatewayConfigEntry(t, client, "api-gateway", namespace)
+	checkTCPRouteConfigEntry(t, client, routeName, namespace)
-	checkTCPRouteConfigEntry(t, client, "api-gateway-route", namespace)
 
 	port, err := gatewayService.GetPort(listenerPortOne)
 	require.NoError(t, err)
@@ -112,72 +141,36 @@ func conditionStatusIsValue(typeName string, statusValue string, conditions []ap
 	return false
 }
 
-// TODO this code is just copy pasted from elsewhere, it is likely we will need to modify it some
-func createCluster(t *testing.T, ports ...int) *libcluster.Cluster {
-	opts := libcluster.BuildOptions{
-		InjectAutoEncryption:   true,
-		InjectGossipEncryption: true,
-		AllowHTTPAnyway:        true,
-	}
-	ctx := libcluster.NewBuildContext(t, opts)
-
-	conf := libcluster.NewConfigBuilder(ctx).
-		ToAgentConfig(t)
-	t.Logf("Cluster config:\n%s", conf.JSON)
-
-	configs := []libcluster.Config{*conf}
-
-	cluster, err := libcluster.New(t, configs, ports...)
-	require.NoError(t, err)
-
-	node := cluster.Agents[0]
-	client := node.GetClient()
-
-	libcluster.WaitForLeader(t, cluster, client)
-	libcluster.WaitForMembers(t, client, 1)
-
-	// Default Proxy Settings
-	ok, err := utils.ApplyDefaultProxySettings(client)
-	require.NoError(t, err)
-	require.True(t, ok)
-
-	require.NoError(t, err)
-
-	return cluster
-}
-
-func createGatewayConfigEntry(gatewayName, protocol, namespace string, listenerPort int) *api.APIGatewayConfigEntry {
-	return &api.APIGatewayConfigEntry{
-		Kind: api.APIGateway,
-		Name: gatewayName,
-		Listeners: []api.APIGatewayListener{
-			{
-				Name:     "listener",
-				Port:     listenerPort,
-				Protocol: protocol,
-			},
-		},
-		Namespace: namespace,
-	}
-}
-
 func checkGatewayConfigEntry(t *testing.T, client *api.Client, gatewayName string, namespace string) {
+	t.Helper()
+
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace})
-		require.NoError(t, err)
+		if err != nil {
-		if entry == nil {
+			t.Log("error constructing request", err)
 			return false
 		}
+		if entry == nil {
+			t.Log("returned entry is nil")
+			return false
+		}
+
 		apiEntry := entry.(*api.APIGatewayConfigEntry)
 		return isAccepted(apiEntry.Status.Conditions)
 	}, time.Second*10, time.Second*1)
 }
 
 func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) {
+	t.Helper()
+
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace})
-		require.NoError(t, err)
+		if err != nil {
+			t.Log("error constructing request", err)
+			return false
+		}
 		if entry == nil {
+			t.Log("returned entry is nil")
 			return false
 		}
 
@@ -187,10 +180,16 @@ func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName strin
 }
 
 func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) {
+	t.Helper()
+
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.TCPRoute, routeName, &api.QueryOptions{Namespace: namespace})
-		require.NoError(t, err)
+		if err != nil {
+			t.Log("error constructing request", err)
+			return false
+		}
 		if entry == nil {
+			t.Log("returned entry is nil")
 			return false
 		}
 
@@ -199,43 +198,6 @@ func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string
 	}, time.Second*10, time.Second*1)
 }
 
-func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libservice.ServiceOpts, containerArgs []string) libservice.Service {
-	node := cluster.Agents[0]
-	client := node.GetClient()
-	// Create a service and proxy instance
-	service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...)
-	require.NoError(t, err)
-
-	libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy", &api.QueryOptions{Namespace: serviceOpts.Namespace})
-	libassert.CatalogServiceExists(t, client, serviceOpts.Name, &api.QueryOptions{Namespace: serviceOpts.Namespace})
-
-	return service
-}
-
-func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (libservice.Service, libservice.Service) {
-	node := cluster.Agents[0]
-	client := node.GetClient()
-	// Create a service and proxy instance
-	serviceOpts := &libservice.ServiceOpts{
-		Name:     libservice.StaticServerServiceName,
-		ID:       "static-server",
-		HTTPPort: 8080,
-		GRPCPort: 8079,
-	}
-
-	clientConnectProxy := createService(t, cluster, serviceOpts, nil)
-	gwCfg := libservice.GatewayConfig{
-		Name: "api-gateway",
-		Kind: "api",
-	}
-
-	gatewayService, err := libservice.NewGatewayService(context.Background(), gwCfg, cluster.Agents[0], ports...)
-	require.NoError(t, err)
-	libassert.CatalogServiceExists(t, client, "api-gateway", nil)
-
-	return clientConnectProxy, gatewayService
-}
-
 type checkOptions struct {
 	debug      bool
 	statusCode int
@@ -244,26 +206,23 @@ type checkOptions struct {
 
 // checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances
 func checkRoute(t *testing.T, port int, path string, headers map[string]string, expected checkOptions) {
-	ip := "localhost"
+	t.Helper()
+
 	if expected.testName != "" {
 		t.Log("running " + expected.testName)
 	}
-	const phrase = "hello"
-
-	failer := func() *retry.Timer {
-		return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60}
-	}
 
 	client := cleanhttp.DefaultClient()
-
 	path = strings.TrimPrefix(path, "/")
-	url := fmt.Sprintf("http://%s:%d/%s", ip, port, path)
+	url := fmt.Sprintf("http://localhost:%d/%s", port, path)
 
-	retry.RunWith(failer(), t, func(r *retry.R) {
+	require.Eventually(t, func() bool {
-		t.Logf("making call to %s", url)
+		reader := strings.NewReader("hello")
-		reader := strings.NewReader(phrase)
 		req, err := http.NewRequest("POST", url, reader)
-		require.NoError(t, err)
+		if err != nil {
+			t.Log("error constructing request", err)
+			return false
+		}
 		headers["content-type"] = "text/plain"
 
 		for k, v := range headers {
@@ -273,39 +232,41 @@ func checkRoute(t *testing.T, port int, path string, headers map[string]string,
 				req.Host = v
 			}
 		}
+
 		res, err := client.Do(req)
 		if err != nil {
-			t.Log(err)
+			t.Log("error sending request", err)
-			r.Fatal("could not make call to service ", url)
+			return false
 		}
 		defer res.Body.Close()
 
 		body, err := io.ReadAll(res.Body)
 		if err != nil {
-			r.Fatal("could not read response body ", url)
+			t.Log("error reading response body", err)
+			return false
 		}
 
-		assert.Equal(t, expected.statusCode, res.StatusCode)
 		if expected.statusCode != res.StatusCode {
-			r.Fatal("unexpected response code returned")
+			t.Logf("bad status code - expected: %d, actual: %d", expected.statusCode, res.StatusCode)
+			return false
+		}
+		if expected.debug {
+			if !strings.Contains(string(body), "debug") {
+				t.Log("body does not contain 'debug'")
+				return false
+			}
+		}
+		if !strings.Contains(string(body), "hello") {
+			t.Log("body does not contain 'hello'")
+			return false
 		}
 
-		// if debug is expected, debug should be in the response body
+		return true
-		assert.Equal(t, expected.debug, strings.Contains(string(body), "debug"))
+	}, time.Second*30, time.Second*1)
-		if expected.statusCode != res.StatusCode {
-			r.Fatal("unexpected response body returned")
-		}
-
-		if !strings.Contains(string(body), phrase) {
-			r.Fatal("received an incorrect response ", string(body))
-		}
-	})
 }
 
 func checkRouteError(t *testing.T, ip string, port int, path string, headers map[string]string, expected string) {
-	failer := func() *retry.Timer {
+	t.Helper()
-		return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60}
-	}
 
 	client := cleanhttp.DefaultClient()
 	url := fmt.Sprintf("http://%s:%d", ip, port)
@@ -314,11 +275,12 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map
 		url += "/" + path
 	}
 
-	retry.RunWith(failer(), t, func(r *retry.R) {
+	require.Eventually(t, func() bool {
-		t.Logf("making call to %s", url)
 		req, err := http.NewRequest("GET", url, nil)
-		assert.NoError(t, err)
+		if err != nil {
-
+			t.Log("error constructing request", err)
+			return false
+		}
 		for k, v := range headers {
 			req.Header.Set(k, v)
 
@@ -327,10 +289,16 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map
 			}
 		}
 		_, err = client.Do(req)
-		assert.Error(t, err)
+		if err == nil {
-
+			t.Log("client request should have errored, but didn't")
-		if expected != "" {
+			return false
-			assert.ErrorContains(t, err, expected)
 		}
-	})
+		if expected != "" {
+			if !strings.Contains(err.Error(), expected) {
+				t.Logf("expected %q to contain %q", err.Error(), expected)
+				return false
+			}
+		}
+		return true
+	}, time.Second*30, time.Second*1)
 }

test/integration/consul-container/test/gateways/http_route_test.go

@@ -36,10 +36,25 @@ func TestHTTPRouteFlattening(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
+
 	t.Parallel()
 
 	// infrastructure set up
-	listenerPort := 6000
+	listenerPort := 6004
+	serviceOneHTTPPort := 6005
+	serviceOneGRPCPort := 6006
+	serviceTwoHTTPPort := 6007
+	serviceTwoGRPCPort := 6008
+
+	serviceOneName := randomName("service", 16)
+	serviceTwoName := randomName("service", 16)
+	serviceOneResponseCode := 200
+	serviceTwoResponseCode := 418
+	gatewayName := randomName("gw", 16)
+	routeOneName := randomName("route", 16)
+	routeTwoName := randomName("route", 16)
+	path1 := "/"
+	path2 := "/v2"
 
 	clusterConfig := &libtopology.ClusterConfig{
 		NumServers: 1,
@@ -50,7 +65,13 @@ func TestHTTPRouteFlattening(t *testing.T) {
 			InjectGossipEncryption: true,
 			AllowHTTPAnyway:        true,
 		},
-		Ports:                     []int{listenerPort},
+		Ports: []int{
+			listenerPort,
+			serviceOneHTTPPort,
+			serviceOneGRPCPort,
+			serviceTwoHTTPPort,
+			serviceTwoGRPCPort,
+		},
 		ApplyDefaultProxySettings: true,
 	}
 
@@ -64,40 +85,34 @@ func TestHTTPRouteFlattening(t *testing.T) {
 		require.NoError(t, err)
 	}
 
-	service1ResponseCode := 200
+	_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
-	service2ResponseCode := 418
+		ID:        serviceOneName,
-	serviceOne := createService(t, cluster, &libservice.ServiceOpts{
+		Name:      serviceOneName,
-		Name:      "service1",
-		ID:        "service1",
-		HTTPPort:  8080,
-		GRPCPort:  8079,
 		Namespace: namespace,
-	}, []string{
+		HTTPPort:  serviceOneHTTPPort,
-		// customizes response code so we can distinguish between which service is responding
+		GRPCPort:  serviceOneGRPCPort,
-		"-echo-server-default-params", fmt.Sprintf("status=%d", service1ResponseCode),
-	})
-	serviceTwo := createService(t, cluster, &libservice.ServiceOpts{
-		Name:      "service2",
-		ID:        "service2",
-		HTTPPort:  8081,
-		GRPCPort:  8082,
-		Namespace: namespace,
-	}, []string{
-		"-echo-server-default-params", fmt.Sprintf("status=%d", service2ResponseCode),
 	},
+		// customizes response code so we can distinguish between which service is responding
+		"-echo-server-default-params", fmt.Sprintf("status=%d", serviceOneResponseCode),
 	)
+	require.NoError(t, err)
 
-	gatewayName := randomName("gw", 16)
+	_, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
-	routeOneName := randomName("route", 16)
+		ID:        serviceTwoName,
-	routeTwoName := randomName("route", 16)
+		Name:      serviceTwoName,
-	path1 := "/"
+		Namespace: namespace,
-	path2 := "/v2"
+		HTTPPort:  serviceTwoHTTPPort,
+		GRPCPort:  serviceTwoGRPCPort,
+	},
+		// customizes response code so we can distinguish between which service is responding
+		"-echo-server-default-params", fmt.Sprintf("status=%d", serviceTwoResponseCode),
+	)
+	require.NoError(t, err)
 
 	// write config entries
 	proxyDefaults := &api.ProxyConfigEntry{
-		Kind:      api.ProxyDefaults,
+		Kind: api.ProxyDefaults,
-		Name:      api.ProxyConfigGlobal,
+		Name: api.ProxyConfigGlobal,
-		Namespace: "", // proxy-defaults can only be set in the default namespace
 		Config: map[string]interface{}{
 			"protocol": "http",
 		},
@@ -119,8 +134,9 @@ func TestHTTPRouteFlattening(t *testing.T) {
 	}
 
 	routeOne := &api.HTTPRouteConfigEntry{
-		Kind: api.HTTPRoute,
+		Kind:      api.HTTPRoute,
-		Name: routeOneName,
+		Name:      routeOneName,
+		Namespace: namespace,
 		Parents: []api.ResourceReference{
 			{
 				Kind:      api.APIGateway,
@@ -132,12 +148,11 @@ func TestHTTPRouteFlattening(t *testing.T) {
 			"test.foo",
 			"test.example",
 		},
-		Namespace: namespace,
 		Rules: []api.HTTPRouteRule{
 			{
 				Services: []api.HTTPService{
 					{
-						Name:      serviceOne.GetServiceName(),
+						Name:      serviceOneName,
 						Namespace: namespace,
 					},
 				},
@@ -154,8 +169,9 @@ func TestHTTPRouteFlattening(t *testing.T) {
 	}
 
 	routeTwo := &api.HTTPRouteConfigEntry{
-		Kind: api.HTTPRoute,
+		Kind:      api.HTTPRoute,
-		Name: routeTwoName,
+		Name:      routeTwoName,
+		Namespace: namespace,
 		Parents: []api.ResourceReference{
 			{
 				Kind:      api.APIGateway,
@@ -166,12 +182,11 @@ func TestHTTPRouteFlattening(t *testing.T) {
 		Hostnames: []string{
 			"test.foo",
 		},
-		Namespace: namespace,
 		Rules: []api.HTTPRouteRule{
 			{
 				Services: []api.HTTPService{
 					{
-						Name:      serviceTwo.GetServiceName(),
+						Name:      serviceTwoName,
 						Namespace: namespace,
 					},
 				},
@@ -210,6 +225,7 @@ func TestHTTPRouteFlattening(t *testing.T) {
 
 	// make sure config entries have been properly created
 	checkGatewayConfigEntry(t, client, gatewayName, namespace)
+	t.Log("checking route one")
 	checkHTTPRouteConfigEntry(t, client, routeOneName, namespace)
 	checkHTTPRouteConfigEntry(t, client, routeTwoName, namespace)
 
@@ -222,31 +238,31 @@ func TestHTTPRouteFlattening(t *testing.T) {
 	checkRoute(t, gatewayPort, "/v2", map[string]string{
 		"Host": "test.foo",
 		"x-v2": "v2",
-	}, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"})
+	}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 header and path"})
 	checkRoute(t, gatewayPort, "/v2", map[string]string{
 		"Host": "test.foo",
-	}, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"})
+	}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just path match"})
 
 	// //v1 path with the header
 	checkRoute(t, gatewayPort, "/check", map[string]string{
 		"Host": "test.foo",
 		"x-v2": "v2",
-	}, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"})
+	}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just header match"})
 
 	checkRoute(t, gatewayPort, "/v2/path/value", map[string]string{
 		"Host": "test.foo",
 		"x-v2": "v2",
-	}, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"})
+	}, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 v2 with path"})
 
 	// hit service 1 by hitting root path
 	checkRoute(t, gatewayPort, "", map[string]string{
 		"Host": "test.foo",
-	}, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"})
+	}, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1 root prefix"})
 
 	// hit service 1 by hitting v2 path with v1 hostname
 	checkRoute(t, gatewayPort, "/v2", map[string]string{
 		"Host": "test.example",
-	}, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"})
+	}, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1, v2 path with v2 hostname"})
 }
 
 func TestHTTPRoutePathRewrite(t *testing.T) {
@@ -257,14 +273,46 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 	t.Parallel()
 
 	// infrastructure set up
-	listenerPort := 6001
+	listenerPort := 6009
+	fooHTTPPort := 6010
+	fooGRPCPort := 6011
+	barHTTPPort := 6012
+	barGRPCPort := 6013
+
+	fooName := randomName("foo", 16)
+	barName := randomName("bar", 16)
+	gatewayName := randomName("gw", 16)
+	invalidRouteName := randomName("route", 16)
+	validRouteName := randomName("route", 16)
+
 	// create cluster
-	cluster := createCluster(t, listenerPort)
+	clusterConfig := &libtopology.ClusterConfig{
-	client := cluster.Agents[0].GetClient()
+		NumServers: 1,
+		NumClients: 1,
+		BuildOpts: &libcluster.BuildOptions{
+			Datacenter:             "dc1",
+			InjectAutoEncryption:   true,
+			InjectGossipEncryption: true,
+			AllowHTTPAnyway:        true,
+		},
+		Ports: []int{
+			listenerPort,
+			fooHTTPPort,
+			fooGRPCPort,
+			barHTTPPort,
+			barGRPCPort,
+		},
+		ApplyDefaultProxySettings: true,
+	}
+
+	cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
+	client := cluster.APIClient(0)
+
 	fooStatusCode := 400
 	barStatusCode := 201
 	fooPath := "/v1/foo"
 	barPath := "/v1/bar"
+
 	namespace := getNamespace()
 	if namespace != "" {
 		ns := &api.Namespace{Name: namespace}
@@ -272,33 +320,32 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 		require.NoError(t, err)
 	}
 
-	fooService := createService(t, cluster, &libservice.ServiceOpts{
+	_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
-		Name:      "foo",
+		ID:        fooName,
-		ID:        "foo",
+		Name:      fooName,
-		HTTPPort:  8080,
-		GRPCPort:  8081,
 		Namespace: namespace,
-	}, []string{
+		HTTPPort:  fooHTTPPort,
+		GRPCPort:  fooGRPCPort,
+	},
 		// customizes response code so we can distinguish between which service is responding
 		"-echo-debug-path", fooPath,
 		"-echo-server-default-params", fmt.Sprintf("status=%d", fooStatusCode),
-	})
+	)
-	barService := createService(t, cluster, &libservice.ServiceOpts{
+	require.NoError(t, err)
-		Name: "bar",
+
-		ID:   "bar",
+	_, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
-		// TODO we can potentially get conflicts if these ports are the same
+		ID:        barName,
-		HTTPPort:  8079,
+		Name:      barName,
-		GRPCPort:  8078,
 		Namespace: namespace,
-	}, []string{
+		HTTPPort:  barHTTPPort,
+		GRPCPort:  barGRPCPort,
+	},
+		// customizes response code so we can distinguish between which service is responding
 		"-echo-debug-path", barPath,
 		"-echo-server-default-params", fmt.Sprintf("status=%d", barStatusCode),
-	},
 	)
+	require.NoError(t, err)
 
-	gatewayName := randomName("gw", 16)
-	invalidRouteName := randomName("route", 16)
-	validRouteName := randomName("route", 16)
 	fooUnrewritten := "/foo"
 	barUnrewritten := "/bar"
 
@@ -314,7 +361,18 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 
 	require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults))
 
-	apiGateway := createGatewayConfigEntry(gatewayName, "http", namespace, listenerPort)
+	apiGateway := &api.APIGatewayConfigEntry{
+		Kind: api.APIGateway,
+		Name: gatewayName,
+		Listeners: []api.APIGatewayListener{
+			{
+				Name:     "listener",
+				Port:     listenerPort,
+				Protocol: "http",
+			},
+		},
+		Namespace: namespace,
+	}
 
 	fooRoute := &api.HTTPRouteConfigEntry{
 		Kind: api.HTTPRoute,
@@ -339,7 +397,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 				},
 				Services: []api.HTTPService{
 					{
-						Name:      fooService.GetServiceName(),
+						Name:      fooName,
 						Namespace: namespace,
 					},
 				},
@@ -378,7 +436,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 				},
 				Services: []api.HTTPService{
 					{
-						Name:      barService.GetServiceName(),
+						Name:      barName,
 						Namespace: namespace,
 					},
 				},
@@ -445,17 +503,43 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
+
 	t.Parallel()
 
 	// infrastructure set up
 	address := "localhost"
 
-	listenerOnePort := 6000
+	listenerOnePort := 6014
-	listenerTwoPort := 6001
+	listenerTwoPort := 6015
+	serviceHTTPPort := 6016
+	serviceGRPCPort := 6017
 
-	// create cluster and service
+	serviceName := randomName("service", 16)
-	cluster := createCluster(t, listenerOnePort, listenerTwoPort)
+	gatewayOneName := randomName("gw1", 16)
-	client := cluster.Agents[0].GetClient()
+	gatewayTwoName := randomName("gw2", 16)
+	routeName := randomName("route", 16)
+
+	// create cluster
+	clusterConfig := &libtopology.ClusterConfig{
+		NumServers: 1,
+		NumClients: 1,
+		BuildOpts: &libcluster.BuildOptions{
+			Datacenter:             "dc1",
+			InjectAutoEncryption:   true,
+			InjectGossipEncryption: true,
+			AllowHTTPAnyway:        true,
+		},
+		Ports: []int{
+			listenerOnePort,
+			listenerTwoPort,
+			serviceHTTPPort,
+			serviceGRPCPort,
+		},
+		ApplyDefaultProxySettings: true,
+	}
+
+	cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
+	client := cluster.APIClient(0)
 
 	// getNamespace() should always return an empty string in Consul OSS
 	namespace := getNamespace()
@@ -465,23 +549,19 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 		require.NoError(t, err)
 	}
 
-	service := createService(t, cluster, &libservice.ServiceOpts{
+	_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{
-		Name:      "service",
+		ID:        serviceName,
-		ID:        "service",
+		Name:      serviceName,
-		HTTPPort:  8080,
-		GRPCPort:  8079,
 		Namespace: namespace,
-	}, []string{})
+		HTTPPort:  serviceHTTPPort,
-
+		GRPCPort:  serviceGRPCPort,
-	gatewayOneName := randomName("gw1", 16)
+	})
-	gatewayTwoName := randomName("gw2", 16)
+	require.NoError(t, err)
-	routeName := randomName("route", 16)
 
 	// write config entries
 	proxyDefaults := &api.ProxyConfigEntry{
-		Kind:      api.ProxyDefaults,
+		Kind: api.ProxyDefaults,
-		Name:      api.ProxyConfigGlobal,
+		Name: api.ProxyConfigGlobal,
-		Namespace: "", // proxy-defaults can only be set in the default namespace
 		Config: map[string]interface{}{
 			"protocol": "http",
 		},
@@ -504,16 +584,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 		Namespace: namespace,
 	}
 	require.NoError(t, cluster.ConfigEntryWrite(gatewayOne))
-	require.Eventually(t, func() bool {
+	checkGatewayConfigEntry(t, client, gatewayOneName, namespace)
-		entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayOneName, &api.QueryOptions{Namespace: namespace})
-		assert.NoError(t, err)
-		if entry == nil {
-			return false
-		}
-		apiEntry := entry.(*api.APIGatewayConfigEntry)
-		t.Log(entry)
-		return isAccepted(apiEntry.Status.Conditions)
-	}, time.Second*10, time.Second*1)
 
 	// create gateway service
 	gwOneCfg := libservice.GatewayConfig{
@@ -539,19 +610,8 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 		},
 		Namespace: namespace,
 	}
-
 	require.NoError(t, cluster.ConfigEntryWrite(gatewayTwo))
-
+	checkGatewayConfigEntry(t, client, gatewayTwoName, namespace)
-	require.Eventually(t, func() bool {
-		entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayTwoName, &api.QueryOptions{Namespace: namespace})
-		assert.NoError(t, err)
-		if entry == nil {
-			return false
-		}
-		apiEntry := entry.(*api.APIGatewayConfigEntry)
-		t.Log(entry)
-		return isAccepted(apiEntry.Status.Conditions)
-	}, time.Second*10, time.Second*1)
 
 	// create gateway service
 	gwTwoCfg := libservice.GatewayConfig{
@@ -583,7 +643,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 			{
 				Services: []api.HTTPService{
 					{
-						Name:      service.GetServiceName(),
+						Name:      serviceName,
 						Namespace: namespace,
 					},
 				},