Merge pull request #4216 from hashicorp/NET-2252-add-assert-fortioname (#16212)

NET-2252: integration tests: add assert.FortioName

test/integration/consul-container/libs/assert/service.go

@@ -3,6 +3,8 @@ package assert
 import (
 	"fmt"
 	"io"
+	"net/http"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -13,6 +15,7 @@ import (
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
+	"github.com/stretchr/testify/assert"
 )
 
 const (
@@ -89,6 +92,30 @@ func ServiceLogContains(t *testing.T, service libservice.Service, target string)
 	return strings.Contains(logs, target)
 }
 
+// AssertFortioName asserts that the fortio service replying at urlbase/debug
+// has a `FORTIO_NAME` env variable set. This validates that the client is sending
+// traffic to the right envoy proxy.
+//
+// It retries with timeout defaultHTTPTimeout and wait defaultHTTPWait.
+func AssertFortioName(t *testing.T, urlbase string, name string) {
+	t.Helper()
+	var fortioNameRE = regexp.MustCompile(("\nFORTIO_NAME=(.+)\n"))
+	var body []byte
+	retry.RunWith(&retry.Timer{Timeout: defaultHTTPTimeout, Wait: defaultHTTPWait}, t, func(r *retry.R) {
+		resp, err := http.Get(fmt.Sprintf("%s/debug?env=dump", urlbase))
+		if err != nil {
+			r.Error(err)
+			return
+		}
+		defer resp.Body.Close()
+		body, err = io.ReadAll(resp.Body)
+		require.NoError(t, err)
+	})
+	m := fortioNameRE.FindStringSubmatch(string(body))
+	require.GreaterOrEqual(t, len(m), 2)
+	assert.Equal(t, name, m[1])
+}
+
 // AssertContainerState validates service container status
 func AssertContainerState(t *testing.T, service libservice.Service, state string) {
 	containerStatus, err := service.GetStatus()

test/integration/consul-container/libs/cluster/container.go

@@ -514,8 +514,13 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR
 
 			"8443/tcp", // Envoy Gateway Listener
 
-			"8079/tcp", // Envoy App Listener
+			"8079/tcp", // Envoy App Listener - grpc port used by static-server
-			"8080/tcp", // Envoy App Listener
+			"8078/tcp", // Envoy App Listener - grpc port used by static-server-v1
+			"8077/tcp", // Envoy App Listener - grpc port used by static-server-v2
+
+			"8080/tcp", // Envoy App Listener - http port used by static-server
+			"8081/tcp", // Envoy App Listener - http port used by static-server-v1
+			"8082/tcp", // Envoy App Listener - http port used by static-server-v2
 			"9998/tcp", // Envoy App Listener
 			"9999/tcp", // Envoy App Listener
 		},

test/integration/consul-container/libs/service/helpers.go

@@ -16,9 +16,11 @@ const (
 )
 
 type ServiceOpts struct {
-	Name string
+	Name     string
-	ID   string
+	ID       string
-	Meta map[string]string
+	Meta     map[string]string
+	HTTPPort int
+	GRPCPort int
 }
 
 func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) {
@@ -32,7 +34,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 	req := &api.AgentServiceRegistration{
 		Name: serviceOpts.Name,
 		ID:   serviceOpts.ID,
-		Port: 8080,
+		Port: serviceOpts.HTTPPort,
 		Connect: &api.AgentServiceConnect{
 			SidecarService: &api.AgentServiceRegistration{
 				Proxy: &api.AgentServiceConnectProxyConfig{},
@@ -40,7 +42,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 		},
 		Check: &api.AgentServiceCheck{
 			Name:     "Static Server Listening",
-			TCP:      fmt.Sprintf("127.0.0.1:%d", 8080),
+			TCP:      fmt.Sprintf("127.0.0.1:%d", serviceOpts.HTTPPort),
 			Interval: "10s",
 			Status:   api.HealthPassing,
 		},
@@ -52,7 +54,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 	}
 
 	// Create a service and proxy instance
-	serverService, err := NewExampleService(context.Background(), StaticServerServiceName, 8080, 8079, node)
+	serverService, err := NewExampleService(context.Background(), serviceOpts.ID, serviceOpts.HTTPPort, serviceOpts.GRPCPort, node)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -60,7 +62,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 		_ = serverService.Terminate()
 	})
 
-	serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticServerServiceName), serviceOpts.ID, 8080, node) // bindPort not used
+	serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticServerServiceName), serviceOpts.ID, serviceOpts.HTTPPort, node) // bindPort not used
 	if err != nil {
 		return nil, nil, err
 	}

test/integration/consul-container/libs/topology/peering_topology.go

@@ -68,9 +68,11 @@ func BasicPeeringTwoClustersSetup(
 		var err error
 		// Create a service and proxy instance
 		serviceOpts := libservice.ServiceOpts{
-			Name: libservice.StaticServerServiceName,
+			Name:     libservice.StaticServerServiceName,
-			ID:   "static-server",
+			ID:       "static-server",
-			Meta: map[string]string{"version": ""},
+			Meta:     map[string]string{"version": ""},
+			HTTPPort: 8080,
+			GRPCPort: 8079,
 		}
 		serverSidecarService, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(clientNode, &serviceOpts)
 		require.NoError(t, err)
@@ -107,6 +109,7 @@ func BasicPeeringTwoClustersSetup(
 	libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", DialingPeerName), "HEALTHY", 1)
 	_, port := clientSidecarService.GetAddr()
 	libassert.HTTPServiceEchoes(t, "localhost", port, "")
+	libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 
 	return &BuiltCluster{
 			Cluster:   acceptingCluster,

test/integration/consul-container/test/basic/connect_service_test.go

@@ -1,6 +1,7 @@
 package basic
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -33,6 +34,7 @@ func TestBasicConnectService(t *testing.T) {
 
 	libassert.AssertContainerState(t, clientService, "running")
 	libassert.HTTPServiceEchoes(t, "localhost", port, "")
+	libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 }
 
 func createCluster(t *testing.T) *libcluster.Cluster {
@@ -72,8 +74,10 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) libservice.Servic
 	client := node.GetClient()
 	// Create a service and proxy instance
 	serviceOpts := &libservice.ServiceOpts{
-		Name: libservice.StaticServerServiceName,
+		Name:     libservice.StaticServerServiceName,
-		ID:   "static-server",
+		ID:       "static-server",
+		HTTPPort: 8080,
+		GRPCPort: 8079,
 	}
 
 	// Create a service and proxy instance

test/integration/consul-container/test/observability/access_logs_test.go

@@ -70,6 +70,7 @@ func TestAccessLogs(t *testing.T) {
 	// Validate Custom JSON
 	require.Eventually(t, func() bool {
 		libassert.HTTPServiceEchoes(t, "localhost", port, "banana")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 		client := libassert.ServiceLogContains(t, clientService, "\"banana_path\":\"/banana\"")
 		server := libassert.ServiceLogContains(t, serverService, "\"banana_path\":\"/banana\"")
 		return client && server
@@ -111,6 +112,7 @@ func TestAccessLogs(t *testing.T) {
 	_, port = clientService.GetAddr()
 	require.Eventually(t, func() bool {
 		libassert.HTTPServiceEchoes(t, "localhost", port, "orange")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 		client := libassert.ServiceLogContains(t, clientService, "Orange you glad I didn't say banana: /orange, -")
 		server := libassert.ServiceLogContains(t, serverService, "Orange you glad I didn't say banana: /orange, -")
 		return client && server
@@ -137,8 +139,10 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Servi
 
 	// Create a service and proxy instance
 	serviceOpts := &libservice.ServiceOpts{
-		Name: libservice.StaticServerServiceName,
+		Name:     libservice.StaticServerServiceName,
-		ID:   "static-server",
+		ID:       "static-server",
+		HTTPPort: 8080,
+		GRPCPort: 8079,
 	}
 
 	// Create a service and proxy instance

test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go

@@ -3,6 +3,7 @@ package peering
 import (
 	"context"
 	"encoding/pem"
+	"fmt"
 	"testing"
 	"time"
 
@@ -93,6 +94,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) {
 
 		_, port := clientSidecarService.GetAddr()
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 	}
 
 	testutil.RunStep(t, "rotate exporting cluster's root CA", func(t *testing.T) {
@@ -142,6 +144,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) {
 		// Connectivity should still be contained
 		_, port := clientSidecarService.GetAddr()
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 
 		verifySidecarHasTwoRootCAs(t, clientSidecarService)
 	})
@@ -163,6 +166,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) {
 
 		_, port := clientSidecarService.GetAddr()
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 	})
 }
 

test/integration/consul-container/test/upgrade/peer_control_plane_mgw_test.go

@@ -110,6 +110,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) {
 		require.NoError(t, clientSidecarService.Restart())
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1)
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 	}
 
 	for _, tc := range tcs {

test/integration/consul-container/test/upgrade/peers_http_test.go

@@ -80,6 +80,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 		require.NoError(t, clientSidecarService.Restart())
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1)
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server")
 	}
 
 	for _, tc := range tcs {

test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go

@@ -76,28 +76,26 @@ func TestTrafficManagement_ServiceWithSubsets(t *testing.T) {
 		err := cluster.ConfigEntryWrite(serviceResolver)
 		require.NoError(t, err)
 
-		serverService, serverServiceV1, serverServiceV2, clientService := createService(t, cluster)
+		serverConnectProxy, serverConnectProxyV1, serverConnectProxyV2, clientConnectProxy := createService(t, cluster)
 
-		_, port := clientService.GetAddr()
+		_, port := clientConnectProxy.GetAddr()
-		_, adminPort := clientService.GetAdminAddr()
+		_, adminPort := clientConnectProxy.GetAdminAddr()
-		_, serverAdminPort := serverService.GetAdminAddr()
+		_, serverAdminPort := serverConnectProxy.GetAdminAddr()
-		_, serverAdminPortV1 := serverServiceV1.GetAdminAddr()
+		_, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr()
-		_, serverAdminPortV2 := serverServiceV2.GetAdminAddr()
+		_, serverAdminPortV2 := serverConnectProxyV2.GetAdminAddr()
 
 		// validate client and proxy is up and running
-		libassert.AssertContainerState(t, clientService, "running")
+		libassert.AssertContainerState(t, clientConnectProxy, "running")
-
-		// TO-DO: static-client upstream should be able to connect to static-server-v2 via upstream s2
 
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1)
 
 		// Upgrade cluster, restart sidecars then begin service traffic validation
 		require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion))
-		require.NoError(t, clientService.Restart())
+		require.NoError(t, clientConnectProxy.Restart())
-		require.NoError(t, serverService.Restart())
+		require.NoError(t, serverConnectProxy.Restart())
-		require.NoError(t, serverServiceV1.Restart())
+		require.NoError(t, serverConnectProxyV1.Restart())
-		require.NoError(t, serverServiceV2.Restart())
+		require.NoError(t, serverConnectProxyV2.Restart())
 
 		// POST upgrade validation; repeat client & proxy validation
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
@@ -129,7 +127,8 @@ func TestTrafficManagement_ServiceWithSubsets(t *testing.T) {
 		libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server")
 		libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV2, "static-server")
 
-		// TO-DO: restart envoy sidecar and validate traffic management
+		// static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-v2")
 	}
 
 	for _, tc := range tcs {
@@ -146,35 +145,41 @@ func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Servic
 	client := node.GetClient()
 
 	serviceOpts := &libservice.ServiceOpts{
-		Name: libservice.StaticServerServiceName,
+		Name:     libservice.StaticServerServiceName,
-		ID:   "static-server",
+		ID:       "static-server",
+		HTTPPort: 8080,
+		GRPCPort: 8079,
 	}
-	_, serverService, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts)
+	_, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts)
 	libassert.CatalogServiceExists(t, client, "static-server")
 	require.NoError(t, err)
 
 	serviceOptsV1 := &libservice.ServiceOpts{
-		Name: libservice.StaticServerServiceName,
+		Name:     libservice.StaticServerServiceName,
-		ID:   "static-server-v1",
+		ID:       "static-server-v1",
-		Meta: map[string]string{"version": "v1"},
+		Meta:     map[string]string{"version": "v1"},
+		HTTPPort: 8081,
+		GRPCPort: 8078,
 	}
-	_, serverServiceV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1)
+	_, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1)
 	libassert.CatalogServiceExists(t, client, "static-server")
 	require.NoError(t, err)
 
 	serviceOptsV2 := &libservice.ServiceOpts{
-		Name: libservice.StaticServerServiceName,
+		Name:     libservice.StaticServerServiceName,
-		ID:   "static-server-v2",
+		ID:       "static-server-v2",
-		Meta: map[string]string{"version": "v2"},
+		Meta:     map[string]string{"version": "v2"},
+		HTTPPort: 8082,
+		GRPCPort: 8077,
 	}
-	_, serverServiceV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2)
+	_, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2)
 	libassert.CatalogServiceExists(t, client, "static-server")
 	require.NoError(t, err)
 
 	// Create a client proxy instance with the server as an upstream
-	clientService, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false)
+	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false)
 	require.NoError(t, err)
 	libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName))
 
-	return serverService, serverServiceV1, serverServiceV2, clientService
+	return serverConnectProxy, serverConnectProxyV1, serverConnectProxyV2, clientConnectProxy
 }