NET-2087: Restart proxy sidecar during cluster upgrade (#16140)

test/integration/consul-container/libs/service/connect.go

@@ -40,7 +40,24 @@ func (g ConnectContainer) GetAddr() (string, int) {
 }
 
 func (g ConnectContainer) Restart() error {
-	return fmt.Errorf("Restart Unimplemented by ConnectContainer")
+	_, err := g.GetStatus()
+	if err != nil {
+		return fmt.Errorf("error fetching sidecar container state %s", err)
+	}
+
+	fmt.Printf("Stopping container: %s\n", g.GetName())
+	err = g.container.Stop(g.ctx, nil)
+
+	if err != nil {
+		return fmt.Errorf("error stopping sidecar container %s", err)
+	}
+
+	fmt.Printf("Starting container: %s\n", g.GetName())
+	err = g.container.Start(g.ctx)
+	if err != nil {
+		return fmt.Errorf("error starting sidecar container %s", err)
+	}
+	return nil
 }
 
 func (g ConnectContainer) GetLogs() (string, error) {

test/integration/consul-container/test/upgrade/peer_control_plane_mgw_test.go

@@ -90,9 +90,9 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) {
 		//  - Register a new static-client service in dialing cluster and
 		//  - set upstream to static-server service in peered cluster
 
-		// Restart the gateway
+		// Restart the gateway & proxy sidecar
-		err = dialing.Gateway.Restart()
+		require.NoError(t, dialing.Gateway.Restart())
-		require.NoError(t, err)
+		require.NoError(t, dialing.Container.Restart())
 
 		// Restarted gateway should not have any measurement on data plane traffic
 		libassert.AssertEnvoyMetricAtMost(t, gatewayAdminPort,
@@ -107,6 +107,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) {
 		require.NoError(t, err)
 		_, port := clientSidecarService.GetAddr()
 		_, adminPort := clientSidecarService.GetAdminAddr()
+		require.NoError(t, clientSidecarService.Restart())
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1)
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
 	}

test/integration/consul-container/test/upgrade/peers_http_test.go

@@ -64,9 +64,10 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 		//  - Register a new static-client service in dialing cluster and
 		//  - set upstream to static-server service in peered cluster
 
-		// Restart the gateway
+		// Restart the gateway & proxy sidecar
-		err = dialing.Gateway.Restart()
+		require.NoError(t, dialing.Gateway.Restart())
-		require.NoError(t, err)
+		require.NoError(t, dialing.Container.Restart())
+
 		// Restarted gateway should not have any measurement on data plane traffic
 		libassert.AssertEnvoyMetricAtMost(t, gatewayAdminPort,
 			"cluster.static-server.default.default.accepting-to-dialer.external",
@@ -76,6 +77,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 		require.NoError(t, err)
 		_, port := clientSidecarService.GetAddr()
 		_, adminPort := clientSidecarService.GetAdminAddr()
+		require.NoError(t, clientSidecarService.Restart())
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1)
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
 	}

test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go

@@ -1,4 +1,4 @@
-package l7trafficmanagement
+package upgrade
 
 import (
 	"context"
@@ -24,11 +24,13 @@ import (
 // Steps:
 //   - Create a single agent cluster.
 //   - Create one static-server and 2 subsets and 1 client and sidecar, then register them with Consul
-//   - Validate static-server and 2 subsets are and proxy admin endpoint is healthy
+//   - Validate static-server and 2 subsets are and proxy admin endpoint is healthy - 3 instances
 //   - Validate static servers proxy listeners should be up and have right certs
-func TestTrafficManagement_SetupServerAndClientWithSubsets(t *testing.T) {
+func TestTrafficManagement_ServiceWithSubsets(t *testing.T) {
 	t.Parallel()
 
+	var responseFormat = map[string]string{"format": "json"}
+
 	type testcase struct {
 		oldversion    string
 		targetVersion string
@@ -87,30 +89,34 @@ func TestTrafficManagement_SetupServerAndClientWithSubsets(t *testing.T) {
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1)
 
-		// Upgrade cluster and begin service validation
+		// Upgrade cluster, restart sidecars then begin service traffic validation
 		require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion))
+		require.NoError(t, clientService.Restart())
+		require.NoError(t, serverService.Restart())
+		require.NoError(t, serverServiceV1.Restart())
+		require.NoError(t, serverServiceV2.Restart())
 
 		// POST upgrade validation; repeat client & proxy validation
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1)
 
 		// validate  static-client proxy admin is up
-		_, statusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", map[string]string{"format": "json"})
+		_, statusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat)
 		require.NoError(t, err)
 		assert.Equal(t, http.StatusOK, statusCode, fmt.Sprintf("service cannot be reached %v", statusCode))
 
 		// validate static-server proxy admin is up
-		_, statusCode1, err := libassert.GetEnvoyOutput(serverAdminPort, "stats", map[string]string{"format": "json"})
+		_, statusCode1, err := libassert.GetEnvoyOutput(serverAdminPort, "stats", responseFormat)
 		require.NoError(t, err)
 		assert.Equal(t, http.StatusOK, statusCode1, fmt.Sprintf("service cannot be reached %v", statusCode1))
 
 		// validate static-server-v1 proxy admin is up
-		_, statusCode2, err := libassert.GetEnvoyOutput(serverAdminPortV1, "stats", map[string]string{"format": "json"})
+		_, statusCode2, err := libassert.GetEnvoyOutput(serverAdminPortV1, "stats", responseFormat)
 		require.NoError(t, err)
 		assert.Equal(t, http.StatusOK, statusCode2, fmt.Sprintf("service cannot be reached %v", statusCode2))
 
 		// validate static-server-v2 proxy admin is up
-		_, statusCode3, err := libassert.GetEnvoyOutput(serverAdminPortV2, "stats", map[string]string{"format": "json"})
+		_, statusCode3, err := libassert.GetEnvoyOutput(serverAdminPortV2, "stats", responseFormat)
 		require.NoError(t, err)
 		assert.Equal(t, http.StatusOK, statusCode3, fmt.Sprintf("service cannot be reached %v", statusCode3))