Container test: fix container test slow image build (#17316)

Container integ test: fix container test slow image build

.github/workflows/test-integrations.yml

@@ -374,6 +374,7 @@ jobs:
               | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
               | jq --compact-output 'map(join(" "))'
           } >> "$GITHUB_OUTPUT"
+
   compatibility-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
     needs:
@@ -387,6 +388,8 @@ jobs:
       fail-fast: false
       matrix:
         test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
+    env:
+      ENVOY_VERSION: "1.25.4"
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -405,6 +408,8 @@ jobs:
 
       - name: Build consul:local image
         run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Build consul-envoy:target-version image
+        run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
       - name: Configure GH workaround for ipv6 loopback
         if: ${{ !endsWith(github.repository, '-enterprise') }}
         run: |
@@ -520,7 +525,8 @@ jobs:
         consul-version: [ "1.14", "1.15"]
         test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }}
     env:
-      CONSUL_VERSION: ${{ matrix.consul-version }}
+      CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
+      ENVOY_VERSION: "1.24.6"
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -538,6 +544,10 @@ jobs:
         run: chmod +x consul
       - name: Build consul:local image
         run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Build consul-envoy:latest-version image
+        run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
+      - name: Build consul-envoy:target-version image
+        run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
       - name: Configure GH workaround for ipv6 loopback
         if: ${{ !endsWith(github.repository, '-enterprise') }}
         run: |
@@ -567,8 +577,8 @@ jobs:
             -run "${{ matrix.test-cases }}" \
             --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
             --target-version local \
-            --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --latest-version "${{ env.CONSUL_VERSION }}"
+            --latest-version "${{ env.CONSUL_LATEST_VERSION }}"
           ls -lrt
         env:
           # this is needed because of incompatibility between RYUK container and GHA

GNUmakefile

@@ -61,6 +61,7 @@ GO_BUILD_TAG?=consul-build-go
 UI_BUILD_TAG?=consul-build-ui
 BUILD_CONTAINER_NAME?=consul-builder
 CONSUL_IMAGE_VERSION?=latest
+ENVOY_VERSION?='1.25.4'
 
 ################
 # CI Variables #
@@ -459,10 +460,13 @@ else
 		--latest-version latest
 endif
 
+# NOTE: Use DOCKER_BUILDKIT=0, if docker build fails to resolve consul:local base image
 .PHONY: test-compat-integ-setup
 test-compat-integ-setup: dev-docker
 	@docker tag consul-dev:latest $(CONSUL_COMPAT_TEST_IMAGE):local
 	@docker run --rm -t $(CONSUL_COMPAT_TEST_IMAGE):local consul version
+	@#  'consul-envoy:target-version' is needed by compatibility integ test
+	@docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg ENVOY_VERSION=${ENVOY_VERSION} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
 
 .PHONY: test-metrics-integ
 test-metrics-integ: test-compat-integ-setup

test/integration/consul-container/go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	fortio.org/fortio v1.54.0
 	github.com/avast/retry-go v3.0.0+incompatible
-	github.com/docker/docker v20.10.22+incompatible
+	github.com/docker/docker v23.0.6+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/hashicorp/consul/api v1.20.0
 	github.com/hashicorp/consul/envoyextensions v0.1.2
@@ -19,11 +19,11 @@ require (
 	github.com/mitchellh/copystructure v1.2.0
 	github.com/otiai10/copy v1.10.0
 	github.com/pkg/errors v0.9.1
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569
-	github.com/testcontainers/testcontainers-go v0.15.0
+	github.com/testcontainers/testcontainers-go v0.20.1
-	golang.org/x/mod v0.8.0
+	golang.org/x/mod v0.10.0
-	google.golang.org/grpc v1.53.0
+	google.golang.org/grpc v1.55.0
 )
 
 require (
@@ -32,21 +32,19 @@ require (
 	fortio.org/sets v1.0.2 // indirect
 	fortio.org/version v1.0.2 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/Microsoft/hcsshim v0.9.7 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
-	github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b // indirect
+	github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/containerd/containerd v1.7.1 // indirect
-	github.com/containerd/containerd v1.6.19 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/envoyproxy/go-control-plane v0.10.3 // indirect
+	github.com/envoyproxy/go-control-plane v0.11.0 // indirect
-	github.com/envoyproxy/protoc-gen-validate v0.9.1 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.10.0 // indirect
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
@@ -58,33 +56,35 @@ require (
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/memberlist v0.5.0 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/itchyny/timefmt-go v0.1.4 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/miekg/dns v1.1.41 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/moby/sys/mount v0.3.3 // indirect
+	github.com/moby/patternmatcher v0.5.0 // indirect
-	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
-	google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.4.0 // indirect
 )
 
 replace github.com/hashicorp/consul/api => ../../../api

test/integration/consul-container/libs/cluster/app.go

@@ -62,8 +62,9 @@ func LaunchContainerOnNode(
 		ContainerRequest: req,
 		Started:          true,
 	})
+	fmt.Printf("creating container with image: %s(%s)\n", req.Name, req.Image)
 	if err != nil {
-		return nil, fmt.Errorf("creating container: %w", err)
+		return nil, fmt.Errorf("creating container: %s(%s), %w", req.Name, req.Image, err)
 	}
 	deferClean.Add(func() {
 		_ = container.Terminate(ctx)

test/integration/consul-container/libs/service/common.go

@@ -4,74 +4,10 @@
 package service
 
 import (
-	"archive/tar"
-	"bytes"
 	_ "embed"
-	"os"
-
-	"github.com/testcontainers/testcontainers-go"
 )
 
 const (
-	envoyEnvKey   = "ENVOY_VERSION"
+	envoyLogLevel        = "debug"
-	envoyLogLevel = "debug"
-	envoyVersion  = "1.23.1"
-
 	hashicorpDockerProxy = "docker.mirror.hashicorp.services"
 )
-
-//go:embed assets/Dockerfile-consul-envoy
-var consulEnvoyDockerfile []byte
-
-//go:embed assets/tproxy-startup.sh
-var tproxyStartupScript []byte
-
-// getDevContainerDockerfile returns the necessary context to build a combined consul and
-// envoy image for running "consul connect envoy ..."
-func getDevContainerDockerfile() (testcontainers.FromDockerfile, error) {
-	var buf bytes.Buffer
-	tw := tar.NewWriter(&buf)
-
-	hdr := &tar.Header{
-		Name: "Dockerfile",
-		Mode: 0600,
-		Size: int64(len(consulEnvoyDockerfile)),
-	}
-	if err := tw.WriteHeader(hdr); err != nil {
-		return testcontainers.FromDockerfile{}, err
-	}
-
-	if _, err := tw.Write(consulEnvoyDockerfile); err != nil {
-		return testcontainers.FromDockerfile{}, err
-	}
-
-	hdr = &tar.Header{
-		Name: "tproxy-startup.sh",
-		Mode: 0600,
-		Size: int64(len(tproxyStartupScript)),
-	}
-	if err := tw.WriteHeader(hdr); err != nil {
-		return testcontainers.FromDockerfile{}, err
-	}
-
-	if _, err := tw.Write(tproxyStartupScript); err != nil {
-		return testcontainers.FromDockerfile{}, err
-	}
-
-	if err := tw.Close(); err != nil {
-		return testcontainers.FromDockerfile{}, err
-	}
-	reader := bytes.NewReader(buf.Bytes())
-	fromDockerfile := testcontainers.FromDockerfile{
-		ContextArchive: reader,
-	}
-
-	return fromDockerfile, nil
-}
-
-func getEnvoyVersion() string {
-	if version, ok := os.LookupEnv(envoyEnvKey); ok && version != "" {
-		return version
-	}
-	return envoyVersion
-}

test/integration/consul-container/libs/service/connect.go

@@ -167,29 +167,19 @@ func NewConnectService(ctx context.Context, sidecarCfg SidecarConfig, serviceBin
 	namePrefix := fmt.Sprintf("%s-service-connect-%s", node.GetDatacenter(), sidecarCfg.Name)
 	containerName := utils.RandName(namePrefix)
 
-	envoyVersion := getEnvoyVersion()
 	agentConfig := node.GetConfig()
-	buildargs := map[string]*string{
-		"ENVOY_VERSION": utils.StringToPointer(envoyVersion),
-		"CONSUL_IMAGE":  utils.StringToPointer(agentConfig.DockerImage()),
-	}
-
-	dockerfileCtx, err := getDevContainerDockerfile()
-	if err != nil {
-		return nil, err
-	}
-	dockerfileCtx.BuildArgs = buildargs
-
 	internalAdminPort, err := node.ClaimAdminPort()
 	if err != nil {
 		return nil, err
 	}
 
+	fmt.Println("agent image name", agentConfig.DockerImage())
+	imageVersion := utils.SideCarVersion(agentConfig.DockerImage())
 	req := testcontainers.ContainerRequest{
-		FromDockerfile: dockerfileCtx,
+		Image:      fmt.Sprintf("consul-envoy:%s", imageVersion),
-		WaitingFor:     wait.ForLog("").WithStartupTimeout(10 * time.Second),
+		WaitingFor: wait.ForLog("").WithStartupTimeout(100 * time.Second),
-		AutoRemove:     false,
+		AutoRemove: false,
-		Name:           containerName,
+		Name:       containerName,
 		Cmd: []string{
 			"consul", "connect", "envoy",
 			"-sidecar-for", sidecarCfg.ServiceID,

test/integration/consul-container/libs/service/examples.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/hashicorp/consul/api"
 
-	"github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
@@ -120,7 +119,7 @@ func (g exampleContainer) Stop() error {
 }
 
 func (c exampleContainer) Terminate() error {
-	return cluster.TerminateContainer(c.ctx, c.container, true)
+	return libcluster.TerminateContainer(c.ctx, c.container, true)
 }
 
 func (c exampleContainer) GetStatus() (string, error) {
@@ -153,14 +152,14 @@ func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort
 
 	req := testcontainers.ContainerRequest{
 		Image:      hashicorpDockerProxy + "/fortio/fortio",
-		WaitingFor: wait.ForLog("").WithStartupTimeout(10 * time.Second),
+		WaitingFor: wait.ForLog("").WithStartupTimeout(60 * time.Second),
 		AutoRemove: false,
 		Name:       containerName,
 		Cmd:        command,
 		Env:        map[string]string{"FORTIO_NAME": name},
 	}
 
-	info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{httpPortStr, grpcPortStr})
+	info, err := libcluster.LaunchContainerOnNode(ctx, node, req, []string{httpPortStr, grpcPortStr})
 	if err != nil {
 		return nil, err
 	}

test/integration/consul-container/libs/service/gateway.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/hashicorp/consul/api"
 
-	"github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
@@ -102,7 +101,7 @@ func (g gatewayContainer) Stop() error {
 }
 
 func (c gatewayContainer) Terminate() error {
-	return cluster.TerminateContainer(c.ctx, c.container, true)
+	return libcluster.TerminateContainer(c.ctx, c.container, true)
 }
 
 func (g gatewayContainer) GetAdminAddr() (string, int) {
@@ -161,19 +160,7 @@ func NewGatewayServiceReg(ctx context.Context, gwCfg GatewayConfig, node libclus
 	namePrefix := fmt.Sprintf("%s-service-gateway-%s", node.GetDatacenter(), gwCfg.Name)
 	containerName := utils.RandName(namePrefix)
 
-	envoyVersion := getEnvoyVersion()
 	agentConfig := node.GetConfig()
-	buildargs := map[string]*string{
-		"ENVOY_VERSION": utils.StringToPointer(envoyVersion),
-		"CONSUL_IMAGE":  utils.StringToPointer(agentConfig.DockerImage()),
-	}
-
-	dockerfileCtx, err := getDevContainerDockerfile()
-	if err != nil {
-		return nil, err
-	}
-	dockerfileCtx.BuildArgs = buildargs
-
 	adminPort, err := node.ClaimAdminPort()
 	if err != nil {
 		return nil, err
@@ -194,13 +181,15 @@ func NewGatewayServiceReg(ctx context.Context, gwCfg GatewayConfig, node libclus
 		"--log-level", envoyLogLevel,
 	}
 
+	fmt.Println("agent image name", agentConfig.DockerImage())
+	imageVersion := utils.SideCarVersion(agentConfig.DockerImage())
 	req := testcontainers.ContainerRequest{
-		FromDockerfile: dockerfileCtx,
+		Image:      fmt.Sprintf("consul-envoy:%s", imageVersion),
-		WaitingFor:     wait.ForLog("").WithStartupTimeout(10 * time.Second),
+		WaitingFor: wait.ForLog("").WithStartupTimeout(100 * time.Second),
-		AutoRemove:     false,
+		AutoRemove: false,
-		Name:           containerName,
+		Name:       containerName,
-		Env:            make(map[string]string),
+		Env:        make(map[string]string),
-		Cmd:            append(cmd, envoyArgs...),
+		Cmd:        append(cmd, envoyArgs...),
 	}
 
 	nodeInfo := node.GetInfo()
@@ -240,7 +229,7 @@ func NewGatewayServiceReg(ctx context.Context, gwCfg GatewayConfig, node libclus
 		extraPorts = append(extraPorts, strconv.Itoa(port))
 	}
 
-	info, err := cluster.LaunchContainerOnNode(ctx, node, req, append(
+	info, err := libcluster.LaunchContainerOnNode(ctx, node, req, append(
 		extraPorts,
 		portStr,
 		adminPortStr,

test/integration/consul-container/libs/utils/version.go

@@ -5,6 +5,7 @@ package utils
 
 import (
 	"flag"
+	"strings"
 
 	"github.com/hashicorp/go-version"
 )
@@ -81,3 +82,17 @@ func VersionLT(a, b string) bool {
 	bv := version.Must(version.NewVersion(b))
 	return av.LessThan(bv)
 }
+
+// SideCarVersion returns version based on the agent
+// version in the test: if agent has local, the sidecar
+// version is target-version; otherwise use "latest-version"
+func SideCarVersion(agentVersion string) string {
+	imageVersion := ""
+	if strings.Contains(agentVersion, "local") {
+		imageVersion = "target-version"
+	} else {
+		imageVersion = "latest-version"
+	}
+
+	return imageVersion
+}

test/integration/consul-container/test/upgrade/README.md

@@ -49,6 +49,7 @@ To run the upgrade test, the following tools are required:
 
 ### Running Upgrade integration tests
 - run `make dev-docker`
+- build a consul-envoy container image `cd test/integration/consul-container docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/consul:1.15  --build-arg ENVOY_VERSION=1.24.6  -f ./assets/Dockerfile-consul-envoy ./assets`
 - run the single test `go test -v -timeout 30m -run ^TestACL_Upgrade_Node_Token$ ./.../upgrade/`
 - run all upgrade tests `go test -v -timeout 30m -run ./.../upgrade`
 

test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go

@@ -21,7 +21,7 @@ import (
 // Test resolver directs traffic to default subset
 // - Create 2 additional static-server instances: one in V1 subset and the other in V2 subset
 // - resolver directs traffic to the default subset, which is V2.
-func TestTrafficManagement_ResolveDefaultSubset(t *testing.T) {
+func TestTrafficManagement_ResolverDefaultSubset(t *testing.T) {
 	t.Parallel()
 
 	cluster, staticServerProxy, staticClientProxy := setup(t)
@@ -113,6 +113,8 @@ func TestTrafficManagement_ResolveDefaultSubset(t *testing.T) {
 // - Create one addtional static-server with checks and V1 subset
 // - resolver directs traffic to "test" service
 func TestTrafficManagement_ResolverDefaultOnlyPassing(t *testing.T) {
+	t.Parallel()
+
 	cluster, staticServerProxy, staticClientProxy := setup(t)
 	node := cluster.Agents[0]
 
@@ -211,6 +213,8 @@ func TestTrafficManagement_ResolverDefaultOnlyPassing(t *testing.T) {
 // - Create 3 static-server-2 server instances: one in V1, one in V2, one without any version
 // - service2Resolver directs traffic to static-server-2-v2V
 func TestTrafficManagement_ResolverSubsetRedirect(t *testing.T) {
+	t.Parallel()
+
 	cluster, staticServerProxy, staticClientProxy := setup(t)
 
 	node := cluster.Agents[0]