add ability to start container tests in debug mode and attach a debugger (#16887)

* add ability to start container tests in debug mode and attach a debugger to consul while running it.

* add a debug message with the debug port

* use pod to get the right port

* fix image used in basic test

* add more data to identify which container to debug.

* fix comment

Co-authored-by: Evan Culver <eculver@users.noreply.github.com>

* rename debugUri to debugURI

---------

Co-authored-by: Evan Culver <eculver@users.noreply.github.com>

GNUmakefile

@@ -163,6 +163,19 @@ dev-build:
 	rm -f ./bin/consul
 	cp ${MAIN_GOPATH}/bin/consul ./bin/consul
 
+
+dev-docker-dbg: dev-docker linux dev-build
+	@echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)"
+	@docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null
+	@echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)"
+	@#  'consul-dbg:local' tag is needed to run the integration tests
+	@#  'consul-dev:latest' is needed by older workflows
+	@docker buildx use default && docker buildx build -t 'consul-dbg:local' -t '$(CONSUL_DEV_IMAGE)' \
+       --platform linux/$(GOARCH) \
+	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+       --load \
+       -f $(CURDIR)/build-support/docker/Consul-Dev-Dbg.dockerfile $(CURDIR)/pkg/bin/
+
 dev-docker: linux dev-build
 	@echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)"
 	@docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null

build-support/docker/Consul-Dev-Dbg.dockerfile

@@ -0,0 +1,10 @@
+FROM consul:local
+EXPOSE 4000
+
+COPY --from=golang:1.20-alpine /usr/local/go/ /usr/local/go/
+
+ENV PATH="/usr/local/go/bin:${PATH}"
+
+RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest
+
+CMD [ "/root/go/bin/dlv", "exec", "/bin/consul", "--listen=:4000", "--headless=true", "", "--accept-multiclient", "--continue", "--api-version=2", "--", "agent", "--advertise=0.0.0.0"]

test/integration/consul-container/libs/cluster/agent.go

@@ -92,4 +92,5 @@ type AgentInfo struct {
 	CACertFile    string
 	UseTLSForAPI  bool
 	UseTLSForGRPC bool
+	DebugURI      string
 }

test/integration/consul-container/libs/cluster/builder.go

@@ -121,7 +121,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext {
 	}
 
 	if ctx.consulImageName == "" {
-		ctx.consulImageName = utils.TargetImageName
+		ctx.consulImageName = utils.GetTargetImageName()
 	}
 	if ctx.consulVersion == "" {
 		ctx.consulVersion = utils.TargetVersion
@@ -311,11 +311,15 @@ func (b *Builder) ToAgentConfig(t *testing.T) *Config {
 	confCopy, err := b.conf.Clone()
 	require.NoError(t, err)
 
+	cmd := []string{"agent"}
+	if utils.Debug {
+		cmd = []string{"/root/go/bin/dlv", "exec", "/bin/consul", "--listen=:4000", "--headless=true", "", "--accept-multiclient", "--continue", "--api-version=2", "--", "agent", "--config-file=/consul/config/config.json"}
+	}
 	return &Config{
 		JSON:          string(out),
 		ConfigBuilder: confCopy,
 
-		Cmd: []string{"agent"},
+		Cmd: cmd,
 
 		Image:   b.context.consulImageName,
 		Version: b.context.consulVersion,

test/integration/consul-container/libs/cluster/cluster.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -170,6 +171,10 @@ func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error)
 		}
 	}
 
+	if utils.Debug {
+		c.PrintDebugInfo(agents)
+	}
+
 	return nil
 }
 
@@ -661,6 +666,26 @@ func (c *Cluster) ConfigEntryDelete(entry api.ConfigEntry) error {
 	return err
 }
 
+func (c *Cluster) PrintDebugInfo(agents []Agent) {
+	for _, a := range agents {
+		uri := a.GetInfo().DebugURI
+		n := a.GetAgentName()
+		s := a.IsServer()
+		l := "NA"
+		if s {
+			leader, err := c.Leader()
+			if err == nil {
+				if leader == a {
+					l = "true"
+				} else {
+					l = "false"
+				}
+			}
+		}
+		fmt.Printf("\ndebug info:: n=%s,s=%t,l=%s,uri=%s\n\n", n, s, l, uri)
+	}
+}
+
 func extractSecretIDFrom(tokenOutput string) (string, error) {
 	lines := strings.Split(tokenOutput, "\n")
 	for _, line := range lines {

test/integration/consul-container/libs/cluster/container.go

@@ -33,6 +33,7 @@ const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED"
 const MaxEnvoyOnNode = 10                  // the max number of Envoy sidecar can run along with the agent, base is 19000
 const ServiceUpstreamLocalBindPort = 5000  // local bind Port of service's upstream
 const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams
+const debugPort = "4000/tcp"
 
 // consulContainerNode implements the Agent interface by running a Consul agent
 // in a container.
@@ -169,6 +170,22 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 
 		info AgentInfo
 	)
+	debugURI := ""
+	if utils.Debug {
+		if err := goretry.Do(
+			func() (err error) {
+				debugURI, err = podContainer.PortEndpoint(ctx, "4000", "tcp")
+				return err
+			},
+			goretry.Delay(10*time.Second),
+			goretry.RetryIf(func(err error) bool {
+				return err != nil
+			}),
+		); err != nil {
+			return nil, fmt.Errorf("container creating: %s", err)
+		}
+		info.DebugURI = debugURI
+	}
 	if httpPort > 0 {
 		for i := 0; i < 10; i++ {
 			uri, err := podContainer.PortEndpoint(ctx, "8500", "http")
@@ -578,6 +595,9 @@ func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRe
 	for _, port := range ports {
 		pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port))
 	}
+	if utils.Debug {
+		pod.ExposedPorts = append(pod.ExposedPorts, debugPort)
+	}
 
 	// For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready".
 	// If we only wait until the log stream starts, subsequent commands to agents will fail.

test/integration/consul-container/libs/utils/version.go

@@ -10,13 +10,15 @@ import (
 )
 
 var (
-	TargetImageName string
+	targetImageName string
 	TargetVersion   string
 
 	LatestImageName string
 	LatestVersion   string
 
 	FollowLog bool
+
+	Debug           bool
 	Version_1_14, _ = version.NewVersion("1.14")
 )
 
@@ -27,13 +29,22 @@ const (
 )
 
 func init() {
-	flag.StringVar(&TargetImageName, "target-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")")
+	flag.BoolVar(&Debug, "debug", false, "run consul with dlv to enable live debugging")
+	flag.StringVar(&targetImageName, "target-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")")
 	flag.StringVar(&TargetVersion, "target-version", "local", "docker image version to be used as UUT (unit under test)")
 
 	flag.StringVar(&LatestImageName, "latest-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")")
 	flag.StringVar(&LatestVersion, "latest-version", "latest", "docker image to be used as latest")
 
 	flag.BoolVar(&FollowLog, "follow-log", true, "follow container log in output (Default: true)")
+
+}
+
+func GetTargetImageName() string {
+	if Debug {
+		return targetImageName + "-dbg"
+	}
+	return targetImageName
 }
 
 func DockerImage(image, version string) string {

test/integration/consul-container/test/snapshot/snapshot_restore_test.go

@@ -38,7 +38,7 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore)
 		NumClients: 0,
 		BuildOpts: &libcluster.BuildOptions{
 			Datacenter:      "dc1",
-			ConsulImageName: utils.TargetImageName,
+			ConsulImageName: utils.GetTargetImageName(),
 			ConsulVersion:   utils.TargetVersion,
 			LogStore:        logStore,
 		},
@@ -67,7 +67,7 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore)
 		NumClients: 0,
 		BuildOpts: &libcluster.BuildOptions{
 			Datacenter:      "dc1",
-			ConsulImageName: utils.TargetImageName,
+			ConsulImageName: utils.GetTargetImageName(),
 			ConsulVersion:   utils.TargetVersion,
 			LogStore:        logStore,
 		},

test/integration/consul-container/test/upgrade/basic_test.go

@@ -19,8 +19,8 @@ func TestBasic(t *testing.T) {
 	t.Parallel()
 
 	configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{
-		ConsulImageName: utils.TargetImageName,
-		ConsulVersion:   utils.LatestVersion,
+		ConsulImageName: utils.GetTargetImageName(),
+		ConsulVersion:   utils.TargetVersion,
 	})
 
 	const numServers = 1
@@ -29,7 +29,7 @@ func TestBasic(t *testing.T) {
 		Bootstrap(numServers).
 		ToAgentConfig(t)
 	t.Logf("Cluster config:\n%s", serverConf.JSON)
-	require.Equal(t, utils.LatestVersion, serverConf.Version) // TODO: remove
+	require.Equal(t, utils.TargetVersion, serverConf.Version) // TODO: remove
 
 	cluster, err := libcluster.NewN(t, *serverConf, numServers)
 	require.NoError(t, err)

test/integration/consul-container/test/upgrade/fullstopupgrade_test.go

@@ -51,7 +51,7 @@ func TestStandardUpgradeToTarget_fromLatest(t *testing.T) {
 
 	run := func(t *testing.T, tc testcase) {
 		configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{
-			ConsulImageName: utils.TargetImageName,
+			ConsulImageName: utils.GetTargetImageName(),
 			ConsulVersion:   tc.oldVersion,
 		})
 

test/integration/consul-container/test/upgrade/healthcheck_test.go

@@ -89,7 +89,7 @@ func testMixedServersGAClient(t *testing.T, majorityIsTarget bool) {
 			ConsulVersion:   utils.LatestVersion,
 		}
 		targetOpts = libcluster.BuildOptions{
-			ConsulImageName: utils.TargetImageName,
+			ConsulImageName: utils.GetTargetImageName(),
 			ConsulVersion:   utils.TargetVersion,
 		}