From 41064eb20ba41cd9d1af0378b99ed9730eca6b29 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Tue, 18 Apr 2023 09:49:53 -0400 Subject: [PATCH] add ability to start container tests in debug mode and attach a debugger (#16887) * add ability to start container tests in debug mode and attach a debugger to consul while running it. * add a debug message with the debug port * use pod to get the right port * fix image used in basic test * add more data to identify which container to debug. * fix comment Co-authored-by: Evan Culver * rename debugUri to debugURI --------- Co-authored-by: Evan Culver --- GNUmakefile | 13 ++++++++++ .../docker/Consul-Dev-Dbg.dockerfile | 10 ++++++++ .../consul-container/libs/cluster/agent.go | 1 + .../consul-container/libs/cluster/builder.go | 8 ++++-- .../consul-container/libs/cluster/cluster.go | 25 +++++++++++++++++++ .../libs/cluster/container.go | 20 +++++++++++++++ .../consul-container/libs/utils/version.go | 17 ++++++++++--- .../test/snapshot/snapshot_restore_test.go | 4 +-- .../test/upgrade/basic_test.go | 6 ++--- .../test/upgrade/fullstopupgrade_test.go | 2 +- .../test/upgrade/healthcheck_test.go | 2 +- 11 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 build-support/docker/Consul-Dev-Dbg.dockerfile diff --git a/GNUmakefile b/GNUmakefile index 9a68a484c..815d56d20 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -163,6 +163,19 @@ dev-build: rm -f ./bin/consul cp ${MAIN_GOPATH}/bin/consul ./bin/consul + +dev-docker-dbg: dev-docker linux dev-build + @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" + @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" + @# 'consul-dbg:local' tag is needed to run the integration tests + @# 'consul-dev:latest' is needed by older workflows + @docker buildx use default && docker buildx build -t 'consul-dbg:local' -t '$(CONSUL_DEV_IMAGE)' \ + --platform linux/$(GOARCH) \ + --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ + --load \ + -f $(CURDIR)/build-support/docker/Consul-Dev-Dbg.dockerfile $(CURDIR)/pkg/bin/ + dev-docker: linux dev-build @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null diff --git a/build-support/docker/Consul-Dev-Dbg.dockerfile b/build-support/docker/Consul-Dev-Dbg.dockerfile new file mode 100644 index 000000000..179e12b99 --- /dev/null +++ b/build-support/docker/Consul-Dev-Dbg.dockerfile @@ -0,0 +1,10 @@ +FROM consul:local +EXPOSE 4000 + +COPY --from=golang:1.20-alpine /usr/local/go/ /usr/local/go/ + +ENV PATH="/usr/local/go/bin:${PATH}" + +RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest + +CMD [ "/root/go/bin/dlv", "exec", "/bin/consul", "--listen=:4000", "--headless=true", "", "--accept-multiclient", "--continue", "--api-version=2", "--", "agent", "--advertise=0.0.0.0"] \ No newline at end of file diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index c0b5bb523..d5a12f481 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -92,4 +92,5 @@ type AgentInfo struct { CACertFile string UseTLSForAPI bool UseTLSForGRPC bool + DebugURI string } diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index b98235037..72228a26d 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -121,7 +121,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { } if ctx.consulImageName == "" { - ctx.consulImageName = utils.TargetImageName + ctx.consulImageName = utils.GetTargetImageName() } if ctx.consulVersion == "" { ctx.consulVersion = utils.TargetVersion @@ -311,11 +311,15 @@ func (b *Builder) ToAgentConfig(t *testing.T) *Config { confCopy, err := b.conf.Clone() require.NoError(t, err) + cmd := []string{"agent"} + if utils.Debug { + cmd = []string{"/root/go/bin/dlv", "exec", "/bin/consul", "--listen=:4000", "--headless=true", "", "--accept-multiclient", "--continue", "--api-version=2", "--", "agent", "--config-file=/consul/config/config.json"} + } return &Config{ JSON: string(out), ConfigBuilder: confCopy, - Cmd: []string{"agent"}, + Cmd: cmd, Image: b.context.consulImageName, Version: b.context.consulVersion, diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index 9b3620e02..98eb2419d 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "os" "path/filepath" "strconv" @@ -170,6 +171,10 @@ func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) } } + if utils.Debug { + c.PrintDebugInfo(agents) + } + return nil } @@ -661,6 +666,26 @@ func (c *Cluster) ConfigEntryDelete(entry api.ConfigEntry) error { return err } +func (c *Cluster) PrintDebugInfo(agents []Agent) { + for _, a := range agents { + uri := a.GetInfo().DebugURI + n := a.GetAgentName() + s := a.IsServer() + l := "NA" + if s { + leader, err := c.Leader() + if err == nil { + if leader == a { + l = "true" + } else { + l = "false" + } + } + } + fmt.Printf("\ndebug info:: n=%s,s=%t,l=%s,uri=%s\n\n", n, s, l, uri) + } +} + func extractSecretIDFrom(tokenOutput string) (string, error) { lines := strings.Split(tokenOutput, "\n") for _, line := range lines { diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index a96d629d4..4f9e8227d 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -33,6 +33,7 @@ const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED" const MaxEnvoyOnNode = 10 // the max number of Envoy sidecar can run along with the agent, base is 19000 const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstream const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams +const debugPort = "4000/tcp" // consulContainerNode implements the Agent interface by running a Consul agent // in a container. @@ -169,6 +170,22 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) + debugURI := "" + if utils.Debug { + if err := goretry.Do( + func() (err error) { + debugURI, err = podContainer.PortEndpoint(ctx, "4000", "tcp") + return err + }, + goretry.Delay(10*time.Second), + goretry.RetryIf(func(err error) bool { + return err != nil + }), + ); err != nil { + return nil, fmt.Errorf("container creating: %s", err) + } + info.DebugURI = debugURI + } if httpPort > 0 { for i := 0; i < 10; i++ { uri, err := podContainer.PortEndpoint(ctx, "8500", "http") @@ -578,6 +595,9 @@ func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRe for _, port := range ports { pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port)) } + if utils.Debug { + pod.ExposedPorts = append(pod.ExposedPorts, debugPort) + } // For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready". // If we only wait until the log stream starts, subsequent commands to agents will fail. diff --git a/test/integration/consul-container/libs/utils/version.go b/test/integration/consul-container/libs/utils/version.go index 62d22e81b..4d1ae5af0 100644 --- a/test/integration/consul-container/libs/utils/version.go +++ b/test/integration/consul-container/libs/utils/version.go @@ -10,13 +10,15 @@ import ( ) var ( - TargetImageName string + targetImageName string TargetVersion string LatestImageName string LatestVersion string - FollowLog bool + FollowLog bool + + Debug bool Version_1_14, _ = version.NewVersion("1.14") ) @@ -27,13 +29,22 @@ const ( ) func init() { - flag.StringVar(&TargetImageName, "target-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")") + flag.BoolVar(&Debug, "debug", false, "run consul with dlv to enable live debugging") + flag.StringVar(&targetImageName, "target-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")") flag.StringVar(&TargetVersion, "target-version", "local", "docker image version to be used as UUT (unit under test)") flag.StringVar(&LatestImageName, "latest-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")") flag.StringVar(&LatestVersion, "latest-version", "latest", "docker image to be used as latest") flag.BoolVar(&FollowLog, "follow-log", true, "follow container log in output (Default: true)") + +} + +func GetTargetImageName() string { + if Debug { + return targetImageName + "-dbg" + } + return targetImageName } func DockerImage(image, version string) string { diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go index 3e989bc7d..d472f6efc 100644 --- a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -38,7 +38,7 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) NumClients: 0, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.TargetVersion, LogStore: logStore, }, @@ -67,7 +67,7 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) NumClients: 0, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.TargetVersion, LogStore: logStore, }, diff --git a/test/integration/consul-container/test/upgrade/basic_test.go b/test/integration/consul-container/test/upgrade/basic_test.go index e65b7a1b2..bd62caae9 100644 --- a/test/integration/consul-container/test/upgrade/basic_test.go +++ b/test/integration/consul-container/test/upgrade/basic_test.go @@ -19,8 +19,8 @@ func TestBasic(t *testing.T) { t.Parallel() configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{ - ConsulImageName: utils.TargetImageName, - ConsulVersion: utils.LatestVersion, + ConsulImageName: utils.GetTargetImageName(), + ConsulVersion: utils.TargetVersion, }) const numServers = 1 @@ -29,7 +29,7 @@ func TestBasic(t *testing.T) { Bootstrap(numServers). ToAgentConfig(t) t.Logf("Cluster config:\n%s", serverConf.JSON) - require.Equal(t, utils.LatestVersion, serverConf.Version) // TODO: remove + require.Equal(t, utils.TargetVersion, serverConf.Version) // TODO: remove cluster, err := libcluster.NewN(t, *serverConf, numServers) require.NoError(t, err) diff --git a/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go b/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go index 2539a66f9..dfe6e1a13 100644 --- a/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go +++ b/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go @@ -51,7 +51,7 @@ func TestStandardUpgradeToTarget_fromLatest(t *testing.T) { run := func(t *testing.T, tc testcase) { configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{ - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: tc.oldVersion, }) diff --git a/test/integration/consul-container/test/upgrade/healthcheck_test.go b/test/integration/consul-container/test/upgrade/healthcheck_test.go index da7572511..1a6116e09 100644 --- a/test/integration/consul-container/test/upgrade/healthcheck_test.go +++ b/test/integration/consul-container/test/upgrade/healthcheck_test.go @@ -89,7 +89,7 @@ func testMixedServersGAClient(t *testing.T, majorityIsTarget bool) { ConsulVersion: utils.LatestVersion, } targetOpts = libcluster.BuildOptions{ - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.TargetVersion, }