tests: enable newer windows (#17401)

* "allow" (don't try to drop) linux capabilities
  in the docker test driver harness (see #15181)
* refactor to allow different busybox images
  since windows containers need to be the same
  version as the underlying OS, and we're
  moving from 2016 to 2019
* one docker test was flaky from apparently
  being a bit slower on windows, so add Wait()

drivers/docker/docklog/docker_logger_test.go

@@ -8,7 +8,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"runtime"
+	"strings"
 	"testing"
 	"time"
 
@@ -22,20 +22,12 @@ import (
 )
 
 func testContainerDetails() (image string, imageName string, imageTag string) {
-	name := "busybox"
-	tag := "1"
+	image = testutil.TestBusyboxImage()
+	parts := strings.Split(image, ":")
+	imageName = parts[0]
+	imageTag = parts[1]
 
-	if runtime.GOOS == "windows" {
-		name = "hashicorpdev/busybox-windows"
-		tag = "server2016-0.1"
-	}
-
-	if testutil.IsCI() {
-		// In CI, use HashiCorp Mirror to avoid DockerHub rate limiting
-		name = "docker.mirror.hashicorp.services/" + name
-	}
-
-	return name + ":" + tag, name, tag
+	return image, imageName, imageTag
 }
 
 func TestDockerLogger_Success(t *testing.T) {

drivers/docker/driver_test.go

@@ -20,9 +20,14 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 	hclog "github.com/hashicorp/go-hclog"
+	"github.com/shoenig/test/must"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/client/testutil"
+	"github.com/hashicorp/nomad/drivers/shared/capabilities"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclspecutils"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
@@ -33,9 +38,6 @@ import (
 	"github.com/hashicorp/nomad/plugins/drivers"
 	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
 	tu "github.com/hashicorp/nomad/testutil"
-	"github.com/shoenig/test/must"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 var (
@@ -190,6 +192,14 @@ func dockerDriverHarness(t *testing.T, cfg map[string]interface{}) *dtestutil.Dr
 			},
 		}
 	}
+
+	// If on windows, "allow" (don't attempt to drop) linux capabilities.
+	// https://github.com/hashicorp/nomad/issues/15181
+	// TODO: this should instead get fixed properly in capabilities package.
+	if _, ok := cfg["allow_caps"]; !ok && runtime.GOOS == "windows" {
+		cfg["allow_caps"] = capabilities.DockerDefaults().Slice(false)
+	}
+
 	plugLoader, err := loader.NewPluginLoader(&loader.PluginLoaderConfig{
 		Logger:            logger,
 		PluginDir:         "./plugins",

drivers/docker/driver_windows_test.go

@@ -9,18 +9,13 @@ import (
 	"testing"
 
 	"github.com/hashicorp/nomad/client/allocdir"
-	tu "github.com/hashicorp/nomad/testutil"
+	"github.com/hashicorp/nomad/testutil"
 )
 
 func newTaskConfig(variant string, command []string) TaskConfig {
 	// busyboxImageID is an id of an image containing nanoserver windows and
 	// a busybox exe.
-	busyboxImageID := "hashicorpdev/busybox-windows:server2016-0.1"
-
-	if tu.IsCI() {
-		// In CI, use HashiCorp Mirror to avoid DockerHub rate limiting
-		busyboxImageID = "docker.mirror.hashicorp.services/" + busyboxImageID
-	}
+	busyboxImageID := testutil.TestBusyboxImage()
 
 	return TaskConfig{
 		Image:            busyboxImageID,

drivers/docker/reconcile_dangling_test.go

@@ -13,11 +13,13 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/hashicorp/go-set"
+	"github.com/shoenig/test/must"
+	"github.com/shoenig/test/wait"
+
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/client/testutil"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/plugins/drivers"
-	"github.com/shoenig/test/must"
 )
 
 func fakeContainerList(t *testing.T) (nomadContainer, nonNomadContainer docker.APIContainers) {
@@ -243,14 +245,26 @@ func TestDanglingContainerRemoval_Stopped(t *testing.T) {
 	tracked := reconciler.trackedContainers()
 	must.NotContains[string](t, container.ID, tracked)
 
-	untracked, err := reconciler.untrackedContainers(set.New[string](0), time.Now())
-	must.NoError(t, err)
-	must.NotContains[string](t, container.ID, untracked)
+	checkUntracked := func() error {
+		untracked, err := reconciler.untrackedContainers(set.New[string](0), time.Now())
+		must.NoError(t, err)
+		if untracked.Contains(container.ID) {
+			return fmt.Errorf("container ID %s in untracked set: %v", container.ID, untracked.Slice())
+		}
+		return nil
+	}
+
+	// retry because it's slower on windows :\
+	must.Wait(t, wait.InitialSuccess(
+		wait.ErrorFunc(checkUntracked),
+		wait.Timeout(time.Second),
+		wait.Gap(100*time.Millisecond),
+	))
 
 	// if we start container again, it'll be marked as untracked
 	must.NoError(t, dockerClient.StartContainer(container.ID, nil))
 
-	untracked, err = reconciler.untrackedContainers(set.New[string](0), time.Now())
+	untracked, err := reconciler.untrackedContainers(set.New[string](0), time.Now())
 	must.NoError(t, err)
 	must.Contains[string](t, container.ID, untracked)
 }

testutil/container_images.go

@@ -0,0 +1,43 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package testutil
+
+import (
+	"os"
+	"runtime"
+)
+
+const (
+	// BusyboxImageEnvName is always used, if set, by TestBusyboxImage()
+	BusyboxImageEnvName = "BUSYBOX_IMAGE"
+)
+
+func TestDockerImage(name, tag string) string {
+	img := name + ":" + tag
+	if IsCI() {
+		// use our mirror to avoid rate-limiting in CI
+		img = "docker.mirror.hashicorp.services/" + img
+	} else {
+		// explicitly include docker.io for podman
+		img = "docker.io/" + img
+	}
+	return img
+}
+
+func TestBusyboxImage() string {
+	// if env is set, use it verbatim
+	if img, ok := os.LookupEnv(BusyboxImageEnvName); ok {
+		return img
+	}
+
+	// otherwise, figure it out
+	name := "busybox"
+	tag := "1"
+	if runtime.GOOS == "windows" {
+		// this image is maintained in https://github.com/hashicorp/busybox-windows
+		name = "hashicorpdev/busybox-windows"
+		tag = "server2016-0.1"
+	}
+	return TestDockerImage(name, tag)
+}