Merge pull request #12604 from hashicorp/b-fixup-chroot-test

ci: fixup task runner chroot test

ci/slow.go

@@ -26,3 +26,41 @@ func Parallel(t *testing.T) {
 		t.Parallel()
 	}
 }
+
+// TinyChroot is useful for testing, where we do not use anything other than
+// trivial /bin commands like sleep and sh. Copying a minimal chroot helps in
+// environments like GHA with very poor [network] disk performance.
+//
+// Note that you cannot chroot a symlink.
+//
+// Do not modify this value.
+var TinyChroot = map[string]string{
+	// destination: /bin
+	"/usr/bin/sleep": "/bin/sleep",
+	"/usr/bin/dash":  "/bin/sh",
+	"/usr/bin/bash":  "/bin/bash",
+	"/usr/bin/cat":   "/bin/cat",
+
+	// destination: /usr/bin
+	"/usr/bin/stty":   "/usr/bin/stty",
+	"/usr/bin/head":   "/usr/bin/head",
+	"/usr/bin/mktemp": "/usr/bin/mktemp",
+	"/usr/bin/echo":   "/usr/bin/echo",
+	"/usr/bin/touch":  "/usr/bin/touch",
+	"/usr/bin/stat":   "/usr/bin/stat",
+
+	// destination: /etc/
+	"/etc/ld.so.cache":  "/etc/ld.so.cache",
+	"/etc/ld.so.conf":   "/etc/ld.so.conf",
+	"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
+	"/etc/passwd":       "/etc/passwd",
+	"/etc/resolv.conf":  "/etc/resolv.conf",
+
+	// others
+	"/lib":                 "/lib",
+	"/lib32":               "/lib32",
+	"/lib64":               "/lib64",
+	"/usr/lib/jvm":         "/usr/lib/jvm",
+	"/run/resolvconf":      "/run/resolvconf",
+	"/run/systemd/resolve": "/run/systemd/resolve",
+}

client/allocrunner/taskrunner/task_runner_test.go

@@ -15,15 +15,12 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/hashicorp/nomad/ci"
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
 	"github.com/hashicorp/nomad/client/config"
 	consulapi "github.com/hashicorp/nomad/client/consul"
 	"github.com/hashicorp/nomad/client/devicemanager"
+	"github.com/hashicorp/nomad/client/lib/cgutil"
 	"github.com/hashicorp/nomad/client/pluginmanager/drivermanager"
 	regMock "github.com/hashicorp/nomad/client/serviceregistration/mock"
 	"github.com/hashicorp/nomad/client/serviceregistration/wrapper"
@@ -41,6 +38,9 @@ import (
 	"github.com/hashicorp/nomad/plugins/device"
 	"github.com/hashicorp/nomad/plugins/drivers"
 	"github.com/hashicorp/nomad/testutil"
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type MockTaskStateUpdater struct {
@@ -92,10 +92,26 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri
 	}
 	taskDir := allocDir.NewTaskDir(taskName)
 
+	// Compute the name of the v2 cgroup in case we need it in creation, configuration, and cleanup
+	cgroup := filepath.Join(cgutil.CgroupRoot, "testing.slice", cgutil.CgroupScope(alloc.ID, taskName))
+
+	// Create the cgroup if we are in v2 mode
+	if cgutil.UseV2 {
+		if err := os.MkdirAll(cgroup, 0755); err != nil {
+			t.Fatalf("failed to setup v2 cgroup for test: %v:", err)
+		}
+	}
+
 	trCleanup := func() {
 		if err := allocDir.Destroy(); err != nil {
 			t.Logf("error destroying alloc dir: %v", err)
 		}
+
+		// Cleanup the cgroup if we are in v2 mode
+		if cgutil.UseV2 {
+			_ = os.RemoveAll(cgroup)
+		}
+
 		cleanup()
 	}
 
@@ -130,6 +146,14 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri
 		ShutdownDelayCancelFn: shutdownDelayCancelFn,
 		ServiceRegWrapper:     wrapperMock,
 	}
+
+	// Set the cgroup path getter if we are in v2 mode
+	if cgutil.UseV2 {
+		conf.CpusetCgroupPathGetter = func(context.Context) (string, error) {
+			return filepath.Join(cgutil.CgroupRoot, "testing.slice", alloc.ID, thisTask.Name), nil
+		}
+	}
+
 	return conf, trCleanup
 }
 
@@ -587,7 +611,6 @@ func TestTaskRunner_TaskEnv_Interpolated(t *testing.T) {
 func TestTaskRunner_TaskEnv_Chroot(t *testing.T) {
 	ctestutil.ExecCompatible(t)
 	ci.Parallel(t)
-	require := require.New(t)
 
 	alloc := mock.BatchAlloc()
 	task := alloc.Job.TaskGroups[0].Tasks[0]
@@ -611,33 +634,27 @@ func TestTaskRunner_TaskEnv_Chroot(t *testing.T) {
 	conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name)
 	defer cleanup()
 
-	// Remove /sbin and /usr from chroot
-	conf.ClientConfig.ChrootEnv = map[string]string{
-		"/bin":            "/bin",
-		"/etc":            "/etc",
-		"/lib":            "/lib",
-		"/lib32":          "/lib32",
-		"/lib64":          "/lib64",
-		"/run/resolvconf": "/run/resolvconf",
-	}
-
 	tr, err := NewTaskRunner(conf)
-	require.NoError(err)
+	require.NoError(t, err)
 	go tr.Run()
 	defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
 
 	// Wait for task to exit
+	timeout := 15 * time.Second
+	if testutil.IsCI() {
+		timeout = 120 * time.Second
+	}
 	select {
 	case <-tr.WaitCh():
-	case <-time.After(15 * time.Second):
-		require.Fail("timeout waiting for task to exit")
+	case <-time.After(timeout):
+		require.Fail(t, "timeout waiting for task to exit")
 	}
 
 	// Read stdout
 	p := filepath.Join(conf.TaskDir.LogDir, task.Name+".stdout.0")
 	stdout, err := ioutil.ReadFile(p)
-	require.NoError(err)
-	require.Equalf(exp, string(stdout), "expected: %s\n\nactual: %s\n", exp, stdout)
+	require.NoError(t, err)
+	require.Equalf(t, exp, string(stdout), "expected: %s\n\nactual: %s\n", exp, stdout)
 }
 
 // TestTaskRunner_TaskEnv_Image asserts image drivers use chroot paths and
@@ -1722,6 +1739,7 @@ func TestTaskRunner_Download_ChrootExec(t *testing.T) {
 	task.Config = map[string]interface{}{
 		"command": "noop.sh",
 	}
+
 	task.Artifacts = []*structs.TaskArtifact{
 		{
 			GetterSource: fmt.Sprintf("%s/testdata/noop.sh", ts.URL),

client/config/testing.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/nomad/mock"
@@ -51,6 +52,12 @@ func TestClientConfig(t testing.T) (*Config, func()) {
 	}
 	conf.StateDir = stateDir
 
+	// Use a minimal chroot environment
+	conf.ChrootEnv = ci.TinyChroot
+
+	// Helps make sure we are respecting configured parent
+	conf.CgroupParent = "testing.slice"
+
 	conf.VaultConfig.Enabled = helper.BoolToPtr(false)
 	conf.DevMode = true
 

plugins/drivers/testutils/testing.go

@@ -12,6 +12,7 @@ import (
 
 	hclog "github.com/hashicorp/go-hclog"
 	plugin "github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/lib/cgutil"
@@ -109,41 +110,6 @@ func (h *DriverHarness) cleanupCgroup() {
 	}
 }
 
-// tinyChroot is useful for testing, where we do not use anything other than
-// trivial /bin commands like sleep and sh.
-//
-// Note that you cannot chroot a symlink.
-var tinyChroot = map[string]string{
-	// destination: /bin
-	"/usr/bin/sleep": "/bin/sleep",
-	"/usr/bin/dash":  "/bin/sh",
-	"/usr/bin/bash":  "/bin/bash",
-	"/usr/bin/cat":   "/bin/cat",
-
-	// destination: /usr/bin
-	"/usr/bin/stty":   "/usr/bin/stty",
-	"/usr/bin/head":   "/usr/bin/head",
-	"/usr/bin/mktemp": "/usr/bin/mktemp",
-	"/usr/bin/echo":   "/usr/bin/echo",
-	"/usr/bin/touch":  "/usr/bin/touch",
-	"/usr/bin/stat":   "/usr/bin/stat",
-
-	// destination: /etc/
-	"/etc/ld.so.cache":  "/etc/ld.so.cache",
-	"/etc/ld.so.conf":   "/etc/ld.so.conf",
-	"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
-	"/etc/passwd":       "/etc/passwd",
-	"/etc/resolv.conf":  "/etc/resolv.conf",
-
-	// others
-	"/lib":                 "/lib",
-	"/lib32":               "/lib32",
-	"/lib64":               "/lib64",
-	"/usr/lib/jvm":         "/usr/lib/jvm",
-	"/run/resolvconf":      "/run/resolvconf",
-	"/run/systemd/resolve": "/run/systemd/resolve",
-}
-
 // MkAllocDir creates a temporary directory and allocdir structure.
 // If enableLogs is set to true a logmon instance will be started to write logs
 // to the LogDir of the task
@@ -165,7 +131,7 @@ func (h *DriverHarness) MkAllocDir(t *drivers.TaskConfig, enableLogs bool) func(
 
 	fsi := caps.FSIsolation
 	h.logger.Trace("FS isolation", "fsi", fsi)
-	require.NoError(h.t, taskDir.Build(fsi == drivers.FSIsolationChroot, tinyChroot))
+	require.NoError(h.t, taskDir.Build(fsi == drivers.FSIsolationChroot, ci.TinyChroot))
 
 	task := &structs.Task{
 		Name: t.Name,