build: upgrade and speedup circleci configuration

This PR upgrades our CI images and fixes some affected tests.

- upgrade go-machine-image to premade latest ubuntu LTS (ubuntu-2004:202111-02)

- eliminate go-machine-recent-image (no longer necessary)

- manage GOPATH in GNUMakefile (see https://discuss.circleci.com/t/gopath-is-set-to-multiple-directories/7174)

- fix tcp dial error check (message seems to be OS specific)

- spot check values measured instead of specifically 'RSS' (rss no longer reported in cgroups v2)

- use safe MkdirTemp for generating tmpfiles

NOT applied: (too flakey)

- eliminate setting GOMAXPROCS=1 (build tools were also affected by this setting)

- upgrade resource type for all imanges to large (2C -> 4C)

.changelog/11889.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+build: upgrade and speedup circleci configuration
+```

.circleci/config.yml

@@ -3,9 +3,7 @@ version: 2.1
 references:
   # environment specific references - aim to avoid conflicts
   go-machine-image: &go_machine_image
-    circleci/classic:201808-01
+    ubuntu-2004:202111-02
-  go-machine-recent-image: &go_machine_recent_image
-    ubuntu-1604:201903-01
   go-windows-image: &go_windows_image
     windows-server-2019-vs2019:stable
 
@@ -289,7 +287,7 @@ jobs:
         default: ""
       executor:
         type: string
-        default: "go-machine-recent"
+        default: "go-machine"
       goarch:
         type: string
         default: "amd64"
@@ -513,6 +511,7 @@ executors:
     working_directory: /go/src/github.com/hashicorp/nomad
     docker:
       - image: docker.mirror.hashicorp.services/golang:1.17.5
+    resource_class: medium
     environment:
       <<: *common_envs
       GOPATH: /go
@@ -521,19 +520,11 @@ executors:
     working_directory: ~/go/src/github.com/hashicorp/nomad
     machine:
       image: *go_machine_image
+      resource_class: medium
     environment: &machine_env
       <<: *common_envs
-      GOPATH: /home/circleci/go
       GOLANG_VERSION: 1.17.5
 
-  # uses a more recent image with unattended upgrades disabled properly
-  # but seems to break docker builds
-  go-machine-recent:
-    working_directory: ~/go/src/github.com/hashicorp/nomad
-    machine:
-      image: *go_machine_recent_image
-    environment: *machine_env
-
   go-macos:
     working_directory: ~/go/src/github.com/hashicorp/nomad
     macos:
@@ -636,9 +627,6 @@ workflows:
       - test-machine:
           name: "test-docker"
           test_packages: "./drivers/docker"
-          # docker is misbehaving in docker-machine-recent image
-          # and we get unexpected failures
-          # e.g. https://circleci.com/gh/hashicorp/nomad/3854
           executor: go-machine
           filters: *backend_test_branches_filter
       - test-machine:

GNUmakefile

@@ -8,6 +8,13 @@ GIT_DIRTY := $(if $(shell git status --porcelain),+CHANGES)
 
 GO_LDFLAGS := "-X github.com/hashicorp/nomad/version.GitCommit=$(GIT_COMMIT)$(GIT_DIRTY)"
 
+ifneq (MSYS_NT,$(THIS_OS))
+# GOPATH supports PATH style multi-paths; assume the first entry is favorable.
+# Necessary because new Circle images override GOPATH with multiple values.
+# See: https://discuss.circleci.com/t/gopath-is-set-to-multiple-directories/7174
+GOPATH=$(shell go env GOPATH | cut -d: -f1)
+endif
+
 GO_TAGS ?=
 
 ifeq ($(CI),true)
@@ -241,7 +248,6 @@ tidy:
 .PHONY: dev
 dev: GOOS=$(shell go env GOOS)
 dev: GOARCH=$(shell go env GOARCH)
-dev: GOPATH=$(shell go env GOPATH)
 dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad
 dev: hclfmt ## Build for the current development platform
 	@echo "==> Removing old development build..."
@@ -297,7 +303,7 @@ test-nomad: dev ## Run Nomad test suites
 
 .PHONY: test-nomad-module
 test-nomad-module: dev ## Run Nomad test suites on a sub-module
-	@echo "==> Running Nomad test suites on sub-module:"
+	@echo "==> Running Nomad test suites on sub-module $(GOTEST_MOD)"
 	@cd $(GOTEST_MOD) && $(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \
 		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
 		-cover \

command/metrics_test.go

@@ -72,7 +72,7 @@ func TestCommand_Metrics_Cases(t *testing.T) {
 			[]string{"-address=http://foo"},
 			1,
 			"",
-			"no such host",
+			"dial tcp: lookup foo: Temporary failure in name resolution",
 		},
 	}
 

drivers/exec/driver_test.go

@@ -425,7 +425,7 @@ func TestExecDriver_Stats(t *testing.T) {
 	require.NoError(err)
 	select {
 	case stats := <-statsCh:
-		require.NotZero(stats.ResourceUsage.MemoryStats.RSS)
+		require.NotEmpty(stats.ResourceUsage.MemoryStats.Measured)
 		require.NotZero(stats.Timestamp)
 		require.WithinDuration(time.Now(), time.Unix(0, stats.Timestamp), time.Second)
 	case <-time.After(time.Second):

drivers/shared/executor/executor_linux.go

@@ -71,6 +71,7 @@ type LibcontainerExecutor struct {
 }
 
 func NewExecutorWithIsolation(logger hclog.Logger) Executor {
+
 	logger = logger.Named("isolated_executor")
 	if err := shelpers.Init(); err != nil {
 		logger.Error("unable to initialize stats", "error", err)

drivers/shared/executor/executor_test.go

@@ -254,7 +254,6 @@ func TestExecutor_WaitExitSignal(pt *testing.T) {
 	pt.Parallel()
 	for name, factory := range executorFactories {
 		pt.Run(name, func(t *testing.T) {
-			require := require.New(t)
 			testExecCmd := testExecutorCommand(t)
 			execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
 			execCmd.Cmd = "/bin/sleep"
@@ -266,8 +265,8 @@ func TestExecutor_WaitExitSignal(pt *testing.T) {
 			executor := factory.new(testlog.HCLogger(t))
 			defer executor.Shutdown("", 0)
 
-			ps, err := executor.Launch(execCmd)
+			pState, err := executor.Launch(execCmd)
-			require.NoError(err)
+			require.NoError(t, err)
 
 			go func() {
 				tu.WaitForResult(func() (bool, error) {
@@ -280,10 +279,14 @@ func TestExecutor_WaitExitSignal(pt *testing.T) {
 						return false, fmt.Errorf("stats failed to send on interval")
 					case ru := <-ch:
 						assert.NotEmpty(t, ru.Pids, "no pids recorded in stats")
-						assert.NotZero(t, ru.ResourceUsage.MemoryStats.RSS)
+
+						// just checking we measured something; each executor type has its own abilities,
+						// and e.g. cgroup v2 provides different information than cgroup v1
+						assert.NotEmpty(t, ru.ResourceUsage.MemoryStats.Measured)
+
 						assert.WithinDuration(t, time.Now(), time.Unix(0, ru.Timestamp), time.Second)
 					}
-					proc, err := os.FindProcess(ps.Pid)
+					proc, err := os.FindProcess(pState.Pid)
 					if err != nil {
 						return false, err
 					}
@@ -298,9 +301,9 @@ func TestExecutor_WaitExitSignal(pt *testing.T) {
 				})
 			}()
 
-			ps, err = executor.Wait(context.Background())
+			pState, err = executor.Wait(context.Background())
-			require.NoError(err)
+			require.NoError(t, err)
-			require.Equal(ps.Signal, int(syscall.SIGKILL))
+			require.Equal(t, pState.Signal, int(syscall.SIGKILL))
 		})
 	}
 }

testutil/wait_test.go

@@ -12,32 +12,37 @@ import (
 
 func TestWait_WaitForFilesUntil(t *testing.T) {
 
-	var files []string
+	N := 10
-	for i := 1; i < 10; i++ {
-		filename := fmt.Sprintf("test%d.txt", i)
-		filepath := filepath.Join(os.TempDir(), filename)
-		files = append(files, filepath)
 
-		defer os.Remove(filepath)
+	tmpDir, err := os.MkdirTemp("", "waiter")
+	require.NoError(t, err)
+
+	defer func() {
+		require.NoError(t, os.RemoveAll(tmpDir))
+	}()
+
+	var files []string
+	for i := 1; i < N; i++ {
+		files = append(files, filepath.Join(
+			tmpDir, fmt.Sprintf("test%d.txt", i),
+		))
 	}
 
 	go func() {
-		for _, filepath := range files {
+		for _, file := range files {
-			t.Logf("Creating file %s...", filepath)
+			t.Logf("Creating file %s ...", file)
-			fh, err := os.Create(filepath)
+			fh, createErr := os.Create(file)
-			fh.Close()
+			require.NoError(t, createErr)
 
-			require.NoError(t, err, "error creating test file")
+			closeErr := fh.Close()
-			require.FileExists(t, filepath)
+			require.NoError(t, closeErr)
+			require.FileExists(t, file)
 
 			time.Sleep(250 * time.Millisecond)
 		}
 	}()
 
 	duration := 5 * time.Second
-	t.Log("Waiting 5 seconds for files...")
+	t.Log("Waiting 5 seconds for files ...")
 	WaitForFilesUntil(t, files, duration)
-
-	t.Log("done")
-
 }