From 2f0cfb57409accc326b3d69e239c07ae5323eb91 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Tue, 18 Jan 2022 12:17:37 -0600 Subject: [PATCH] build: upgrade and speedup circleci configuration This PR upgrades our CI images and fixes some affected tests. - upgrade go-machine-image to premade latest ubuntu LTS (ubuntu-2004:202111-02) - eliminate go-machine-recent-image (no longer necessary) - manage GOPATH in GNUMakefile (see https://discuss.circleci.com/t/gopath-is-set-to-multiple-directories/7174) - fix tcp dial error check (message seems to be OS specific) - spot check values measured instead of specifically 'RSS' (rss no longer reported in cgroups v2) - use safe MkdirTemp for generating tmpfiles NOT applied: (too flakey) - eliminate setting GOMAXPROCS=1 (build tools were also affected by this setting) - upgrade resource type for all imanges to large (2C -> 4C) --- .changelog/11889.txt | 3 ++ .circleci/config.yml | 20 +++--------- GNUmakefile | 10 ++++-- command/metrics_test.go | 2 +- drivers/exec/driver_test.go | 2 +- drivers/shared/executor/executor_linux.go | 1 + drivers/shared/executor/executor_test.go | 19 +++++++----- testutil/wait_test.go | 37 +++++++++++++---------- 8 files changed, 50 insertions(+), 44 deletions(-) create mode 100644 .changelog/11889.txt diff --git a/.changelog/11889.txt b/.changelog/11889.txt new file mode 100644 index 000000000..b45ac0403 --- /dev/null +++ b/.changelog/11889.txt @@ -0,0 +1,3 @@ +```release-note:improvement +build: upgrade and speedup circleci configuration +``` diff --git a/.circleci/config.yml b/.circleci/config.yml index bf2f74f9e..047772df1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,9 +3,7 @@ version: 2.1 references: # environment specific references - aim to avoid conflicts go-machine-image: &go_machine_image - circleci/classic:201808-01 - go-machine-recent-image: &go_machine_recent_image - ubuntu-1604:201903-01 + ubuntu-2004:202111-02 go-windows-image: &go_windows_image windows-server-2019-vs2019:stable @@ -289,7 +287,7 @@ jobs: default: "" executor: type: string - default: "go-machine-recent" + default: "go-machine" goarch: type: string default: "amd64" @@ -513,6 +511,7 @@ executors: working_directory: /go/src/github.com/hashicorp/nomad docker: - image: docker.mirror.hashicorp.services/golang:1.17.5 + resource_class: medium environment: <<: *common_envs GOPATH: /go @@ -521,19 +520,11 @@ executors: working_directory: ~/go/src/github.com/hashicorp/nomad machine: image: *go_machine_image + resource_class: medium environment: &machine_env <<: *common_envs - GOPATH: /home/circleci/go GOLANG_VERSION: 1.17.5 - # uses a more recent image with unattended upgrades disabled properly - # but seems to break docker builds - go-machine-recent: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: - image: *go_machine_recent_image - environment: *machine_env - go-macos: working_directory: ~/go/src/github.com/hashicorp/nomad macos: @@ -636,9 +627,6 @@ workflows: - test-machine: name: "test-docker" test_packages: "./drivers/docker" - # docker is misbehaving in docker-machine-recent image - # and we get unexpected failures - # e.g. https://circleci.com/gh/hashicorp/nomad/3854 executor: go-machine filters: *backend_test_branches_filter - test-machine: diff --git a/GNUmakefile b/GNUmakefile index 84a02085e..162021832 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -8,6 +8,13 @@ GIT_DIRTY := $(if $(shell git status --porcelain),+CHANGES) GO_LDFLAGS := "-X github.com/hashicorp/nomad/version.GitCommit=$(GIT_COMMIT)$(GIT_DIRTY)" +ifneq (MSYS_NT,$(THIS_OS)) +# GOPATH supports PATH style multi-paths; assume the first entry is favorable. +# Necessary because new Circle images override GOPATH with multiple values. +# See: https://discuss.circleci.com/t/gopath-is-set-to-multiple-directories/7174 +GOPATH=$(shell go env GOPATH | cut -d: -f1) +endif + GO_TAGS ?= ifeq ($(CI),true) @@ -241,7 +248,6 @@ tidy: .PHONY: dev dev: GOOS=$(shell go env GOOS) dev: GOARCH=$(shell go env GOARCH) -dev: GOPATH=$(shell go env GOPATH) dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad dev: hclfmt ## Build for the current development platform @echo "==> Removing old development build..." @@ -297,7 +303,7 @@ test-nomad: dev ## Run Nomad test suites .PHONY: test-nomad-module test-nomad-module: dev ## Run Nomad test suites on a sub-module - @echo "==> Running Nomad test suites on sub-module:" + @echo "==> Running Nomad test suites on sub-module $(GOTEST_MOD)" @cd $(GOTEST_MOD) && $(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \ $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ -cover \ diff --git a/command/metrics_test.go b/command/metrics_test.go index cdf18c334..a537f3997 100644 --- a/command/metrics_test.go +++ b/command/metrics_test.go @@ -72,7 +72,7 @@ func TestCommand_Metrics_Cases(t *testing.T) { []string{"-address=http://foo"}, 1, "", - "no such host", + "dial tcp: lookup foo: Temporary failure in name resolution", }, } diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index 8651eda20..2188dc8be 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -425,7 +425,7 @@ func TestExecDriver_Stats(t *testing.T) { require.NoError(err) select { case stats := <-statsCh: - require.NotZero(stats.ResourceUsage.MemoryStats.RSS) + require.NotEmpty(stats.ResourceUsage.MemoryStats.Measured) require.NotZero(stats.Timestamp) require.WithinDuration(time.Now(), time.Unix(0, stats.Timestamp), time.Second) case <-time.After(time.Second): diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index f40863a74..989c4a062 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -71,6 +71,7 @@ type LibcontainerExecutor struct { } func NewExecutorWithIsolation(logger hclog.Logger) Executor { + logger = logger.Named("isolated_executor") if err := shelpers.Init(); err != nil { logger.Error("unable to initialize stats", "error", err) diff --git a/drivers/shared/executor/executor_test.go b/drivers/shared/executor/executor_test.go index 6a396ea6a..0b7e8b000 100644 --- a/drivers/shared/executor/executor_test.go +++ b/drivers/shared/executor/executor_test.go @@ -254,7 +254,6 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { pt.Parallel() for name, factory := range executorFactories { pt.Run(name, func(t *testing.T) { - require := require.New(t) testExecCmd := testExecutorCommand(t) execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir execCmd.Cmd = "/bin/sleep" @@ -266,8 +265,8 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { executor := factory.new(testlog.HCLogger(t)) defer executor.Shutdown("", 0) - ps, err := executor.Launch(execCmd) - require.NoError(err) + pState, err := executor.Launch(execCmd) + require.NoError(t, err) go func() { tu.WaitForResult(func() (bool, error) { @@ -280,10 +279,14 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { return false, fmt.Errorf("stats failed to send on interval") case ru := <-ch: assert.NotEmpty(t, ru.Pids, "no pids recorded in stats") - assert.NotZero(t, ru.ResourceUsage.MemoryStats.RSS) + + // just checking we measured something; each executor type has its own abilities, + // and e.g. cgroup v2 provides different information than cgroup v1 + assert.NotEmpty(t, ru.ResourceUsage.MemoryStats.Measured) + assert.WithinDuration(t, time.Now(), time.Unix(0, ru.Timestamp), time.Second) } - proc, err := os.FindProcess(ps.Pid) + proc, err := os.FindProcess(pState.Pid) if err != nil { return false, err } @@ -298,9 +301,9 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { }) }() - ps, err = executor.Wait(context.Background()) - require.NoError(err) - require.Equal(ps.Signal, int(syscall.SIGKILL)) + pState, err = executor.Wait(context.Background()) + require.NoError(t, err) + require.Equal(t, pState.Signal, int(syscall.SIGKILL)) }) } } diff --git a/testutil/wait_test.go b/testutil/wait_test.go index bcc7ff8c6..290105dc4 100644 --- a/testutil/wait_test.go +++ b/testutil/wait_test.go @@ -12,32 +12,37 @@ import ( func TestWait_WaitForFilesUntil(t *testing.T) { - var files []string - for i := 1; i < 10; i++ { - filename := fmt.Sprintf("test%d.txt", i) - filepath := filepath.Join(os.TempDir(), filename) - files = append(files, filepath) + N := 10 - defer os.Remove(filepath) + tmpDir, err := os.MkdirTemp("", "waiter") + require.NoError(t, err) + + defer func() { + require.NoError(t, os.RemoveAll(tmpDir)) + }() + + var files []string + for i := 1; i < N; i++ { + files = append(files, filepath.Join( + tmpDir, fmt.Sprintf("test%d.txt", i), + )) } go func() { - for _, filepath := range files { - t.Logf("Creating file %s...", filepath) - fh, err := os.Create(filepath) - fh.Close() + for _, file := range files { + t.Logf("Creating file %s ...", file) + fh, createErr := os.Create(file) + require.NoError(t, createErr) - require.NoError(t, err, "error creating test file") - require.FileExists(t, filepath) + closeErr := fh.Close() + require.NoError(t, closeErr) + require.FileExists(t, file) time.Sleep(250 * time.Millisecond) } }() duration := 5 * time.Second - t.Log("Waiting 5 seconds for files...") + t.Log("Waiting 5 seconds for files ...") WaitForFilesUntil(t, files, duration) - - t.Log("done") - }