From e85886d33c9410fbc455c5deabe0275bd44593a4 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:24:13 -0500 Subject: [PATCH 1/5] git: only .circleci/config.yml is a generated one --- .circleci/.gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/.gitattributes b/.circleci/.gitattributes index 2dd06ee5f..f7c6b31eb 100644 --- a/.circleci/.gitattributes +++ b/.circleci/.gitattributes @@ -1 +1 @@ -config.yml linguist-generated +/config.yml linguist-generated From 5c17bf36b889ed21ec47e71f5776b9084ec2a9ed Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:29:06 -0500 Subject: [PATCH 2/5] ci: ignore docs only changes Skip running backend and UI changes for docs/website updates, as indicated by branch name. --- .circleci/config.yml | 13 +++++++++++++ .circleci/config/workflows/build-test.yml | 2 ++ 2 files changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5a887dc9..973d2e825 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -805,77 +805,90 @@ workflows: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-client: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-nomad: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-api: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-devices: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-other: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-docker: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-exec: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-shared-exec: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-32bit: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-e2e: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-ui: filters: branches: ignore: - stable-website + - /^docs-.*/ - test-website: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website website: jobs: diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml index 136fd4087..3cbc270d7 100644 --- a/.circleci/config/workflows/build-test.yml +++ b/.circleci/config/workflows/build-test.yml @@ -5,6 +5,7 @@ jobs: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-machine: name: "test-client" @@ -58,5 +59,6 @@ jobs: branches: ignore: - stable-website + - /^docs-.*/ - test-website: filters: *backend_branches_filter From 57ff36df5c2ea95fe2929ada44a1f78c9688bd93 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:51:10 -0500 Subject: [PATCH 3/5] ci: avoid paging --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ .circleci/config/config.yml | 23 ++++++++++++++--------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 973d2e825..02c78ce4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,11 +9,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./nomad/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -83,11 +85,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./api/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -157,11 +161,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./drivers/exec - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -231,11 +237,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./client/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -305,11 +313,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./drivers/shared/executor - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -421,10 +431,12 @@ jobs: - run: command: make checkscripts environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat build-website: docker: - image: hashicorp/middleman-hashicorp:0.3.35 @@ -452,11 +464,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: '' - GOTEST_PKGS_EXCLUDE: ./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices - GOTESTARCH: amd64 @@ -526,10 +540,12 @@ jobs: - image: golang:1.12.13 working_directory: /go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./devices/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -587,21 +603,25 @@ jobs: - run: command: make test-website environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat test-docker: machine: image: circleci/classic:201808-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./drivers/docker - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -671,10 +691,12 @@ jobs: - image: golang:1.12.13 working_directory: /go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GO_TAGS: codegen_generated steps: - checkout @@ -718,20 +740,24 @@ jobs: - run: command: sudo -E -H -u circleci PATH=${PATH} make e2e-test environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat test-32bit: machine: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./client/fingerprint - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: '386' diff --git a/.circleci/config/config.yml b/.circleci/config/config.yml index 9d3168971..c18e455f2 100644 --- a/.circleci/config/config.yml +++ b/.circleci/config/config.yml @@ -7,29 +7,34 @@ references: go-machine-recent-image: &go_machine_recent_image ubuntu-1604:201903-01 + # common references + common_envs: &common_envs + GOMAXPROCS: 1 + NOMAD_SLOW_TEST: 1 + GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + # disable implicit git paging. CircleCI runs commands with in a tty + # making git assume it's an interactive session. + PAGER: cat + GIT_PAGER: cat + executors: go: working_directory: /go/src/github.com/hashicorp/nomad docker: - image: golang:1.12.13 - environment: &common_envs - GOMAXPROCS: 1 - NOMAD_SLOW_TEST: 1 - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + environment: + <<: *common_envs GOPATH: /go - + go-machine: working_directory: ~/go/src/github.com/hashicorp/nomad machine: image: *go_machine_image environment: &machine_env - GOMAXPROCS: 1 - NOMAD_SLOW_TEST: 1 - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + <<: *common_envs GOPATH: /home/circleci/go GOLANG_VERSION: "1.12.13" - # uses a more recent image with unattended upgrades disabled properly # but seems to break docker builds go-machine-recent: From 500fb6f8ced60e92c90dfe2d7f705cbbf7e23066 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 14:50:49 -0500 Subject: [PATCH 4/5] ci: verify .circleci/config.yml is up to date --- .circleci/config.yml | 6 ++++ .../commands/install-circleci-local-cli.yml | 31 +++++++++++++++++++ .circleci/config/jobs/lint-go.yml | 4 +++ 3 files changed, 41 insertions(+) create mode 100644 .circleci/config/commands/install-circleci-local-cli.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 02c78ce4e..5e28eec14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -424,12 +424,18 @@ jobs: sudo rm -rf /usr/bin/protoc sudo ./scripts/vagrant-linux-priv-protoc.sh name: install protoc + - run: + command: "CCI_VERSION=\"0.1.5879\"\nCCI_SHA256=\"f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6\"\n\nURL=\"https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz\"\n\nmkdir -p /tmp/circleci-cli/\ncurl --fail --show-error --location \\\n -o /tmp/circleci-cli/cli.tar.gz \"${URL}\"\n\necho \"$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz\" | sha256sum -c\n\ntar -xz --strip-components=1 \\\n -C /tmp/circleci-cli \\\n -f /tmp/circleci-cli/cli.tar.gz \\\n \"circleci-cli_${CCI_VERSION}_linux_amd64/circleci\" \n\nsudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli\n\ncircleci-local-cli version\n" + name: Install CircleCI CLI 0.1.5879 - run: command: make deps lint-deps - run: command: make check - run: command: make checkscripts + - run: + command: make -C .circleci CIRCLECI="circleci-local-cli --skip-update-check" ci-verify + name: check .circleci/config.yml is up-to-date environment: - GIT_PAGER: cat - GOMAXPROCS: 1 diff --git a/.circleci/config/commands/install-circleci-local-cli.yml b/.circleci/config/commands/install-circleci-local-cli.yml new file mode 100644 index 000000000..5b89a3f69 --- /dev/null +++ b/.circleci/config/commands/install-circleci-local-cli.yml @@ -0,0 +1,31 @@ +parameters: + version: + type: string + default: 0.1.5879 + + sha256: + type: string + default: f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6 +steps: + - run: + name: Install CircleCI CLI << parameters.version >> + command: | + CCI_VERSION="<< parameters.version >>" + CCI_SHA256="<< parameters.sha256 >>" + + URL="https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz" + + mkdir -p /tmp/circleci-cli/ + curl --fail --show-error --location \ + -o /tmp/circleci-cli/cli.tar.gz "${URL}" + + echo "$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz" | sha256sum -c + + tar -xz --strip-components=1 \ + -C /tmp/circleci-cli \ + -f /tmp/circleci-cli/cli.tar.gz \ + "circleci-cli_${CCI_VERSION}_linux_amd64/circleci" + + sudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli + + circleci-local-cli version diff --git a/.circleci/config/jobs/lint-go.yml b/.circleci/config/jobs/lint-go.yml index 90a9abcab..72e955010 100644 --- a/.circleci/config/jobs/lint-go.yml +++ b/.circleci/config/jobs/lint-go.yml @@ -3,6 +3,10 @@ steps: - checkout - run: apt-get update; apt-get install -y shellcheck sudo unzip - install-protoc + - install-circleci-local-cli - run: make deps lint-deps - run: make check - run: make checkscripts + - run: + name: check .circleci/config.yml is up-to-date + command: make -C .circleci CIRCLECI="circleci-local-cli --skip-update-check" ci-verify From e8dc686019d855ac1a081832e8e4bf5137564c47 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 15:32:48 -0500 Subject: [PATCH 5/5] ci: avoid building binaries in stable-website This speeds up building and pushing releases --- .circleci/config.yml | 6 +++++- .circleci/config/workflows/build-test.yml | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e28eec14..73ae04eed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -831,7 +831,11 @@ jobs: workflows: build-test: jobs: - - build-binaries + - build-binaries: + filters: + branches: + ignore: + - stable-website - lint-go: filters: branches: diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml index 3cbc270d7..f3d7f7b76 100644 --- a/.circleci/config/workflows/build-test.yml +++ b/.circleci/config/workflows/build-test.yml @@ -1,5 +1,11 @@ jobs: -- build-binaries +- build-binaries: + # almost always build binaries as they may be needed + # for e2e tests + filters: + branches: + ignore: + - stable-website - lint-go: filters: &backend_branches_filter branches: