Merge pull request #6745 from hashicorp/c-circleci-tweaks-20191120

ci: Revamp CircleCI config

.circleci/.gitattributes

@@ -0,0 +1 @@
+config.yml linguist-generated

.circleci/.gitignore

@@ -0,0 +1 @@
+.tmp/

.circleci/Makefile

@@ -0,0 +1,80 @@
+# Set SHELL to 'strict mode' without using .SHELLFLAGS for max compatibility.
+# See https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/
+SHELL := /usr/bin/env bash -euo pipefail -c
+
+CIRCLECI := circleci --skip-update-check
+
+# Set up some documentation/help message variables.
+# We do not attempt to install the CircleCI CLI from this Makefile.
+CCI_INSTALL_LINK := https://circleci.com/docs/2.0/local-cli/\#installation
+CCI_INSTALL_MSG := Please install CircleCI CLI. See $(CCI_INSTALL_LINK)
+CCI_VERSION := $(shell $(CIRCLECI) version 2> /dev/null)
+ifeq ($(CCI_VERSION),)
+# Attempting to use the CLI fails with installation instructions.
+CIRCLECI := echo '$(CCI_INSTALL_MSG)'; exit 1; \#
+endif
+
+SOURCE_DIR     := config
+SOURCE_YML     := $(shell [ ! -d $(SOURCE_DIR) ] || find $(SOURCE_DIR) -name '*.yml')
+CONFIG_SOURCE  := Makefile $(SOURCE_YML) | $(SOURCE_DIR)
+OUT            := config.yml
+TMP            := .tmp/config-processed
+CONFIG_PACKED  := .tmp/config-packed
+
+default: help
+
+help:
+	@echo "Usage:"
+	@echo "  make ci-config: recompile config.yml from $(SOURCE_DIR)/"
+	@echo "  make ci-verify: verify that config.yml is a true mapping from $(SOURCE_DIR)/"
+	@echo
+	@echo "Diagnostics:"
+	@[ -z "$(CCI_VERSION)" ] || echo "  circleci-cli version $(CCI_VERSION)"
+	@[ -n "$(CCI_VERSION)" ] || echo "  $(CCI_INSTALL_MSG)"
+
+$(SOURCE_DIR):
+	@echo No source directory $(SOURCE_DIR) found.; exit 1
+
+# Make sure our .tmp dir exists.
+$(shell [ -d .tmp ] || mkdir .tmp)
+
+.PHONY: ci-config
+ci-config: $(OUT)
+
+.PHONY: ci-verify
+ci-verify: config-up-to-date
+	@$(CIRCLECI) config validate $(OUT)
+
+define GENERATED_FILE_HEADER
+### ***
+### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'.
+### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make ci-config'.
+### ***
+endef
+export GENERATED_FILE_HEADER
+
+# GEN_CONFIG writes the config to a temporary file. If the whole process succeeds,
+# it then moves that file to $@. This makes is an atomic operation, so if it fails
+# make doesn't consider a half-baked file up to date.
+define GEN_CONFIG
+	@$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED)
+	@echo "$$GENERATED_FILE_HEADER" > $@.tmp || { rm -f $@; exit 1; }
+	@$(CIRCLECI) config process $(CONFIG_PACKED) >> $@.tmp || { rm -f $@.tmp; exit 1; }
+	@mv -f $@.tmp $@
+endef
+
+$(OUT): $(CONFIG_SOURCE)
+	$(GEN_CONFIG)
+	@echo "$@ updated"
+
+$(TMP): $(CONFIG_SOURCE)
+	$(GEN_CONFIG)
+
+.PHONY: config-up-to-date
+config-up-to-date: $(TMP) # Note this must not depend on $(OUT)!
+	@if diff config.yml $<; then \
+		echo "Generated $(OUT) is up to date!"; \
+	else \
+		echo "Generated $(OUT) is out of date, run make $(CONFIG) to update."; \
+		exit 1; \
+	fi

.circleci/README.md

@@ -0,0 +1,3 @@
+# How to use CircleCI multi-file config
+
+Refer to https://github.com/hashicorp/vault/blob/master/.circleci/README.md .

.circleci/config/commands/install-consul.yml

@@ -0,0 +1,11 @@
+parameters:
+  version:
+    type: string
+    default: 1.6.1
+steps:
+  - run:
+      name: Install Consul << parameters.version >>
+      command: |
+        wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip
+        sudo unzip -d /usr/local/bin /tmp/consul.zip
+        rm -rf /tmp/consul*

.circleci/config/commands/install-golang.yml

@@ -0,0 +1,10 @@
+steps:
+  - run:
+      name: install golang
+      command: |
+        set -x
+        echo installing golang ${GOLANG_VERSION}
+        sudo rm -rf /usr/local/go
+        wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz
+        sudo tar -C /usr/local -xzf /tmp/golang.tar.gz
+        rm -rf /tmp/golang.tar.gz

.circleci/config/commands/install-protoc.yml

@@ -0,0 +1,6 @@
+steps:
+  - run:
+      name: install protoc
+      command: |
+        sudo rm -rf /usr/bin/protoc
+        sudo ./scripts/vagrant-linux-priv-protoc.sh

.circleci/config/commands/install-vault.yml

@@ -0,0 +1,11 @@
+parameters:
+  version:
+    type: string
+    default: 1.2.3
+steps:
+  - run:
+      name: Install Vault << parameters.version >>
+      command: |
+        wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/<< parameters.version >>/vault_<< parameters.version>>_linux_amd64.zip
+        sudo unzip -d /usr/local/bin /tmp/vault.zip
+        rm -rf /tmp/vault*

.circleci/config/commands/run-tests.yml

@@ -0,0 +1,18 @@
+steps:
+  - run:
+      name: Running Nomad Tests
+      command: |
+        if [ -z $GOTEST_PKGS_EXCLUDE ];
+        then
+          unset GOTEST_PKGS_EXCLUDE
+        else
+          unset GOTEST_PKGS
+        fi
+
+        if [ ! -z $GOTESTARCH ]; then
+          export GOARCH="$GOTESTARCH";
+        fi
+
+        mkdir -p /tmp/test-reports
+        sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs
+        sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad

.circleci/config/config.yml

@@ -0,0 +1,39 @@
+version: 2.1
+
+references:
+  # environment specific references - aim to avoid conflicts
+  go-machine-image: &go_machine_image
+    circleci/classic:201808-01
+  go-machine-recent-image: &go_machine_recent_image
+    ubuntu-1604:201903-01
+
+executors:
+  go:
+    working_directory: /go/src/github.com/hashicorp/nomad
+    docker:
+      - image: golang:1.12.13
+    environment: &common_envs
+      GOMAXPROCS: 1
+      NOMAD_SLOW_TEST: 1
+      GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml
+      GOPATH: /go
+ 
+  go-machine:
+    working_directory: ~/go/src/github.com/hashicorp/nomad
+    machine:
+      image: *go_machine_image
+    environment: &machine_env
+      GOMAXPROCS: 1
+      NOMAD_SLOW_TEST: 1
+      GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml
+      GOPATH: /home/circleci/go
+      GOLANG_VERSION: "1.12.13"
+
+
+  # uses a more recent image with unattended upgrades disabled properly
+  # but seems to break docker builds
+  go-machine-recent:
+    working_directory: ~/go/src/github.com/hashicorp/nomad
+    machine:
+      image: *go_machine_recent_image
+    environment: *machine_env

.circleci/config/jobs/build-binaries.yml

@@ -0,0 +1,17 @@
+executor: go
+environment:
+  # TODO: add ui tag here
+  GO_TAGS: "codegen_generated"
+steps:
+  - checkout
+  - run: apt-get update; apt-get install -y sudo unzip zip
+  - run: make deps
+  - install-protoc
+  - run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs
+  - run: make pkg/windows_amd64.zip pkg/linux_amd64.zip
+  - store_artifacts:
+      path: pkg/windows_amd64.zip
+      destination: /builds/nomad_windows_amd64.zip
+  - store_artifacts:
+      path: pkg/linux_amd64.zip
+      destination: /builds/nomad_linux_amd64.zip

.circleci/config/jobs/build-website.yml

@@ -0,0 +1,30 @@
+# setting the working_directory along with the checkout path allows us to not have
+# to cd into the website/ directory for commands
+working_directory: ~/project/website
+docker:
+  - image: hashicorp/middleman-hashicorp:0.3.35
+steps:
+  - checkout:
+      path: ~/project
+
+  # restores gem cache
+  - restore_cache:
+      key: static-site-gems-v1-{{ checksum "Gemfile.lock" }}
+
+  - run:
+      name: install gems
+      command: bundle check || bundle install --path vendor/bundle --retry=3
+
+  # saves gem cache if we have changed the Gemfile
+  - save_cache:
+      key: static-site-gems-v1-{{ checksum "Gemfile.lock" }}
+      paths:
+        - ~/project/website/vendor/bundle
+
+  - run:
+      name: middleman build
+      command: bundle exec middleman build
+
+  - run:
+      name: website deploy
+      command: ./scripts/deploy.sh

.circleci/config/jobs/lint-go.yml

@@ -0,0 +1,8 @@
+executor: go
+steps:
+  - checkout
+  - run: apt-get update; apt-get install -y shellcheck sudo unzip
+  - install-protoc
+  - run: make deps lint-deps
+  - run: make check
+  - run: make checkscripts

.circleci/config/jobs/test-container.yml

@@ -0,0 +1,27 @@
+executor: go
+parameters:
+  test_packages:
+    type: string
+    default: ""
+  exclude_packages:
+    type: string
+    default: ""
+  goarch:
+    type: string
+    default: "amd64"
+environment:
+  GOTEST_PKGS: "<< parameters.test_packages >>"
+  GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>"
+  GOTESTARCH: "<< parameters.goarch >>"
+steps:
+  - checkout
+  - run: apt-get update; apt-get install -y shellcheck sudo unzip
+  - run: make deps
+  - install-protoc
+  - install-consul
+  - install-vault
+  - run-tests
+  - store_test_results:
+      path: /tmp/test-reports
+  - store_artifacts:
+      path: /tmp/test-reports

.circleci/config/jobs/test-e2e.yml

@@ -0,0 +1,17 @@
+executor: go
+steps:
+  - checkout
+  - run: apt-get update; apt-get install -y sudo unzip
+  # e2e tests require privileged mount/umount permissions when running as root
+  # TODO: switch to using machine executor and run as root to test e2e path
+  - run:
+      name: prepare non-root user
+      command: |
+        groupadd --gid 3434 circleci
+        useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci
+        echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci
+        echo 'Defaults    env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep
+        chown -R circleci:circleci /go
+
+  - run: sudo -E -H -u circleci PATH=${PATH} make deps
+  - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test

.circleci/config/jobs/test-machine.yml

@@ -0,0 +1,39 @@
+executor: "<< parameters.executor >>"
+parameters:
+  test_packages:
+    type: string
+    default: ""
+  exclude_packages:
+    type: string
+    default: ""
+  executor:
+    type: string
+    default: "go-machine-recent"
+  goarch:
+    type: string
+    default: "amd64"
+environment:
+  GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>"
+  GOTEST_PKGS: "<< parameters.test_packages >>"
+  GOTESTARCH: "<< parameters.goarch >>"
+steps:
+  - checkout
+  - install-golang
+  - install-protoc
+  - install-consul
+  - install-vault
+  - run:
+      name: Install 32bit gcc libs
+      command: |
+        if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then
+          sudo apt-get update
+          sudo apt-get install -y gcc-multilib
+        else
+          echo "Skipping 32bit lib installation while building for not 386"
+        fi
+  - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap
+  - run-tests
+  - store_test_results:
+      path: /tmp/test-reports
+  - store_artifacts:
+      path: /tmp/test-reports

.circleci/config/jobs/test-ui.yml

@@ -0,0 +1,27 @@
+docker:
+  - image: circleci/node:10-browsers
+    environment:
+      # See https://git.io/vdao3 for details.
+      JOBS: 2
+steps:
+  - checkout
+  - restore_cache:
+      keys:
+        - v1-deps-{{ checksum "ui/yarn.lock" }}
+        - v1-deps-
+  - run:
+      name: yarn install
+      command: cd ui && yarn install
+  - save_cache:
+      key: v1-deps-{{ checksum "ui/yarn.lock" }}
+      paths:
+        - ./ui/node_modules
+  - run:
+      name: lint:js
+      command: cd ui && yarn run lint:js
+  - run:
+      name: lint:hbs
+      command: cd ui && yarn run lint:hbs
+  - run:
+      name: Ember tests
+      command: cd ui && yarn test

.circleci/config/jobs/test-website.yml

@@ -0,0 +1,4 @@
+executor: go-machine-recent
+steps:
+  - checkout
+  - run: make test-website

.circleci/config/workflows/build-test.yml

@@ -0,0 +1,62 @@
+jobs:
+- build-binaries
+- lint-go:
+    filters: &backend_branches_filter
+      branches:
+        ignore:
+          - /^.-ui\b.*/
+          - stable-website
+- test-machine:
+    name: "test-client"
+    test_packages: "./client/..."
+    filters: *backend_branches_filter
+- test-machine:
+    name: "test-nomad"
+    test_packages: "./nomad/..."
+    filters: *backend_branches_filter
+- test-machine:
+    # API Tests run in a VM rather than container due to the FS tests
+    # requiring `mount` priviliges.
+    name: "test-api"
+    test_packages: "./api/..."
+    filters: *backend_branches_filter
+- test-container:
+    name: "test-devices"
+    test_packages: "./devices/..."
+    filters: *backend_branches_filter
+- test-machine:
+    name: "test-other"
+    exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices"
+    filters: *backend_branches_filter
+- test-machine:
+    name: "test-docker"
+    test_packages: "./drivers/docker"
+    # docker is misbehaving in docker-machine-recent image
+    # and we get unexpected failures
+    # e.g. https://circleci.com/gh/hashicorp/nomad/3854
+    executor: go-machine
+    filters: *backend_branches_filter
+- test-machine:
+    name: "test-exec"
+    test_packages: "./drivers/exec"
+    filters: *backend_branches_filter
+- test-machine:
+    name: "test-shared-exec"
+    test_packages: "./drivers/shared/executor"
+    filters: *backend_branches_filter
+- test-machine:
+    name: "test-32bit"
+    # Currently we only explicitly test fingerprinting on 32bit
+    # architectures.
+    test_packages: "./client/fingerprint"
+    goarch: "386"
+    filters: *backend_branches_filter
+- test-e2e:
+    filters: *backend_branches_filter
+- test-ui:
+    filters:
+      branches:
+        ignore:
+        - stable-website
+- test-website:
+    filters: *backend_branches_filter

.circleci/config/workflows/website.yml

@@ -0,0 +1,6 @@
+jobs:
+  - build-website:
+      context: static-sites
+      filters:
+        branches:
+          only: stable-website