Merge pull request #6745 from hashicorp/c-circleci-tweaks-20191120
ci: Revamp CircleCI config
This commit is contained in:
Mahmood Ali
GitHub
commit
0947155ddb
|
|
@ -0,0 +1 @@
|
|||
config.yml linguist-generated
|
||||
|
|
@ -0,0 +1 @@
|
|||
.tmp/
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
# Set SHELL to 'strict mode' without using .SHELLFLAGS for max compatibility.
|
||||
# See https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/
|
||||
SHELL := /usr/bin/env bash -euo pipefail -c
|
||||
|
||||
CIRCLECI := circleci --skip-update-check
|
||||
|
||||
# Set up some documentation/help message variables.
|
||||
# We do not attempt to install the CircleCI CLI from this Makefile.
|
||||
CCI_INSTALL_LINK := https://circleci.com/docs/2.0/local-cli/\#installation
|
||||
CCI_INSTALL_MSG := Please install CircleCI CLI. See $(CCI_INSTALL_LINK)
|
||||
CCI_VERSION := $(shell $(CIRCLECI) version 2> /dev/null)
|
||||
ifeq ($(CCI_VERSION),)
|
||||
# Attempting to use the CLI fails with installation instructions.
|
||||
CIRCLECI := echo '$(CCI_INSTALL_MSG)'; exit 1; \#
|
||||
endif
|
||||
|
||||
SOURCE_DIR := config
|
||||
SOURCE_YML := $(shell [ ! -d $(SOURCE_DIR) ] || find $(SOURCE_DIR) -name '*.yml')
|
||||
CONFIG_SOURCE := Makefile $(SOURCE_YML) | $(SOURCE_DIR)
|
||||
OUT := config.yml
|
||||
TMP := .tmp/config-processed
|
||||
CONFIG_PACKED := .tmp/config-packed
|
||||
|
||||
default: help
|
||||
|
||||
help:
|
||||
@echo "Usage:"
|
||||
@echo " make ci-config: recompile config.yml from $(SOURCE_DIR)/"
|
||||
@echo " make ci-verify: verify that config.yml is a true mapping from $(SOURCE_DIR)/"
|
||||
@echo
|
||||
@echo "Diagnostics:"
|
||||
@[ -z "$(CCI_VERSION)" ] || echo " circleci-cli version $(CCI_VERSION)"
|
||||
@[ -n "$(CCI_VERSION)" ] || echo " $(CCI_INSTALL_MSG)"
|
||||
|
||||
$(SOURCE_DIR):
|
||||
@echo No source directory $(SOURCE_DIR) found.; exit 1
|
||||
|
||||
# Make sure our .tmp dir exists.
|
||||
$(shell [ -d .tmp ] || mkdir .tmp)
|
||||
|
||||
.PHONY: ci-config
|
||||
ci-config: $(OUT)
|
||||
|
||||
.PHONY: ci-verify
|
||||
ci-verify: config-up-to-date
|
||||
@$(CIRCLECI) config validate $(OUT)
|
||||
|
||||
define GENERATED_FILE_HEADER
|
||||
### ***
|
||||
### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'.
|
||||
### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make ci-config'.
|
||||
### ***
|
||||
endef
|
||||
export GENERATED_FILE_HEADER
|
||||
|
||||
# GEN_CONFIG writes the config to a temporary file. If the whole process succeeds,
|
||||
# it then moves that file to $@. This makes is an atomic operation, so if it fails
|
||||
# make doesn't consider a half-baked file up to date.
|
||||
define GEN_CONFIG
|
||||
@$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED)
|
||||
@echo "$$GENERATED_FILE_HEADER" > $@.tmp || { rm -f $@; exit 1; }
|
||||
@$(CIRCLECI) config process $(CONFIG_PACKED) >> $@.tmp || { rm -f $@.tmp; exit 1; }
|
||||
@mv -f $@.tmp $@
|
||||
endef
|
||||
|
||||
$(OUT): $(CONFIG_SOURCE)
|
||||
$(GEN_CONFIG)
|
||||
@echo "$@ updated"
|
||||
|
||||
$(TMP): $(CONFIG_SOURCE)
|
||||
$(GEN_CONFIG)
|
||||
|
||||
.PHONY: config-up-to-date
|
||||
config-up-to-date: $(TMP) # Note this must not depend on $(OUT)!
|
||||
@if diff config.yml $<; then \
|
||||
echo "Generated $(OUT) is up to date!"; \
|
||||
else \
|
||||
echo "Generated $(OUT) is out of date, run make $(CONFIG) to update."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# How to use CircleCI multi-file config
|
||||
|
||||
Refer to https://github.com/hashicorp/vault/blob/master/.circleci/README.md .
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,11 @@
|
|||
parameters:
|
||||
version:
|
||||
type: string
|
||||
default: 1.6.1
|
||||
steps:
|
||||
- run:
|
||||
name: Install Consul << parameters.version >>
|
||||
command: |
|
||||
wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip
|
||||
sudo unzip -d /usr/local/bin /tmp/consul.zip
|
||||
rm -rf /tmp/consul*
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
steps:
|
||||
- run:
|
||||
name: install golang
|
||||
command: |
|
||||
set -x
|
||||
echo installing golang ${GOLANG_VERSION}
|
||||
sudo rm -rf /usr/local/go
|
||||
wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz
|
||||
sudo tar -C /usr/local -xzf /tmp/golang.tar.gz
|
||||
rm -rf /tmp/golang.tar.gz
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
steps:
|
||||
- run:
|
||||
name: install protoc
|
||||
command: |
|
||||
sudo rm -rf /usr/bin/protoc
|
||||
sudo ./scripts/vagrant-linux-priv-protoc.sh
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
parameters:
|
||||
version:
|
||||
type: string
|
||||
default: 1.2.3
|
||||
steps:
|
||||
- run:
|
||||
name: Install Vault << parameters.version >>
|
||||
command: |
|
||||
wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/<< parameters.version >>/vault_<< parameters.version>>_linux_amd64.zip
|
||||
sudo unzip -d /usr/local/bin /tmp/vault.zip
|
||||
rm -rf /tmp/vault*
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
steps:
|
||||
- run:
|
||||
name: Running Nomad Tests
|
||||
command: |
|
||||
if [ -z $GOTEST_PKGS_EXCLUDE ];
|
||||
then
|
||||
unset GOTEST_PKGS_EXCLUDE
|
||||
else
|
||||
unset GOTEST_PKGS
|
||||
fi
|
||||
|
||||
if [ ! -z $GOTESTARCH ]; then
|
||||
export GOARCH="$GOTESTARCH";
|
||||
fi
|
||||
|
||||
mkdir -p /tmp/test-reports
|
||||
sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs
|
||||
sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
version: 2.1
|
||||
|
||||
references:
|
||||
# environment specific references - aim to avoid conflicts
|
||||
go-machine-image: &go_machine_image
|
||||
circleci/classic:201808-01
|
||||
go-machine-recent-image: &go_machine_recent_image
|
||||
ubuntu-1604:201903-01
|
||||
|
||||
executors:
|
||||
go:
|
||||
working_directory: /go/src/github.com/hashicorp/nomad
|
||||
docker:
|
||||
- image: golang:1.12.13
|
||||
environment: &common_envs
|
||||
GOMAXPROCS: 1
|
||||
NOMAD_SLOW_TEST: 1
|
||||
GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml
|
||||
GOPATH: /go
|
||||
|
||||
go-machine:
|
||||
working_directory: ~/go/src/github.com/hashicorp/nomad
|
||||
machine:
|
||||
image: *go_machine_image
|
||||
environment: &machine_env
|
||||
GOMAXPROCS: 1
|
||||
NOMAD_SLOW_TEST: 1
|
||||
GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml
|
||||
GOPATH: /home/circleci/go
|
||||
GOLANG_VERSION: "1.12.13"
|
||||
|
||||
|
||||
# uses a more recent image with unattended upgrades disabled properly
|
||||
# but seems to break docker builds
|
||||
go-machine-recent:
|
||||
working_directory: ~/go/src/github.com/hashicorp/nomad
|
||||
machine:
|
||||
image: *go_machine_recent_image
|
||||
environment: *machine_env
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
executor: go
|
||||
environment:
|
||||
# TODO: add ui tag here
|
||||
GO_TAGS: "codegen_generated"
|
||||
steps:
|
||||
- checkout
|
||||
- run: apt-get update; apt-get install -y sudo unzip zip
|
||||
- run: make deps
|
||||
- install-protoc
|
||||
- run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs
|
||||
- run: make pkg/windows_amd64.zip pkg/linux_amd64.zip
|
||||
- store_artifacts:
|
||||
path: pkg/windows_amd64.zip
|
||||
destination: /builds/nomad_windows_amd64.zip
|
||||
- store_artifacts:
|
||||
path: pkg/linux_amd64.zip
|
||||
destination: /builds/nomad_linux_amd64.zip
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# setting the working_directory along with the checkout path allows us to not have
|
||||
# to cd into the website/ directory for commands
|
||||
working_directory: ~/project/website
|
||||
docker:
|
||||
- image: hashicorp/middleman-hashicorp:0.3.35
|
||||
steps:
|
||||
- checkout:
|
||||
path: ~/project
|
||||
|
||||
# restores gem cache
|
||||
- restore_cache:
|
||||
key: static-site-gems-v1-{{ checksum "Gemfile.lock" }}
|
||||
|
||||
- run:
|
||||
name: install gems
|
||||
command: bundle check || bundle install --path vendor/bundle --retry=3
|
||||
|
||||
# saves gem cache if we have changed the Gemfile
|
||||
- save_cache:
|
||||
key: static-site-gems-v1-{{ checksum "Gemfile.lock" }}
|
||||
paths:
|
||||
- ~/project/website/vendor/bundle
|
||||
|
||||
- run:
|
||||
name: middleman build
|
||||
command: bundle exec middleman build
|
||||
|
||||
- run:
|
||||
name: website deploy
|
||||
command: ./scripts/deploy.sh
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
executor: go
|
||||
steps:
|
||||
- checkout
|
||||
- run: apt-get update; apt-get install -y shellcheck sudo unzip
|
||||
- install-protoc
|
||||
- run: make deps lint-deps
|
||||
- run: make check
|
||||
- run: make checkscripts
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
executor: go
|
||||
parameters:
|
||||
test_packages:
|
||||
type: string
|
||||
default: ""
|
||||
exclude_packages:
|
||||
type: string
|
||||
default: ""
|
||||
goarch:
|
||||
type: string
|
||||
default: "amd64"
|
||||
environment:
|
||||
GOTEST_PKGS: "<< parameters.test_packages >>"
|
||||
GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>"
|
||||
GOTESTARCH: "<< parameters.goarch >>"
|
||||
steps:
|
||||
- checkout
|
||||
- run: apt-get update; apt-get install -y shellcheck sudo unzip
|
||||
- run: make deps
|
||||
- install-protoc
|
||||
- install-consul
|
||||
- install-vault
|
||||
- run-tests
|
||||
- store_test_results:
|
||||
path: /tmp/test-reports
|
||||
- store_artifacts:
|
||||
path: /tmp/test-reports
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
executor: go
|
||||
steps:
|
||||
- checkout
|
||||
- run: apt-get update; apt-get install -y sudo unzip
|
||||
# e2e tests require privileged mount/umount permissions when running as root
|
||||
# TODO: switch to using machine executor and run as root to test e2e path
|
||||
- run:
|
||||
name: prepare non-root user
|
||||
command: |
|
||||
groupadd --gid 3434 circleci
|
||||
useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci
|
||||
echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci
|
||||
echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep
|
||||
chown -R circleci:circleci /go
|
||||
|
||||
- run: sudo -E -H -u circleci PATH=${PATH} make deps
|
||||
- run: sudo -E -H -u circleci PATH=${PATH} make e2e-test
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
executor: "<< parameters.executor >>"
|
||||
parameters:
|
||||
test_packages:
|
||||
type: string
|
||||
default: ""
|
||||
exclude_packages:
|
||||
type: string
|
||||
default: ""
|
||||
executor:
|
||||
type: string
|
||||
default: "go-machine-recent"
|
||||
goarch:
|
||||
type: string
|
||||
default: "amd64"
|
||||
environment:
|
||||
GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>"
|
||||
GOTEST_PKGS: "<< parameters.test_packages >>"
|
||||
GOTESTARCH: "<< parameters.goarch >>"
|
||||
steps:
|
||||
- checkout
|
||||
- install-golang
|
||||
- install-protoc
|
||||
- install-consul
|
||||
- install-vault
|
||||
- run:
|
||||
name: Install 32bit gcc libs
|
||||
command: |
|
||||
if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-multilib
|
||||
else
|
||||
echo "Skipping 32bit lib installation while building for not 386"
|
||||
fi
|
||||
- run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap
|
||||
- run-tests
|
||||
- store_test_results:
|
||||
path: /tmp/test-reports
|
||||
- store_artifacts:
|
||||
path: /tmp/test-reports
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
docker:
|
||||
- image: circleci/node:10-browsers
|
||||
environment:
|
||||
# See https://git.io/vdao3 for details.
|
||||
JOBS: 2
|
||||
steps:
|
||||
- checkout
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-deps-{{ checksum "ui/yarn.lock" }}
|
||||
- v1-deps-
|
||||
- run:
|
||||
name: yarn install
|
||||
command: cd ui && yarn install
|
||||
- save_cache:
|
||||
key: v1-deps-{{ checksum "ui/yarn.lock" }}
|
||||
paths:
|
||||
- ./ui/node_modules
|
||||
- run:
|
||||
name: lint:js
|
||||
command: cd ui && yarn run lint:js
|
||||
- run:
|
||||
name: lint:hbs
|
||||
command: cd ui && yarn run lint:hbs
|
||||
- run:
|
||||
name: Ember tests
|
||||
command: cd ui && yarn test
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
executor: go-machine-recent
|
||||
steps:
|
||||
- checkout
|
||||
- run: make test-website
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
jobs:
|
||||
- build-binaries
|
||||
- lint-go:
|
||||
filters: &backend_branches_filter
|
||||
branches:
|
||||
ignore:
|
||||
- /^.-ui\b.*/
|
||||
- stable-website
|
||||
- test-machine:
|
||||
name: "test-client"
|
||||
test_packages: "./client/..."
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
name: "test-nomad"
|
||||
test_packages: "./nomad/..."
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
# API Tests run in a VM rather than container due to the FS tests
|
||||
# requiring `mount` priviliges.
|
||||
name: "test-api"
|
||||
test_packages: "./api/..."
|
||||
filters: *backend_branches_filter
|
||||
- test-container:
|
||||
name: "test-devices"
|
||||
test_packages: "./devices/..."
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
name: "test-other"
|
||||
exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices"
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
name: "test-docker"
|
||||
test_packages: "./drivers/docker"
|
||||
# docker is misbehaving in docker-machine-recent image
|
||||
# and we get unexpected failures
|
||||
# e.g. https://circleci.com/gh/hashicorp/nomad/3854
|
||||
executor: go-machine
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
name: "test-exec"
|
||||
test_packages: "./drivers/exec"
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
name: "test-shared-exec"
|
||||
test_packages: "./drivers/shared/executor"
|
||||
filters: *backend_branches_filter
|
||||
- test-machine:
|
||||
name: "test-32bit"
|
||||
# Currently we only explicitly test fingerprinting on 32bit
|
||||
# architectures.
|
||||
test_packages: "./client/fingerprint"
|
||||
goarch: "386"
|
||||
filters: *backend_branches_filter
|
||||
- test-e2e:
|
||||
filters: *backend_branches_filter
|
||||
- test-ui:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- stable-website
|
||||
- test-website:
|
||||
filters: *backend_branches_filter
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
jobs:
|
||||
- build-website:
|
||||
context: static-sites
|
||||
filters:
|
||||
branches:
|
||||
only: stable-website
|
||||
Loading…
Reference in New Issue