open-consul/.circleci/config.yml
John Cowen 1e26c1fe11
ui: App-ify Lock Sessions (#12482)
This commit moves our in-app LockSessions code into an external 'app', which can theoretically be side-loaded but for now it just makes for good isolation/code hygiene.

Functionally, there is kind of one change here, and that is we only show the 'Lock Session' tab if you have permissions to see them. Currently as our UI authorization endpoint needs to be changed slightly to suit our usecase, you will always have permissions to see Lock Sessions as we hardcode the session:read to true (obvs this is a frontend thing, not a backend thing), so it doesn't really change anything from a user perspective.

Also added very bare docs while I was here.

Small note here, ideally we need to add the each individual tab depending on whether an 'app' is enabled or not instead of just permissions, ideally it would be done totally from The Outside rather than a can based conditional on the inside, just something else to be thinking about for the future.
2022-03-14 16:54:49 +00:00

1239 lines
40 KiB
YAML

---
version: 2.1
parameters:
commit:
type: string
default: ""
description: "Commit to run load tests against"
trigger-load-test:
type: boolean
default: false
description: "Boolean whether to run the load test workflow"
references:
images:
# When updating the Go version, remember to also update the versions in the
# workflows section for go-test-lib jobs.
go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.17.5
ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:14-browsers
paths:
test-results: &TEST_RESULTS_DIR /tmp/test-results
cache:
yarn: &YARN_CACHE_KEY consul-ui-v7-{{ checksum "ui/yarn.lock" }}
environment: &ENVIRONMENT
TEST_RESULTS_DIR: *TEST_RESULTS_DIR
EMAIL: noreply@hashicorp.com
GIT_AUTHOR_NAME: circleci-consul
GIT_COMMITTER_NAME: circleci-consul
S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2
BASH_ENV: .circleci/bash_env.sh
VAULT_BINARY_VERSION: 1.2.2
steps:
install-gotestsum: &install-gotestsum
name: install gotestsum
environment:
GOTESTSUM_RELEASE: 1.6.4
command: |
ARCH=`uname -m`
if [[ "$ARCH" == "aarch64" ]]; then
ARCH="arm64"
else
ARCH="amd64"
fi
url=https://github.com/gotestyourself/gotestsum/releases/download
curl -sSL "${url}/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_${ARCH}.tar.gz" | \
sudo tar -xz --overwrite -C /usr/local/bin gotestsum
get-aws-cli: &get-aws-cli
run:
name: download and install AWS CLI
command: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
echo -e "${AWS_CLI_GPG_KEY}" | gpg --import
curl -o awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig
gpg --verify awscliv2.sig awscliv2.zip
unzip awscliv2.zip
sudo ./aws/install
# This step MUST be at the end of any set of steps due to the 'when' condition
notify-slack-failure: &notify-slack-failure
name: notify-slack-failure
when: on_fail
command: |
if [[ $CIRCLE_BRANCH == "main" ]]; then
CIRCLE_ENDPOINT="https://app.circleci.com/pipelines/github/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}?branch=${CIRCLE_BRANCH}"
GITHUB_ENDPOINT="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}"
COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1)
SHORT_REF=$(git rev-parse --short "${CIRCLE_SHA1}")
curl -X POST -H 'Content-type: application/json' \
--data \
"{ \
\"attachments\": [ \
{ \
\"fallback\": \"CircleCI job failed!\", \
\"text\": \"❌ Failed: \`${CIRCLE_USERNAME}\`'s <${CIRCLE_BUILD_URL}|${CIRCLE_STAGE}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${CIRCLE_BRANCH}\`!\n\n- <${COMMIT_MESSAGE}\", \
\"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
\"ts\": \"$(date +%s)\", \
\"color\": \"danger\" \
} \
] \
}" "${FEED_CONSUL_GH_URL}"
else
echo "Not posting slack failure notifications for non-main branch"
fi
commands:
assume-role:
description: "Assume role to an ARN"
parameters:
access-key:
type: env_var_name
default: AWS_ACCESS_KEY_ID
secret-key:
type: env_var_name
default: AWS_SECRET_ACCESS_KEY
role-arn:
type: env_var_name
default: ROLE_ARN
steps:
- run: |
export AWS_ACCESS_KEY_ID="${<< parameters.access-key >>}"
export AWS_SECRET_ACCESS_KEY="${<< parameters.secret-key >>}"
export ROLE_ARN="${<< parameters.role-arn >>}"
# assume role has duration of 15 min (the minimum allowed)
CREDENTIALS="$(aws sts assume-role --duration-seconds 900 --role-arn ${ROLE_ARN} --role-session-name build-${CIRCLE_SHA1} | jq '.Credentials')"
echo "export AWS_ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.AccessKeyId')" >> $BASH_ENV
echo "export AWS_SECRET_ACCESS_KEY=$(echo $CREDENTIALS | jq -r '.SecretAccessKey')" >> $BASH_ENV
echo "export AWS_SESSION_TOKEN=$(echo $CREDENTIALS | jq -r '.SessionToken')" >> $BASH_ENV
run-go-test-full:
parameters:
go_test_flags:
type: string
default: ""
steps:
- attach_workspace:
at: /home/circleci/go/bin
- run: sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y rsyslog
- run: sudo service rsyslog start
- run: go mod download
- run:
name: go test
command: |
mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./... | circleci tests split --split-by=timings --timings-type=classname)
echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
echo $PACKAGE_NAMES
# some tests expect this umask, and arm images have a different default
umask 0022
<< parameters.go_test_flags >>
gotestsum \
--format=short-verbose \
--jsonfile /tmp/jsonfile/go-test-${CIRCLE_NODE_INDEX}.log \
--debug \
--rerun-fails=3 \
--rerun-fails-max-failures=40 \
--rerun-fails-report=/tmp/gotestsum-rerun-fails \
--packages="$PACKAGE_NAMES" \
--junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
-tags="$GOTAGS" -p 2 \
${GO_TEST_FLAGS-} \
-cover -coverprofile=coverage.txt
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: /tmp/jsonfile
- run: &rerun-fails-report
name: "Re-run fails report"
command: |
.circleci/scripts/rerun-fails-report.sh /tmp/gotestsum-rerun-fails
- run: *notify-slack-failure
jobs:
# lint consul tests
lint-consul-retry:
docker:
- image: *GOLANG_IMAGE
steps:
- checkout
- run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry
- run: *notify-slack-failure
lint:
description: "Run golangci-lint"
parameters:
go-arch:
type: string
default: ""
docker:
- image: *GOLANG_IMAGE
environment:
GOTAGS: "" # No tags for OSS but there are for enterprise
GOARCH: "<<parameters.go-arch>>"
steps:
- checkout
- run: go env
- run:
name: Install golangci-lint
command: |
download=https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh
wget -O- -q $download | sh -x -s -- -d -b /home/circleci/go/bin v1.40.1
- run: go mod download
- run:
name: lint
command: &lintcmd |
golangci-lint run --build-tags="$GOTAGS" -v --concurrency 2
- run:
name: lint api
working_directory: api
command: *lintcmd
- run:
name: lint sdk
working_directory: sdk
command: *lintcmd
- run: *notify-slack-failure
check-go-mod:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- checkout
- run: go mod tidy
- run: |
if [[ -n $(git status -s) ]]; then
echo "Git directory has changes"
git status -s
exit 1
fi
- run: *notify-slack-failure
check-generated-protobuf:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- checkout
- run:
name: Install protobuf
command: |
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protoc-3.12.3-linux-x86_64.zip
sudo unzip -d /usr/local protoc-*.zip
sudo chmod +x /usr/local/bin/protoc
rm protoc-*.zip
- run:
name: Install gogo/protobuf
command: |
gogo_version=$(go list -m github.com/gogo/protobuf | awk '{print $2}')
go install -v github.com/hashicorp/protoc-gen-go-binary@master
go install -v github.com/gogo/protobuf/protoc-gen-gofast@${gogo_version}
- run:
command: make --always-make proto
- run: |
if ! git diff --exit-code; then
echo "Generated code was not updated correctly"
exit 1
fi
go-test-arm64:
machine:
image: ubuntu-2004:202101-01
resource_class: arm.medium
parallelism: 4
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
# GOMAXPROCS defaults to number of cores on underlying hardware, set
# explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
GOMAXPROCS: 4
steps:
- checkout
- run:
command: |
sudo rm -rf /usr/local/go
wget https://golang.org/dl/go1.17.5.linux-arm64.tar.gz
sudo tar -C /usr/local -xzvf go1.17.5.linux-arm64.tar.gz
- run: *install-gotestsum
- run: go mod download
- run:
name: make dev
command: |
if [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then
make dev
mkdir -p /home/circleci/bin
cp ./bin/consul /home/circleci/bin/consul
fi
- run-go-test-full:
go_test_flags: 'if ! [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi'
go-test:
docker:
- image: *GOLANG_IMAGE
parallelism: 4
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
# GOMAXPROCS defaults to number of cores on underlying hardware, set
# explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
GOMAXPROCS: 4
steps:
- checkout
- run-go-test-full
go-test-race:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
# GOMAXPROCS defaults to number of cores on underlying hardware, set
# explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
GOMAXPROCS: 4
# The medium resource class (default) boxes are 2 vCPUs, 4GB RAM
# https://circleci.com/docs/2.0/configuration-reference/#docker-executor
# but we can run a little over that limit.
steps:
- checkout
- run: go mod download
- run:
name: go test -race
command: |
mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
pkgs="$(go list ./... | \
grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | \
grep -E -v '^github.com/hashicorp/consul/command/')"
gotestsum \
--jsonfile /tmp/jsonfile/go-test-race.log \
--junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
-tags="$GOTAGS" -p 2 \
-race -gcflags=all=-d=checkptr=0 \
$pkgs
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: /tmp/jsonfile
- run: *notify-slack-failure
# go-test-32bit is to catch problems where 64-bit ints must be 64-bit aligned
# to use them with sync/atomic. See https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
# Running tests with GOARCH=386 seems to be the best way to detect this
# problem. Only runs tests that are -short to limit the time we spend checking
# for these bugs.
go-test-32bit:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
steps:
- checkout
- run: go mod download
- run:
name: go test 32-bit
environment:
GOARCH: 386
command: |
mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
go env
PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./...)
gotestsum \
--jsonfile /tmp/jsonfile/go-test-32bit.log \
--rerun-fails=3 \
--rerun-fails-max-failures=40 \
--rerun-fails-report=/tmp/gotestsum-rerun-fails \
--packages="$PACKAGE_NAMES" \
--junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
-tags="$GOTAGS" -p 2 \
-short
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
- run: *notify-slack-failure
go-test-lib:
description: "test a library against a specific Go version"
parameters:
go-version:
type: string
path:
type: string
docker:
- image: "docker.mirror.hashicorp.services/cimg/go:<<parameters.go-version>>"
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
steps:
- checkout
- attach_workspace:
at: /home/circleci/go/bin
- run:
working_directory: <<parameters.path>>
command: go mod download
- run:
working_directory: <<parameters.path>>
name: go test
command: |
mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
gotestsum \
--format=short-verbose \
--jsonfile /tmp/jsonfile/go-test-<<parameters.path>>.log \
--junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
-tags="$GOTAGS" -cover -coverprofile=coverage.txt \
./...
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: /tmp/jsonfile
- run: *notify-slack-failure
# build is a templated job for build-x
build-distros: &build-distros
docker:
- image: *GOLANG_IMAGE
environment: &build-env
<<: *ENVIRONMENT
steps:
- checkout
- run:
name: Build
command: |
for os in $XC_OS; do
target="./pkg/bin/${GOOS}_${GOARCH}/"
GOOS="$os" CGO_ENABLED=0 go build -o "$target" -ldflags "$(GOLDFLAGS)" -tags "$(GOTAGS)"
done
# save dev build to CircleCI
- store_artifacts:
path: ./pkg/bin
- run: *notify-slack-failure
# build all 386 architecture supported OS binaries
build-386:
<<: *build-distros
environment:
<<: *build-env
XC_OS: "freebsd linux windows"
GOARCH: "386"
# build all amd64 architecture supported OS binaries
build-amd64:
<<: *build-distros
environment:
<<: *build-env
XC_OS: "darwin freebsd linux solaris windows"
GOARCH: "amd64"
# build all arm/arm64 architecture supported OS binaries
build-arm:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
CGO_ENABLED: 1
GOOS: linux
steps:
- checkout
- run: sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
- run:
environment:
GOARM: 5
CC: arm-linux-gnueabi-gcc
GOARCH: arm
command: go build -o ./pkg/bin/linux_armel/consul -ldflags="-linkmode=external ${GOLDFLAGS}"
- run:
environment:
GOARM: 6
CC: arm-linux-gnueabihf-gcc
GOARCH: arm
command: go build -o ./pkg/bin/linux_armhf/consul -ldflags="-linkmode=external ${GOLDFLAGS}"
- run:
environment:
CC: aarch64-linux-gnu-gcc
GOARCH: arm64
command: go build -o ./pkg/bin/linux_aarch64/consul -ldflags="-linkmode=external ${GOLDFLAGS}"
- store_artifacts:
path: ./pkg/bin
- run: *notify-slack-failure
# create a development build
dev-build:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- checkout
- attach_workspace: # this normally runs as the first job and has nothing to attach; only used in main branch after rebuilding UI
at: .
- run:
name: Build
command: |
make dev
mkdir -p /home/circleci/go/bin
cp ./bin/consul /home/circleci/go/bin/consul
# save dev build to pass to downstream jobs
- persist_to_workspace:
root: /home/circleci/go/bin
paths:
- consul
- run: *notify-slack-failure
# upload development build to s3
dev-upload-s3:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- checkout
- *get-aws-cli
- assume-role:
access-key: AWS_ACCESS_KEY_ID_S3_UPLOAD
secret-key: AWS_SECRET_ACCESS_KEY_S3_UPLOAD
role-arn: ROLE_ARN_S3_UPLOAD
# get consul binary
- attach_workspace:
at: bin/
- run:
name: package binary
command: zip -j consul.zip bin/consul
- run:
name: Upload to s3
command: |
if [ -n "${S3_ARTIFACT_PATH}" ]; then
aws s3 cp \
--metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \
"consul.zip" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.zip" --acl public-read
else
echo "CircleCI - S3_ARTIFACT_PATH was not set"
exit 1
fi
- run: *notify-slack-failure
# upload dev docker image
dev-upload-docker:
docker:
- image: *GOLANG_IMAGE # use a circleci image so the attach_workspace step works (has ca-certs installed)
environment:
<<: *ENVIRONMENT
steps:
- checkout
# get consul binary
- attach_workspace:
at: bin/
- setup_remote_docker
- run: make ci.dev-docker
- run: *notify-slack-failure
# Nomad 0.8 builds on go1.10
# Run integration tests on nomad/v0.8.7
nomad-integration-0_8:
docker:
- image: docker.mirror.hashicorp.services/circleci/golang:1.10
environment:
<<: *ENVIRONMENT
NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /go/src/github.com/hashicorp/nomad
NOMAD_VERSION: v0.8.7
steps: &NOMAD_INTEGRATION_TEST_STEPS
- run: git clone https://github.com/hashicorp/nomad.git --branch ${NOMAD_VERSION} ${NOMAD_WORKING_DIR}
# get consul binary
- attach_workspace:
at: /go/bin
# make dev build of nomad
- run:
command: make pkg/linux_amd64/nomad
working_directory: *NOMAD_WORKING_DIR
- run: *install-gotestsum
# run integration tests
- run:
name: go test
command: |
mkdir -p $TEST_RESULTS_DIR
gotestsum \
--format=short-verbose \
--junitfile $TEST_RESULTS_DIR/results.xml -- \
./command/agent/consul -run TestConsul
working_directory: *NOMAD_WORKING_DIR
# store test results for CircleCI
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
- run: *notify-slack-failure
# run integration tests on nomad/main
nomad-integration-main:
docker:
- image: docker.mirror.hashicorp.services/circleci/golang:1.17 # TODO: replace with cimg/go (requires steps update)
environment:
<<: *ENVIRONMENT
NOMAD_WORKING_DIR: /go/src/github.com/hashicorp/nomad
NOMAD_VERSION: main
steps: *NOMAD_INTEGRATION_TEST_STEPS
build-website-docker-image:
docker:
- image: docker.mirror.hashicorp.services/circleci/buildpack-deps
shell: /usr/bin/env bash -euo pipefail -c
steps:
- checkout
- setup_remote_docker
- run:
name: Build Docker Image if Necessary
command: |
# Ignore job if running an enterprise build
IMAGE_TAG=$(cat website/Dockerfile website/package-lock.json | sha256sum | awk '{print $1;}')
echo "Using $IMAGE_TAG"
if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/consul.git" ]; then
echo "Not Consul OSS Repo, not building website docker image"
elif curl https://hub.docker.com/v2/repositories/hashicorp/consul-website/tags/$IMAGE_TAG -fsL > /dev/null; then
echo "Dependencies have not changed, not building a new website docker image."
else
cd website/
docker build -t hashicorp/consul-website:$IMAGE_TAG .
docker tag hashicorp/consul-website:$IMAGE_TAG hashicorp/consul-website:latest
docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS
docker push hashicorp/consul-website
fi
- run: *notify-slack-failure
algolia-index:
docker:
- image: docker.mirror.hashicorp.services/node:14
steps:
- checkout
- run:
name: Push content to Algolia Index
command: |
if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/consul.git" ]; then
echo "Not Consul OSS Repo, not indexing Algolia"
exit 0
fi
cd website/
npm install -g npm@latest
npm install
node scripts/index_search_content.js
- run: *notify-slack-failure
# build frontend yarn cache
frontend-cache:
docker:
- image: *EMBER_IMAGE
steps:
- checkout
# cache yarn deps
- restore_cache:
key: *YARN_CACHE_KEY
- run:
name: install yarn packages
command: cd ui && make deps
- save_cache:
key: *YARN_CACHE_KEY
paths:
- ui/node_modules
- ui/packages/consul-ui/node_modules
- run: *notify-slack-failure
# build ember so frontend tests run faster
ember-build-oss: &ember-build-oss
docker:
- image: *EMBER_IMAGE
environment:
JOBS: 2 # limit parallelism for broccoli-babel-transpiler
CONSUL_NSPACES_ENABLED: 0
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- run: cd ui/packages/consul-ui && make build-ci
# saves the build to a workspace to be passed to a downstream job
- persist_to_workspace:
root: ui
paths:
- packages/consul-ui/dist
- run: *notify-slack-failure
# build ember so frontend tests run faster
ember-build-ent:
<<: *ember-build-oss
environment:
JOBS: 2 # limit parallelism for broccoli-babel-transpiler
CONSUL_NSPACES_ENABLED: 1
# rebuild UI for packaging
ember-build-prod:
docker:
- image: *EMBER_IMAGE
environment:
JOBS: 2 # limit parallelism for broccoli-babel-transpiler
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- run: cd ui && make
# saves the build to a workspace to be passed to a downstream job
- persist_to_workspace:
root: ui
paths:
- packages/consul-ui/dist
- run: *notify-slack-failure
# build static-assets file
build-static-assets:
docker:
- image: *GOLANG_IMAGE
steps:
- checkout
- attach_workspace:
at: ./pkg
- run: mv pkg/packages/consul-ui/dist pkg/web_ui # 'make static-assets' looks for the 'pkg/web_ui' path
- run: make tools
- run: make static-assets
- persist_to_workspace:
root: .
paths:
- ./agent/uiserver/bindata_assetfs.go
- run: *notify-slack-failure
# commits static assets to git
publish-static-assets:
docker:
- image: *GOLANG_IMAGE
steps:
- checkout
- add_ssh_keys: # needs a key to push updated static asset commit back to github
fingerprints:
- "fc:55:84:15:0a:1d:c8:e9:06:d0:e8:9c:7b:a9:b7:31"
- attach_workspace:
at: .
- run:
name: commit agent/uiserver/bindata_assetfs.go if there are UI changes
command: |
# check if there are any changes in ui/
# if there are, we commit the ui static asset file
# HEAD^! is shorthand for HEAD^..HEAD (parent of HEAD and HEAD)
if ! git diff --quiet --exit-code HEAD^! ui/; then
git config --local user.email "github-team-consul-core@hashicorp.com"
git config --local user.name "hc-github-team-consul-core"
# -B resets the CI branch to main which may diverge history
# but we will force push anyways.
git checkout -B ci/main-assetfs-build main
short_sha=$(git rev-parse --short HEAD)
git add agent/uiserver/bindata_assetfs.go
git commit -m "auto-updated agent/uiserver/bindata_assetfs.go from commit ${short_sha}"
git push --force origin ci/main-assetfs-build
else
echo "no UI changes so no static assets to publish"
fi
- run: *notify-slack-failure
# run node tests
node-tests:
docker:
- image: *EMBER_IMAGE
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- attach_workspace:
at: ui
- run:
working_directory: ui/packages/consul-ui
command: make test-node
- run: *notify-slack-failure
# run yarn workspace wide checks/tests
workspace-tests:
docker:
- image: *EMBER_IMAGE
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- attach_workspace:
at: ui
- run:
working_directory: ui
command: make test-workspace
- run: *notify-slack-failure
# run ember frontend tests
ember-test-oss:
docker:
- image: *EMBER_IMAGE
environment:
EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary
EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam
CONSUL_NSPACES_ENABLED: 0
parallelism: 4
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- attach_workspace:
at: ui
- run:
working_directory: ui/packages/consul-ui
command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit
- store_test_results:
path: ui/packages/consul-ui/test-results
- run: *notify-slack-failure
# run ember frontend tests
ember-test-ent:
docker:
- image: *EMBER_IMAGE
environment:
EMBER_TEST_REPORT: test-results/report-ent.xml #outputs test report for CircleCI test summary
EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam
CONSUL_NSPACES_ENABLED: 1
parallelism: 4
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- attach_workspace:
at: ui
- run:
working_directory: ui/packages/consul-ui
command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit
- store_test_results:
path: ui/packages/consul-ui/test-results
- run: *notify-slack-failure
# run ember frontend unit tests to produce coverage report
ember-coverage:
docker:
- image: *EMBER_IMAGE
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- attach_workspace:
at: ui
- run:
working_directory: ui/packages/consul-ui
command: make test-coverage-ci
- run: *notify-slack-failure
envoy-integration-test-1_17_4: &ENVOY_TESTS
docker:
# We only really need bash and docker-compose which is installed on all
# Circle images but pick Go since we have to pick one of them.
- image: *GOLANG_IMAGE
parallelism: 2
environment:
ENVOY_VERSION: "1.17.4"
steps: &ENVOY_INTEGRATION_TEST_STEPS
- checkout
# Get go binary from workspace
- attach_workspace:
at: .
- setup_remote_docker
# Build the consul-dev image from the already built binary
- run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile .
- run:
name: Envoy Integration Tests
command: |
subtests=$(ls -d test/integration/connect/envoy/*/ | xargs -n 1 basename | circleci tests split)
echo "Running $(echo $subtests | wc -w) subtests"
echo "$subtests"
subtests_pipe_sepr=$(echo "$subtests" | xargs | sed 's/ /|/g')
mkdir -p /tmp/test-results/
gotestsum -- -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/($subtests_pipe_sepr)"
environment:
GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
# tput complains if this isn't set to something.
TERM: ansi
- store_artifacts:
path: ./test/integration/connect/envoy/workdir/logs
destination: container-logs
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
- run: *notify-slack-failure
envoy-integration-test-1_18_6:
<<: *ENVOY_TESTS
environment:
ENVOY_VERSION: "1.18.6"
envoy-integration-test-1_19_3:
<<: *ENVOY_TESTS
environment:
ENVOY_VERSION: "1.19.3"
envoy-integration-test-1_20_2:
<<: *ENVOY_TESTS
environment:
ENVOY_VERSION: "1.20.2"
# run integration tests for the connect ca providers
test-connect-ca-providers:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- run:
name: Install vault
command: |
wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/${VAULT_BINARY_VERSION}/vault_${VAULT_BINARY_VERSION}_linux_amd64.zip
sudo unzip -d /usr/local/bin /tmp/vault.zip
rm -rf /tmp/vault*
- checkout
- run: go mod download
- run:
name: go test
command: |
mkdir -p $TEST_RESULTS_DIR
make test-connect-ca-providers
- store_test_results:
path: *TEST_RESULTS_DIR
- run: *notify-slack-failure
# only runs on main: checks latest commit to see if the PR associated has a backport/* or docs* label to cherry-pick
cherry-picker:
docker:
- image: docker.mirror.hashicorp.services/alpine:3.12
steps:
- run: apk add --no-cache --no-progress git bash curl ncurses jq openssh-client
- checkout
- add_ssh_keys: # needs a key to push cherry-picked commits back to github
fingerprints:
- "fc:55:84:15:0a:1d:c8:e9:06:d0:e8:9c:7b:a9:b7:31"
- run: .circleci/scripts/cherry-picker.sh
- run: *notify-slack-failure
trigger-oss-merge:
docker:
- image: docker.mirror.hashicorp.services/alpine:3.12
steps:
- run: apk add --no-cache --no-progress curl jq
- run:
name: trigger oss merge
command: |
curl -s -X POST \
--header "Circle-Token: ${CIRCLECI_API_TOKEN}" \
--header "Content-Type: application/json" \
-d '{"build_parameters": {"CIRCLE_JOB": "oss-merge"}}' \
"https://circleci.com/api/v1.1/project/github/hashicorp/consul-enterprise/tree/${CIRCLE_BRANCH}" | jq -r '.build_url'
- run: *notify-slack-failure
# Run load tests against a commit
load-test:
docker:
- image: hashicorp/terraform:latest
environment:
AWS_DEFAULT_REGION: us-east-2
BUCKET: consul-ci-load-tests
BASH_ENV: /etc/profile
shell: /bin/sh -leo pipefail
steps:
- checkout
- run: apk add jq curl bash
- run:
name: export load-test credentials
command: |
echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_LOAD_TEST" >> $BASH_ENV
echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_LOAD_TEST" >> $BASH_ENV
- run:
name: export role arn
command: |
echo "export TF_VAR_role_arn=$ROLE_ARN_LOAD_TEST" >> $BASH_ENV
- run:
name: setup TF_VARs
command: |
# if pipeline.parameters.commit="" it was not triggered/set through the API
# so we use the latest commit from _this_ branch. This is the case for nightly tests.
if [ "<< pipeline.parameters.commit >>" = "" ]; then
LOCAL_COMMIT_SHA=$(git rev-parse HEAD)
else
LOCAL_COMMIT_SHA="<< pipeline.parameters.commit >>"
fi
echo "export LOCAL_COMMIT_SHA=${LOCAL_COMMIT_SHA}" >> $BASH_ENV
short_ref=$(git rev-parse --short ${LOCAL_COMMIT_SHA})
echo "export TF_VAR_ami_owners=$LOAD_TEST_AMI_OWNERS" >> $BASH_ENV
echo "export TF_VAR_vpc_name=$short_ref" >> $BASH_ENV
echo "export TF_VAR_cluster_name=$short_ref" >> $BASH_ENV
echo "export TF_VAR_consul_download_url=https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" >> $BASH_ENV
- run:
name: wait for dev build from test-integrations workflow
command: |
echo "curl-ing https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip"
until [ $SECONDS -ge 300 ] && exit 1; do
curl -o /dev/null --fail --silent "https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" && exit
echo -n "."
sleep 2
done
- run:
working_directory: .circleci/terraform/load-test
name: terraform init
command: |
echo "commit is ${LOCAL_COMMIT_SHA}"
terraform init \
-backend-config="bucket=${BUCKET}" \
-backend-config="key=${LOCAL_COMMIT_SHA}" \
-backend-config="region=${AWS_DEFAULT_REGION}" \
-backend-config="role_arn=${ROLE_ARN_LOAD_TEST}"
- run:
working_directory: .circleci/terraform/load-test
name: run terraform apply
command: |
terraform apply -auto-approve
- run:
working_directory: .circleci/terraform/load-test
when: always
name: terraform destroy
command: |
terraform destroy -auto-approve
- run: *notify-slack-failure
# The noop job is a used as a very fast job in the verify-ci workflow because every workflow
# requires at least one job. It does nothing.
noop:
docker:
- image: docker.mirror.hashicorp.services/alpine:latest
steps:
- run: "echo ok"
workflows:
version: 2
# verify-ci is a no-op workflow that must run on every PR. It is used in a
# branch protection rule to detect when CI workflows are not running.
verify-ci:
jobs: [ noop ]
go-tests:
unless: << pipeline.parameters.trigger-load-test >>
jobs:
- check-go-mod: &filter-ignore-non-go-branches
filters:
branches:
ignore:
- stable-website
- /^docs\/.*/
- /^ui\/.*/
- check-generated-protobuf: *filter-ignore-non-go-branches
- lint-consul-retry: *filter-ignore-non-go-branches
- lint: *filter-ignore-non-go-branches
- lint:
name: "lint-32bit"
go-arch: "386"
<<: *filter-ignore-non-go-branches
- test-connect-ca-providers: *filter-ignore-non-go-branches
- go-test-arm64: *filter-ignore-non-go-branches
- dev-build: *filter-ignore-non-go-branches
- go-test:
requires: [ dev-build ]
- go-test-lib:
name: "go-test-api go1.16"
path: api
go-version: "1.16"
requires: [ dev-build ]
- go-test-lib:
name: "go-test-api go1.17"
path: api
go-version: "1.17"
requires: [ dev-build ]
- go-test-lib:
name: "go-test-sdk go1.16"
path: sdk
go-version: "1.16"
<<: *filter-ignore-non-go-branches
- go-test-lib:
name: "go-test-sdk go1.17"
path: sdk
go-version: "1.17"
<<: *filter-ignore-non-go-branches
- go-test-race: *filter-ignore-non-go-branches
- go-test-32bit: *filter-ignore-non-go-branches
build-distros:
unless: << pipeline.parameters.trigger-load-test >>
jobs:
- check-go-mod: *filter-ignore-non-go-branches
- build-386: &require-check-go-mod
requires:
- check-go-mod
- build-amd64: *require-check-go-mod
- build-arm: *require-check-go-mod
# every commit on main will have a rebuilt UI
- frontend-cache:
filters:
branches:
only:
- main
- ember-build-prod:
requires:
- frontend-cache
- build-static-assets:
requires:
- ember-build-prod
- publish-static-assets:
filters:
branches:
only:
- main
- /release\/\d+\.\d+\.x$/
requires:
- build-static-assets
- dev-build:
requires:
- build-static-assets
- dev-upload-s3:
requires:
- dev-build
- dev-upload-docker:
requires:
- dev-build
context: consul-ci
test-integrations:
unless: << pipeline.parameters.trigger-load-test >>
jobs:
- dev-build: *filter-ignore-non-go-branches
- dev-upload-s3: &dev-upload
requires:
- dev-build
filters:
branches:
ignore:
- /^pull\/.*$/ # only push dev builds from non forks
- main # all main dev uploads will include a UI rebuild in build-distros
- dev-upload-docker:
<<: *dev-upload
context: consul-ci
- nomad-integration-main:
requires:
- dev-build
- nomad-integration-0_8:
requires:
- dev-build
- envoy-integration-test-1_17_4:
requires:
- dev-build
- envoy-integration-test-1_18_6:
requires:
- dev-build
- envoy-integration-test-1_19_3:
requires:
- dev-build
- envoy-integration-test-1_20_2:
requires:
- dev-build
website:
unless: << pipeline.parameters.trigger-load-test >>
jobs:
- build-website-docker-image:
context: website-docker-image
filters:
branches:
only:
- main
- algolia-index:
filters:
branches:
only:
- stable-website
frontend:
unless: << pipeline.parameters.trigger-load-test >>
jobs:
- frontend-cache:
filters:
branches:
only:
- main
- /^ui\/.*/
- workspace-tests:
requires:
- frontend-cache
- node-tests:
requires:
- frontend-cache
- ember-build-oss:
requires:
- frontend-cache
- ember-build-ent:
requires:
- frontend-cache
- ember-test-oss:
requires:
- ember-build-oss
- ember-test-ent:
requires:
- ember-build-ent
# ember-coverage in CI uses the dist/ folder to run tests so it requires
# either/or ent/oss to be built first
- ember-coverage:
requires:
- ember-build-ent
workflow-automation:
unless: << pipeline.parameters.trigger-load-test >>
jobs:
- trigger-oss-merge:
context: team-consul
filters:
branches:
only:
- main
- /release\/\d+\.\d+\.x$/
- cherry-picker:
context: team-consul
filters:
branches:
only:
- main
- /release\/\d+\.\d+\.x$/
load-test:
when: << pipeline.parameters.trigger-load-test >>
jobs:
- load-test
nightly-jobs:
triggers:
- schedule:
cron: "0 4 * * *" # 4AM UTC <> 12AM EST <> 9PM PST should have no impact
filters:
branches:
only:
- main
jobs:
- load-test