open-consul/.circleci/config.yml

---
version: 2.1

parameters:
  commit:
    type: string
    default: ""
    description: "Commit to run load tests against"
  trigger-load-test:
    type: boolean
    default: false
    description: "Boolean whether to run the load test workflow"

references:
  paths:
    test-results: &TEST_RESULTS_DIR /tmp/test-results
  environment: &ENVIRONMENT
    TEST_RESULTS_DIR: *TEST_RESULTS_DIR
    EMAIL: noreply@hashicorp.com
    GIT_AUTHOR_NAME: circleci-consul
    GIT_COMMITTER_NAME: circleci-consul
    S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2
    BASH_ENV: .circleci/bash_env.sh
    VAULT_BINARY_VERSION: 1.9.4
    GO_VERSION: 1.19.2
  envoy-versions: &supported_envoy_versions
    - &default_envoy_version "1.21.5"
    - "1.22.5"
    - "1.23.2"
    - "1.24.0"
  nomad-versions: &supported_nomad_versions
    - &default_nomad_version "1.3.3"
    - "1.2.10"
    - "1.1.16"
  images:
    # When updating the Go version, remember to also update the versions in the
    # workflows section for go-test-lib jobs.
    go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.19.2
    ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:14-browsers
    ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02
  cache:
    yarn: &YARN_CACHE_KEY consul-ui-v9-{{ checksum "ui/yarn.lock" }}

steps:
  install-gotestsum: &install-gotestsum
    name: install gotestsum
    environment:
      GOTESTSUM_RELEASE: 1.6.4
    command: |
      ARCH=`uname -m`
      if [[ "$ARCH" == "aarch64" ]]; then
         ARCH="arm64"
      else
         ARCH="amd64"
      fi
      url=https://github.com/gotestyourself/gotestsum/releases/download
      curl -sSL "${url}/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_${ARCH}.tar.gz" | \
      sudo tar -xz --overwrite -C /usr/local/bin gotestsum
  get-aws-cli: &get-aws-cli
    run:
      name: download and install AWS CLI
      command: |
        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
        echo -e "${AWS_CLI_GPG_KEY}" | gpg --import
        curl -o awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig
        gpg --verify awscliv2.sig awscliv2.zip
        unzip awscliv2.zip
        sudo ./aws/install
  # This step MUST be at the end of any set of steps due to the 'when' condition
  notify-slack-failure: &notify-slack-failure
    name: notify-slack-failure
    when: on_fail
    command: |
      if [[ $CIRCLE_BRANCH == "main" ]]; then
        CIRCLE_ENDPOINT="https://app.circleci.com/pipelines/github/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}?branch=${CIRCLE_BRANCH}"
        GITHUB_ENDPOINT="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}"
        COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1)
        SHORT_REF=$(git rev-parse --short "${CIRCLE_SHA1}")
        curl -X POST -H 'Content-type: application/json' \
        --data \
        "{ \
        \"attachments\": [ \
            { \
            \"fallback\": \"CircleCI job failed!\", \
            \"text\": \"❌ Failed: \`${CIRCLE_USERNAME}\`'s <${CIRCLE_BUILD_URL}|${CIRCLE_STAGE}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${CIRCLE_BRANCH}\`!\n\n- <${COMMIT_MESSAGE}\", \
            \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
            \"ts\": \"$(date +%s)\", \
            \"color\": \"danger\" \
            } \
        ] \
        }" "${FEED_CONSUL_GH_URL}"
      else
        echo "Not posting slack failure notifications for non-main branch"
      fi
commands:
  assume-role:
    description: "Assume role to an ARN"
    parameters:
      access-key:
        type: env_var_name
        default: AWS_ACCESS_KEY_ID
      secret-key:
        type: env_var_name
        default: AWS_SECRET_ACCESS_KEY
      role-arn:
        type: env_var_name
        default: ROLE_ARN
    steps:
      # Only run the assume-role command for the main repo. The AWS credentials aren't available for forks.
      - run: |
          if [[ "${CIRCLE_BRANCH%%/*}/" != "pull/" ]]; then
            export AWS_ACCESS_KEY_ID="${<< parameters.access-key >>}"
            export AWS_SECRET_ACCESS_KEY="${<< parameters.secret-key >>}"
            export ROLE_ARN="${<< parameters.role-arn >>}"
            # assume role has duration of 15 min (the minimum allowed)
            CREDENTIALS="$(aws sts assume-role --duration-seconds 900 --role-arn ${ROLE_ARN} --role-session-name build-${CIRCLE_SHA1} | jq '.Credentials')"
            echo "export AWS_ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.AccessKeyId')" >> $BASH_ENV
            echo "export AWS_SECRET_ACCESS_KEY=$(echo $CREDENTIALS | jq -r '.SecretAccessKey')" >> $BASH_ENV
            echo "export AWS_SESSION_TOKEN=$(echo $CREDENTIALS | jq -r '.SessionToken')" >> $BASH_ENV
          fi
  run-go-test-full:
    parameters:
      go_test_flags:
        type: string
        default: ""
    steps:
      - attach_workspace:
          at: /home/circleci/go/bin
      - run: go mod download
      - run:
          name: go test
          command: |
            mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile /tmp/coverage
            PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./... | circleci tests split --split-by=timings --timings-type=classname)
            echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
            echo $PACKAGE_NAMES
            # some tests expect this umask, and arm images have a different default
            umask 0022
            << parameters.go_test_flags >>
            gotestsum \
              --format=short-verbose \
              --jsonfile /tmp/jsonfile/go-test-${CIRCLE_NODE_INDEX}.log \
              --debug \
              --rerun-fails=3 \
              --rerun-fails-max-failures=40 \
              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
              --packages="$PACKAGE_NAMES" \
              --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
              -tags="$GOTAGS" -p 2 \
              ${GO_TEST_FLAGS-} \
              -cover -coverprofile=/tmp/coverage/test-coverage-$CIRCLE_NODE_INDEX || true
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: /tmp/jsonfile
      - persist_to_workspace:
          root: /tmp
          paths:
            - coverage
      - run: &rerun-fails-report
          name: "Re-run fails report"
          command: |
            .circleci/scripts/rerun-fails-report.sh /tmp/gotestsum-rerun-fails
      - run: *notify-slack-failure

jobs:
  # lint consul tests
  lint-consul-retry:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry
      - run: *notify-slack-failure

  lint-enums:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./...
      - run: *notify-slack-failure

  lint:
    description: "Run golangci-lint"
    parameters:
      go-arch:
        type: string
        default: ""
    docker:
      - image: *GOLANG_IMAGE
    resource_class: xlarge
    environment:
      GOTAGS: "" # No tags for OSS but there are for enterprise
      GOARCH: "<<parameters.go-arch>>"
    steps:
      - checkout
      - run: go env
      - run:
          name: Install golangci-lint
          command: make lint-tools
      - run: go mod download
      - run:
          name: lint
          command: &lintcmd |
            golangci-lint run --build-tags="$GOTAGS" -v
      - run:
          name: lint api
          working_directory: api
          command: *lintcmd
      - run:
          name: lint sdk
          working_directory: sdk
          command: *lintcmd
      - run: *notify-slack-failure

  check-go-mod:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      - run: go mod tidy
      - run: |
          if [[ -n $(git status -s) ]]; then
            echo "Git directory has changes"
            git status -s
            exit 1
          fi
      - run: *notify-slack-failure

  check-generated-protobuf:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      # tput complains if this isn't set to something.
      TERM: ansi
    steps:
      - checkout
      - run:
          name: Install protobuf
          command: make proto-tools
      - run:
          name: "Protobuf Format"
          command: make proto-format
      - run:
          command: make --always-make proto
      - run: |
          if ! git diff --exit-code; then
            echo "Generated code was not updated correctly"
            exit 1
          fi
      - run:
          name: "Protobuf Lint"
          command: make proto-lint

  check-generated-deep-copy:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      # tput complains if this isn't set to something.
      TERM: ansi
    steps:
      - checkout
      - run:
          name: Install deep-copy
          command: make codegen-tools
      - run:
          command: make --always-make deep-copy
      - run: |
          if ! git diff --exit-code; then
            echo "Generated code was not updated correctly"
            exit 1
          fi
  go-test-arm64:
    machine:
      image: *UBUNTU_CI_IMAGE
    resource_class: arm.large
    parallelism: 4
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
      # GOMAXPROCS defaults to number of cores on underlying hardware, set
      # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
      GOMAXPROCS: 4
    steps:
      - checkout
      - run:
          command: |
            sudo rm -rf /usr/local/go
            wget https://dl.google.com/go/go${GO_VERSION}.linux-arm64.tar.gz
            sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-arm64.tar.gz
      - run: *install-gotestsum
      - run: go mod download
      - run:
          name: make dev
          command: |
            if [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then
              make dev
              mkdir -p /home/circleci/bin
              cp ./bin/consul /home/circleci/bin/consul
            fi
      - run-go-test-full:
          go_test_flags: 'if ! [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi'

  go-test:
    docker:
      - image: *GOLANG_IMAGE
    resource_class: large
    parallelism: 4
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
      # GOMAXPROCS defaults to number of cores on underlying hardware, set
      # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
      GOMAXPROCS: 4
    steps:
      - checkout
      - run-go-test-full

  go-test-race:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
      # GOMAXPROCS defaults to number of cores on underlying hardware, set
      # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
      GOMAXPROCS: 4
      # The medium resource class (default) boxes are 2 vCPUs, 4GB RAM
      # https://circleci.com/docs/2.0/configuration-reference/#docker-executor
      # but we can run a little over that limit.
    steps:
      - checkout
      - run: go mod download
      - run:
          name: go test -race
          command: |
            mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
            pkgs="$(go list ./... | \
              grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | \
              grep -E -v '^github.com/hashicorp/consul/command/')"
            gotestsum \
              --jsonfile /tmp/jsonfile/go-test-race.log \
              --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
              -tags="$GOTAGS" -p 2 \
              -race -gcflags=all=-d=checkptr=0 \
              $pkgs
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: /tmp/jsonfile
      - run: *notify-slack-failure

  # go-test-32bit is to catch problems where 64-bit ints must be 64-bit aligned
  # to use them with sync/atomic. See https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
  # Running tests with GOARCH=386 seems to be the best way to detect this
  # problem. Only runs tests that are -short to limit the time we spend checking
  # for these bugs.
  go-test-32bit:
    docker:
      - image: *GOLANG_IMAGE
    resource_class: large
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
    steps:
      - checkout
      - run: go mod download
      - run:
          name: go test 32-bit
          environment:
            GOARCH: 386
          command: |
            mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
            go env
            PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./...)
            gotestsum \
              --jsonfile /tmp/jsonfile/go-test-32bit.log \
              --rerun-fails=3 \
              --rerun-fails-max-failures=40 \
              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
              --packages="$PACKAGE_NAMES" \
              --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
              -tags="$GOTAGS" -p 2 \
              -short
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - run: *notify-slack-failure

  go-test-lib:
    description: "test a library against a specific Go version"
    parameters:
      go-version:
        type: string
      path:
        type: string
    docker:
      - image: "docker.mirror.hashicorp.services/cimg/go:<<parameters.go-version>>"
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
    steps:
      - checkout
      - attach_workspace:
          at: /home/circleci/go/bin
      - run:
          working_directory: <<parameters.path>>
          command: go mod download
      - run:
          working_directory: <<parameters.path>>
          name: go test
          command: |
            mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
            gotestsum \
              --format=short-verbose \
              --jsonfile /tmp/jsonfile/go-test-<<parameters.path>>.log \
              --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
              -tags="$GOTAGS" -cover -coverprofile=coverage.txt \
              ./...
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: /tmp/jsonfile
      - run: *notify-slack-failure

  # build is a templated job for build-x
  build-distros: &build-distros
    docker:
      - image: *GOLANG_IMAGE
    resource_class: large
    environment: &build-env
      <<: *ENVIRONMENT
    steps:
      - checkout
      - run:
          name: Build
          command: |
            for os in $XC_OS; do
              target="./pkg/bin/${GOOS}_${GOARCH}/"
              GOOS="$os" CGO_ENABLED=0 go build -o "${target}" -ldflags "${GOLDFLAGS}" -tags "${GOTAGS}"
            done
      # save dev build to CircleCI
      - store_artifacts:
          path: ./pkg/bin
      - run: *notify-slack-failure

  # build all 386 architecture supported OS binaries
  build-386:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: "freebsd linux windows"
      GOARCH: "386"

  # build all amd64 architecture supported OS binaries
  build-amd64:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: "darwin freebsd linux solaris windows"
      GOARCH: "amd64"

  # build all arm/arm64 architecture supported OS binaries
  build-arm:
    docker:
      - image: *GOLANG_IMAGE
    resource_class: large
    environment:
      <<: *ENVIRONMENT
      CGO_ENABLED: 1
      GOOS: linux
    steps:
      - checkout
      - run: sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
      - run:
          environment:
            GOARM: 5
            CC: arm-linux-gnueabi-gcc
            GOARCH: arm
          command: go build -o ./pkg/bin/linux_armel/consul  -ldflags="-linkmode=external ${GOLDFLAGS}"
      - run:
          environment:
            GOARM: 6
            CC: arm-linux-gnueabihf-gcc
            GOARCH: arm
          command: go build -o ./pkg/bin/linux_armhf/consul  -ldflags="-linkmode=external ${GOLDFLAGS}"
      - run:
          environment:
            CC: aarch64-linux-gnu-gcc
            GOARCH: arm64
          command: go build -o ./pkg/bin/linux_aarch64/consul  -ldflags="-linkmode=external ${GOLDFLAGS}"
      - store_artifacts:
          path: ./pkg/bin
      - run: *notify-slack-failure

  # create a development build
  dev-build:
    docker:
      - image: *GOLANG_IMAGE
    resource_class: large
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      - attach_workspace: # this normally runs as the first job and has nothing to attach; only used in main branch after rebuilding UI
          at: .
      - run:
          name: Build
          command: |
            make dev
            mkdir -p /home/circleci/go/bin
            cp ./bin/consul /home/circleci/go/bin/consul
      # save dev build to pass to downstream jobs
      - persist_to_workspace:
          root: /home/circleci/go/bin
          paths:
            - consul
      - run: *notify-slack-failure

  # upload development build to s3
  dev-upload-s3:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      - *get-aws-cli
      - assume-role:
          access-key: AWS_ACCESS_KEY_ID_S3_UPLOAD
          secret-key: AWS_SECRET_ACCESS_KEY_S3_UPLOAD
          role-arn: ROLE_ARN_S3_UPLOAD
      # get consul binary
      - attach_workspace:
          at: bin/
      - run:
          name: package binary
          command: zip -j consul.zip bin/consul
      - run:
          name: Upload to s3
          command: |
            if [ -n "${S3_ARTIFACT_PATH}" ]; then
              aws s3 cp \
              --metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \
              "consul.zip" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.zip" --acl public-read
            else
              echo "CircleCI - S3_ARTIFACT_PATH was not set"
              exit 1
            fi
      - run: *notify-slack-failure

  # upload dev docker image
  dev-upload-docker:
    docker:
      - image: *GOLANG_IMAGE # use a circleci image so the attach_workspace step works (has ca-certs installed)
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      # get consul binary
      - attach_workspace:
          at: bin/
      - setup_remote_docker
      - run: make ci.dev-docker
      - run: *notify-slack-failure

  nomad-integration-test: &NOMAD_TESTS
    docker:
      - image: docker.mirror.hashicorp.services/cimg/go:1.19
    parameters:
      nomad-version:
        type: enum
        enum: *supported_nomad_versions
        default: *default_nomad_version
    environment:
      <<: *ENVIRONMENT
      NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /home/circleci/go/src/github.com/hashicorp/nomad
      NOMAD_VERSION: << parameters.nomad-version >>
    steps: &NOMAD_INTEGRATION_TEST_STEPS
      - run: git clone https://github.com/hashicorp/nomad.git --branch v${NOMAD_VERSION} ${NOMAD_WORKING_DIR}

      # get consul binary
      - attach_workspace:
          at: /home/circleci/go/bin

      # make dev build of nomad
      - run:
          command: make pkg/linux_amd64/nomad
          working_directory: *NOMAD_WORKING_DIR

      - run: *install-gotestsum

      # run integration tests
      - run:
          name: go test
          command: |
            mkdir -p $TEST_RESULTS_DIR
            gotestsum \
              --format=short-verbose \
              --junitfile $TEST_RESULTS_DIR/results.xml -- \
              ./command/agent/consul -run TestConsul
          working_directory: *NOMAD_WORKING_DIR

      # store test results for CircleCI
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - run: *notify-slack-failure

  # build frontend yarn cache
  frontend-cache:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout

      # cache yarn deps
      - restore_cache:
          key: *YARN_CACHE_KEY

      - run:
          name: install yarn packages
          command: cd ui && make deps

      - save_cache:
          key: *YARN_CACHE_KEY
          paths:
            - ui/node_modules
            - ui/packages/consul-ui/node_modules
      - run: *notify-slack-failure

  # build ember so frontend tests run faster
  ember-build-oss: &ember-build-oss
    docker:
      - image: *EMBER_IMAGE
    environment:
      JOBS: 2 # limit parallelism for broccoli-babel-transpiler
      CONSUL_NSPACES_ENABLED: 0
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - run: cd ui/packages/consul-ui && make build-ci

      # saves the build to a workspace to be passed to a downstream job
      - persist_to_workspace:
          root: ui
          paths:
            - packages/consul-ui/dist
      - run: *notify-slack-failure

  # build ember so frontend tests run faster
  ember-build-ent:
    <<: *ember-build-oss
    environment:
      JOBS: 2 # limit parallelism for broccoli-babel-transpiler
      CONSUL_NSPACES_ENABLED: 1

  # rebuild UI for packaging
  ember-build-prod:
    docker:
      - image: *EMBER_IMAGE
    environment:
      JOBS: 2 # limit parallelism for broccoli-babel-transpiler
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - run: cd ui && make

      # saves the build to a workspace to be passed to a downstream job
      - persist_to_workspace:
          root: ui
          paths:
            - packages/consul-ui/dist
      - run: *notify-slack-failure

  # commits static assets to git
  publish-static-assets:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - add_ssh_keys: # needs a key to push updated static asset commit back to github
          fingerprints:
            - "fc:55:84:15:0a:1d:c8:e9:06:d0:e8:9c:7b:a9:b7:31"
      - attach_workspace:
          at: .
      - run:
          name: move compiled ui files to agent/uiserver
          command: |
            rm -rf agent/uiserver/dist
            mv packages/consul-ui/dist agent/uiserver
      - run:
          name: commit agent/uiserver/dist/ if there are UI changes
          command: |
            # check if there are any changes in ui/
            # if there are, we commit the ui static asset file
            # HEAD^! is shorthand for HEAD^..HEAD (parent of HEAD and HEAD)
            if ! git diff --quiet --exit-code HEAD^! ui/; then
              git config --local user.email "github-team-consul-core@hashicorp.com"
              git config --local user.name "hc-github-team-consul-core"
              # -B resets the CI branch to main which may diverge history
              # but we will force push anyways.
              git checkout -B ci/main-assetfs-build main
              short_sha=$(git rev-parse --short HEAD)
              git add agent/uiserver/dist/
              git commit -m "auto-updated agent/uiserver/dist/ from commit ${short_sha}"
              git push --force origin ci/main-assetfs-build
            else
              echo "no UI changes so no static assets to publish"
            fi
      - run: *notify-slack-failure

  # run node tests
  node-tests:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui
      - run:
          working_directory: ui/packages/consul-ui
          command: make test-node
      - run: *notify-slack-failure
  # run yarn workspace wide checks/tests
  workspace-tests:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui
      - run:
          working_directory: ui
          command: make test-workspace
      - run: *notify-slack-failure

  # run ember frontend tests
  ember-test-oss:
    docker:
      - image: *EMBER_IMAGE
    environment:
      EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary
      EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam
      CONSUL_NSPACES_ENABLED: 0
    parallelism: 4
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui
      - run:
          working_directory: ui/packages/consul-ui
          command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit
      - store_test_results:
          path: ui/packages/consul-ui/test-results
      - run: *notify-slack-failure

  # run ember frontend tests
  ember-test-ent:
    docker:
      - image: *EMBER_IMAGE
    environment:
      EMBER_TEST_REPORT: test-results/report-ent.xml #outputs test report for CircleCI test summary
      EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam
      CONSUL_NSPACES_ENABLED: 1
    parallelism: 4
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui
      - run:
          working_directory: ui/packages/consul-ui
          command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit
      - store_test_results:
          path: ui/packages/consul-ui/test-results
      - run: *notify-slack-failure

  # run ember frontend unit tests to produce coverage report
  ember-coverage:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui
      - run:
          working_directory: ui/packages/consul-ui
          command: make test-coverage-ci
      - run: *notify-slack-failure

  compatibility-integration-test:
    machine:
      image: *UBUNTU_CI_IMAGE
      docker_layer_caching: true
    parallelism: 1
    steps:
      - checkout
      # Get go binary from workspace
      - attach_workspace:
          at: .
      # Build the consul:local image from the already built binary
      - run:
          command: |
            sudo rm -rf /usr/local/go
            wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
            sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
          environment:
            <<: *ENVIRONMENT
      - run: *install-gotestsum
      - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
      - run:
          name: Compatibility Integration Tests
          command: |
            mkdir -p /tmp/test-results/
            cd ./test/integration/consul-container
            docker run --rm consul:local consul version
            gotestsum \
              --format=short-verbose \
              --debug \
              --rerun-fails=3 \
              --packages="./..." \
              -- \
              -timeout=30m \
              ./... \
              --target-version local \
              --latest-version latest
            ls -lrt
          environment:
            # this is needed because of incompatibility between RYUK container and circleci
            GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml
            GOTESTSUM_FORMAT: standard-verbose
            COMPOSE_INTERACTIVE_NO_CLI: 1
            # tput complains if this isn't set to something.
            TERM: ansi
      - store_artifacts:
          path: ./test/integration/consul-container/test/upgrade/workdir/logs
          destination: container-logs
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - run: *notify-slack-failure

  envoy-integration-test: &ENVOY_TESTS
    machine:
      image: *UBUNTU_CI_IMAGE
    parallelism: 4
    resource_class: medium
    parameters:
      envoy-version:
        type: enum
        enum: *supported_envoy_versions
        default: *default_envoy_version
      xds-target:
        type: enum
        enum: ["server", "client"]
        default: "server"
    environment:
      ENVOY_VERSION: << parameters.envoy-version >>
      XDS_TARGET: << parameters.xds-target >>
      AWS_LAMBDA_REGION: us-west-2
    steps: &ENVOY_INTEGRATION_TEST_STEPS
      - checkout
      - assume-role:
          access-key: AWS_ACCESS_KEY_ID_LAMBDA
          secret-key: AWS_SECRET_ACCESS_KEY_LAMBDA
          role-arn: ROLE_ARN_LAMBDA
      # Get go binary from workspace
      - attach_workspace:
          at: .
      - run: *install-gotestsum
      # Build the consul:local image from the already built binary
      - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
      - run:
          name: Envoy Integration Tests
          command: |
            subtests=$(ls -d test/integration/connect/envoy/*/ | xargs -n 1 basename | circleci tests split)
            echo "Running $(echo $subtests | wc -w) subtests"
            echo "$subtests"
            subtests_pipe_sepr=$(echo "$subtests" | xargs | sed 's/ /|/g')
            mkdir -p /tmp/test-results/
            gotestsum -- -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/($subtests_pipe_sepr)"
          environment:
            GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml
            GOTESTSUM_FORMAT: standard-verbose
            COMPOSE_INTERACTIVE_NO_CLI: 1
            LAMBDA_TESTS_ENABLED: "true"
            # tput complains if this isn't set to something.
            TERM: ansi
      - store_artifacts:
          path: ./test/integration/connect/envoy/workdir/logs
          destination: container-logs
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR
      - run: *notify-slack-failure

  # run integration tests for the connect ca providers
  test-connect-ca-providers:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      - run:
          name: Install vault
          command: |
            wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/${VAULT_BINARY_VERSION}/vault_${VAULT_BINARY_VERSION}_linux_amd64.zip
            sudo unzip -d /usr/local/bin /tmp/vault.zip
            rm -rf /tmp/vault*
      - checkout
      - run: go mod download
      - run:
          name: go test
          command: |
            mkdir -p $TEST_RESULTS_DIR
            make test-connect-ca-providers
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - run: *notify-slack-failure

  # Run load tests against a commit
  load-test:
    docker:
      - image: hashicorp/terraform:latest
    environment:
      AWS_DEFAULT_REGION: us-east-2
      BUCKET: consul-ci-load-tests
      BASH_ENV: /etc/profile
    shell: /bin/sh -leo pipefail
    steps:
      - checkout
      - run: apk add jq curl bash
      - run:
          name: export load-test credentials
          command: |
            echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_LOAD_TEST" >> $BASH_ENV
            echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_LOAD_TEST" >> $BASH_ENV
      - run:
          name: export role arn
          command: |
            echo "export TF_VAR_role_arn=$ROLE_ARN_LOAD_TEST" >> $BASH_ENV
      - run:
          name: setup TF_VARs
          command: |
            # if pipeline.parameters.commit="" it was not triggered/set through the API
            # so we use the latest commit from _this_ branch. This is the case for nightly tests.
            if [ "<< pipeline.parameters.commit >>" = "" ]; then
              LOCAL_COMMIT_SHA=$(git rev-parse HEAD)
            else
              LOCAL_COMMIT_SHA="<< pipeline.parameters.commit >>"
            fi
            echo "export LOCAL_COMMIT_SHA=${LOCAL_COMMIT_SHA}" >> $BASH_ENV
            git checkout ${LOCAL_COMMIT_SHA}
            short_ref=$(git rev-parse --short ${LOCAL_COMMIT_SHA})
            echo "export TF_VAR_ami_owners=$LOAD_TEST_AMI_OWNERS" >> $BASH_ENV
            echo "export TF_VAR_vpc_name=$short_ref" >> $BASH_ENV
            echo "export TF_VAR_cluster_name=$short_ref" >> $BASH_ENV
            echo "export TF_VAR_consul_download_url=https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" >> $BASH_ENV
      - run:
          name: wait for dev build from test-integrations workflow
          command: |
            echo "curl-ing https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip"
            until [ $SECONDS -ge 300 ] && exit 1; do
              curl -o /dev/null --fail --silent "https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" && exit
              echo -n "."
              sleep 2
            done
      - run:
          working_directory: .circleci/terraform/load-test
          name: terraform init
          command: |
            short_ref=$(git rev-parse --short HEAD)
            echo "Testing commit id: $short_ref"
            terraform init \
            -backend-config="bucket=${BUCKET}" \
            -backend-config="key=${LOCAL_COMMIT_SHA}" \
            -backend-config="region=${AWS_DEFAULT_REGION}" \
            -backend-config="role_arn=${ROLE_ARN_LOAD_TEST}"
      - run:
          working_directory: .circleci/terraform/load-test
          name: run terraform apply
          command: |
            terraform apply -auto-approve
      - run:
          working_directory: .circleci/terraform/load-test
          when: always
          name: terraform destroy
          command: |
            for i in $(seq 1 5); do terraform destroy -auto-approve && s=0 && break || s=$? && sleep 20; done; (exit $s)
      - run: *notify-slack-failure

  # The noop job is a used as a very fast job in the verify-ci workflow because every workflow
  # requires at least one job. It does nothing.
  noop:
    docker:
      - image: docker.mirror.hashicorp.services/alpine:latest
    steps:
      - run: "echo ok"

  show-coverage:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - attach_workspace:
          at: /tmp
      - run:
          name: collate coverage output into single file and convert to readable format
          command: |
            mkdir -p /tmp/output
            echo "mode: set" > cov.txt
            for f in /tmp/coverage/*; do tail -n +2 $f >> cov.txt; done
            go tool cover -func=cov.txt -o /tmp/output/coverage_by_function.txt
            cp cov.txt /tmp/output/cov.txt
      - run: sudo apt-get install jq
      - run:
          name: get diff of test coverage from changes
          command: |
            repository="consul"
            file_url_response=$(curl --location --request GET "https://api.github.com/repos/hashicorp/$repository/contents/.github/cover.out?ref=$TEST_COVERAGE_GOLDEN_BRANCH" \
            -u $GH_USER:$GH_TOKEN)
            if [ $(echo $file_url_response | jq length) -eq 0 ]; then
              echo "No coverage report file found"
              exit 1
            else
              file_download_url=$(echo $file_url_response | jq -r '.download_url')
            fi
            echo "Downloading $file_download_url"
            curl "$file_download_url" > main_cov.txt
            echo "mode: set" > diff_cov.txt
            comm -13 <(sort main_cov.txt) <(sort cov.txt) >> diff_cov.txt
            go tool cover -html=diff_cov.txt -o /tmp/output/diff_coverage_heatmap.html
      - store_artifacts:
          path: /tmp/output
      - run:
          name: post results to GitHub PR
          command: |
            repository="consul"
            pr_response=$(curl --location --request GET "https://api.github.com/repos/hashicorp/$repository/pulls?head=hashicorp:$CIRCLE_BRANCH&state=open" \
            -u $GH_USER:$GH_TOKEN)
            if [ $(echo $pr_response | jq length) -eq 0 ]; then
              echo "No PR found to update"
              exit 1
            else
              pr_comment_url=$(echo $pr_response | jq -r ".[]._links.comments.href")
            fi
            
            new_cov_percent=$(tail -1 /tmp/output/coverage_by_function.txt | awk -F' ' '{print $NF}')
            go tool cover -func=main_cov.txt -o main_cov_by_func.txt
            main_go_test_cov_percent=$(tail -1 main_cov_by_func.txt | awk -F' ' '{print $NF}')
            artifacts=$(curl -X GET "https://circleci.com/api/v2/project/github/hashicorp/$repository/$CIRCLE_BUILD_NUM/artifacts" \
            -H "Accept: application/json" \
            -u "$CIRCLE_API_TOKEN:") 
            
            heatmap_url=$(echo $artifacts | jq -r '.items|.[]|select(.path | endswith("heatmap.html"))|.url')
            
            message="All unit tests passed! Your changes bring test coverage over affected code to $new_cov_percent. Total coverage in main is $main_go_test_cov_percent. [See coverage delta for your changes]($heatmap_url)"
            
            curl --location --request POST "$pr_comment_url" \
            -u hc-github-team-consul-core:$HC_GITHUB_TEAM_CONSUL_CORE_GITHUB_TOKEN \
            --header 'Content-Type: application/json' \
            --data-raw '{
             "body": "'"$message"'"
            }'
      - add_ssh_keys:
          fingerprints:
            - "b1:4d:ed:a5:4f:e1:39:28:37:ab:bd:4d:06:e5:24:ca"
      - run:
          name: push latest coverage results to github
          command: |
            cp cov.txt .github/go_test_coverage.txt
            git config user.email "github-team-consul-core@hashicorp.com"
            git config user.name "hc-github-team-consul-core"
            git add .github/go_test_coverage.txt
            git commit -m "[skip ci] update test coverage"
            git push --set-upstream origin $CIRCLE_BRANCH
workflows:
  version: 2
  # verify-ci is a no-op workflow that must run on every PR. It is used in a
  # branch protection rule to detect when CI workflows are not running.
  verify-ci:
    jobs: [noop]

  go-tests:
    unless: << pipeline.parameters.trigger-load-test >>
    jobs:
      - check-go-mod: &filter-ignore-non-go-branches
          filters:
            branches:
              ignore:
                - stable-website
                - /^docs\/.*/
                - /^ui\/.*/
      - check-generated-protobuf: *filter-ignore-non-go-branches
      - check-generated-deep-copy: *filter-ignore-non-go-branches
      - lint-enums: *filter-ignore-non-go-branches
      - lint-consul-retry: *filter-ignore-non-go-branches
      - lint: *filter-ignore-non-go-branches
      - lint:
          name: "lint-32bit"
          go-arch: "386"
          <<: *filter-ignore-non-go-branches
      - test-connect-ca-providers: *filter-ignore-non-go-branches
      - go-test-arm64: *filter-ignore-non-go-branches
      - dev-build: *filter-ignore-non-go-branches
      - go-test:
          requires: [dev-build]
      - go-test-lib:
          name: "go-test-api go1.18"
          path: api
          go-version: "1.18"
          requires: [dev-build]
      - go-test-lib:
          name: "go-test-api go1.19"
          path: api
          go-version: "1.19"
          requires: [ dev-build ]
      - go-test-lib:
          name: "go-test-sdk go1.18"
          path: sdk
          go-version: "1.18"
          <<: *filter-ignore-non-go-branches
      - go-test-lib:
          name: "go-test-sdk go1.19"
          path: sdk
          go-version: "1.19"
          <<: *filter-ignore-non-go-branches
      - go-test-race: *filter-ignore-non-go-branches
      - go-test-32bit: *filter-ignore-non-go-branches
      - show-coverage:
          requires:
            - go-test
          filters:
            branches:
              ignore:
                - main # we don't want tests run in main to overwrite coverage reports from the latest merge
                - /^release\/.*$/ # we don't want to compare coverage on release branches

      - noop
  build-distros:
    unless: << pipeline.parameters.trigger-load-test >>
    jobs:
      - check-go-mod: *filter-ignore-non-go-branches
      - build-386: &require-check-go-mod
          requires:
            - check-go-mod
      - build-amd64: *require-check-go-mod
      - build-arm: *require-check-go-mod
      # every commit on main will have a rebuilt UI
      - frontend-cache:
          filters:
            branches:
              only:
                - main
      - ember-build-prod:
          requires:
            - frontend-cache
      - publish-static-assets:
          requires:
            - ember-build-prod
      - dev-build:
          requires:
            - ember-build-prod
      - dev-upload-s3:
          requires:
            - dev-build
      - dev-upload-docker:
          requires:
            - dev-build
          context: consul-ci
      - noop
  test-integrations:
    unless: << pipeline.parameters.trigger-load-test >>
    jobs:
      - dev-build: *filter-ignore-non-go-branches
      - dev-upload-s3: &dev-upload
          requires:
            - dev-build
          filters:
            branches:
              ignore:
                - /^pull\/.*$/ # only push dev builds from non forks
                - main # all main dev uploads will include a UI rebuild in build-distros
      - dev-upload-docker:
          <<: *dev-upload
          context: consul-ci
      - nomad-integration-test:
          requires:
            - dev-build
          matrix:
            parameters:
              nomad-version: *supported_nomad_versions
      - envoy-integration-test:
          requires:
            - dev-build
          matrix:
            parameters:
              envoy-version: *supported_envoy_versions
              xds-target: ["server", "client"]
      - compatibility-integration-test:
          requires:
            - dev-build
      - noop
  frontend:
    unless: << pipeline.parameters.trigger-load-test >>
    jobs:
      - frontend-cache:
          filters:
            branches:
              only:
                - main
                - /^ui\/.*/
      - workspace-tests:
          requires:
            - frontend-cache
      - node-tests:
          requires:
            - frontend-cache
      - ember-build-oss:
          requires:
            - frontend-cache
      - ember-build-ent:
          requires:
            - frontend-cache
      - ember-test-oss:
          requires:
            - ember-build-oss
      - ember-test-ent:
          requires:
            - ember-build-ent
      # ember-coverage in CI uses the dist/ folder to run tests so it requires
      # either/or ent/oss to be built first
      - ember-coverage:
          requires:
            - ember-build-ent
      - noop

  load-test:
    when: << pipeline.parameters.trigger-load-test >>
    jobs:
      - load-test

  nightly-jobs:
    triggers:
      - schedule:
          cron: "0 4 * * *" # 4AM UTC <> 12AM EST <> 9PM PST should have no impact
          filters:
            branches:
              only:
                - main
    jobs:
      - load-test