open-vault/.circleci/config.yml

### ***
### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'.
### INSTEAD: Edit or merge the source in config/ then run 'make ci-config'.
### ***
# Orb 'circleci/slack@3.2.0' resolved to 'circleci/slack@3.2.0'
version: 2
jobs:
  install-ui-dependencies:
    docker:
    - image: docker.mirror.hashicorp.services/node:14-buster
    shell: /usr/bin/env bash -euo pipefail -c
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    steps:
    - checkout
    - restore_cache:
        key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }}
        name: Restore yarn cache
    - run:
        command: |
          cd ui
          yarn install
          npm rebuild node-sass
        name: Install UI dependencies
    - save_cache:
        key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }}
        name: Save yarn cache
        paths:
        - ui/node_modules
  test-ui:
    docker:
    - image: docker.mirror.hashicorp.services/node:14-buster
    shell: /usr/bin/env bash -euo pipefail -c
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    resource_class: xlarge
    steps:
    - run:
        command: |
          case "$CIRCLE_BRANCH" in
          main|ui/*|backport/ui/*|release/*|merge*) ;;
          *) # If the branch being tested doesn't match one of the above patterns,
             # we don't need to run test-ui and can abort the job.
             circleci-agent step halt
             ;;
          esac

          # exit with success either way
          exit 0
        name: Check branch name
        working_directory: ~/
    - checkout
    - restore_cache:
        key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }}
        name: Restore yarn cache
    - attach_workspace:
        at: .
    - run:
        command: |
          set -x

          # Install Chrome
          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub \
            | apt-key add -
          echo "deb http://dl.google.com/linux/chrome/deb/ stable main" \
            | tee /etc/apt/sources.list.d/google-chrome.list
          apt-get update
          apt-get -y install google-chrome-stable
          rm /etc/apt/sources.list.d/google-chrome.list
          rm -rf /var/lib/apt/lists/* /var/cache/apt/*

          # Add ./bin to the PATH so vault binary can be run by Ember tests
          export PATH="${PWD}/bin:${PATH}"

          # Run Ember tests
          cd ui
          mkdir -p test-results/qunit
          yarn test:oss
        name: Test UI
    - store_artifacts:
        path: ui/test-results
    - store_test_results:
        path: ui/test-results
  test-ui-browserstack:
    docker:
    - image: docker.mirror.hashicorp.services/node:14-buster
    shell: /usr/bin/env bash -euo pipefail -c
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    resource_class: xlarge
    steps:
    - checkout
    - restore_cache:
        key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }}
        name: Restore yarn cache
    - attach_workspace:
        at: .
    - run:
        command: |
          # Add ./bin to the PATH so vault binary can be found.
          export PATH="${PWD}"/bin:${PATH}
          make test-ui-browserstack
        name: Run Browserstack Tests
  build-go-dev:
    machine:
      image: ubuntu-2004:202201-02
    shell: /usr/bin/env bash -euo pipefail -c
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    steps:
    - run:
        command: |
          [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; }
          # Install Go
          curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz"
          sudo rm -rf /usr/local/go
          sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
          rm -f "go${GO_VERSION}.linux-amd64.tar.gz"
          GOPATH="/home/circleci/go"
          mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
          mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; }
          echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
          echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
          echo "export GOPROXY=off" >> "$BASH_ENV"
          echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV"

          echo "$ go version"
          go version
        name: Setup Go
        working_directory: ~/
    - checkout
    - restore_cache:
        keys:
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        name: Restore exact go modules cache
    - attach_workspace:
        at: .
    - run:
        command: |
          # Move dev UI assets to expected location
          rm -rf ./pkg
          mkdir ./pkg

          # Build dev binary
          make ci-bootstrap dev
        name: Build dev binary
    - persist_to_workspace:
        paths:
        - bin
        root: .
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
    - GO_VERSION: 1.17.11
    - GOFUMPT_VERSION: 0.2.1
    - GOTESTSUM_VERSION: 0.5.2
  test-go-remote-docker:
    docker:
    - image: docker.mirror.hashicorp.services/cimg/go:1.17.11
    resource_class: medium
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    parallelism: 8
    steps:
    - run:
        command: |
          # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing
          [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && {
            # stop the job from this step
            circleci-agent step halt
          }
          # exit with success either way
          exit 0
        name: Check branch name
        working_directory: ~/
    - checkout
    - setup_remote_docker:
        docker_layer_caching: true
        version: 18.09.3
    - add_ssh_keys:
        fingerprints:
        - 0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06
    - run:
        command: |
          git config --global url."git@github.com:".insteadOf https://github.com/
    - run:
        command: |
          TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key
        name: Compute test cache key
    - restore_cache:
        keys:
        - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
    - restore_cache:
        keys:
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        name: Restore exact go modules cache
    - run:
        command: |
          set -exo pipefail

          EXTRA_TAGS=
          case "" in
          *-race*) export VAULT_CI_GO_TEST_RACE=1;;
          *)       EXTRA_TAGS=deadlock;;
          esac

          # Install CircleCI CLI
          curl -sSL \
              "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \
            | sudo tar --overwrite -xz \
              -C /usr/local/bin \
              "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci"

          USE_DOCKER=0
          USE_DOCKER=1

          # Check all directories with a go.mod file
          modules=("." "api" "sdk")
          all_package_names=""

          for dir in "${modules[@]}"
          do
            pushd "$dir"
            # On its own line so that -e will fail the tests if we detect errors here.
            go list -test -json ./... > test-list.json
            # Split Go tests by prior test times.  If use_docker is true, only run
            # tests that depend on docker, otherwise only those that don't.
            # The appended true condition ensures the command will succeed if no packages are found
            if [ $USE_DOCKER == 1 ]; then
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            else
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            fi
            # Move back into root directory
            popd
            # Append the test packages into the global list, if any are found
            if [ -n "$package_names" ]; then
              all_package_names+=" ${package_names}"
            fi
          done

          # After running tests split step, we are now running the following steps
          # in multiple different containers, each getting a different subset of
          # the test packages in their package_names variable.  Each container
          # has its own remote docker VM.

          make prep
          mkdir -p test-results/go-test

          # We don't want VAULT_LICENSE set when running Go tests, because that's
          # not what developers have in their environments and it could break some
          # tests; it would be like setting VAULT_TOKEN.  However some non-Go
          # CI commands, like the UI tests, shouldn't have to worry about licensing.
          # So we set VAULT_LICENSE in CI, and here we unset it.  Instead of
          # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want
          # an externally supplied license can opt-in to using it.
          export VAULT_LICENSE_CI="$VAULT_LICENSE"
          VAULT_LICENSE=

          # Create a docker network for our testcontainer
          if [ $USE_DOCKER == 1 ]; then
            # Despite the fact that we're using a circleci image (thus getting the
            # version they chose for the docker cli) and that we're specifying a
            # docker version to use for the remote docker instances, we occasionally
            # see "client version too new, max supported version 1.39" errors for
            # reasons unclear.
            export DOCKER_API_VERSION=1.39

            export TEST_DOCKER_NETWORK_ID=$(docker network list -q -f 'name=vaulttest')
            if [ -z $TEST_DOCKER_NETWORK_ID ]; then
              TEST_DOCKER_NETWORK_ID=$(docker network create vaulttest)
            fi



            # Start a docker testcontainer to run the tests in
            docker run -d \
              -e TEST_DOCKER_NETWORK_ID \
              -e GOPRIVATE \
              -e DOCKER_CERT_PATH \
              -e DOCKER_HOST \
              -e DOCKER_MACHINE_NAME \
              -e DOCKER_TLS_VERIFY \
              -e NO_PROXY \
              -e VAULT_TEST_LOG_DIR=/tmp/testlogs \
              --network vaulttest --name \
              testcontainer docker.mirror.hashicorp.services/cimg/go:1.17.11 \
              tail -f /dev/null

            # Run tests
            test -d /tmp/go-cache && docker cp /tmp/go-cache testcontainer:/tmp/gocache
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault'
            docker cp . testcontainer:/home/circleci/go/src/github.com/hashicorp/vault/
            docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH

            # Copy the downloaded modules inside the container.
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/pkg'
            docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/home/circleci/go/pkg/mod

            docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \
              -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
              -e GOCACHE=/tmp/gocache \
              -e GO_TAGS \
              -e GOPROXY="off" \
              -e VAULT_LICENSE_CI \
              -e GOARCH=amd64 \
              testcontainer \
                gotestsum --format=short-verbose \
                  --junitfile test-results/go-test/results.xml \
                  --jsonfile test-results/go-test/results.json \
                  -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                  -timeout=60m \
                  -parallel=20 \
                   \
                  ${all_package_names}
          else
            GOARCH=amd64 \
              GOCACHE=/tmp/go-cache \
              gotestsum --format=short-verbose \
                --junitfile test-results/go-test/results.xml \
                --jsonfile test-results/go-test/results.json \
                -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                -timeout=60m \
                -parallel=20 \
                 \
                ${all_package_names}
          fi
        environment:
          GOPRIVATE: github.com/hashicorp/*
        name: Run Go tests
        no_output_timeout: 60m
    - run:
        command: |
          docker cp testcontainer:/home/circleci/go/src/github.com/hashicorp/vault/test-results .
          docker cp testcontainer:/tmp/gocache /tmp/go-cache
        name: Copy test results
        when: always
    - store_artifacts:
        path: test-results
    - store_test_results:
        path: test-results
    - store_artifacts:
        path: /tmp/testlogs
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
  fmt:
    machine:
      image: ubuntu-2004:202201-02
    shell: /usr/bin/env bash -euo pipefail -c
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    steps:
    - run:
        command: |
          [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; }
          # Install Go
          curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz"
          sudo rm -rf /usr/local/go
          sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
          rm -f "go${GO_VERSION}.linux-amd64.tar.gz"
          GOPATH="/home/circleci/go"
          mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
          mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; }
          echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
          echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
          echo "export GOPROXY=https://proxy.golang.org,direct" >> "$BASH_ENV"
          echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV"

          echo "$ go version"
          go version
        name: Setup Go
        working_directory: ~/
    - checkout
    - run:
        command: |
          go install "mvdan.cc/gofumpt@v${GOFUMPT_VERSION}"
          make fmt
          if ! git diff --exit-code; then
            echo "Code has formatting errors. Run 'make fmt' to fix"
            exit 1
          fi
        name: make fmt
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
    - GO_VERSION: 1.17.11
    - GOFUMPT_VERSION: 0.2.1
    - GOTESTSUM_VERSION: 0.5.2
  test-go-race:
    docker:
    - image: docker.mirror.hashicorp.services/cimg/go:1.17.11
    resource_class: xlarge
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    parallelism: 8
    steps:
    - run:
        command: |
          # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing
          [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && {
            # stop the job from this step
            circleci-agent step halt
          }
          # exit with success either way
          exit 0
        name: Check branch name
        working_directory: ~/
    - checkout
    - add_ssh_keys:
        fingerprints:
        - 0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06
    - run:
        command: |
          git config --global url."git@github.com:".insteadOf https://github.com/
    - run:
        command: |
          TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key
        name: Compute test cache key
    - restore_cache:
        keys:
        - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
    - restore_cache:
        keys:
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        name: Restore exact go modules cache
    - run:
        command: |
          set -exo pipefail

          EXTRA_TAGS=
          case "-race" in
          *-race*) export VAULT_CI_GO_TEST_RACE=1;;
          *)       EXTRA_TAGS=deadlock;;
          esac

          # Install CircleCI CLI
          curl -sSL \
              "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \
            | sudo tar --overwrite -xz \
              -C /usr/local/bin \
              "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci"

          USE_DOCKER=0

          # Check all directories with a go.mod file
          modules=("." "api" "sdk")
          all_package_names=""

          for dir in "${modules[@]}"
          do
            pushd "$dir"
            # On its own line so that -e will fail the tests if we detect errors here.
            go list -test -json ./... > test-list.json
            # Split Go tests by prior test times.  If use_docker is true, only run
            # tests that depend on docker, otherwise only those that don't.
            # The appended true condition ensures the command will succeed if no packages are found
            if [ $USE_DOCKER == 1 ]; then
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            else
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            fi
            # Move back into root directory
            popd
            # Append the test packages into the global list, if any are found
            if [ -n "$package_names" ]; then
              all_package_names+=" ${package_names}"
            fi
          done

          # After running tests split step, we are now running the following steps
          # in multiple different containers, each getting a different subset of
          # the test packages in their package_names variable.  Each container
          # has its own remote docker VM.

          make prep
          mkdir -p test-results/go-test

          # We don't want VAULT_LICENSE set when running Go tests, because that's
          # not what developers have in their environments and it could break some
          # tests; it would be like setting VAULT_TOKEN.  However some non-Go
          # CI commands, like the UI tests, shouldn't have to worry about licensing.
          # So we set VAULT_LICENSE in CI, and here we unset it.  Instead of
          # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want
          # an externally supplied license can opt-in to using it.
          export VAULT_LICENSE_CI="$VAULT_LICENSE"
          VAULT_LICENSE=

          # Create a docker network for our testcontainer
          if [ $USE_DOCKER == 1 ]; then
            # Despite the fact that we're using a circleci image (thus getting the
            # version they chose for the docker cli) and that we're specifying a
            # docker version to use for the remote docker instances, we occasionally
            # see "client version too new, max supported version 1.39" errors for
            # reasons unclear.
            export DOCKER_API_VERSION=1.39

            export TEST_DOCKER_NETWORK_ID=$(docker network list -q -f 'name=vaulttest')
            if [ -z $TEST_DOCKER_NETWORK_ID ]; then
              TEST_DOCKER_NETWORK_ID=$(docker network create vaulttest)
            fi



            # Start a docker testcontainer to run the tests in
            docker run -d \
              -e TEST_DOCKER_NETWORK_ID \
              -e GOPRIVATE \
              -e DOCKER_CERT_PATH \
              -e DOCKER_HOST \
              -e DOCKER_MACHINE_NAME \
              -e DOCKER_TLS_VERIFY \
              -e NO_PROXY \
              -e VAULT_TEST_LOG_DIR=/tmp/testlogs \
              --network vaulttest --name \
              testcontainer docker.mirror.hashicorp.services/cimg/go:1.17.11 \
              tail -f /dev/null

            # Run tests
            test -d /tmp/go-cache && docker cp /tmp/go-cache testcontainer:/tmp/gocache
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault'
            docker cp . testcontainer:/home/circleci/go/src/github.com/hashicorp/vault/
            docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH

            # Copy the downloaded modules inside the container.
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/pkg'
            docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/home/circleci/go/pkg/mod

            docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \
              -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
              -e GOCACHE=/tmp/gocache \
              -e GO_TAGS \
              -e GOPROXY="off" \
              -e VAULT_LICENSE_CI \
              -e GOARCH=amd64 \
              testcontainer \
                gotestsum --format=short-verbose \
                  --junitfile test-results/go-test/results.xml \
                  --jsonfile test-results/go-test/results.json \
                  -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                  -timeout=60m \
                  -parallel=20 \
                  -race \
                  ${all_package_names}
          else
            GOARCH=amd64 \
              GOCACHE=/tmp/go-cache \
              gotestsum --format=short-verbose \
                --junitfile test-results/go-test/results.xml \
                --jsonfile test-results/go-test/results.json \
                -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                -timeout=60m \
                -parallel=20 \
                -race \
                ${all_package_names}
          fi
        environment:
          GOPRIVATE: github.com/hashicorp/*
        name: Run Go tests
        no_output_timeout: 60m
    - store_artifacts:
        path: test-results
    - store_test_results:
        path: test-results
    - store_artifacts:
        path: /tmp/testlogs
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
  test-go:
    docker:
    - image: docker.mirror.hashicorp.services/cimg/go:1.17.11
    resource_class: large
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    parallelism: 8
    steps:
    - run:
        command: |
          # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing
          [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && {
            # stop the job from this step
            circleci-agent step halt
          }
          # exit with success either way
          exit 0
        name: Check branch name
        working_directory: ~/
    - checkout
    - add_ssh_keys:
        fingerprints:
        - 0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06
    - run:
        command: |
          git config --global url."git@github.com:".insteadOf https://github.com/
    - run:
        command: |
          TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key
        name: Compute test cache key
    - restore_cache:
        keys:
        - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
    - restore_cache:
        keys:
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        name: Restore exact go modules cache
    - run:
        command: |
          set -exo pipefail

          EXTRA_TAGS=
          case "" in
          *-race*) export VAULT_CI_GO_TEST_RACE=1;;
          *)       EXTRA_TAGS=deadlock;;
          esac

          # Install CircleCI CLI
          curl -sSL \
              "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \
            | sudo tar --overwrite -xz \
              -C /usr/local/bin \
              "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci"

          USE_DOCKER=0

          # Check all directories with a go.mod file
          modules=("." "api" "sdk")
          all_package_names=""

          for dir in "${modules[@]}"
          do
            pushd "$dir"
            # On its own line so that -e will fail the tests if we detect errors here.
            go list -test -json ./... > test-list.json
            # Split Go tests by prior test times.  If use_docker is true, only run
            # tests that depend on docker, otherwise only those that don't.
            # The appended true condition ensures the command will succeed if no packages are found
            if [ $USE_DOCKER == 1 ]; then
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            else
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            fi
            # Move back into root directory
            popd
            # Append the test packages into the global list, if any are found
            if [ -n "$package_names" ]; then
              all_package_names+=" ${package_names}"
            fi
          done

          # After running tests split step, we are now running the following steps
          # in multiple different containers, each getting a different subset of
          # the test packages in their package_names variable.  Each container
          # has its own remote docker VM.

          make prep
          mkdir -p test-results/go-test

          # We don't want VAULT_LICENSE set when running Go tests, because that's
          # not what developers have in their environments and it could break some
          # tests; it would be like setting VAULT_TOKEN.  However some non-Go
          # CI commands, like the UI tests, shouldn't have to worry about licensing.
          # So we set VAULT_LICENSE in CI, and here we unset it.  Instead of
          # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want
          # an externally supplied license can opt-in to using it.
          export VAULT_LICENSE_CI="$VAULT_LICENSE"
          VAULT_LICENSE=

          # Create a docker network for our testcontainer
          if [ $USE_DOCKER == 1 ]; then
            # Despite the fact that we're using a circleci image (thus getting the
            # version they chose for the docker cli) and that we're specifying a
            # docker version to use for the remote docker instances, we occasionally
            # see "client version too new, max supported version 1.39" errors for
            # reasons unclear.
            export DOCKER_API_VERSION=1.39

            export TEST_DOCKER_NETWORK_ID=$(docker network list -q -f 'name=vaulttest')
            if [ -z $TEST_DOCKER_NETWORK_ID ]; then
              TEST_DOCKER_NETWORK_ID=$(docker network create vaulttest)
            fi



            # Start a docker testcontainer to run the tests in
            docker run -d \
              -e TEST_DOCKER_NETWORK_ID \
              -e GOPRIVATE \
              -e DOCKER_CERT_PATH \
              -e DOCKER_HOST \
              -e DOCKER_MACHINE_NAME \
              -e DOCKER_TLS_VERIFY \
              -e NO_PROXY \
              -e VAULT_TEST_LOG_DIR=/tmp/testlogs \
              --network vaulttest --name \
              testcontainer docker.mirror.hashicorp.services/cimg/go:1.17.11 \
              tail -f /dev/null

            # Run tests
            test -d /tmp/go-cache && docker cp /tmp/go-cache testcontainer:/tmp/gocache
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault'
            docker cp . testcontainer:/home/circleci/go/src/github.com/hashicorp/vault/
            docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH

            # Copy the downloaded modules inside the container.
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/pkg'
            docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/home/circleci/go/pkg/mod

            docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \
              -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
              -e GOCACHE=/tmp/gocache \
              -e GO_TAGS \
              -e GOPROXY="off" \
              -e VAULT_LICENSE_CI \
              -e GOARCH=amd64 \
              testcontainer \
                gotestsum --format=short-verbose \
                  --junitfile test-results/go-test/results.xml \
                  --jsonfile test-results/go-test/results.json \
                  -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                  -timeout=60m \
                  -parallel=20 \
                   \
                  ${all_package_names}
          else
            GOARCH=amd64 \
              GOCACHE=/tmp/go-cache \
              gotestsum --format=short-verbose \
                --junitfile test-results/go-test/results.xml \
                --jsonfile test-results/go-test/results.json \
                -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                -timeout=60m \
                -parallel=20 \
                 \
                ${all_package_names}
          fi
        environment:
          GOPRIVATE: github.com/hashicorp/*
        name: Run Go tests
        no_output_timeout: 60m
    - store_artifacts:
        path: test-results
    - store_test_results:
        path: test-results
    - store_artifacts:
        path: /tmp/testlogs
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
  semgrep:
    docker:
    - image: docker.mirror.hashicorp.services/alpine:3.13
    shell: /bin/sh
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    steps:
    - run:
        command: |
          apk add --no-cache python3 py3-pip make
          python3 -m pip install --user semgrep==0.86.5
          export PATH="$HOME/.local/bin:$PATH"

          echo "$ semgrep --version"
          semgrep --version
        name: Setup Semgrep
        working_directory: ~/
    - checkout
    - attach_workspace:
        at: .
    - run:
        command: "# Alpine images can't run the make file due to a bash requirement. Run\n# semgrep explicitly here. \nexport PATH=\"$HOME/.local/bin:$PATH\" \nsemgrep --error --include '*.go' --exclude 'vendor' -f tools/semgrep/ci .\n"
        name: Run Semgrep Rules
  pre-flight-checks:
    machine:
      image: ubuntu-2004:202201-02
    shell: /usr/bin/env bash -euo pipefail -c
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    steps:
    - run:
        command: |
          [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; }
          # Install Go
          curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz"
          sudo rm -rf /usr/local/go
          sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
          rm -f "go${GO_VERSION}.linux-amd64.tar.gz"
          GOPATH="/home/circleci/go"
          mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
          mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; }
          echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
          echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
          echo "export GOPROXY=https://proxy.golang.org,direct" >> "$BASH_ENV"
          echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV"

          echo "$ go version"
          go version
        name: Setup Go
        working_directory: ~/
    - checkout
    - run:
        command: |
          export CCI_PATH=/tmp/circleci-cli/$CIRCLECI_CLI_VERSION
          mkdir -p $CCI_PATH
          NAME=circleci-cli_${CIRCLECI_CLI_VERSION}_${ARCH}
          URL=$BASE/v${CIRCLECI_CLI_VERSION}/${NAME}.tar.gz
          curl -sSL $URL \
            | tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci"
          # Add circleci to the path for subsequent steps.
          echo "export PATH=$CCI_PATH:\$PATH" >> $BASH_ENV
          # Done, print some debug info.
          set -x
          . $BASH_ENV
          which circleci
          circleci version
        environment:
          ARCH: linux_amd64
          BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download
        name: Install CircleCI CLI
    - run:
        command: |
          set -x
          . $BASH_ENV
          make ci-verify
        name: Verify CircleCI
    - add_ssh_keys:
        fingerprints:
        - 0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06
    - run:
        command: |
          git config --global url."git@github.com:".insteadOf https://github.com/
    - restore_cache:
        keys:
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}
        - v1.5-{{checksum "go.sum"}}
        name: Restore closest matching go modules cache
    - run:
        command: |
          # set GOPATH explicitly to download to the right cache
          export GOPATH=$HOME/go
          # go list ./... forces downloading some additional versions of modules that 'go mod
          # download' misses. We need this because we make use of go list itself during
          # code generation in later builds that rely on this module cache.
          go list ./...
          go mod download -json
          ( cd sdk && go mod download -json; )
          ( cd api && go mod download -json; )
        name: go mod download
    - run:
        command: |
          git --no-pager diff --exit-code || {
            echo "ERROR: Files modified by go mod download, see above."
            exit 1
          }
        name: Verify downloading modules did not modify any files
    - save_cache:
        key: v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        name: Save go modules cache
        paths:
        - /home/circleci/go/pkg/mod
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
    - GO_VERSION: 1.17.11
    - GOFUMPT_VERSION: 0.2.1
    - GOTESTSUM_VERSION: 0.5.2
  test-go-race-remote-docker:
    docker:
    - image: docker.mirror.hashicorp.services/cimg/go:1.17.11
    resource_class: medium
    working_directory: /home/circleci/go/src/github.com/hashicorp/vault
    parallelism: 8
    steps:
    - run:
        command: |
          # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing
          [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && {
            # stop the job from this step
            circleci-agent step halt
          }
          # exit with success either way
          exit 0
        name: Check branch name
        working_directory: ~/
    - checkout
    - setup_remote_docker:
        docker_layer_caching: true
        version: 18.09.3
    - add_ssh_keys:
        fingerprints:
        - 0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06
    - run:
        command: |
          git config --global url."git@github.com:".insteadOf https://github.com/
    - run:
        command: |
          TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key
        name: Compute test cache key
    - restore_cache:
        keys:
        - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
    - restore_cache:
        keys:
        - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
        name: Restore exact go modules cache
    - run:
        command: |
          set -exo pipefail

          EXTRA_TAGS=
          case "-race" in
          *-race*) export VAULT_CI_GO_TEST_RACE=1;;
          *)       EXTRA_TAGS=deadlock;;
          esac

          # Install CircleCI CLI
          curl -sSL \
              "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \
            | sudo tar --overwrite -xz \
              -C /usr/local/bin \
              "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci"

          USE_DOCKER=0
          USE_DOCKER=1

          # Check all directories with a go.mod file
          modules=("." "api" "sdk")
          all_package_names=""

          for dir in "${modules[@]}"
          do
            pushd "$dir"
            # On its own line so that -e will fail the tests if we detect errors here.
            go list -test -json ./... > test-list.json
            # Split Go tests by prior test times.  If use_docker is true, only run
            # tests that depend on docker, otherwise only those that don't.
            # The appended true condition ensures the command will succeed if no packages are found
            if [ $USE_DOCKER == 1 ]; then
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            else
              package_names=$(< test-list.json jq -r 'select(.Deps != null) |
                  select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) |
                  .ForTest | select(. != null)' |
                  sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
            fi
            # Move back into root directory
            popd
            # Append the test packages into the global list, if any are found
            if [ -n "$package_names" ]; then
              all_package_names+=" ${package_names}"
            fi
          done

          # After running tests split step, we are now running the following steps
          # in multiple different containers, each getting a different subset of
          # the test packages in their package_names variable.  Each container
          # has its own remote docker VM.

          make prep
          mkdir -p test-results/go-test

          # We don't want VAULT_LICENSE set when running Go tests, because that's
          # not what developers have in their environments and it could break some
          # tests; it would be like setting VAULT_TOKEN.  However some non-Go
          # CI commands, like the UI tests, shouldn't have to worry about licensing.
          # So we set VAULT_LICENSE in CI, and here we unset it.  Instead of
          # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want
          # an externally supplied license can opt-in to using it.
          export VAULT_LICENSE_CI="$VAULT_LICENSE"
          VAULT_LICENSE=

          # Create a docker network for our testcontainer
          if [ $USE_DOCKER == 1 ]; then
            # Despite the fact that we're using a circleci image (thus getting the
            # version they chose for the docker cli) and that we're specifying a
            # docker version to use for the remote docker instances, we occasionally
            # see "client version too new, max supported version 1.39" errors for
            # reasons unclear.
            export DOCKER_API_VERSION=1.39

            export TEST_DOCKER_NETWORK_ID=$(docker network list -q -f 'name=vaulttest')
            if [ -z $TEST_DOCKER_NETWORK_ID ]; then
              TEST_DOCKER_NETWORK_ID=$(docker network create vaulttest)
            fi



            # Start a docker testcontainer to run the tests in
            docker run -d \
              -e TEST_DOCKER_NETWORK_ID \
              -e GOPRIVATE \
              -e DOCKER_CERT_PATH \
              -e DOCKER_HOST \
              -e DOCKER_MACHINE_NAME \
              -e DOCKER_TLS_VERIFY \
              -e NO_PROXY \
              -e VAULT_TEST_LOG_DIR=/tmp/testlogs \
              --network vaulttest --name \
              testcontainer docker.mirror.hashicorp.services/cimg/go:1.17.11 \
              tail -f /dev/null

            # Run tests
            test -d /tmp/go-cache && docker cp /tmp/go-cache testcontainer:/tmp/gocache
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault'
            docker cp . testcontainer:/home/circleci/go/src/github.com/hashicorp/vault/
            docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH

            # Copy the downloaded modules inside the container.
            docker exec testcontainer sh -c 'mkdir -p /home/circleci/go/pkg'
            docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/home/circleci/go/pkg/mod

            docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \
              -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
              -e GOCACHE=/tmp/gocache \
              -e GO_TAGS \
              -e GOPROXY="off" \
              -e VAULT_LICENSE_CI \
              -e GOARCH=amd64 \
              testcontainer \
                gotestsum --format=short-verbose \
                  --junitfile test-results/go-test/results.xml \
                  --jsonfile test-results/go-test/results.json \
                  -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                  -timeout=60m \
                  -parallel=20 \
                  -race \
                  ${all_package_names}
          else
            GOARCH=amd64 \
              GOCACHE=/tmp/go-cache \
              gotestsum --format=short-verbose \
                --junitfile test-results/go-test/results.xml \
                --jsonfile test-results/go-test/results.json \
                -- \
                -tags "${GO_TAGS} ${EXTRA_TAGS}" \
                -timeout=60m \
                -parallel=20 \
                -race \
                ${all_package_names}
          fi
        environment:
          GOPRIVATE: github.com/hashicorp/*
        name: Run Go tests
        no_output_timeout: 60m
    - run:
        command: |
          docker cp testcontainer:/home/circleci/go/src/github.com/hashicorp/vault/test-results .
          docker cp testcontainer:/tmp/gocache /tmp/go-cache
        name: Copy test results
        when: always
    - store_artifacts:
        path: test-results
    - store_test_results:
        path: test-results
    - store_artifacts:
        path: /tmp/testlogs
    environment:
    - CIRCLECI_CLI_VERSION: 0.1.5546
    - GO_TAGS: ''
workflows:
  ci:
    jobs:
    - pre-flight-checks
    - fmt
    - install-ui-dependencies:
        requires:
        - pre-flight-checks
    - build-go-dev:
        requires:
        - pre-flight-checks
    - test-ui:
        requires:
        - install-ui-dependencies
        - build-go-dev
    - test-ui-browserstack:
        filters:
          branches:
            ignore: /pull\/[0-9]+/
        requires:
        - install-ui-dependencies
        - build-go-dev
    - test-go:
        requires:
        - pre-flight-checks
    - test-go-remote-docker:
        requires:
        - pre-flight-checks
    - test-go-race:
        requires:
        - pre-flight-checks
    - test-go-race-remote-docker:
        requires:
        - pre-flight-checks
    - semgrep:
        requires:
        - pre-flight-checks
  version: 2