# Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: MPL-2.0 name: test-integrations on: pull_request: branches-ignore: - stable-website - 'docs/**' - 'ui/**' - 'mktg-**' # Digital Team Terraform-generated branch prefix - 'backport/docs/**' - 'backport/ui/**' - 'backport/mktg-**' env: TEST_RESULTS_DIR: /tmp/test-results TEST_RESULTS_ARTIFACT_NAME: test-results CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} GOTESTSUM_VERSION: "1.9.0" CONSUL_BINARY_UPLOAD_NAME: consul-bin # strip the hashicorp/ off the front of github.repository for consul CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }} jobs: setup: runs-on: ubuntu-latest name: Setup outputs: compute-small: ${{ steps.runners.outputs.compute-small }} compute-medium: ${{ steps.runners.outputs.compute-medium }} compute-large: ${{ steps.runners.outputs.compute-large }} compute-xl: ${{ steps.runners.outputs.compute-xl }} enterprise: ${{ steps.runners.outputs.enterprise }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - id: runners run: .github/scripts/get_runner_classes.sh dev-build: needs: [setup] uses: ./.github/workflows/reusable-dev-build.yml with: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} uploaded-binary-name: 'consul-bin' secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} nomad-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} needs: - setup - dev-build strategy: matrix: nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] steps: - name: Checkout Nomad uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 with: repository: hashicorp/nomad ref: ${{ matrix.nomad-version }} - name: Install Go uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' - name: Fetch Consul binary uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' path: ./bin - name: Restore Consul permissions run: | chmod +x ./bin/consul echo "$(pwd)/bin" >> $GITHUB_PATH - name: Make Nomad dev build run: make pkg/linux_amd64/nomad - name: Run integration tests run: | go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \ gotestsum \ --format=short-verbose \ --rerun-fails \ --rerun-fails-report=/tmp/gotestsum-rerun-fails \ --packages="./command/agent/consul" \ --junitfile $TEST_RESULTS_DIR/results.xml -- \ -run TestConsul vault-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} needs: - setup - dev-build strategy: matrix: vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"] env: VAULT_BINARY_VERSION: ${{ matrix.vault-version }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. - name: Setup Git if: ${{ endsWith(github.repository, '-enterprise') }} run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' - name: Install Vault run: | wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip" unzip -d /tmp /tmp/vault.zip echo "/tmp" >> $GITHUB_PATH - name: Run Connect CA Provider Tests run: | mkdir -p "${{ env.TEST_RESULTS_DIR }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --format=short-verbose \ --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \ -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca # Run leader tests that require Vault go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --format=short-verbose \ --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \ -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul # Run agent tests that require Vault go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --format=short-verbose \ --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \ -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent generate-envoy-job-matrices: needs: [setup] runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} name: Generate Envoy Job Matrices outputs: envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - name: Generate Envoy Job Matrix id: set-matrix env: # this is further going to multiplied in envoy-integration tests by the # other dimensions in the matrix. Currently TOTAL_RUNNERS would be # multiplied by 8 based on these values: # envoy-version: ["1.23.8", "1.24.6", "1.25.4", "1.26.0"] # xds-target: ["server", "client"] TOTAL_RUNNERS: 4 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | NUM_RUNNERS=$TOTAL_RUNNERS NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l) if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." NUM_RUNNERS=$((NUM_DIRS-1)) fi # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "envoy-matrix=" find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ | xargs -0 -n 1 basename \ | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join("|"))' } >> "$GITHUB_OUTPUT" envoy-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: - setup - generate-envoy-job-matrices - dev-build strategy: fail-fast: false matrix: envoy-version: ["1.23.8", "1.24.6", "1.25.4", "1.26.0"] xds-target: ["server", "client"] test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} env: ENVOY_VERSION: ${{ matrix.envoy-version }} XDS_TARGET: ${{ matrix.xds-target }} AWS_LAMBDA_REGION: us-west-2 steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' - name: fetch binary uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' path: ./bin - name: restore mode+x run: chmod +x ./bin/consul - name: Set up Docker Buildx uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1 - name: Docker build run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin - name: Envoy Integration Tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 LAMBDA_TESTS_ENABLED: "true" # tput complains if this isn't set to something. TERM: ansi run: | # shellcheck disable=SC2001 echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" # shellcheck disable=SC2001 sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --debug \ --rerun-fails \ --rerun-fails-report=/tmp/gotestsum-rerun-fails \ --jsonfile /tmp/jsonfile/go-test.log \ --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} path: ${{ env.TEST_RESULTS_DIR }} generate-compatibility-job-matrices: needs: [setup] runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} name: Generate Compatibility Job Matrices outputs: compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - name: Generate Compatibility Job Matrix id: set-matrix env: TOTAL_RUNNERS: 6 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | cd ./test/integration/consul-container NUM_RUNNERS=$TOTAL_RUNNERS NUM_DIRS=$(find ./test -mindepth 1 -maxdepth 2 -type d | wc -l) if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." NUM_RUNNERS=$((NUM_DIRS-1)) fi # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "compatibility-matrix=" find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ | grep -v util | grep -v upgrade \ | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join(" "))' } >> "$GITHUB_OUTPUT" compatibility-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: - setup - dev-build - generate-compatibility-job-matrices strategy: fail-fast: false matrix: test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' - run: go env # Build the consul:local image from the already built binary - name: fetch binary uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' path: . - name: restore mode+x run: chmod +x consul - name: Build consul:local image run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . - name: Configure GH workaround for ipv6 loopback if: ${{ !endsWith(github.repository, '-enterprise') }} run: | cat /etc/hosts && echo "-----------" sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts cat /etc/hosts - name: Compatibility Integration Tests run: | mkdir -p "/tmp/test-results" cd ./test/integration/consul-container docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" # shellcheck disable=SC2001 sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ --format=short-verbose \ --debug \ --rerun-fails=3 \ -- \ go test \ -p=4 \ -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json \ "${{ matrix.test-cases }}" \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --latest-version latest ls -lrt env: # this is needed because of incompatibility between RYUK container and GHA GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 # tput complains if this isn't set to something. TERM: ansi - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} path: ${{ env.TEST_RESULTS_DIR }} generate-upgrade-job-matrices: needs: [setup] runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} name: Generate Upgrade Job Matrices outputs: upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' - name: Generate Updgrade Job Matrix id: set-matrix env: TOTAL_RUNNERS: 5 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | cd ./test/integration/consul-container/test/upgrade NUM_RUNNERS=$TOTAL_RUNNERS NUM_DIRS=$(go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' | wc -l) if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." NUM_RUNNERS=$((NUM_DIRS-1)) fi # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "upgrade-matrix=" go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \ | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join("|"))' } >> "$GITHUB_OUTPUT" upgrade-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. contents: read needs: - setup - dev-build - generate-upgrade-job-matrices strategy: fail-fast: false matrix: consul-version: [ "1.14", "1.15"] test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }} env: CONSUL_VERSION: ${{ matrix.consul-version }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' - run: go env # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ endsWith(github.repository, '-enterprise') }} id: vault-auth run: vault-auth # NOTE: ENT specific step as we store secrets in Vault. - name: Fetch Secrets if: ${{ endsWith(github.repository, '-enterprise') }} id: secrets uses: hashicorp/vault-action@v2.5.0 with: url: ${{ steps.vault-auth.outputs.addr }} caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} token: ${{ steps.vault-auth.outputs.token }} secrets: | kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME; kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN; # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS. - name: Login to Docker Hub uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0 with: username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }} # Get go binary from workspace - name: fetch binary uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' path: . - name: restore mode+x run: chmod +x consul - name: Build consul:local image run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . - name: Configure GH workaround for ipv6 loopback if: ${{ !endsWith(github.repository, '-enterprise') }} run: | cat /etc/hosts && echo "-----------" sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts cat /etc/hosts - name: Upgrade Integration Tests run: | mkdir -p "${{ env.TEST_RESULTS_DIR }}" cd ./test/integration/consul-container/test/upgrade docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" # shellcheck disable=SC2001 sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ --format=short-verbose \ --debug \ --rerun-fails=3 \ --packages="./..." \ -- \ go test \ -p=4 \ -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json ./... \ -run "${{ matrix.test-cases }}" \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --latest-version "${{ env.CONSUL_VERSION }}" ls -lrt env: # this is needed because of incompatibility between RYUK container and GHA GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 # tput complains if this isn't set to something. TERM: ansi - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} path: ${{ env.TEST_RESULTS_DIR }} test-integrations-success: needs: - setup - dev-build - nomad-integration-test - vault-integration-test - generate-envoy-job-matrices - envoy-integration-test - generate-compatibility-job-matrices - compatibility-integration-test - generate-upgrade-job-matrices - upgrade-integration-test runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} if: ${{ always() }} steps: - name: evaluate upstream job results run: | # exit 1 if failure or cancelled result for any upstream job if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" exit 1 fi