open-consul/.github/workflows/test-integrations.yml

# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0

name: test-integrations

on:
  pull_request:
    branches-ignore:
      - stable-website
      - 'docs/**'
      - 'ui/**'
      - 'mktg-**' # Digital Team Terraform-generated branch prefix
      - 'backport/docs/**'
      - 'backport/ui/**'
      - 'backport/mktg-**'

env:
  TEST_RESULTS_DIR: /tmp/test-results
  TEST_RESULTS_ARTIFACT_NAME: test-results
  CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }}
  GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
  GOTESTSUM_VERSION: "1.9.0"
  CONSUL_BINARY_UPLOAD_NAME: consul-bin
  # strip the hashicorp/ off the front of github.repository for consul
  CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }}

jobs:
  setup:
    runs-on: ubuntu-latest
    name: Setup
    outputs:
      compute-small: ${{ steps.runners.outputs.compute-small }}
      compute-medium: ${{ steps.runners.outputs.compute-medium }}
      compute-large: ${{ steps.runners.outputs.compute-large }}
      compute-xl: ${{ steps.runners.outputs.compute-xl }}
      enterprise: ${{ steps.runners.outputs.enterprise }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - id: runners
        run: .github/scripts/get_runner_classes.sh

  dev-build:
    needs: [setup]
    uses: ./.github/workflows/reusable-dev-build.yml
    with:
      runs-on: ${{ needs.setup.outputs.compute-xl }}
      repository-name: ${{ github.repository }}
      uploaded-binary-name: 'consul-bin'
    secrets:
      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}

  nomad-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
    needs:
      - setup
      - dev-build
    strategy:
      matrix:
        nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16']
    steps:
      - name: Checkout Nomad
        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
        with:
          repository: hashicorp/nomad
          ref: ${{ matrix.nomad-version }}

      - name: Install Go
        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
          go-version-file: 'go.mod'

      - name: Fetch Consul binary
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
          path: ./bin
      - name: Restore Consul permissions
        run: |
          chmod +x ./bin/consul
          echo "$(pwd)/bin" >> $GITHUB_PATH          

      - name: Make Nomad dev build
        run: make pkg/linux_amd64/nomad

      - name: Run integration tests
        run: |
          go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \
          gotestsum \
            --format=short-verbose \
            --rerun-fails \
            --rerun-fails-report=/tmp/gotestsum-rerun-fails \
            --packages="./command/agent/consul" \
            --junitfile $TEST_RESULTS_DIR/results.xml -- \
            -run TestConsul          

  vault-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
    needs:
      - setup
      - dev-build
    strategy:
      matrix:
        vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"]
    env:
      VAULT_BINARY_VERSION: ${{ matrix.vault-version }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0

      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
      - name: Setup Git
        if: ${{ endsWith(github.repository, '-enterprise') }}
        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"

      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
          go-version-file: 'go.mod'

      - name: Install Vault
        run: |
          wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip"
          unzip -d /tmp /tmp/vault.zip
          echo "/tmp" >> $GITHUB_PATH          

      - name: Run Connect CA Provider Tests
        run: |
          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
            --format=short-verbose \
            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \
            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca
          # Run leader tests that require Vault
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
            --format=short-verbose \
            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \
            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul
          # Run agent tests that require Vault
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
            --format=short-verbose \
            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \
            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent          

  generate-envoy-job-matrices:
    needs: [setup]
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
    name: Generate Envoy Job Matrices
    outputs:
      envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - name: Generate Envoy Job Matrix
        id: set-matrix
        env:
          # this is further going to multiplied in envoy-integration tests by the 
          # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
          # multiplied by 8 based on these values:
          # envoy-version: ["1.23.8", "1.24.6", "1.25.4", "1.26.0"]
          # xds-target: ["server", "client"]
          TOTAL_RUNNERS: 4 
          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
        run: |
          NUM_RUNNERS=$TOTAL_RUNNERS
          NUM_DIRS=$(find ./test/integration/connect/envoy -maxdepth 1 -type d | wc -l)
          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
            NUM_RUNNERS=$NUM_DIRS
          fi
          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
          NUM_RUNNERS=$((NUM_RUNNERS-1))
          {
            echo -n "envoy-matrix="
            find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \
              | xargs -0 -n 1 basename \
              | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
              | jq --compact-output 'map(join("|"))'
          } >> "$GITHUB_OUTPUT"          
  
  envoy-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
    needs:
      - setup
      - generate-envoy-job-matrices
      - dev-build
    strategy:
      fail-fast: false
      matrix:
        envoy-version: ["1.23.8", "1.24.6", "1.25.4", "1.26.0"]
        xds-target: ["server", "client"]
        test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
    env:
      ENVOY_VERSION: ${{ matrix.envoy-version }}
      XDS_TARGET: ${{ matrix.xds-target }}
      AWS_LAMBDA_REGION: us-west-2
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
          go-version-file: 'go.mod'

      - name: fetch binary
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
          path: ./bin
      - name: restore mode+x
        run: chmod +x ./bin/consul

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1

      - name: Docker build
        run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin

      - name: Envoy Integration Tests
        env:
          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
          GOTESTSUM_FORMAT: standard-verbose
          COMPOSE_INTERACTIVE_NO_CLI: 1
          LAMBDA_TESTS_ENABLED: "true"
          # tput complains if this isn't set to something.
          TERM: ansi
        run: |
          # shellcheck disable=SC2001
          echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
          # shellcheck disable=SC2001
          sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
              --debug \
              --rerun-fails \
              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
              --jsonfile /tmp/jsonfile/go-test.log \
              --packages=./test/integration/connect/envoy \
              -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})"          

      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
          path: ${{ env.TEST_RESULTS_DIR }}

  generate-compatibility-job-matrices:
    needs: [setup]
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
    name: Generate Compatibility Job Matrices
    outputs:
      compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - name: Generate Compatibility Job Matrix
        id: set-matrix
        env:
          TOTAL_RUNNERS: 6
          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
        run: |
          cd ./test/integration/consul-container
          NUM_RUNNERS=$TOTAL_RUNNERS
          NUM_DIRS=$(find ./test -maxdepth 2 -type d | wc -l)
          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
            NUM_RUNNERS=$NUM_DIRS
          fi
          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
          NUM_RUNNERS=$((NUM_RUNNERS-1))
          {
            echo -n "compatibility-matrix="
            find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \
              | grep -v util | grep -v upgrade \
              | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
              | jq --compact-output 'map(join(" "))'
          } >> "$GITHUB_OUTPUT"          
  compatibility-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
    needs:
      - setup
      - dev-build
      - generate-compatibility-job-matrices
    strategy:
      fail-fast: false
      matrix:
        test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
          go-version-file: 'go.mod'
      - run: go env

      # Build the consul:local image from the already built binary
      - name: fetch binary
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
          path: .
      - name: restore mode+x
        run: chmod +x consul

      - name: Build consul:local image
        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
      - name: Configure GH workaround for ipv6 loopback
        if: ${{ !endsWith(github.repository, '-enterprise') }}
        run: |
          cat /etc/hosts && echo "-----------"
          sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
          cat /etc/hosts          
      - name: Compatibility Integration Tests
        run: |
            mkdir -p "/tmp/test-results"
            cd ./test/integration/consul-container
            docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
            echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
            # shellcheck disable=SC2001
            sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
            go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
              --raw-command \
              --format=short-verbose \
              --debug \
              --rerun-fails=3 \
              -- \
              go test \
              -p=4 \
              -tags "${{ env.GOTAGS }}" \
              -timeout=30m \
              -json \
              "${{ matrix.test-cases }}" \
              --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
              --target-version local \
              --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
              --latest-version latest
            ls -lrt            
        env:
          # this is needed because of incompatibility between RYUK container and GHA
          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
          GOTESTSUM_FORMAT: standard-verbose
          COMPOSE_INTERACTIVE_NO_CLI: 1
          # tput complains if this isn't set to something.
          TERM: ansi

      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
          path: ${{ env.TEST_RESULTS_DIR }}

  generate-upgrade-job-matrices:
    needs: [setup]
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
    name: Generate Upgrade Job Matrices
    outputs:
      upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
          go-version-file: 'go.mod'
      - name: Generate Updgrade Job Matrix
        id: set-matrix
        env:
          TOTAL_RUNNERS: 5
          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
        run: |
          cd ./test/integration/consul-container/test/upgrade
          NUM_RUNNERS=$TOTAL_RUNNERS
          NUM_DIRS=$(go test ./...  -list=. -json | jq -r '.Output | select (. !=null) |  select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' | wc -l)
          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
            NUM_RUNNERS=$NUM_DIRS
          fi
          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
          NUM_RUNNERS=$((NUM_RUNNERS-1))
          {
            echo -n "upgrade-matrix="
            go test ./...  -list=. -json | jq -r '.Output | select (. !=null) |  select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \
            | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
            | jq --compact-output 'map(join("|"))'
          } >> "$GITHUB_OUTPUT"          
  
  upgrade-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
    permissions:
      id-token: write # NOTE: this permission is explicitly required for Vault auth.
      contents: read
    needs:
      - setup
      - dev-build
      - generate-upgrade-job-matrices
    strategy:
      fail-fast: false
      matrix:
        consul-version: [ "1.14", "1.15"]
        test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }}
    env:
      CONSUL_VERSION: ${{ matrix.consul-version }}
    steps:
      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
          go-version-file: 'go.mod'
      - run: go env

      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Authenticate to Vault
        if: ${{ endsWith(github.repository, '-enterprise') }}
        id: vault-auth
        run: vault-auth

      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Fetch Secrets
        if: ${{ endsWith(github.repository, '-enterprise') }}
        id: secrets
        uses: hashicorp/vault-action@v2.5.0
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
              kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME;
              kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN;              

      # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS.
      - name: Login to Docker Hub
        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0
        with:
          username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
          password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}


      # Get go binary from workspace
      - name: fetch binary
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
          path: .
      - name: restore mode+x
        run: chmod +x consul
      - name: Build consul:local image
        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
      - name: Configure GH workaround for ipv6 loopback
        if: ${{ !endsWith(github.repository, '-enterprise') }}
        run: |
          cat /etc/hosts && echo "-----------"
          sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
          cat /etc/hosts          
      - name: Upgrade Integration Tests
        run: |
          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
          cd ./test/integration/consul-container/test/upgrade
          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
          echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
          # shellcheck disable=SC2001
          sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
            --raw-command \
            --format=short-verbose \
            --debug \
            --rerun-fails=3 \
            --packages="./..." \
            -- \
            go test \
            -p=4 \
            -tags "${{ env.GOTAGS }}" \
            -timeout=30m \
            -json ./... \
            -run "${{ matrix.test-cases }}" \
            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
            --target-version local \
            --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
            --latest-version "${{ env.CONSUL_VERSION }}"
          ls -lrt          
        env:
          # this is needed because of incompatibility between RYUK container and GHA
          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
          GOTESTSUM_FORMAT: standard-verbose
          COMPOSE_INTERACTIVE_NO_CLI: 1
          # tput complains if this isn't set to something.
          TERM: ansi
      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
          path: ${{ env.TEST_RESULTS_DIR }}

  test-integrations-success:
    needs: 
    - setup
    - dev-build
    - nomad-integration-test
    - vault-integration-test
    - generate-envoy-job-matrices
    - envoy-integration-test
    - generate-compatibility-job-matrices
    - compatibility-integration-test
    - generate-upgrade-job-matrices
    - upgrade-integration-test
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
    if: |
      (always() && ! cancelled()) &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled')      
    steps:
      - run: echo "test-integrations succeeded"