ci: add test-integrations (#16915)

* add test-integrations workflow

* add test-integrations success job

* update vault integration testing versions (#16949)

* change parallelism to 4 forgotestsum.  use env.CONSUL_VERSION so we can see the version.

* use env for repeated values

* match test to circleci

* fix envvar

* fix envvar 2

* fix envvar 3

* fix envvar 4

* fix envvar 5

* make upgrade and compatibility tests match circleci

* run go env to check environment

* debug docker

Signed-off-by: Dan Bond <danbond@protonmail.com>

* debug docker

Signed-off-by: Dan Bond <danbond@protonmail.com>

* revert debug docker

Signed-off-by: Dan Bond <danbond@protonmail.com>

* going back to command that worked 5 days ago for compatibility tests

* Update Envoy versions to reflect changes in #16889

* cd to test dir

* try running ubuntu latest

* update PR with latest changes that work in enterprise

* yaml still sucks

* test GH fix (localhost resolution)

* change for testing

* test splitting and ipv6 lookup for compatibility and upgrade tests

* fix indention

* consul as image name

* remove the on push

* add gotestsum back in

* removing the use of the gotestsum download action

* yaml sucks today just like yesterday

* fixing nomad tests

* worked out the kinks on enterprise

---------

Signed-off-by: Dan Bond <danbond@protonmail.com>
Co-authored-by: John Eikenberry <jae@zhar.net>
Co-authored-by: Dan Bond <danbond@protonmail.com>
Co-authored-by: Nathan Coleman <nathan.coleman@hashicorp.com>
Co-authored-by: Sarah <sthompson@hashicorp.com>
This commit is contained in:
John Murret 2023-04-18 20:45:30 -06:00 committed by GitHub
parent 9345ce89d4
commit 9a77ec0b6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 503 additions and 0 deletions

View File

@ -346,6 +346,7 @@ jobs:
sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
environment: environment:
<<: *ENVIRONMENT <<: *ENVIRONMENT
- run: go env
- run: *install-gotestsum - run: *install-gotestsum
- run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
- run: - run:
@ -409,6 +410,7 @@ jobs:
sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
environment: environment:
<<: *ENVIRONMENT <<: *ENVIRONMENT
- run: go env
- run: *install-gotestsum - run: *install-gotestsum
- run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
- run: - run:

501
.github/workflows/test-integrations.yml vendored Normal file
View File

@ -0,0 +1,501 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
name: test-integrations
on:
pull_request:
branches-ignore:
- stable-website
- 'docs/**'
- 'ui/**'
- 'mktg-**' # Digital Team Terraform-generated branch prefix
- 'backport/docs/**'
- 'backport/ui/**'
- 'backport/mktg-**'
env:
TEST_RESULTS_DIR: /tmp/test-results
TEST_RESULTS_ARTIFACT_NAME: test-results
CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }}
GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
GOTESTSUM_VERSION: "1.9.0"
CONSUL_BINARY_UPLOAD_NAME: consul-bin
# strip the hashicorp/ off the front of github.repository for consul
CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }}
jobs:
setup:
runs-on: ubuntu-latest
name: Setup
outputs:
compute-small: ${{ steps.runners.outputs.compute-small }}
compute-medium: ${{ steps.runners.outputs.compute-medium }}
compute-large: ${{ steps.runners.outputs.compute-large }}
compute-xl: ${{ steps.runners.outputs.compute-xl }}
enterprise: ${{ steps.runners.outputs.enterprise }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- id: runners
run: .github/scripts/get_runner_classes.sh
dev-build:
needs: [setup]
uses: ./.github/workflows/reusable-dev-build.yml
with:
runs-on: ${{ needs.setup.outputs.compute-xl }}
repository-name: ${{ github.repository }}
uploaded-binary-name: 'consul-bin'
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
nomad-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- dev-build
strategy:
matrix:
nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16']
steps:
- name: Checkout Nomad
uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
with:
repository: hashicorp/nomad
ref: ${{ matrix.nomad-version }}
- name: Install Go
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
with:
go-version-file: 'go.mod'
- name: Fetch Consul binary
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: ./bin
- name: Restore Consul permissions
run: |
chmod +x ./bin/consul
echo "$(pwd)/bin" >> $GITHUB_PATH
- name: Make Nomad dev build
run: make pkg/linux_amd64/nomad
- name: Run integration tests
run: |
go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \
gotestsum \
--format=short-verbose \
--rerun-fails \
--rerun-fails-report=/tmp/gotestsum-rerun-fails \
--packages="./command/agent/consul" \
--junitfile $TEST_RESULTS_DIR/results.xml -- \
-run TestConsul
vault-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- dev-build
strategy:
matrix:
vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"]
env:
VAULT_BINARY_VERSION: ${{ matrix.vault-version }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
with:
go-version-file: 'go.mod'
- name: Install Vault
run: |
wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip"
unzip -d /tmp /tmp/vault.zip
echo "/tmp" >> $GITHUB_PATH
- name: Run Connect CA Provider Tests
run: |
mkdir -p "${{ env.TEST_RESULTS_DIR }}"
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--format=short-verbose \
--junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \
-- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca
# Run leader tests that require Vault
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--format=short-verbose \
--junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \
-- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul
# Run agent tests that require Vault
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--format=short-verbose \
--junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \
-- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent
generate-envoy-job-matrices:
needs: [setup]
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
name: Generate Envoy Job Matrices
outputs:
envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- name: Generate Envoy Job Matrix
id: set-matrix
env:
# this is further going to multiplied in envoy-integration tests by the
# other dimensions in the matrix. Currently TOTAL_RUNNERS would be
# multiplied by 8 based on these values:
# envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"]
# xds-target: ["server", "client"]
TOTAL_RUNNERS: 3
JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
run: |
{
echo -n "envoy-matrix="
find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \
| xargs -0 -n 1 basename \
| jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
| jq --compact-output 'map(join("|"))'
} >> "$GITHUB_OUTPUT"
cat "$GITHUB_OUTPUT"
envoy-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
needs:
- setup
- generate-envoy-job-matrices
- dev-build
strategy:
fail-fast: false
matrix:
envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"]
xds-target: ["server", "client"]
test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
env:
ENVOY_VERSION: ${{ matrix.envoy-version }}
XDS_TARGET: ${{ matrix.xds-target }}
AWS_LAMBDA_REGION: us-west-2
steps:
# NOTE: ENT specific step as we store secrets in Vault.
- name: Authenticate to Vault
if: ${{ endsWith(github.repository, '-enterprise') }}
id: vault-auth
run: vault-auth
# NOTE: ENT specific step as we store secrets in Vault.
- name: Fetch Secrets
if: ${{ endsWith(github.repository, '-enterprise') }}
id: secrets
uses: hashicorp/vault-action@v2.5.0
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/aws arn | AWS_ROLE_ARN ;
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
with:
go-version-file: 'go.mod'
- name: fetch binary
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: ./bin
- name: restore mode+x
run: chmod +x ./bin/consul
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1
- name: Docker build
run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin
- name: Envoy Integration Tests
env:
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
LAMBDA_TESTS_ENABLED: "true"
# tput complains if this isn't set to something.
TERM: ansi
run: |
# shellcheck disable=SC2001
echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
# shellcheck disable=SC2001
sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--debug \
--rerun-fails \
--rerun-fails-report=/tmp/gotestsum-rerun-fails \
--jsonfile /tmp/jsonfile/go-test.log \
--packages=./test/integration/connect/envoy \
-- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})"
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: container-logs
path: ./test/integration/connect/envoy/workdir/logs
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
path: ${{ env.TEST_RESULTS_DIR }}
generate-compatibility-job-matrices:
needs: [setup]
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
name: Generate Compatibility Job Matrices
outputs:
compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- name: Generate Compatibility Job Matrix
id: set-matrix
env:
TOTAL_RUNNERS: 5
JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
run: |
cd ./test/integration/consul-container
{
echo -n "compatibility-matrix="
find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \
| grep -v util | grep -v upgrade \
| jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
| jq --compact-output 'map(join(" "))'
} >> "$GITHUB_OUTPUT"
cat "$GITHUB_OUTPUT"
compatibility-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
needs:
- setup
- dev-build
- generate-compatibility-job-matrices
strategy:
fail-fast: false
matrix:
test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
with:
go-version-file: 'go.mod'
- run: go env
# Build the consul:local image from the already built binary
- name: fetch binary
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: .
- name: restore mode+x
run: chmod +x consul
- name: Build consul:local image
run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
- name: Configure GH workaround for ipv6 loopback
if: ${{ !endsWith(github.repository, '-enterprise') }}
run: |
cat /etc/hosts && echo "-----------"
sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
cat /etc/hosts
- name: Compatibility Integration Tests
run: |
mkdir -p "/tmp/test-results"
cd ./test/integration/consul-container
docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
# shellcheck disable=SC2001
sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--raw-command \
--format=short-verbose \
--debug \
--rerun-fails=3 \
-- \
go test \
-p=4 \
-tags "${{ env.GOTAGS }}" \
-timeout=30m \
-json \
"${{ matrix.test-cases }}" \
--target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--target-version local \
--latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--latest-version latest
ls -lrt
env:
# this is needed because of incompatibility between RYUK container and circleci
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
# tput complains if this isn't set to something.
TERM: ansi
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
path: ${{ env.TEST_RESULTS_DIR }}
generate-upgrade-job-matrices:
needs: [setup]
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
name: Generate Upgrade Job Matrices
outputs:
upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
with:
go-version-file: 'go.mod'
- name: Generate Updgrade Job Matrix
id: set-matrix
env:
TOTAL_RUNNERS: 4
JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
run: |
cd ./test/integration/consul-container/test/upgrade
{
echo -n "upgrade-matrix="
go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \
| jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
| jq --compact-output 'map(join("|"))'
} >> "$GITHUB_OUTPUT"
cat "$GITHUB_OUTPUT"
upgrade-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
needs:
- setup
- dev-build
- generate-upgrade-job-matrices
strategy:
fail-fast: false
matrix:
consul-version: [ "1.14", "1.15"]
test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }}
env:
CONSUL_VERSION: ${{ matrix.consul-version }}
steps:
- uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
with:
go-version-file: 'go.mod'
- run: go env
# NOTE: ENT specific step as we store secrets in Vault.
- name: Authenticate to Vault
if: ${{ endsWith(github.repository, '-enterprise') }}
id: vault-auth
run: vault-auth
# NOTE: ENT specific step as we store secrets in Vault.
- name: Fetch Secrets
if: ${{ endsWith(github.repository, '-enterprise') }}
id: secrets
uses: hashicorp/vault-action@v2.5.0
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME;
kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN;
# NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS.
- name: Login to Docker Hub
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0
with:
username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}
# Get go binary from workspace
- name: fetch binary
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: .
- name: restore mode+x
run: chmod +x consul
- name: Build consul:local image
run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
- name: Configure GH workaround for ipv6 loopback
if: ${{ !endsWith(github.repository, '-enterprise') }}
run: |
cat /etc/hosts && echo "-----------"
sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
cat /etc/hosts
- name: Upgrade Integration Tests
run: |
mkdir -p "${{ env.TEST_RESULTS_DIR }}"
cd ./test/integration/consul-container/test/upgrade
docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
# shellcheck disable=SC2001
sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--raw-command \
--format=short-verbose \
--debug \
--rerun-fails=3 \
--packages="./..." \
-- \
go test \
-p=4 \
-tags "${{ env.GOTAGS }}" \
-timeout=30m \
-json ./... \
-run "${{ matrix.test-cases }}" \
--target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--target-version local \
--latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--latest-version "${{ env.CONSUL_VERSION }}"
ls -lrt
env:
# this is needed because of incompatibility between RYUK container and circleci
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
# tput complains if this isn't set to something.
TERM: ansi
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
path: ${{ env.TEST_RESULTS_DIR }}
test-integrations-success:
needs:
- setup
- dev-build
- nomad-integration-test
- vault-integration-test
- generate-envoy-job-matrices
- envoy-integration-test
- generate-compatibility-job-matrices
- compatibility-integration-test
- generate-upgrade-job-matrices
- upgrade-integration-test
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
if: |
(always() && ! cancelled()) &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
steps:
- run: echo "test-integrations succeeded"