From b5794c803feadbdf0fa94efea1e889c1307d2096 Mon Sep 17 00:00:00 2001 From: John Murret Date: Tue, 28 Mar 2023 15:29:27 -0600 Subject: [PATCH] Github Actions Migration - move go-tests workflows to GHA (#16761) * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * making on pull_request * Update .github/scripts/rerun_fails_report.sh Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * make runs-on required * removing go-version param that is not used. * removing go-version param that is not used. * Modify build-distros to use medium runners (#16773) * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * trying mediums * adding in script * fixing runs-on to be parameter * fixing merge conflict * changing to on push * removing whitespace * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * changing back to on pull_request --------- Co-authored-by: Dan Bond * Github Actions Migration - move verify-ci workflows to GHA (#16777) * add verify-ci workflow * adding comment and changing to on pull request. * changing to pull_requests * changing to pull_request * Apply suggestions from code review Co-authored-by: Dan Bond * [NET-3029] Migrate frontend to GHA (#16731) * changing set up to a small * using consuls own custom runner pool. --------- Co-authored-by: Dan Bond --- .github/scripts/get_runner_classes.sh | 22 ++ .github/scripts/notify_slack.sh | 27 ++ .github/scripts/rerun_fails_report.sh | 24 ++ .github/scripts/set_test_package_matrix.sh | 8 + .github/workflows/build-distros.yml | 48 ++-- .github/workflows/frontend.yml | 94 +++++++ .github/workflows/go-tests.yml | 267 ++++++++++++++++++++ .github/workflows/reusable-check-go-mod.yml | 28 ++ .github/workflows/reusable-dev-build.yml | 31 +++ .github/workflows/reusable-lint.yml | 47 ++++ .github/workflows/reusable-unit-split.yml | 128 ++++++++++ .github/workflows/reusable-unit.yml | 107 ++++++++ .github/workflows/verify-ci.yml | 17 ++ 13 files changed, 830 insertions(+), 18 deletions(-) create mode 100755 .github/scripts/get_runner_classes.sh create mode 100755 .github/scripts/notify_slack.sh create mode 100755 .github/scripts/rerun_fails_report.sh create mode 100755 .github/scripts/set_test_package_matrix.sh create mode 100644 .github/workflows/frontend.yml create mode 100644 .github/workflows/go-tests.yml create mode 100644 .github/workflows/reusable-check-go-mod.yml create mode 100644 .github/workflows/reusable-dev-build.yml create mode 100644 .github/workflows/reusable-lint.yml create mode 100644 .github/workflows/reusable-unit-split.yml create mode 100644 .github/workflows/reusable-unit.yml create mode 100644 .github/workflows/verify-ci.yml diff --git a/.github/scripts/get_runner_classes.sh b/.github/scripts/get_runner_classes.sh new file mode 100755 index 000000000..4834e9e92 --- /dev/null +++ b/.github/scripts/get_runner_classes.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# This script generates tag-sets that can be used as runs-on: values to select runners. + +set -euo pipefail + +case "$GITHUB_REPOSITORY" in + *-enterprise) + # shellcheck disable=SC2129 + echo "compute-small=['self-hosted', 'linux', 'small']" >> "$GITHUB_OUTPUT" + echo "compute-medium=['self-hosted', 'linux', 'medium']" >> "$GITHUB_OUTPUT" + echo "compute-large=['self-hosted', 'linux', 'large']" >> "$GITHUB_OUTPUT" + echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5.2xlarge']" >> "$GITHUB_OUTPUT" + ;; + *) + # shellcheck disable=SC2129 + echo "compute-small=['custom-linux-s-consul-latest']" >> "$GITHUB_OUTPUT" + echo "compute-medium=['custom-linux-m-consul-latest']" >> "$GITHUB_OUTPUT" + echo "compute-large=['custom-linux-l-consul-latest']" >> "$GITHUB_OUTPUT" + echo "compute-xl=['custom-linux-xl-consul-latest']" >> "$GITHUB_OUTPUT" + ;; +esac diff --git a/.github/scripts/notify_slack.sh b/.github/scripts/notify_slack.sh new file mode 100755 index 000000000..b3dcdb210 --- /dev/null +++ b/.github/scripts/notify_slack.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -uo pipefail + +# This script is used in GitHub Actions pipelines to notify Slack of a job failure. + +if [[ $GITHUB_REF_NAME == "main" ]]; then + GITHUB_ENDPOINT="https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" + GITHUB_ACTIONS_ENDPOINT="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1) + SHORT_REF=$(git rev-parse --short "${GITHUB_SHA}") + curl -X POST -H 'Content-type: application/json' \ + --data \ + "{ \ + \"attachments\": [ \ + { \ + \"fallback\": \"GitHub Actions workflow failed!\", \ + \"text\": \"❌ Failed: \`${GITHUB_ACTOR}\`'s <${GITHUB_ACTIONS_ENDPOINT}|${GITHUB_JOB}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${GITHUB_REF_NAME}\`!\n\n- <${COMMIT_MESSAGE}\", \ + \"footer\": \"${GITHUB_REPOSITORY}\", \ + \"ts\": \"$(date +%s)\", \ + \"color\": \"danger\" \ + } \ + ] \ + }" "${FEED_CONSUL_GH_URL}" +else + echo "Not posting slack failure notifications for non-main branch" +fi diff --git a/.github/scripts/rerun_fails_report.sh b/.github/scripts/rerun_fails_report.sh new file mode 100755 index 000000000..ac6b7cf2f --- /dev/null +++ b/.github/scripts/rerun_fails_report.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# +# Add a comment on the github PR if there were any rerun tests. +# +set -eu -o pipefail + +report_filename="${1?report filename is required}" +if [ ! -s "$report_filename" ]; then + echo "gotestsum rerun report file is empty or missing" + exit 0 +fi + +function report { + echo ":repeat: gotestsum re-ran some tests in https://github.com/hashicorp/consul/actions/run/$GITHUB_RUN_ID" + echo + echo '```' + cat "$report_filename" + echo '```' +} + +report diff --git a/.github/scripts/set_test_package_matrix.sh b/.github/scripts/set_test_package_matrix.sh new file mode 100755 index 000000000..a9a21c747 --- /dev/null +++ b/.github/scripts/set_test_package_matrix.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +export RUNNER_COUNT=$1 + +# set matrix var to list of unique packages containing tests +matrix="$(go list -json="ImportPath,TestGoFiles" ./... | jq --compact-output '. | select(.TestGoFiles != null) | .ImportPath' | jq --slurp --compact-output '.' | jq --argjson runnercount $RUNNER_COUNT -cM '[_nwise(length / $runnercount | floor)]'))" + +echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 87d6f9605..620504e4e 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -8,26 +8,34 @@ permissions: contents: read jobs: - check-go-mod: - runs-on: ubuntu-22.04 + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 - with: - go-version-file: 'go.mod' - - run: go mod tidy - - run: | - if [[ -n $(git status -s) ]]; then - echo "Git directory has changes" - git status -s - exit 1 - fi + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + check-go-mod: + needs: + - setup + uses: ./.github/workflows/reusable-check-go-mod.yml + with: + runs-on: ${{ needs.setup.outputs.compute-medium }} build-386: - needs: check-go-mod + needs: + - setup + - check-go-mod env: XC_OS: "freebsd linux windows" - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 @@ -40,10 +48,12 @@ jobs: done build-amd64: - needs: check-go-mod + needs: + - setup + - check-go-mod env: XC_OS: "darwin freebsd linux solaris windows" - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 @@ -56,8 +66,10 @@ jobs: done build-arm: - needs: check-go-mod - runs-on: ubuntu-22.04 + needs: + - setup + - check-go-mod + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} env: CGO_ENABLED: 1 GOOS: linux diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000..e7d9e3855 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,94 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: frontend + +on: + push: + branches: + - main + - ui/** + - backport/ui/** + +permissions: + contents: read + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + workspace-tests: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + defaults: + run: + working-directory: ui + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - run: make test-workspace + + node-tests: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - run: make test-node + working-directory: ui/packages/consul-ui + + ember-build-test: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + env: + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam + CONSUL_NSPACES_ENABLED: 0 # NOTE: this should be 1 in ENT. + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - working-directory: ui/packages/consul-ui + run: | + make build-ci + node_modules/.bin/ember exam --path dist --silent -r xunit + + - working-directory: ui/packages/consul-ui + run: make test-coverage-ci diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml new file mode 100644 index 000000000..9c799fa9a --- /dev/null +++ b/.github/workflows/go-tests.yml @@ -0,0 +1,267 @@ +name: go-tests + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branches' prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + +permissions: + contents: read + +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: 1.8.2 + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + check-go-mod: + needs: + - setup + uses: ./.github/workflows/reusable-check-go-mod.yml + with: + runs-on: ${{ needs.setup.outputs.compute-small }} + + check-generated-protobuf: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: make proto-tools + name: Install protobuf + - run: make proto-format + name: "Protobuf Format" + - run: make --always-make proto + - run: | + if ! git diff --exit-code; then + echo "Generated code was not updated correctly" + exit 1 + fi + - run: make proto-lint + name: "Protobuf Lint" + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + check-generated-deep-copy: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: make codegen-tools + name: Install deep-copy + - run: make --always-make deep-copy + - run: | + if ! git diff --exit-code; then + echo "Generated code was not updated correctly" + exit 1 + fi + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + lint-enums: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + + lint-container-test-deps: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: make lint-container-test-deps + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + + lint-consul-retry: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + + lint: + needs: + - setup + uses: ./.github/workflows/reusable-lint.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + + lint-32bit: + needs: + - setup + uses: ./.github/workflows/reusable-lint.yml + with: + go-arch: "386" + runs-on: ${{ needs.setup.outputs.compute-xl }} + + + # create a development build + dev-build: + needs: + - setup + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + + # TODO(JM): - linux arm64 is not available in our self-hosted runners + # they are currently on the roadmap. + # # create a development build for arm64 + # dev-build-arm64: + # needs: + # - setup + # uses: ./.github/workflows/reusable-dev-build.yml + # with: + # uploaded-binary-name: 'consul-bin-arm64' + # # runs-on: ${{ needs.setup.outputs.compute-xl-arm64 }} + + # go-test-arm64: + # # TODO(JM): Fix to run on arm64 + # needs: + # - setup + # - dev-build-arm64 + # uses: ./.github/workflows/reusable-unit-split.yml + # with: + # directory: . + # uploaded-binary-name: 'consul-bin-arm64' + # runner-count: 12 + # # runs-on: ${{ needs.setup.outputs.compute-xl-arm64 }} + # go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' + + go-test: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit-split.yml + with: + directory: . + runner-count: 12 + runs-on: ${{ needs.setup.outputs.compute-xl }} + + + go-test-race: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: . + go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"' + package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul/command/'" + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-32bit: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: . + go-arch: "386" + go-test-flags: 'export GO_TEST_FLAGS="-short"' + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-envoyextensions: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: envoyextensions + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-troubleshoot: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: troubleshoot + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-api-1-19: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: api + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-api-1-20: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: api + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-sdk-1-19: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: sdk + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-sdk-1-20: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: sdk + runs-on: ${{ needs.setup.outputs.compute-xl }} + + noop: + runs-on: ubuntu-latest + steps: + - run: "echo ok" diff --git a/.github/workflows/reusable-check-go-mod.yml b/.github/workflows/reusable-check-go-mod.yml new file mode 100644 index 000000000..ab6ef2420 --- /dev/null +++ b/.github/workflows/reusable-check-go-mod.yml @@ -0,0 +1,28 @@ +name: check-go-mod + +on: + workflow_call: + inputs: + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string +jobs: + check-go-mod: + runs-on: ${{ fromJSON(inputs.runs-on) }} + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go mod tidy + - run: | + if [[ -n $(git status -s) ]]; then + echo "Git directory has changes" + git status -s + exit 1 + fi + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml new file mode 100644 index 000000000..d4832fb0a --- /dev/null +++ b/.github/workflows/reusable-dev-build.yml @@ -0,0 +1,31 @@ +name: reusable-dev-build + +on: + workflow_call: + inputs: + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string +jobs: + build: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - name: Build + run: make dev + # save dev build to pass to downstream jobs + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: ./bin/consul + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-lint.yml b/.github/workflows/reusable-lint.yml new file mode 100644 index 000000000..121d9231f --- /dev/null +++ b/.github/workflows/reusable-lint.yml @@ -0,0 +1,47 @@ +name: reusable-lint + +on: + workflow_call: + inputs: + go-arch: + required: false + type: string + default: "" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + +env: + GOTAGS: "" # No tags for OSS but there are for enterprise + GOARCH: ${{inputs.go-arch}} + +jobs: + lint: + runs-on: ${{ fromJSON(inputs.runs-on) }} + strategy: + matrix: + directory: + - "" + - "api" + - "sdk" + - "envoyextensions" + - "troubleshoot" + - "test/integration/consul-container" + fail-fast: true + name: lint ${{ matrix.directory }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + - name: lint-${{ matrix.directory }} + uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # pin@v3.4.0 + with: + working-directory: ${{ matrix.directory }} + version: v1.51 + args: --build-tags="${{ env.GOTAGS }}" -v + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml new file mode 100644 index 000000000..fbd4d859b --- /dev/null +++ b/.github/workflows/reusable-unit-split.yml @@ -0,0 +1,128 @@ +name: reusable-unit-split + +on: + workflow_call: + inputs: + directory: + required: true + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + go-arch: + required: false + type: string + default: "" + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + args: + required: false + type: string + default: "" + runner-count: + required: false + type: number + default: 1 + go-test-flags: + required: false + type: string + default: "" + +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: 1.8.2 + GOARCH: ${{inputs.go-arch}} + TOTAL_RUNNERS: ${{inputs.runner-count}} + +jobs: + set-test-package-matrix: + runs-on: ubuntu-latest + outputs: + package-matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - id: set-matrix + run: ./.github/scripts/set_test_package_matrix.sh ${{env.TOTAL_RUNNERS}} + + go-test: + runs-on: ${{ fromJSON(inputs.runs-on) }} + name: "go-test" + needs: + - set-test-package-matrix + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.set-test-package-matrix.outputs.package-matrix) }} + steps: + - name: ulimit + run: | + echo "Soft limits" + ulimit -Sa + echo "Hard limits" + ulimit -Ha + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + cache: true + - name: Install gotestsum + run: | + wget https://github.com/gotestyourself/gotestsum/releases/download/v${{env.GOTESTSUM_VERSION}}/gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + sudo tar -C /usr/local/bin -xzf gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + rm gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + - run: mkdir -p ${{env.TEST_RESULTS}} + - name: go mod download + working-directory: ${{inputs.directory}} + run: go mod download + - name: Download consul + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: /usr/local/bin + - name: Make sure consul is executable + run: sudo chmod +x /usr/local/bin/consul + - run: go env + - name: Run tests + working-directory: ${{inputs.directory}} + run: | + # separate the list + PACKAGE_NAMES="${{ join(matrix.package, ' ') }}" + # PACKAGE_NAMES="${{ matrix.package }}" + + ${{inputs.go-test-flags}} + + # some tests expect this umask, and arm images have a different default + umask 0022 + + gotestsum \ + --format=short-verbose \ + --jsonfile /tmp/jsonfile/go-test.log \ + --debug \ + --rerun-fails=3 \ + --rerun-fails-max-failures=40 \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="$PACKAGE_NAMES" \ + --junitfile ${{env.TEST_RESULTS}}/gotestsum-report.xml -- \ + -tags="${{env.GOTAGS}}" -p 2 \ + ${GO_TEST_FLAGS-} \ + -cover -coverprofile=coverage.txt + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: test-results + path: ${{env.TEST_RESULTS}} + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: jsonfile + path: /tmp/jsonfile + - name: "Re-run fails report" + run: | + .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml new file mode 100644 index 000000000..24c20abe5 --- /dev/null +++ b/.github/workflows/reusable-unit.yml @@ -0,0 +1,107 @@ +name: reusable-unit + +on: + workflow_call: + inputs: + directory: + required: true + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + go-arch: + required: false + type: string + default: "" + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + package-names-command: + required: false + type: string + default: 'go list -tags "$GOTAGS" ./...' + go-test-flags: + required: false + type: string + default: "" + +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: 1.8.2 + GOARCH: ${{inputs.go-arch}} + +jobs: + go-test: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - name: Setup go mod cache + uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # pin@v3.2.6 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Install gotestsum + run: | + wget https://github.com/gotestyourself/gotestsum/releases/download/v${{env.GOTESTSUM_VERSION}}/gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + sudo tar -C /usr/local/bin -xzf gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + rm gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + - run: mkdir -p ${{env.TEST_RESULTS}} + - name: go mod download + working-directory: ${{inputs.directory}} + run: go mod download + - name: Download consul + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: /usr/local/bin + - name: Make sure consul is executable + run: sudo chmod +x /usr/local/bin/consul + - name: Display downloaded file + run: ls -ld consul + working-directory: /usr/local/bin + - run: go env + - name: Run tests + working-directory: ${{inputs.directory}} + run: | + PACKAGE_NAMES=$(${{inputs.package-names-command}}) + + # some tests expect this umask, and arm images have a different default + umask 0022 + + ${{inputs.go-test-flags}} + + gotestsum \ + --format=short-verbose \ + --jsonfile /tmp/jsonfile/go-test.log \ + --debug \ + --rerun-fails=3 \ + --rerun-fails-max-failures=40 \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="$PACKAGE_NAMES" \ + --junitfile ${{env.TEST_RESULTS}}/gotestsum-report.xml -- \ + -tags="${{env.GOTAGS}}" \ + ${GO_TEST_FLAGS-} \ + -cover -coverprofile=coverage.txt + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: test-results + path: ${{env.TEST_RESULTS}} + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: jsonfile + path: /tmp/jsonfile + - name: "Re-run fails report" + run: | + .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/verify-ci.yml b/.github/workflows/verify-ci.yml new file mode 100644 index 000000000..16fb4db25 --- /dev/null +++ b/.github/workflows/verify-ci.yml @@ -0,0 +1,17 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# verify-ci is a no-op workflow that must run on every PR. It is used in a +# branch protection rule to detect when CI workflows are not running. +name: verify-ci + +permissions: + contents: read + +on: [pull_request] + +jobs: + noop: + runs-on: ubuntu-latest + steps: + - run: echo "ok"