Migrating CircleCI Jobs to GHA Workflow (#19662)

* address lint reports

* add diff-oss-ci and test-ui jobs to ci GHA workflow

* Add actions linter workflow

* Fix actions linter errors

* pin 3rd party components with SHA hash and limit actionlint workflow to pull requests touching paths under .github directory

* Fix actionlint runner

* pin SHA hash of 3rd party components
use .go-version file to provide go version to setup-go action
remove unncessary ref parameter in checkout action

---------

Co-authored-by: Brian Shore <bshore@hashicorp.com>
This commit is contained in:
Marc Boudreau 2023-03-22 15:02:06 -04:00 committed by GitHub
parent 06e3f971ef
commit 4ad111b3dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 224 additions and 75 deletions

10
.github/actionlint.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
self-hosted-runner:
# Labels of self-hosted runner in array of string
labels:
- small
- large
- ondemand

15
.github/workflows/actionlint.yml vendored Normal file
View File

@ -0,0 +1,15 @@
name: Lint GitHub Actions Workflows
on:
push:
pull_request:
paths:
- '.github/**'
jobs:
actionlint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
- name: "Check workflow files"
uses: docker://docker.mirror.hashicorp.services/rhysd/actionlint@sha256:93834930f56ca380be3e9a3377670d7aa5921be251b9c774891a39b3629b83b8

View File

@ -63,7 +63,7 @@ jobs:
env:
GOARCH: ${{ inputs.goarch }}
GOOS: ${{ inputs.goos }}
run: echo "ARTIFACT_BASENAME=$(make ci-get-artifact-basename)" >> $GITHUB_ENV
run: echo "ARTIFACT_BASENAME=$(make ci-get-artifact-basename)" >> "$GITHUB_ENV"
- name: Bundle Vault
env:
BUNDLE_PATH: out/${{ env.ARTIFACT_BASENAME }}.zip
@ -93,8 +93,8 @@ jobs:
- if: ${{ inputs.create-packages }}
name: Determine package file names
run: |
echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV
echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV
echo "RPM_PACKAGE=$(basename out/*.rpm)" >> "$GITHUB_ENV"
echo "DEB_PACKAGE=$(basename out/*.deb)" >> "$GITHUB_ENV"
- if: ${{ inputs.create-packages }}
uses: actions/upload-artifact@v3
with:

View File

@ -31,13 +31,14 @@ jobs:
# enos-run-matrices.
MATRIX_MAX_TEST_GROUPS: 5
run: |
echo "build-date=$(make ci-get-date)" >> $GITHUB_OUTPUT
echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT
echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> $GITHUB_OUTPUT
echo "package-name=vault" >> $GITHUB_OUTPUT
echo "vault-base-version=$(make ci-get-version-base)" >> $GITHUB_OUTPUT
echo "vault-revision=$(make ci-get-revision)" >> $GITHUB_OUTPUT
echo "vault-version=$(make ci-get-version)" >> $GITHUB_OUTPUT
# shellcheck disable=SC2129
echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT"
echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT"
echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT"
echo "package-name=vault" >> "$GITHUB_OUTPUT"
echo "vault-base-version=$(make ci-get-version-base)" >> "$GITHUB_OUTPUT"
echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT"
echo "vault-version=$(make ci-get-version)" >> "$GITHUB_OUTPUT"
- uses: hashicorp/actions-generate-metadata@v1
id: generate-metadata-file
with:

View File

@ -56,9 +56,9 @@ jobs:
# Else, we found some toolchain files. Let's make sure the contents are correct.
if ! grep -q 'release-note:change' "$toolchain_files" || ! grep -q '^core: Bump Go version to' "$toolchain_files"; then
echo "Invalid format for changelog. Expected format:"
echo "```release-note:change"
echo '```release-note:change'
echo "core: Bump Go version to x.y.z."
echo "```"
echo '```'
exit 1
else
echo "Found Go toolchain changelog entry in PR!"

View File

@ -21,15 +21,17 @@ jobs:
github_repository="${{ github.repository }}"
if [ "${github_repository##*/}" == "vault-enterprise" ] ; then
echo 'compute-standard=["self-hosted","ondemand","linux","type=m5.xlarge"]' >> $GITHUB_OUTPUT
echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> $GITHUB_OUTPUT
echo 'enterprise=1' >> $GITHUB_OUTPUT
echo 'go-build-tags=ent,enterprise' >> $GITHUB_OUTPUT
# shellcheck disable=SC2129
echo 'compute-standard=["self-hosted","ondemand","linux","type=m5.xlarge"]' >> "$GITHUB_OUTPUT"
echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> "$GITHUB_OUTPUT"
echo 'enterprise=1' >> "$GITHUB_OUTPUT"
echo 'go-build-tags=ent,enterprise' >> "$GITHUB_OUTPUT"
else
echo 'compute-standard=["custom", "linux", "small"]' >> $GITHUB_OUTPUT
echo 'compute-larger=["custom", "linux", "medium"]' >> $GITHUB_OUTPUT
echo 'enterprise=' >> $GITHUB_OUTPUT
echo 'go-build-tags=' >> $GITHUB_OUTPUT
# shellcheck disable=SC2129
echo 'compute-standard=["custom", "linux", "small"]' >> "$GITHUB_OUTPUT"
echo 'compute-larger=["custom", "linux", "medium"]' >> "$GITHUB_OUTPUT"
echo 'enterprise=' >> "$GITHUB_OUTPUT"
echo 'go-build-tags=' >> "$GITHUB_OUTPUT"
fi
semgrep:
name: Semgrep
@ -70,6 +72,38 @@ jobs:
echo "Code has formatting errors. Run 'make fmt' to fix"
exit 1
fi
diff-oss-ci:
name: Diff OSS
needs:
- setup
if: ${{ needs.setup.outputs.enterprise != '' && github.base_ref != '' }}
runs-on: ['self-hosted','ondemand','small']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
with:
fetch-depth: 0
- id: determine-branch
run: |
branch="${{ github.base_ref }}"
if [[ $branch = release/* ]] ; then
branch=${branch%%+ent}
# Add OSS remote
git config --global user.email "github-team-secret-vault-core@hashicorp.com"
git config --global user.name "hc-github-team-secret-vault-core"
git remote add oss https://github.com/hashicorp/vault.git
git fetch oss "$branch"
branch="oss/$branch"
else
branch="origin/$branch"
fi
echo "BRANCH=$branch" >> "$GITHUB_OUTPUT"
- id: diff
run: |
./.github/scripts/oss-diff.sh ${{ steps.determine-branch.outputs.BRANCH }} HEAD
test-go:
name: Run Go tests
needs:
@ -101,3 +135,96 @@ jobs:
runs-on: ${{ needs.setup.outputs.compute-larger }}
enterprise: ${{ needs.setup.outputs.enterprise }}
secrets: inherit
test-ui:
name: Test UI
# The test-ui Job is only run for pushes to main, ui/*, backport/ui/*, release/*, and merge*
if: github.ref_name == 'main' || startsWith(github.ref_name, 'ui/') || startsWith(github.ref_name, 'backport/ui/') || startsWith(github.ref_name, 'release/') || startsWith(github.ref_name, 'merge')
needs:
- setup
permissions:
id-token: write
contents: read
runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
- uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613
with:
go-version-file: ./.go-version
cache: true
# Setup node.js without caching to allow running npm install -g yarn (next step)
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
with:
node-version: 14
- id: install-yarn
run: |
npm install -g yarn
# Setup node.js with caching using the yarn.lock file
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
with:
node-version: 14
cache: yarn
cache-dependency-path: ui/yarn.lock
- id: install-browser-libraries
run: sudo apt install -y libnss3-dev libgdk-pixbuf2.0-dev libgtk-3-dev libxss-dev libasound2
- id: install-browser
uses: browser-actions/setup-chrome@597130847c84cdac5acceccbd676d612e6f8beb8
- id: ui-dependencies
name: ui-dependencies
working-directory: ./ui
run: |
yarn install --frozen-lockfile
npm rebuild node-sass
- id: vault-auth
name: Authenticate to Vault
if: github.repository == 'hashicorp/vault-enterprise'
run: vault-auth
- id: secrets
name: Fetch secrets
if: github.repository == 'hashicorp/vault-enterprise'
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/hashicorp/vault-enterprise/github-token token | PRIVATE_REPO_GITHUB_TOKEN;
kv/data/github/hashicorp/vault-enterprise/license license_1 | VAULT_LICENSE;
- id: setup-git
name: Setup Git
if: github.repository == 'hashicorp/vault-enterprise'
env:
PRIVATE_REPO_GITHUB_TOKEN: ${{ steps.secrets.outputs.PRIVATE_REPO_GITHUB_TOKEN }}
run: |
git config --global url."https://hc-github-team-secure-vault-core:${PRIVATE_REPO_GITHUB_TOKEN}@github.com".insteadOf https://github.com
- id: build-go-dev
name: build-go-dev
run: |
rm -rf ./pkg
mkdir ./pkg
make ci-bootstrap dev
- id: test-ui
name: test-ui
env:
VAULT_LICENSE: ${{ steps.secrets.outputs.VAULT_LICENSE }}
run: |
export PATH="${PWD}/bin:${PATH}"
if [ "${{ github.repository }}" == 'hashicorp/vault' ] ; then
export VAULT_LICENSE="${{ secrets.VAULT_LICENSE }}"
fi
# Run Ember tests
cd ui
mkdir -p test-results/qunit
yarn test:oss
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: test-results-ui
path: ui/test-results
if: always()
- uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f
with:
paths: "ui/test-results/qunit/results.xml"
show: "fail"
if: always()

View File

@ -27,9 +27,10 @@ jobs:
# enos-run-matrices.
MATRIX_MAX_TEST_GROUPS: 2
run: |
echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> $GITHUB_OUTPUT
echo "vault-revision=$(make ci-get-revision)" >> $GITHUB_OUTPUT
echo "vault-version=$(make ci-get-version)" >> $GITHUB_OUTPUT
# shellcheck disable=SC2129
echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT"
echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT"
echo "vault-version=$(make ci-get-version)" >> "$GITHUB_OUTPUT"
test:
name: Test ${{ matrix.build-artifact-name }}

View File

@ -47,7 +47,7 @@ jobs:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
- name: Download Docker Image
id: download
uses: actions/download-artifact@v3
uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
with:
name: ${{ inputs.artifact-name }}
path: ./enos/support/downloads
@ -56,16 +56,16 @@ jobs:
IS_ENT: ${{ startsWith(env.ARTIFACT_NAME, 'vault-enterprise' ) }}
run: |
mkdir -p ./enos/support/terraform-plugin-cache
if ${IS_ENT} == true; then
if [ "$IS_ENT" == true ]; then
echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
echo "edition=ent" >> $GITHUB_ENV
echo "edition=ent" >> "$GITHUB_ENV"
echo "edition set to 'ent'"
echo "image_repo=hashicorp/vault-enterprise" >> $GITHUB_ENV
echo "image_repo=hashicorp/vault-enterprise" >> "$GITHUB_ENV"
echo "image repo set to 'hashicorp/vault-enterprise'"
else
echo "edition=oss" >> $GITHUB_ENV
echo "edition=oss" >> "$GITHUB_ENV"
echo "edition set to 'oss'"
echo "image_repo=hashicorp/vault" >> $GITHUB_ENV
echo "image_repo=hashicorp/vault" >> "$GITHUB_ENV"
echo "image repo set to 'hashicorp/vault'"
fi
- name: Run Enos scenario

View File

@ -11,17 +11,13 @@ jobs:
godoc-test-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: get metadata
id: get-metadata
run: echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT
- name: Set Up Go
uses: actions/setup-go@v3
uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613
with:
cache: true
go-version: ${{ steps.get-metadata.outputs.go-version }}
go-version-file: ./.go-version
- name: Verify new tests have go docs
run: make ci-vet-godoctests
run: make ci-vet-godoctests

View File

@ -58,15 +58,15 @@ jobs:
- 'ui/**'
- name: "Default to core board"
run: echo "PROJECT=170" >> $GITHUB_ENV
run: echo "PROJECT=170" >> "$GITHUB_ENV"
- if: github.event.pull_request != null && steps.changes.outputs.cryptosec == 'true'
run: echo "PROJECT=172" >> $GITHUB_ENV
run: echo "PROJECT=172" >> "$GITHUB_ENV"
- if: github.event.pull_request != null && steps.changes.outputs.ecosystem == 'true'
run: echo "PROJECT=169" >> $GITHUB_ENV
run: echo "PROJECT=169" >> "$GITHUB_ENV"
- if: github.event.pull_request != null && steps.changes.outputs.devex == 'true'
run: echo "PROJECT=176" >> $GITHUB_ENV
run: echo "PROJECT=176" >> "$GITHUB_ENV"
- if: github.event.pull_request != null && steps.changes.outputs.ui == 'true'
run: echo "PROJECT=171" >> $GITHUB_ENV
run: echo "PROJECT=171" >> "$GITHUB_ENV"
- uses: actions/add-to-project@v0.3.0
with:
@ -125,4 +125,4 @@ jobs:
# ) {
# deletedItemId
# }
# }' -f project_id=$PROJECT_ID -f item_id=$item_id || true
# }' -f project_id=$PROJECT_ID -f item_id=$item_id || true

View File

@ -10,8 +10,7 @@ on:
jobs:
scan:
runs-on:
labels: ['linux', 'large']
runs-on: ['linux', 'large']
if: ${{ github.actor != 'dependabot[bot]' || github.actor != 'hc-github-team-secure-vault-core' }}
steps:
- uses: actions/checkout@v3
@ -39,14 +38,14 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mkdir $HOME/.bin
cd $GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-semgrep
mkdir "$HOME/.bin"
cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-semgrep"
go build -o scan-plugin-semgrep .
mv scan-plugin-semgrep $HOME/.bin
mv scan-plugin-semgrep "$HOME/.bin"
cd $GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-codeql
cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-codeql"
go build -o scan-plugin-codeql .
mv scan-plugin-codeql $HOME/.bin
mv scan-plugin-codeql "$HOME/.bin"
# Semgrep
python3 -m pip install semgrep
@ -54,11 +53,11 @@ jobs:
# CodeQL
LATEST=$(gh release list --repo https://github.com/github/codeql-action | cut -f 3 | sort --version-sort | tail -n1)
gh release download --repo https://github.com/github/codeql-action --pattern codeql-bundle-linux64.tar.gz "$LATEST"
tar xf codeql-bundle-linux64.tar.gz -C $HOME/.bin
tar xf codeql-bundle-linux64.tar.gz -C "$HOME/.bin"
# Add to PATH
echo "$HOME/.bin" >> $GITHUB_PATH
echo "$HOME/.bin/codeql" >> $GITHUB_PATH
echo "$HOME/.bin" >> "$GITHUB_PATH"
echo "$HOME/.bin/codeql" >> "$GITHUB_PATH"
- name: Scan
id: scan

View File

@ -22,7 +22,7 @@ jobs:
- name: Get all regions
id: setup
run: |
echo "regions=$(aws ec2 describe-regions --region us-east-1 --output json --query 'Regions[].RegionName' | tr -d '\n ')" >> $GITHUB_OUTPUT
echo "regions=$(aws ec2 describe-regions --region us-east-1 --output json --query 'Regions[].RegionName' | tr -d '\n ')" >> "$GITHUB_OUTPUT"
aws-nuke:
needs: setup

View File

@ -42,16 +42,16 @@ jobs:
env:
IS_ENT: ${{ startsWith(github.event.repository.name, 'vault-enterprise' ) }}
run: |
echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT
echo "node-version=$(cat ./ui/.nvmrc)" >> $GITHUB_OUTPUT
if ${IS_ENT} == true; then
echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT"
echo "node-version=$(cat ./ui/.nvmrc)" >> "$GITHUB_OUTPUT"
if [ "$IS_ENT" == true ]; then
echo "detected vault_edition=ent"
echo "runs-on=['self-hosted', 'ondemand', 'os=linux', 'type=m5d.4xlarge']" >> $GITHUB_OUTPUT
echo "vault_edition=ent" >> $GITHUB_OUTPUT
echo "runs-on=['self-hosted', 'ondemand', 'os=linux', 'type=m5d.4xlarge']" >> "$GITHUB_OUTPUT"
echo "vault_edition=ent" >> "$GITHUB_OUTPUT"
else
echo "detected vault_edition=oss"
echo "runs-on=\"custom-linux-xl-vault-latest\"" >> $GITHUB_OUTPUT
echo "vault_edition=oss" >> $GITHUB_OUTPUT
echo "runs-on=\"custom-linux-xl-vault-latest\"" >> "$GITHUB_OUTPUT"
echo "vault_edition=oss" >> "$GITHUB_OUTPUT"
fi
run-ui-tests:
@ -103,7 +103,7 @@ jobs:
run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
- name: Check Chrome Installed
id: chrome-check
run: echo "chrome-version=$(chrome --version 2> /dev/null || google-chrome --version 2> /dev/null || google-chrome-stable --version 2> /dev/null || echo 'not-installed')" >> $GITHUB_OUTPUT
run: echo "chrome-version=$(chrome --version 2> /dev/null || google-chrome --version 2> /dev/null || google-chrome-stable --version 2> /dev/null || echo 'not-installed')" >> "$GITHUB_OUTPUT"
- name: Install Chrome Dependencies
if: steps.chrome-check.outputs.chrome-version == 'not-installed'
run: |

View File

@ -52,9 +52,9 @@ jobs:
steps:
- id: generate-index-list
run: |
INDEX_LIST=$(seq 1 ${{ env.total-runners }})
INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST})
echo "indexes=${INDEX_JSON}" >> ${GITHUB_OUTPUT}
INDEX_LIST="$(seq 1 ${{ env.total-runners }})"
INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")"
echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}"
test-go:
permissions:
id-token: write # Note: this permission is explicitly required for Vault auth
@ -134,6 +134,7 @@ jobs:
# CI commands, like the UI tests, shouldn't have to worry about licensing.
# So we provide the tests which want an externally supplied license with licenses
# via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it.
# shellcheck disable=SC2034
VAULT_LICENSE=
# Assign test licenses to relevant variables if they aren't already
@ -150,6 +151,7 @@ jobs:
#export HCP_SCADA_ADDRESS=${{ secrets.HCP_SCADA_ADDRESS }}
fi
# shellcheck disable=SC2086 # can't quote package list
GOARCH=${{ inputs.go-arch }} \
go run gotest.tools/gotestsum --format=short-verbose \
--junitfile test-results/go-test/results.xml \

View File

@ -20,15 +20,13 @@ jobs:
go-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- id: get-metadata
run: echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
- name: Set Up Go
uses: actions/setup-go@v3
uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613
with:
go-version: ${{ steps.get-metadata.outputs.go-version }}
go-version-file: ./.go-version
- run: go test -v ./${{ inputs.path }}/... 2>&1 | tee ${{ inputs.name }}.txt
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: ${{ inputs.name }}-output
path: ${{ inputs.name }}.txt

View File

@ -77,10 +77,10 @@ jobs:
ref: ${{ inputs.vault-revision }}
- id: metadata
run: |
echo "build-date=$(make ci-get-date)" >> $GITHUB_OUTPUT
echo "version=$(make ci-get-version)" >> $GITHUB_OUTPUT
filtered=$(make ci-filter-matrix)
echo "matrix=$(echo $filtered)}" >> $GITHUB_OUTPUT
# shellcheck disable=SC2129
echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT"
echo "version=$(make ci-get-version)" >> "$GITHUB_OUTPUT"
echo "matrix=$(make ci-filter-matrix)" >> "$GITHUB_OUTPUT"
# Run the Enos test scenarios
run: