In order to reliably store Go test times in the Github Actions cache we need to reduce our cache thrashing by not using more than 10gb over all of our caches. This change reduces our cache usage significantly by sharing Go module cache between our Go CI workflows and our build workflows. We lose our per-builder cache which will result in a bit of performance hit, but we'll enable better automatic rebalancing of our CI workflows. Overall we should see a per branch reduction in cache sizes from ~17gb to ~850mb. Some preliminary investigation into this new strategy: Prior build workflow strategy on a cache miss: Download modules: ~20s Build Vault: ~40s Upload cache: ~30s Total: ~1m30s Prior build workflow strategy on a cache hit: Download and decompress modules and build cache: ~12s Build Vault: ~15s Total: ~28s New build workflow strategy on a cache miss: Download modules: ~20 Build Vault: ~40s Upload cache: ~6s Total: ~1m6s New build workflow strategy on a cache hit: Download and decompress modules: ~3s Build Vault: ~40s Total: ~43s Expected time if we used no Go caching: Download modules: ~20 Build Vault: ~40s Total: ~1m Signed-off-by: Ryan Cragun <me@ryan.ec> Co-authored-by: Ryan Cragun <me@ryan.ec>
This commit is contained in:
hc-github-team-secure-vault-core
GitHub
parent
841507c314
commit
59cbdcda39
|
|
@ -0,0 +1,73 @@
|
|||
---
|
||||
name: Set up Go with a shared module cache
|
||||
description: Set up Go with a shared module cache
|
||||
|
||||
inputs:
|
||||
github-token:
|
||||
description: "An elevated Github token to access private modules if necessary"
|
||||
type: string
|
||||
no-restore:
|
||||
description: "Whether or not to restore the Go module cache on a cache hit"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
outputs:
|
||||
cache-key:
|
||||
description: "The Go modules cache key"
|
||||
value: ${{ steps.metadata.outputs.cache-key }}
|
||||
cache-path:
|
||||
description: "The GOMODCACHE path"
|
||||
value: ${{ steps.metadata.outputs.cache-path }}
|
||||
go-version:
|
||||
description: "The version of Go in the .go-version file"
|
||||
value: ${{ steps.go-version.outputs.go-version }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- id: go-version
|
||||
shell: bash
|
||||
run: echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
with:
|
||||
go-version: ${{ steps.go-version.outputs.go-version }}
|
||||
cache: false # We use our own caching strategy
|
||||
- id: metadata
|
||||
shell: bash
|
||||
run: |
|
||||
echo "cache-path=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
|
||||
echo "cache-key=go-modules-${{ hashFiles('**/go.sum') }}" >> "$GITHUB_OUTPUT"
|
||||
- id: cache-modules
|
||||
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
||||
with:
|
||||
enableCrossOsArchive: true
|
||||
lookup-only: ${{ inputs.no-restore }}
|
||||
# We need to be very considerate of our caching strategy because Github only allows 10gb
|
||||
# of caches per repository before it starts to evict older caches. This is usually fine
|
||||
# if you only use the actions cache for cache, but we also use it for Go test time results.
|
||||
# These results are used to balance our Go test groups, without which we could have
|
||||
# painfully unbalanced Go test execution times. We have to ensure current caches for all
|
||||
# active release branches and main do not exceed 10gb. Ideally we'd cache Go modules
|
||||
# and Go build cache on a per version/platform/architecture/tag/module basis, but that
|
||||
# would result in several hungred gb over all of our build workflows and release branches.
|
||||
# Instead, we've chosen a middle ground approach where were share Go modules between build
|
||||
# workflows but lose the Go build cache.
|
||||
# We intentionally do not use partial restore keys. If we get dont get an exact cache hit
|
||||
# we only want to download the latest modules, not append them to a prior cache. This
|
||||
# keeps cache upload time, download time, and storage size to a minimum.
|
||||
path: ${{ steps.metadata.outputs.cache-path }}
|
||||
key: ${{ steps.metadata.outputs.cache-key }}
|
||||
- if: steps.cache-modules.outputs.cache-hit != 'true'
|
||||
name: Download go modules
|
||||
shell: bash
|
||||
run: |
|
||||
git config --global url."https://${{ inputs.github-token }}@github.com".insteadOf https://github.com
|
||||
# go list ./... forces downloading some additional versions of modules that 'go mod
|
||||
# download' misses. We need this because we make use of go list itself during
|
||||
# code generation in later builds that rely on this module cache.
|
||||
go list ./...
|
||||
go list -test ./...
|
||||
go mod download
|
||||
( cd sdk && go mod download )
|
||||
( cd api && go mod download )
|
||||
|
|
@ -24,16 +24,8 @@ on:
|
|||
goarch:
|
||||
required: true
|
||||
type: string
|
||||
go-cache:
|
||||
required: true
|
||||
type: string
|
||||
go-mod-cache:
|
||||
required: true
|
||||
type: string
|
||||
go-tags:
|
||||
type: string
|
||||
go-version:
|
||||
type: string
|
||||
package-name:
|
||||
type: string
|
||||
default: vault
|
||||
|
|
@ -50,33 +42,16 @@ jobs:
|
|||
name: Vault ${{ inputs.goos }} ${{ inputs.goarch }} v${{ inputs.vault-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version: ${{ inputs.go-version }}
|
||||
cache: false # Use our own caching strategy for better cross platform support
|
||||
- name: Set up Go cache key tags
|
||||
id: cache-key-tags
|
||||
run: echo "gotags=$(echo ${{ inputs.go-tags }} | tr ' ' '-')" >> "$GITHUB_ENV"
|
||||
- name: Set up Go cache
|
||||
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
||||
with:
|
||||
path: |
|
||||
${{ inputs.go-cache }}
|
||||
${{ inputs.go-mod-cache }}
|
||||
# Manage the Go cache for each build workflow individually. This ensures that only relevant
|
||||
# module and build cache for that specific combination kept. This helps reduce our cache
|
||||
# download and speeds up compiling because the build cache is always preserved.
|
||||
key: go-${{ inputs.go-version }}-${{ inputs.goos }}-${{ inputs.goarch }}-${{ env.gotags }}-${{ hashFiles('**/go.sum') }}
|
||||
# We intentionally omit partial restore keys to ensure that we always create a new cache
|
||||
# if we don't get a hit. That ensures that we only keep up-to-date modules and build cache.
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
- name: Restore UI from cache
|
||||
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
||||
with:
|
||||
# Restore the UI asset from the UI build workflow. Never use a partial restore key.
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
path: http/web_ui
|
||||
# Only restore the UI asset cache if we haven't modified anything in the ui directory.
|
||||
# Never do a partial restore of the web_ui if we don't get a cache hit.
|
||||
key: ${{ inputs.web-ui-cache-key }}
|
||||
- name: Build Vault
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -8,14 +8,16 @@ on:
|
|||
# This is insufficient for our needs, since we're skipping stuff on PRs in
|
||||
# draft mode. By adding the ready_for_review type, when a draft pr is marked
|
||||
# ready, we run everything, including the stuff we'd have skipped up until now.
|
||||
types: [ opened, synchronize, reopened, ready_for_review ]
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- release/**
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.run_id }}-build
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# verify-changes determines if the changes are only for docs (website)
|
||||
verify-changes:
|
||||
|
|
@ -46,24 +48,19 @@ jobs:
|
|||
outputs:
|
||||
build-date: ${{ steps.get-metadata.outputs.build-date }}
|
||||
filepath: ${{ steps.generate-metadata-file.outputs.filepath }}
|
||||
go-cache: ${{ steps.get-metadata.outputs.go-cache }}
|
||||
go-mod-cache: ${{ steps.get-metadata.outputs.go-mod-cache }}
|
||||
go-version: ${{ steps.go-version.outputs.go-version }}
|
||||
matrix-test-group: ${{ steps.get-metadata.outputs.matrix-test-group }}
|
||||
package-name: ${{ steps.get-metadata.outputs.package-name }}
|
||||
vault-revision: ${{ steps.get-metadata.outputs.vault-revision }}
|
||||
vault-version: ${{ steps.get-metadata.outputs.vault-version }}
|
||||
vault-base-version: ${{ steps.get-metadata.outputs.vault-base-version }}
|
||||
web-ui-cache-key: ui-${{ steps.get-metadata.outputs.web-ui-cache-key }}
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- name: Determine Go version
|
||||
id: go-version
|
||||
run: echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- name: Ensure Go modules are cached
|
||||
uses: ./.github/actions/set-up-go
|
||||
id: set-up-go
|
||||
with:
|
||||
go-version: ${{ steps.go-version.outputs.go-version }}
|
||||
cache: false
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
no-restore: true # don't download them on a cache hit
|
||||
- name: Get metadata
|
||||
id: get-metadata
|
||||
env:
|
||||
|
|
@ -74,13 +71,10 @@ jobs:
|
|||
run: |
|
||||
# shellcheck disable=SC2129
|
||||
echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT"
|
||||
echo "go-cache=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
|
||||
echo "go-mod-cache=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
|
||||
echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT"
|
||||
echo "package-name=vault" >> "$GITHUB_OUTPUT"
|
||||
echo "vault-base-version=$(make ci-get-version-base)" >> "$GITHUB_OUTPUT"
|
||||
echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT"
|
||||
echo "web-ui-cache-key=$(git ls-tree HEAD ui --object-only)" >> "$GITHUB_OUTPUT"
|
||||
echo "vault-version=$(make ci-get-version)" >> "$GITHUB_OUTPUT"
|
||||
- uses: hashicorp/actions-generate-metadata@v1
|
||||
id: generate-metadata-file
|
||||
|
|
@ -146,10 +140,7 @@ jobs:
|
|||
create-packages: false
|
||||
goarch: ${{ matrix.goarch }}
|
||||
goos: ${{ matrix.goos }}
|
||||
go-cache: ${{ needs.product-metadata.outputs.go-cache }}
|
||||
go-mod-cache: ${{ needs.product-metadata.outputs.go-mod-cache }}
|
||||
go-tags: ui
|
||||
go-version: ${{ needs.product-metadata.outputs.go-version }}
|
||||
package-name: ${{ needs.product-metadata.outputs.package-name }}
|
||||
web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }}
|
||||
vault-version: ${{ needs.product-metadata.outputs.vault-version }}
|
||||
|
|
@ -169,10 +160,7 @@ jobs:
|
|||
with:
|
||||
goarch: ${{ matrix.goarch }}
|
||||
goos: ${{ matrix.goos }}
|
||||
go-cache: ${{ needs.product-metadata.outputs.go-cache }}
|
||||
go-mod-cache: ${{ needs.product-metadata.outputs.go-mod-cache }}
|
||||
go-tags: ui
|
||||
go-version: ${{ needs.product-metadata.outputs.go-version }}
|
||||
package-name: ${{ needs.product-metadata.outputs.package-name }}
|
||||
web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }}
|
||||
vault-version: ${{ needs.product-metadata.outputs.vault-version }}
|
||||
|
|
@ -193,10 +181,7 @@ jobs:
|
|||
create-packages: false
|
||||
goarch: ${{ matrix.goarch }}
|
||||
goos: ${{ matrix.goos }}
|
||||
go-cache: ${{ needs.product-metadata.outputs.go-cache }}
|
||||
go-mod-cache: ${{ needs.product-metadata.outputs.go-mod-cache }}
|
||||
go-tags: ui
|
||||
go-version: ${{ needs.product-metadata.outputs.go-version }}
|
||||
package-name: ${{ needs.product-metadata.outputs.package-name }}
|
||||
web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }}
|
||||
vault-version: ${{ needs.product-metadata.outputs.vault-version }}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,11 @@ on:
|
|||
- main
|
||||
- release/**
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.run_id }}-ci
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
name: Setup
|
||||
|
|
@ -24,8 +26,9 @@ jobs:
|
|||
compute-larger: ${{ steps.setup-outputs.outputs.compute-larger }}
|
||||
compute-huge: ${{ steps.setup-outputs.outputs.compute-huge }}
|
||||
enterprise: ${{ steps.setup-outputs.outputs.enterprise }}
|
||||
go-build-tags: ${{ steps.setup-outputs.outputs.go-build-tags }}
|
||||
go-tags: ${{ steps.setup-outputs.outputs.go-tags }}
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- id: setup-outputs
|
||||
name: Setup outputs
|
||||
run: |
|
||||
|
|
@ -38,7 +41,7 @@ jobs:
|
|||
echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> "$GITHUB_OUTPUT"
|
||||
echo 'compute-huge=["self-hosted","ondemand","linux","type=m5.4xlarge"]' >> "$GITHUB_OUTPUT"
|
||||
echo 'enterprise=1' >> "$GITHUB_OUTPUT"
|
||||
echo 'go-build-tags=ent,enterprise' >> "$GITHUB_OUTPUT"
|
||||
echo 'go-tags=ent,enterprise' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
# shellcheck disable=SC2129
|
||||
echo 'compute-tiny="ubuntu-latest"' >> "$GITHUB_OUTPUT" # 2 cores, 7 GB RAM, 14 GB SSD
|
||||
|
|
@ -46,16 +49,14 @@ jobs:
|
|||
echo 'compute-larger="custom-linux-medium-vault-latest"' >> "$GITHUB_OUTPUT" # 16 cores, 64 GB RAM, 600 GB SSD
|
||||
echo 'compute-huge="custom-linux-xl-vault-latest"' >> "$GITHUB_OUTPUT" # 32-cores, 128 GB RAM, 1200 GB SSD
|
||||
echo 'enterprise=' >> "$GITHUB_OUTPUT"
|
||||
echo 'go-build-tags=' >> "$GITHUB_OUTPUT"
|
||||
echo 'go-tags=' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
setup-go-cache:
|
||||
name: Go Caches
|
||||
needs:
|
||||
- setup
|
||||
uses: ./.github/workflows/setup-go-cache.yml
|
||||
with:
|
||||
runs-on: ${{ needs.setup.outputs.compute-standard }}
|
||||
secrets: inherit
|
||||
- name: Ensure Go modules are cached
|
||||
uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
no-restore: true # don't download them on a cache hit
|
||||
|
||||
diff-oss-ci:
|
||||
name: Diff OSS
|
||||
needs:
|
||||
|
|
@ -88,11 +89,11 @@ jobs:
|
|||
- id: diff
|
||||
run: |
|
||||
./.github/scripts/oss-diff.sh ${{ steps.determine-branch.outputs.BRANCH }} HEAD
|
||||
|
||||
test-go:
|
||||
name: Run Go tests
|
||||
needs:
|
||||
- setup
|
||||
- setup-go-cache
|
||||
# Don't run this job for PR branches starting with:
|
||||
# 'ui/', 'backport/ui/', 'docs/', or 'backport/docs/'
|
||||
# OR
|
||||
|
|
@ -110,15 +111,15 @@ jobs:
|
|||
# other tests aren't slowed down waiting for a binary build.
|
||||
total-runners: 17
|
||||
go-arch: amd64
|
||||
go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock'
|
||||
go-tags: '${{ needs.setup.outputs.go-tags }},deadlock'
|
||||
runs-on: ${{ needs.setup.outputs.compute-larger }}
|
||||
enterprise: ${{ needs.setup.outputs.enterprise }}
|
||||
secrets: inherit
|
||||
|
||||
test-go-race:
|
||||
name: Run Go tests with data race detection
|
||||
needs:
|
||||
- setup
|
||||
- setup-go-cache
|
||||
# Don't run this job for PR branches starting with:
|
||||
# 'ui/', 'backport/ui/', 'docs/', or 'backport/docs/'
|
||||
# OR
|
||||
|
|
@ -139,11 +140,12 @@ jobs:
|
|||
}
|
||||
extra-flags: '-race'
|
||||
go-arch: amd64
|
||||
go-build-tags: ${{ needs.setup.outputs.go-build-tags }}
|
||||
go-tags: ${{ needs.setup.outputs.go-tags }}
|
||||
runs-on: ${{ needs.setup.outputs.compute-huge }}
|
||||
enterprise: ${{ needs.setup.outputs.enterprise }}
|
||||
name: "-race"
|
||||
secrets: inherit
|
||||
|
||||
test-go-fips:
|
||||
name: Run Go tests with FIPS configuration
|
||||
# Only run this job for the enterprise repo if the PR branch doesn't start with:
|
||||
|
|
@ -160,7 +162,6 @@ jobs:
|
|||
!contains(github.event.pull_request.labels.*.name, 'docs')
|
||||
needs:
|
||||
- setup
|
||||
- setup-go-cache
|
||||
uses: ./.github/workflows/test-go.yml
|
||||
with:
|
||||
total-runners: 16
|
||||
|
|
@ -169,11 +170,12 @@ jobs:
|
|||
"GOEXPERIMENT": "boringcrypto"
|
||||
}
|
||||
go-arch: amd64
|
||||
go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock,cgo,fips,fips_140_2'
|
||||
go-tags: '${{ needs.setup.outputs.go-tags }},deadlock,cgo,fips,fips_140_2'
|
||||
runs-on: ${{ needs.setup.outputs.compute-larger }}
|
||||
enterprise: ${{ needs.setup.outputs.enterprise }}
|
||||
name: "-fips"
|
||||
secrets: inherit
|
||||
|
||||
test-ui:
|
||||
name: Test UI
|
||||
# The test-ui job is only run on:
|
||||
|
|
@ -195,10 +197,9 @@ jobs:
|
|||
runs-on: ${{ fromJSON(needs.setup.outputs.compute-larger) }}
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
# Setup node.js without caching to allow running npm install -g yarn (next step)
|
||||
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
|
||||
with:
|
||||
|
|
@ -272,17 +273,18 @@ jobs:
|
|||
paths: "ui/test-results/qunit/results.xml"
|
||||
show: "fail"
|
||||
if: always()
|
||||
|
||||
tests-completed:
|
||||
needs:
|
||||
- setup
|
||||
- setup-go-cache
|
||||
- test-go
|
||||
- test-ui
|
||||
if: always()
|
||||
if: always()
|
||||
runs-on: ${{ fromJSON(needs.setup.outputs.compute-tiny) }}
|
||||
steps:
|
||||
- run: |
|
||||
tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)'
|
||||
|
||||
notify-tests-completed-failures-oss:
|
||||
if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -411,4 +413,4 @@ jobs:
|
|||
cat "$temp_file_name" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ on:
|
|||
branches:
|
||||
- main
|
||||
- release/**
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.run_id }}-lint
|
||||
cancel-in-progress: true
|
||||
|
|
@ -20,38 +21,37 @@ jobs:
|
|||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
- run: make ci-deprecations
|
||||
name: Check deprecations
|
||||
|
||||
codechecker:
|
||||
name: Code checks
|
||||
name: Code checks
|
||||
runs-on: ubuntu-latest
|
||||
if: github.base_ref == 'main'
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
# Note: if there is a function we want to ignore the nilnil check for,
|
||||
# You can add 'ignore-nil-nil-function-check' somewhere in the
|
||||
# godoc for the function.
|
||||
- run: make ci-vet-codechecker
|
||||
name: Check custom linters
|
||||
|
||||
format:
|
||||
name: Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
- name: Go format
|
||||
run: |
|
||||
make ci-bootstrap
|
||||
|
|
@ -61,6 +61,7 @@ jobs:
|
|||
echo "Code has formatting errors. Run 'make fmt' to fix"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
semgrep:
|
||||
name: Semgrep
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
with:
|
||||
cache: false # save cache space for vault builds: https://github.com/hashicorp/vault/pull/21764
|
||||
go-version: 1.18
|
||||
|
||||
- name: Set up Python
|
||||
|
|
@ -42,19 +43,19 @@ jobs:
|
|||
cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-semgrep"
|
||||
go build -o scan-plugin-semgrep .
|
||||
mv scan-plugin-semgrep "$HOME/.bin"
|
||||
|
||||
|
||||
cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-codeql"
|
||||
go build -o scan-plugin-codeql .
|
||||
mv scan-plugin-codeql "$HOME/.bin"
|
||||
|
||||
|
||||
# Semgrep
|
||||
python3 -m pip install semgrep
|
||||
|
||||
|
||||
# CodeQL
|
||||
LATEST=$(gh release list --repo https://github.com/github/codeql-action | cut -f 3 | sort --version-sort | tail -n1)
|
||||
gh release download --repo https://github.com/github/codeql-action --pattern codeql-bundle-linux64.tar.gz "$LATEST"
|
||||
tar xf codeql-bundle-linux64.tar.gz -C "$HOME/.bin"
|
||||
|
||||
|
||||
# Add to PATH
|
||||
echo "$HOME/.bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.bin/codeql" >> "$GITHUB_PATH"
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runs-on:
|
||||
required: true
|
||||
type: string
|
||||
jobs:
|
||||
setup-go-cache:
|
||||
runs-on: ${{ fromJSON(inputs.runs-on) }}
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- id: setup-go
|
||||
name: Setup go
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
- id: setup-git
|
||||
name: Setup Git configuration
|
||||
run: |
|
||||
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com".insteadOf https://github.com
|
||||
- id: download-modules
|
||||
name: Download go modules
|
||||
run: |
|
||||
# go list ./... forces downloading some additional versions of modules that 'go mod
|
||||
# download' misses. We need this because we make use of go list itself during
|
||||
# code generation in later builds that rely on this module cache.
|
||||
go list ./...
|
||||
go list -test ./...
|
||||
|
||||
go mod download
|
||||
( cd sdk && go mod download )
|
||||
( cd api && go mod download )
|
||||
|
|
@ -68,10 +68,9 @@ jobs:
|
|||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- name: Set Up Go
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
- uses: hashicorp/action-setup-enos@v1
|
||||
with:
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ on:
|
|||
required: false
|
||||
type: string
|
||||
default: ubuntu-latest
|
||||
go-build-tags:
|
||||
go-tags:
|
||||
description: A comma-separated list of additional build tags to consider satisfied during the build.
|
||||
required: false
|
||||
type: string
|
||||
|
|
@ -48,7 +48,6 @@ on:
|
|||
default: 60
|
||||
type: number
|
||||
|
||||
|
||||
env: ${{ fromJSON(inputs.env-vars) }}
|
||||
|
||||
jobs:
|
||||
|
|
@ -60,10 +59,10 @@ jobs:
|
|||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
no-restore: true # We don't need the vault Go modules when generating indices
|
||||
- name: Authenticate to Vault
|
||||
id: vault-auth
|
||||
if: github.repository == 'hashicorp/vault-enterprise'
|
||||
|
|
@ -130,10 +129,9 @@ jobs:
|
|||
TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
cache: true
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
- name: Authenticate to Vault
|
||||
id: vault-auth
|
||||
if: github.repository == 'hashicorp/vault-enterprise'
|
||||
|
|
@ -201,7 +199,7 @@ jobs:
|
|||
# not what developers have in their environments and it could break some
|
||||
# tests; it would be like setting VAULT_TOKEN. However some non-Go
|
||||
# CI commands, like the UI tests, shouldn't have to worry about licensing.
|
||||
# So we provide the tests which want an externally supplied license with licenses
|
||||
# So we provide the tests which want an externally supplied license with licenses
|
||||
# via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it.
|
||||
# shellcheck disable=SC2034
|
||||
VAULT_LICENSE=
|
||||
|
|
@ -224,7 +222,7 @@ jobs:
|
|||
VAULT_BINARY="$(pwd)/bin/vault"
|
||||
export VAULT_BINARY
|
||||
fi
|
||||
|
||||
|
||||
# shellcheck disable=SC2086 # can't quote package list
|
||||
GOARCH=${{ inputs.go-arch }} \
|
||||
go run gotest.tools/gotestsum --format=short-verbose \
|
||||
|
|
@ -232,7 +230,7 @@ jobs:
|
|||
--jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \
|
||||
--jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \
|
||||
-- \
|
||||
-tags "${{ inputs.go-build-tags }}" \
|
||||
-tags "${{ inputs.go-tags }}" \
|
||||
-timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
|
||||
-parallel=${{ inputs.go-test-parallelism }} \
|
||||
${{ inputs.extra-flags }} \
|
||||
|
|
|
|||
|
|
@ -21,10 +21,9 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
||||
- name: Set Up Go
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
|
||||
- uses: ./.github/actions/set-up-go
|
||||
with:
|
||||
go-version-file: ./.go-version
|
||||
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
|
||||
- run: go test -v ./${{ inputs.path }}/... 2>&1 | tee ${{ inputs.name }}.txt
|
||||
- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
|
||||
with:
|
||||
|
|
|
|||
Loading…
Reference in New Issue