luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

backport of commit 1a46088afb0d5e442273350c6793d1216b6be4d1 (#21985) 

Co-authored-by: Ryan Cragun <me@ryan.ec>
This commit is contained in:
hc-github-team-secure-vault-core 2023-07-20 18:39:29 -04:00 committed by GitHub
parent f155d1fe2c
commit 83b95d3efe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 111 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
.github/actions/set-up-gotestsum/action.yml vendored Normal file
View File

@ -0,0 +1,52 @@
---
name: Set up gotestsum from Github releases
description: Set up gotestsum from Github releases
inputs:
destination:
description: "Where to install the gotestsum binary (default: $HOME/bin/gotestsum)"
type: boolean
default: "$HOME/bin"
version:
description: "The version to install (default: latest)"
type: string
default: Latest
outputs:
destination:
description: Where the installed gotestsum binary is
value: ${{ steps.install.outputs.destination }}
destination-dir:
description: The directory where the installed gotestsum binary is
value: ${{ steps.install.outputs.destination-dir }}
version:
description: The installed version of gotestsum
value: ${{ steps.install.outputs.version }}
runs:
using: composite
steps:
- id: install
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
VERSION=$(gh release list -R gotestyourself/gotestsum --exclude-drafts --exclude-pre-releases | grep Latest | cut -f1)
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
mkdir -p "$HOME/bin"
DESTINATION="$(readlink -f "$HOME/bin")"
echo "destination=$DESTINATION" >> "GITHUB_OUTPUT"
DESTINATION_DIR="$(dirname "$DESTINATION")"
echo "$DESTINATION_DIR" >> "$GITHUB_PATH"
echo "destination-dir=$DESTINATION_DIR" >> "GITHUB_OUTPUT"
OS="$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')"
ARCH="$(echo "$RUNNER_ARCH" | tr '[:upper:]' '[:lower:]')"
if [ "$ARCH" = "x64" ]; then
export ARCH="amd64"
fi
gh release download "$VERSION" -p "*${OS}_${ARCH}.tar.gz" -O gotestsum.tgz -R gotestyourself/gotestsum
tar -xvf gotestsum.tgz
mv gotestsum "${DESTINATION_DIR}/gotestsum"

2
.github/workflows/build.yml vendored
View File

@ -109,7 +109,7 @@ jobs:
key: ui-${{ steps.ui-hash.outputs.ui-hash }}
- if: steps.cache-ui-assets.outputs.cache-hit != 'true'
name: Set up node and yarn
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
with:
node-version-file: ui/package.json
cache: yarn

51
.github/workflows/ci.yml vendored
View File

@ -21,10 +21,11 @@ jobs:
name: Setup
runs-on: ubuntu-latest
outputs:
compute-tiny: ${{ steps.setup-outputs.outputs.compute-tiny }}
compute-standard: ${{ steps.setup-outputs.outputs.compute-standard }}
compute-larger: ${{ steps.setup-outputs.outputs.compute-larger }}
compute-huge: ${{ steps.setup-outputs.outputs.compute-huge }}
compute-small: ${{ steps.setup-outputs.outputs.compute-small }}
compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }}
compute-large: ${{ steps.setup-outputs.outputs.compute-large }}
compute-largem: ${{ steps.setup-outputs.outputs.compute-largem }}
compute-xlarge: ${{ steps.setup-outputs.outputs.compute-xlarge }}
enterprise: ${{ steps.setup-outputs.outputs.enterprise }}
go-tags: ${{ steps.setup-outputs.outputs.go-tags }}
steps:
@ -36,18 +37,20 @@ jobs:
if [ "${github_repository##*/}" == "vault-enterprise" ] ; then
# shellcheck disable=SC2129
echo 'compute-tiny=["self-hosted","ondemand","linux","type=m5.large"]' >> "$GITHUB_OUTPUT"
echo 'compute-standard=["self-hosted","ondemand","linux","type=m5.xlarge"]' >> "$GITHUB_OUTPUT"
echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> "$GITHUB_OUTPUT"
echo 'compute-huge=["self-hosted","ondemand","linux","type=m5.4xlarge"]' >> "$GITHUB_OUTPUT"
echo 'compute-small=["self-hosted","ondemand","linux","type=c6a.large"]' >> "$GITHUB_OUTPUT" # 2x vCPUs, 4 GiB RAM,
echo 'compute-medium=["self-hosted","ondemand","linux","type=c6a.xlarge"]' >> "$GITHUB_OUTPUT" # 4x vCPUs, 8 GiB RAM,
echo 'compute-large=["self-hosted","ondemand","linux","type=c6a.2xlarge"]' >> "$GITHUB_OUTPUT" # 8x vCPUs, 16 GiB RAM,
echo 'compute-largem=["self-hosted","ondemand","linux","type=m6a.2xlarge"]' >> "$GITHUB_OUTPUT" # 8x vCPUs, 32 GiB RAM,
echo 'compute-xlarge=["self-hosted","ondemand","linux","type=c6a.4xlarge"]' >> "$GITHUB_OUTPUT" # 16x vCPUs, 32 GiB RAM,
echo 'enterprise=1' >> "$GITHUB_OUTPUT"
echo 'go-tags=ent,enterprise' >> "$GITHUB_OUTPUT"
else
# shellcheck disable=SC2129
echo 'compute-tiny="ubuntu-latest"' >> "$GITHUB_OUTPUT" # 2 cores, 7 GB RAM, 14 GB SSD
echo 'compute-standard="custom-linux-small-vault-latest"' >> "$GITHUB_OUTPUT" # 8 cores, 32 GB RAM, 300 GB SSD
echo 'compute-larger="custom-linux-medium-vault-latest"' >> "$GITHUB_OUTPUT" # 16 cores, 64 GB RAM, 600 GB SSD
echo 'compute-huge="custom-linux-xl-vault-latest"' >> "$GITHUB_OUTPUT" # 32-cores, 128 GB RAM, 1200 GB SSD
echo 'compute-small="ubuntu-latest"' >> "$GITHUB_OUTPUT" # 2x vCPUs, 7 GiB RAM, 14 GB SSD
echo 'compute-medium="custom-linux-small-vault-latest"' >> "$GITHUB_OUTPUT" # 8x vCPUs, 32 GiB RAM, 300 GB SSD
echo 'compute-large="custom-linux-medium-vault-latest"' >> "$GITHUB_OUTPUT" # 16x vCPUs, 64 GiB RAM, 600 GB SSD
echo 'compute-largem="custom-linux-medium-vault-latest"' >> "$GITHUB_OUTPUT" # 16x vCPUs, 64 GiB RAM, 600 GB SSD
echo 'compute-xlarge="custom-linux-xl-vault-latest"' >> "$GITHUB_OUTPUT" # 32x vCPUs, 128 GiB RAM, 1200 GB SSD
echo 'enterprise=' >> "$GITHUB_OUTPUT"
echo 'go-tags=' >> "$GITHUB_OUTPUT"
fi
@ -62,7 +65,7 @@ jobs:
needs:
- setup
if: ${{ needs.setup.outputs.enterprise != '' && github.base_ref != '' }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-tiny) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
@ -113,7 +116,7 @@ jobs:
total-runners: 16
go-arch: amd64
go-tags: '${{ needs.setup.outputs.go-tags }},deadlock'
runs-on: ${{ needs.setup.outputs.compute-larger }}
runs-on: ${{ needs.setup.outputs.compute-large }}
enterprise: ${{ needs.setup.outputs.enterprise }}
secrets: inherit
@ -142,7 +145,7 @@ jobs:
extra-flags: '-race'
go-arch: amd64
go-tags: ${{ needs.setup.outputs.go-tags }}
runs-on: ${{ needs.setup.outputs.compute-huge }}
runs-on: ${{ needs.setup.outputs.compute-large }}
enterprise: ${{ needs.setup.outputs.enterprise }}
name: "race"
secrets: inherit
@ -172,7 +175,7 @@ jobs:
}
go-arch: amd64
go-tags: '${{ needs.setup.outputs.go-tags }},deadlock,cgo,fips,fips_140_2'
runs-on: ${{ needs.setup.outputs.compute-larger }}
runs-on: ${{ needs.setup.outputs.compute-large }}
enterprise: ${{ needs.setup.outputs.enterprise }}
name: "fips"
secrets: inherit
@ -195,21 +198,21 @@ jobs:
permissions:
id-token: write
contents: read
runs-on: ${{ fromJSON(needs.setup.outputs.compute-larger) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-largem) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: ./.github/actions/set-up-go
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# Setup node.js without caching to allow running npm install -g yarn (next step)
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
- uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
with:
node-version-file: './ui/package.json'
- id: install-yarn
run: |
npm install -g yarn
# Setup node.js with caching using the yarn.lock file
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
- uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
with:
node-version-file: './ui/package.json'
cache: yarn
@ -281,7 +284,7 @@ jobs:
- test-go
- test-ui
if: always()
runs-on: ${{ fromJSON(needs.setup.outputs.compute-tiny) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
steps:
- run: |
tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)'
@ -364,7 +367,7 @@ jobs:
# Sort all of the summary table rows and push them to a temp file.
temp_file_name=temp-$(date +%s)
cat failure-summary-*.md | sort >> "$temp_file_name"
# If there are test failures, present them in a format of a GitHub Markdown table.
if [ -s "$temp_file_name" ]; then
# shellcheck disable=SC2129
@ -376,13 +379,13 @@ jobs:
else
echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"
fi
# the random EOF is needed for a multiline environment variable
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
# shellcheck disable=SC2129
echo "TABLE_TEST_RESULTS<<$EOF" >> "$GITHUB_ENV"
cat "$temp_file_name" >> "$GITHUB_ENV"
echo "$EOF" >> "$GITHUB_ENV"
echo "$EOF" >> "$GITHUB_ENV"
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Create comment
if: github.head_ref != ''
@ -391,5 +394,5 @@ jobs:
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ github.run_id }}
REPO: ${{ github.event.repository.name }}
TABLE_DATA: ${{ env.TABLE_TEST_RESULTS }}
TABLE_DATA: ${{ env.TABLE_TEST_RESULTS }}
run: ./.github/scripts/report_failed_tests.sh

8
.github/workflows/security-scan.yml vendored
View File

@ -19,7 +19,7 @@ jobs:
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
cache: false # save cache space for vault builds: https://github.com/hashicorp/vault/pull/21764
go-version: 1.18
go-version-file: .go-version
- name: Set up Python
uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1
@ -32,7 +32,7 @@ jobs:
repository: hashicorp/security-scanner
token: ${{ secrets.HASHIBOT_PRODSEC_GITHUB_TOKEN }}
path: security-scanner
ref: 5a491479f4131d343afe0a4f18f6fcd36639f3fa
ref: 52d94588851f38a416f11c1e727131b3c8b0dd4d
- name: Install dependencies
shell: bash
@ -69,6 +69,8 @@ jobs:
#SEMGREP_BASELINE_REF: ${{ github.base_ref }}
with:
repository: "$PWD"
cache-build: true
cache-go-modules: false
- name: SARIF Output
shell: bash
@ -78,6 +80,6 @@ jobs:
cat results.sarif
- name: Upload SARIF file
uses: github/codeql-action/upload-sarif@cdcdbb579706841c47f7063dda365e292e5cad7a # codeql-bundle-v2.13.4
uses: github/codeql-action/upload-sarif@46a6823b81f2d7c67ddf123851eea88365bc8a67 # codeql-bundle-v2.13.5
with:
sarif_file: results.sarif

2
.github/workflows/test-enos-scenario-ui.yml vendored
View File

@ -77,7 +77,7 @@ jobs:
- name: Set Up Git
run: git config --global url."https://${{ secrets.elevated_github_token }}:@github.com".insteadOf "https://github.com"
- name: Set Up Node
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
with:
node-version-file: './ui/package.json'
- name: Set Up Terraform

49
.github/workflows/test-go.yml vendored
View File

@ -17,7 +17,7 @@ on:
description: Whether to run the binary tests.
required: false
default: false
type: boolean
type: boolean
env-vars:
description: A map of environment variables as JSON.
required: false
@ -66,7 +66,6 @@ jobs:
- uses: ./.github/actions/set-up-go
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
no-restore: true # We don't need the vault Go modules when generating indices
- name: Authenticate to Vault
id: vault-auth
if: github.repository == 'hashicorp/vault-enterprise'
@ -99,20 +98,12 @@ jobs:
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- run: go install gotest.tools/gotestsum@v1.9.0
- uses: ./.github/actions/set-up-gotestsum
- run: mkdir -p test-results/go-test
# We use a unique "read-" prefix to guarantee that we're not scribbling on
# the aggregated test data in the event of test failure. This key is
# unique for every test run and just used to restore the previous
# aggregated data. We persist all test data after a successful run and
# store that in the go-test-reports- cache.
- id: restore-from-cache
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
- uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: test-results/go-test
key: read-go-test-reports-${{ github.run_number }}
key: go-test-reports-${{ github.run_number }}
restore-keys: go-test-reports-
- name: List cached results
id: list-cached-results
@ -147,13 +138,13 @@ jobs:
"packages": $BINARY,
"description": "partition $BINARY_INDEX - binary test packages"
}]' matrix.json > new-matrix.json
matrix_file="new-matrix.json"
matrix_file="new-matrix.json"
fi
# convert the json to a map keyed by id
(
echo -n "matrix="
jq -c \
'.include | map( { (.id|tostring): . } ) | add' "$matrix_file"
'.include | map( { (.id|tostring): . } ) | add' "$matrix_file"
) >> "$GITHUB_OUTPUT"
# extract an array of ids from the json
(
@ -163,7 +154,7 @@ jobs:
) >> "$GITHUB_OUTPUT"
outputs:
matrix: ${{ steps.build.outputs.matrix }}
matrix_ids: ${{ steps.build.outputs.matrix_ids }}
matrix_ids: ${{ steps.build.outputs.matrix_ids }}
test-go:
needs: test-matrix
@ -174,7 +165,7 @@ jobs:
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
matrix:
id: ${{ fromJSON(needs.test-matrix.outputs.matrix_ids) }}
env:
GOPRIVATE: github.com/hashicorp/*
@ -217,10 +208,11 @@ jobs:
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- id: build
if: inputs.binary-tests && matrix.id == inputs.total-runners
if: inputs.binary-tests && matrix.id == inputs.total-runners
env:
GOPRIVATE: github.com/hashicorp/*
run: time make ci-bootstrap dev
- uses: ./.github/actions/set-up-gotestsum
- id: run-go-tests
name: Run Go tests
timeout-minutes: ${{ fromJSON(env.TIMEOUT_IN_MINUTES) }}
@ -228,12 +220,12 @@ jobs:
COMMIT_SHA: ${{ github.sha }}
run: |
set -exo pipefail
# Build the dynamically generated source files.
make prep
packages=$(echo "${{ toJSON(needs.test-matrix.outputs.matrix) }}" | jq -c -r --arg id "${{ matrix.id }}" '.[$id] | .packages')
packages=$(echo "${{ toJSON(needs.test-matrix.outputs.matrix) }}" | jq -c -r --arg id "${{ matrix.id }}" '.[$id] | .packages')
if [ -z "$packages" ]; then
echo "no test packages to run"
exit 1
@ -267,7 +259,7 @@ jobs:
fi
# On a release branch, add a flag to rerun failed tests
# shellcheck disable=SC2193 # can get false positive for this comparision
# shellcheck disable=SC2193 # can get false positive for this comparision
if [[ "${{ github.base_ref }}" == release/* ]] || [[ -z "${{ github.base_ref }}" && "${{ github.ref_name }}" == release/* ]]
then
RERUN_FAILS="--rerun-fails"
@ -275,7 +267,7 @@ jobs:
# shellcheck disable=SC2086 # can't quote RERUN_FAILS
GOARCH=${{ inputs.go-arch }} \
go run gotest.tools/gotestsum --format=short-verbose \
gotestsum --format=short-verbose \
--junitfile test-results/go-test/results-${{ matrix.id }}.xml \
--jsonfile test-results/go-test/results-${{ matrix.id }}.json \
--jsonfile-timing-events failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}.json \
@ -383,11 +375,6 @@ jobs:
needs: test-go
runs-on: ${{ fromJSON(inputs.runs-on) }}
steps:
- uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: test-results/go-test
key: go-test-reports-${{ github.run_number }}
restore-keys: go-test-reports-
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: test-results
@ -396,3 +383,7 @@ jobs:
ls -lhR test-results/go-test
find test-results/go-test -mindepth 1 -mtime +3 -delete
ls -lhR test-results/go-test
- uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: test-results/go-test
key: go-test-reports-${{ github.run_number }}

10
Makefile
View File

@ -113,7 +113,7 @@ vet:
echo "and fix them if necessary before submitting the code for reviewal."; \
fi
# deprecations runs staticcheck tool to look for deprecations. Checks entire code to see if it
# deprecations runs staticcheck tool to look for deprecations. Checks entire code to see if it
# has deprecated function, variable, constant or field
deprecations: bootstrap prep
@BUILD_TAGS='$(BUILD_TAGS)' ./scripts/deprecations-checker.sh ""
@ -128,13 +128,13 @@ tools/codechecker/.bin/codechecker:
# vet-codechecker runs our custom linters on the test functions. All output gets
# piped to revgrep which will only return an error if new piece of code violates
# the check
# the check
vet-codechecker: bootstrap tools/codechecker/.bin/codechecker prep
@$(GO_CMD) vet -vettool=./tools/codechecker/.bin/codechecker -tags=$(BUILD_TAGS) ./... 2>&1 | revgrep
# vet-codechecker runs our custom linters on the test functions. All output gets
# piped to revgrep which will only return an error if new piece of code that is
# not on main violates the check
# piped to revgrep which will only return an error if new piece of code that is
# not on main violates the check
ci-vet-codechecker: ci-bootstrap tools/codechecker/.bin/codechecker prep
@$(GO_CMD) vet -vettool=./tools/codechecker/.bin/codechecker -tags=$(BUILD_TAGS) ./... 2>&1 | revgrep origin/main
@ -159,7 +159,7 @@ ci-lint:
# dependency.
prep:
@sh -c "'$(CURDIR)/scripts/goversioncheck.sh' '$(GO_VERSION_MIN)'"
@$(GO_CMD) generate $($(GO_CMD) list ./... | grep -v /vendor/)
@$(GO_CMD) generate $$($(GO_CMD) list ./... | grep -v /vendor/)
@if [ -d .git/hooks ]; then cp .hooks/* .git/hooks/; fi
# bootstrap the build by downloading additional tools needed to build