luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

VAULT-17592 Extract failed Go test results across runners (#21625) (#21672) 

Co-authored-by: Kuba Wieczorek <kuba.wieczorek@hashicorp.com>
This commit is contained in:
hc-github-team-secure-vault-core 2023-07-07 13:52:01 -04:00 committed by GitHub
parent f3f97c9658
commit 93d2fc099f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 161 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
.github/workflows/ci.yml vendored
View File

@ -298,7 +298,7 @@ jobs:
name: test-results-ui
path: ui/test-results
if: success() || failure()
- uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action"
- uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action"
with:
paths: "ui/test-results/qunit/results.xml"
show: "fail"
@ -319,9 +319,8 @@ jobs:
steps:
- run: |
tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)'
notify-tests-completed-failures:
if: ${{ always() && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
notify-tests-completed-failures-oss:
if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
runs-on: ubuntu-latest
permissions:
id-token: write
@ -341,3 +340,106 @@ jobs:
channel-id: "C05AABYEA9Y" # sent to #feed-vault-ci-official
payload: |
{"text":"OSS test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: OSS test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]}
notify-tests-completed-failures-ent:
if: ${{ always() && github.repository == 'hashicorp/vault-enterprise' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
runs-on: ['self-hosted', 'linux', 'small']
permissions:
id-token: write
contents: read
strategy:
fail-fast: false
needs:
- tests-completed
steps:
- id: vault-auth
name: Vault Authenticate
run: vault-auth
- id: secrets
name: Fetch Vault Secrets
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/github_actions_notifications_bot token | SLACK_BOT_TOKEN;
- name: send-notification
uses: hashicorp/cloud-gha-slack-notifier@730a033037b8e603adf99ebd3085f0fdfe75e2f4 #v1
with:
channel-id: "C05AABYEA9Y" # sent to #feed-vault-ci-official
slack-bot-token: ${{ steps.secrets.outputs.SLACK_BOT_TOKEN }}
payload: |
{"text":"Enterprise test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: Enterprise test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]}
test-summary:
name: Go test failures
runs-on: ubuntu-latest
if: success() || failure() || needs.tests-completed.result == 'skipped'
needs:
- test-go
- test-go-fips
- test-go-race
- tests-completed
steps:
- name: Download failure summary
uses: actions/download-artifact@v3
with:
name: failure-summary
- name: Prepare failure summary
run: |
# We will store the jq query results in a temp file
temp_file_name=temp-$(date +%s)
# The 'jq' command below filters and formats JSON data from input files to generate a failure summary report.
# The query is a bit of a nightmare, though, so let's have a closer look at it.
#
# The command takes input from files matching the pattern "failure-summary-*.json".
# The input files should contain streams of JSON objects(with no commas in between),
# one per line, representing test results.
# Each object should have the "Action" and "Package" keys.
#
# We invoke the command with two flags:
# - '-r' specifies that the output should be in raw format,
# without any JSON formatting. (I.e. no quotes).
# - '-n' tells 'jq' not to read any input from the command line.
# It is used when input is provided through the 'inputs' function or other methods.
#
# 'inputs':
# Read JSON objects from the input files specified after the 'jq' command.
# We assume that the input files contain one JSON object per line.
#
# 'select(.Action == "fail") | select(.Test != null)':
# Filter JSON array to contain only objects where the value of "Action" is "fail"
# and the value of "Test" key is not null.
#
# The remaining part of the query constructs a formatted string for each filtered JSON object`:
# - '\(.Package)' and '\(.Test)' insert the values of the "Package" and "Test" keys into the string,
# respectively.
# - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed.
# - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the
# resulting parts.
# - '.sub(".json"; "")' removes the ".json" extension from the string.
# - The third part of the filename is extracted using '.split("-") | .[2]'.
# - If the fourth part of the filename exists, it contains the test type.
# Otherwise, the default value "normal" is used.
# - The '.sub(".json"; "")' removes the ".json" extension from the string.
#
# The filtered and formatted data is outputted as rows of a Markdown table, like this:
# | pkg1 | test1 | 1 | normal |
# | pkg2 | test2 | 4 | race |
# | pkg3 | test3 | 6 | fips |
jq -r -n 'inputs | select(.Action == "fail") | select(.Test != null) | "| \(.Package) | \(.Test) | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name"
# if there are test failures, present them in a format of a GH Markdown table
if [ -s "$temp_file_name" ]; then
# shellcheck disable=SC2129
# Here we create the headings for the summary table
echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY"
echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY"
echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY"
cat "$temp_file_name" >> "$GITHUB_STEP_SUMMARY"
else
echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"
fi

103
.github/workflows/test-go.yml vendored
View File

@ -43,9 +43,9 @@ on:
default: 20
type: number
timeout-minutes:
description: The maximum number of minutes that this workflow should run
description: The maximum number of minutes that this workflow should run
required: false
default: 60
default: 60
type: number
@ -59,41 +59,41 @@ jobs:
id-token: write # Note: this permission is explicitly required for Vault auth
contents: read
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: ./.go-version
cache: true
- name: Authenticate to Vault
id: vault-auth
if: github.repository == 'hashicorp/vault-enterprise'
run: vault-auth
- name: Fetch Secrets
id: secrets
if: github.repository == 'hashicorp/vault-enterprise'
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
- id: setup-git-private
name: Setup Git configuration (private)
if: github.repository == 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
- id: setup-git-public
name: Setup Git configuration (public)
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- id: test
working-directory: .github/scripts
env:
GOPRIVATE: github.com/hashicorp/*
run: |
ENTERPRISE=${{ inputs.enterprise }} ./test-generate-test-package-lists.sh
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: ./.go-version
cache: true
- name: Authenticate to Vault
id: vault-auth
if: github.repository == 'hashicorp/vault-enterprise'
run: vault-auth
- name: Fetch Secrets
id: secrets
if: github.repository == 'hashicorp/vault-enterprise'
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
- id: setup-git-private
name: Setup Git configuration (private)
if: github.repository == 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
- id: setup-git-public
name: Setup Git configuration (public)
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- id: test
working-directory: .github/scripts
env:
GOPRIVATE: github.com/hashicorp/*
run: |
ENTERPRISE=${{ inputs.enterprise }} ./test-generate-test-package-lists.sh
runner-indexes:
runs-on: ${{ fromJSON(inputs.runs-on) }}
name: Generate runner indexes
@ -104,18 +104,18 @@ jobs:
outputs:
runner-indexes: ${{ steps.generate-index-list.outputs.indexes }}
steps:
- id: generate-index-list
run: |
INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})"
INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")"
echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}"
- id: generate-index-list
run: |
INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})"
INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")"
echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}"
test-go:
permissions:
id-token: write # Note: this permission is explicitly required for Vault auth
contents: read
name: "${{ matrix.runner-index }}"
needs:
- runner-indexes
- runner-indexes
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
@ -127,7 +127,7 @@ jobs:
runner-index: ${{ fromJSON(needs.runner-indexes.outputs.runner-indexes) }}
env:
GOPRIVATE: github.com/hashicorp/*
TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
@ -230,6 +230,7 @@ jobs:
go run gotest.tools/gotestsum --format=short-verbose \
--junitfile test-results/go-test/results-${{ matrix.runner-index }}.xml \
--jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \
--jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \
-- \
-tags "${{ inputs.go-build-tags }}" \
-timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
@ -238,7 +239,7 @@ jobs:
\
${test_packages[${{ matrix.runner-index }}]}
- name: Prepare datadog-ci
if: github.repository == 'hashicorp/vault' && (success() || failure())
if: github.repository == 'hashicorp/vault' && (success() || failure())
continue-on-error: true
run: |
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
@ -258,10 +259,10 @@ jobs:
with:
name: test-results${{ inputs.name }}
path: test-results/
if: success() || failure()
- name: Create a summary of tests
uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action"
if: success() || failure()
- name: Upload failure summary
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
if: success() || failure()
with:
paths: "test-results/go-test/results-${{ matrix.runner-index }}.xml"
show: "fail"
if: success() || failure()
name: failure-summary
path: failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json

2
go.mod
View File

@ -218,7 +218,7 @@ require (
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
google.golang.org/protobuf v1.30.0
gopkg.in/ory-am/dockertest.v3 v3.3.4
gotest.tools/gotestsum v1.9.0
gotest.tools/gotestsum v1.10.0
honnef.co/go/tools v0.4.3
k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
layeh.com/radius v0.0.0-20190322222518-890bc1058917

4
go.sum
View File

@ -3825,8 +3825,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A=
gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w=
gotest.tools/gotestsum v1.10.0 h1:lVO4uQJoxdsJb7jgmr1fg8QW7zGQ/tuqvsq5fHKyoHQ=
gotest.tools/gotestsum v1.10.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=