From 93d2fc099faf0c3795f20d3f83725bf6bab6fd59 Mon Sep 17 00:00:00 2001 From: hc-github-team-secure-vault-core <82990506+hc-github-team-secure-vault-core@users.noreply.github.com> Date: Fri, 7 Jul 2023 13:52:01 -0400 Subject: [PATCH] VAULT-17592 Extract failed Go test results across runners (#21625) (#21672) Co-authored-by: Kuba Wieczorek --- .github/workflows/ci.yml | 110 ++++++++++++++++++++++++++++++++-- .github/workflows/test-go.yml | 103 +++++++++++++++---------------- go.mod | 2 +- go.sum | 4 +- 4 files changed, 161 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fd4c51a9..5774cc290 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -298,7 +298,7 @@ jobs: name: test-results-ui path: ui/test-results if: success() || failure() - - uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action" + - uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action" with: paths: "ui/test-results/qunit/results.xml" show: "fail" @@ -319,9 +319,8 @@ jobs: steps: - run: | tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)' - - notify-tests-completed-failures: - if: ${{ always() && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }} + notify-tests-completed-failures-oss: + if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }} runs-on: ubuntu-latest permissions: id-token: write @@ -341,3 +340,106 @@ jobs: channel-id: "C05AABYEA9Y" # sent to #feed-vault-ci-official payload: | {"text":"OSS test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: OSS test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]} + + notify-tests-completed-failures-ent: + if: ${{ always() && github.repository == 'hashicorp/vault-enterprise' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }} + runs-on: ['self-hosted', 'linux', 'small'] + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + needs: + - tests-completed + steps: + - id: vault-auth + name: Vault Authenticate + run: vault-auth + - id: secrets + name: Fetch Vault Secrets + uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/github_actions_notifications_bot token | SLACK_BOT_TOKEN; + - name: send-notification + uses: hashicorp/cloud-gha-slack-notifier@730a033037b8e603adf99ebd3085f0fdfe75e2f4 #v1 + with: + channel-id: "C05AABYEA9Y" # sent to #feed-vault-ci-official + slack-bot-token: ${{ steps.secrets.outputs.SLACK_BOT_TOKEN }} + payload: | + {"text":"Enterprise test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: Enterprise test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]} + + test-summary: + name: Go test failures + runs-on: ubuntu-latest + if: success() || failure() || needs.tests-completed.result == 'skipped' + needs: + - test-go + - test-go-fips + - test-go-race + - tests-completed + steps: + - name: Download failure summary + uses: actions/download-artifact@v3 + with: + name: failure-summary + - name: Prepare failure summary + run: | + # We will store the jq query results in a temp file + temp_file_name=temp-$(date +%s) + + # The 'jq' command below filters and formats JSON data from input files to generate a failure summary report. + # The query is a bit of a nightmare, though, so let's have a closer look at it. + # + # The command takes input from files matching the pattern "failure-summary-*.json". + # The input files should contain streams of JSON objects(with no commas in between), + # one per line, representing test results. + # Each object should have the "Action" and "Package" keys. + # + # We invoke the command with two flags: + # - '-r' specifies that the output should be in raw format, + # without any JSON formatting. (I.e. no quotes). + # - '-n' tells 'jq' not to read any input from the command line. + # It is used when input is provided through the 'inputs' function or other methods. + # + # 'inputs': + # Read JSON objects from the input files specified after the 'jq' command. + # We assume that the input files contain one JSON object per line. + # + # 'select(.Action == "fail") | select(.Test != null)': + # Filter JSON array to contain only objects where the value of "Action" is "fail" + # and the value of "Test" key is not null. + # + # The remaining part of the query constructs a formatted string for each filtered JSON object`: + # - '\(.Package)' and '\(.Test)' insert the values of the "Package" and "Test" keys into the string, + # respectively. + # - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed. + # - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the + # resulting parts. + # - '.sub(".json"; "")' removes the ".json" extension from the string. + # - The third part of the filename is extracted using '.split("-") | .[2]'. + # - If the fourth part of the filename exists, it contains the test type. + # Otherwise, the default value "normal" is used. + # - The '.sub(".json"; "")' removes the ".json" extension from the string. + # + # The filtered and formatted data is outputted as rows of a Markdown table, like this: + # | pkg1 | test1 | 1 | normal | + # | pkg2 | test2 | 4 | race | + # | pkg3 | test3 | 6 | fips | + + jq -r -n 'inputs | select(.Action == "fail") | select(.Test != null) | "| \(.Package) | \(.Test) | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name" + + # if there are test failures, present them in a format of a GH Markdown table + if [ -s "$temp_file_name" ]; then + # shellcheck disable=SC2129 + # Here we create the headings for the summary table + echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY" + echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY" + echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY" + cat "$temp_file_name" >> "$GITHUB_STEP_SUMMARY" + else + echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY" + fi \ No newline at end of file diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 4367e7a75..184e28701 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -43,9 +43,9 @@ on: default: 20 type: number timeout-minutes: - description: The maximum number of minutes that this workflow should run + description: The maximum number of minutes that this workflow should run required: false - default: 60 + default: 60 type: number @@ -59,41 +59,41 @@ jobs: id-token: write # Note: this permission is explicitly required for Vault auth contents: read steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true - - name: Authenticate to Vault - id: vault-auth - if: github.repository == 'hashicorp/vault-enterprise' - run: vault-auth - - name: Fetch Secrets - id: secrets - if: github.repository == 'hashicorp/vault-enterprise' - uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e - with: - url: ${{ steps.vault-auth.outputs.addr }} - caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} - token: ${{ steps.vault-auth.outputs.token }} - secrets: | - kv/data/github/${{ github.repository }}/github-token username-and-token | github-token; - - id: setup-git-private - name: Setup Git configuration (private) - if: github.repository == 'hashicorp/vault-enterprise' - run: | - git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com - - id: setup-git-public - name: Setup Git configuration (public) - if: github.repository != 'hashicorp/vault-enterprise' - run: | - git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com - - id: test - working-directory: .github/scripts - env: - GOPRIVATE: github.com/hashicorp/* - run: | - ENTERPRISE=${{ inputs.enterprise }} ./test-generate-test-package-lists.sh + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: ./.go-version + cache: true + - name: Authenticate to Vault + id: vault-auth + if: github.repository == 'hashicorp/vault-enterprise' + run: vault-auth + - name: Fetch Secrets + id: secrets + if: github.repository == 'hashicorp/vault-enterprise' + uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/github-token username-and-token | github-token; + - id: setup-git-private + name: Setup Git configuration (private) + if: github.repository == 'hashicorp/vault-enterprise' + run: | + git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com + - id: setup-git-public + name: Setup Git configuration (public) + if: github.repository != 'hashicorp/vault-enterprise' + run: | + git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com + - id: test + working-directory: .github/scripts + env: + GOPRIVATE: github.com/hashicorp/* + run: | + ENTERPRISE=${{ inputs.enterprise }} ./test-generate-test-package-lists.sh runner-indexes: runs-on: ${{ fromJSON(inputs.runs-on) }} name: Generate runner indexes @@ -104,18 +104,18 @@ jobs: outputs: runner-indexes: ${{ steps.generate-index-list.outputs.indexes }} steps: - - id: generate-index-list - run: | - INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})" - INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")" - echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}" + - id: generate-index-list + run: | + INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})" + INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")" + echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}" test-go: permissions: id-token: write # Note: this permission is explicitly required for Vault auth contents: read name: "${{ matrix.runner-index }}" needs: - - runner-indexes + - runner-indexes runs-on: ${{ fromJSON(inputs.runs-on) }} strategy: fail-fast: false @@ -127,7 +127,7 @@ jobs: runner-index: ${{ fromJSON(needs.runner-indexes.outputs.runner-indexes) }} env: GOPRIVATE: github.com/hashicorp/* - TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }} + TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 @@ -230,6 +230,7 @@ jobs: go run gotest.tools/gotestsum --format=short-verbose \ --junitfile test-results/go-test/results-${{ matrix.runner-index }}.xml \ --jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \ + --jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \ -- \ -tags "${{ inputs.go-build-tags }}" \ -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \ @@ -238,7 +239,7 @@ jobs: \ ${test_packages[${{ matrix.runner-index }}]} - name: Prepare datadog-ci - if: github.repository == 'hashicorp/vault' && (success() || failure()) + if: github.repository == 'hashicorp/vault' && (success() || failure()) continue-on-error: true run: | curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" @@ -258,10 +259,10 @@ jobs: with: name: test-results${{ inputs.name }} path: test-results/ - if: success() || failure() - - name: Create a summary of tests - uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action" + if: success() || failure() + - name: Upload failure summary + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: success() || failure() with: - paths: "test-results/go-test/results-${{ matrix.runner-index }}.xml" - show: "fail" - if: success() || failure() + name: failure-summary + path: failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \ No newline at end of file diff --git a/go.mod b/go.mod index e2392645a..aa6d11fc3 100644 --- a/go.mod +++ b/go.mod @@ -218,7 +218,7 @@ require ( google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.30.0 gopkg.in/ory-am/dockertest.v3 v3.3.4 - gotest.tools/gotestsum v1.9.0 + gotest.tools/gotestsum v1.10.0 honnef.co/go/tools v0.4.3 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 layeh.com/radius v0.0.0-20190322222518-890bc1058917 diff --git a/go.sum b/go.sum index 58fe58adf..fe6edbc6a 100644 --- a/go.sum +++ b/go.sum @@ -3825,8 +3825,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A= -gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= +gotest.tools/gotestsum v1.10.0 h1:lVO4uQJoxdsJb7jgmr1fg8QW7zGQ/tuqvsq5fHKyoHQ= +gotest.tools/gotestsum v1.10.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=