VAULT-17592 Extract failed Go test results across runners (#21625) (#21672)

Co-authored-by: Kuba Wieczorek <kuba.wieczorek@hashicorp.com>
2023-07-07 13:52:01 -04:00 · 2023-07-07 13:52:01 -04:00 · 93d2fc099f
parent f3f97c9658
commit 93d2fc099f
4 changed files with 161 additions and 58 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -319,9 +319,8 @@ jobs:
    steps:
      - run: |
          tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)'
-
-  notify-tests-completed-failures:
-    if: ${{ always() && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
+  notify-tests-completed-failures-oss:
+    if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
    runs-on: ubuntu-latest
    permissions:
      id-token: write
@ -341,3 +340,106 @@ jobs:
          channel-id: "C05AABYEA9Y" # sent to #feed-vault-ci-official
          payload: |
            {"text":"OSS test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: OSS test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]}
+
+  notify-tests-completed-failures-ent:
+    if: ${{ always() && github.repository == 'hashicorp/vault-enterprise' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
+    runs-on: ['self-hosted', 'linux', 'small']
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+    needs:
+      - tests-completed
+    steps:
+      - id: vault-auth
+        name: Vault Authenticate
+        run: vault-auth
+      - id: secrets
+        name: Fetch Vault Secrets
+        uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+            kv/data/github/${{ github.repository }}/github_actions_notifications_bot token | SLACK_BOT_TOKEN;
+      - name: send-notification
+        uses: hashicorp/cloud-gha-slack-notifier@730a033037b8e603adf99ebd3085f0fdfe75e2f4 #v1
+        with:
+          channel-id: "C05AABYEA9Y" # sent to #feed-vault-ci-official
+          slack-bot-token: ${{ steps.secrets.outputs.SLACK_BOT_TOKEN }}
+          payload: |
+            {"text":"Enterprise test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: Enterprise test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]}
+
+  test-summary:
+    name: Go test failures
+    runs-on: ubuntu-latest
+    if: success() || failure() || needs.tests-completed.result == 'skipped'
+    needs:
+      - test-go
+      - test-go-fips
+      - test-go-race
+      - tests-completed
+    steps:
+      - name: Download failure summary
+        uses: actions/download-artifact@v3
+        with:
+          name: failure-summary
+      - name: Prepare failure summary
+        run: |
+          # We will store the jq query results in a temp file
+          temp_file_name=temp-$(date +%s)
+
+          # The 'jq' command below filters and formats JSON data from input files to generate a failure summary report.
+          # The query is a bit of a nightmare, though, so let's have a closer look at it.
+          #
+          # The command takes input from files matching the pattern "failure-summary-*.json".
+          # The input files should contain streams of JSON objects(with no commas in between),
+          # one per line, representing test results.
+          # Each object should have the "Action" and "Package" keys.
+          #
+          # We invoke the command with two flags:
+          # - '-r' specifies that the output should be in raw format,
+          #   without any JSON formatting. (I.e. no quotes).
+          # - '-n' tells 'jq' not to read any input from the command line.
+          #   It is used when input is provided through the 'inputs' function or other methods.
+          #
+          # 'inputs':
+          # Read JSON objects from the input files specified after the 'jq' command.
+          # We assume that the input files contain one JSON object per line.
+          #
+          # 'select(.Action == "fail") | select(.Test != null)':
+          # Filter JSON array to contain only objects where the value of "Action" is "fail"
+          # and the value of "Test" key is not null.
+          #
+          # The remaining part of the query constructs a formatted string for each filtered JSON object`:
+          # - '\(.Package)' and '\(.Test)' insert the values of the "Package" and "Test" keys into the string,
+          #   respectively.
+          # - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed.
+          # - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the
+          #   resulting parts.
+          # - '.sub(".json"; "")' removes the ".json" extension from the string.
+          # - The third part of the filename is extracted using '.split("-") | .[2]'.
+          # - If the fourth part of the filename exists, it contains the test type.
+          #   Otherwise, the default value "normal" is used.
+          # - The '.sub(".json"; "")' removes the ".json" extension from the string.
+          #
+          # The filtered and formatted data is outputted as rows of a Markdown table, like this:
+          # | pkg1 | test1 | 1 | normal |
+          # | pkg2 | test2 | 4 | race |
+          # | pkg3 | test3 | 6 | fips |
+
+          jq -r -n 'inputs | select(.Action == "fail") | select(.Test != null) | "| \(.Package) | \(.Test) | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name"
+
+          # if there are test failures, present them in a format of a GH Markdown table
+          if [ -s "$temp_file_name" ]; then
+            # shellcheck disable=SC2129
+            # Here we create the headings for the summary table
+            echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY"
+            echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY"
+            cat "$temp_file_name" >>  "$GITHUB_STEP_SUMMARY"
+          else
+            echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"
+          fi
--- a/.github/workflows/test-go.yml
+++ b/.github/workflows/test-go.yml
@ -230,6 +230,7 @@ jobs:
            go run gotest.tools/gotestsum --format=short-verbose \
              --junitfile test-results/go-test/results-${{ matrix.runner-index }}.xml \
              --jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \
+              --jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \
              -- \
              -tags "${{ inputs.go-build-tags }}" \
              -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
@ -259,9 +260,9 @@ jobs:
          name: test-results${{ inputs.name }}
          path: test-results/
        if: success() || failure()
-      - name: Create a summary of tests
-        uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f    # TSCCR: no entry for repository "test-summary/action"
-        with:
-          paths: "test-results/go-test/results-${{ matrix.runner-index }}.xml"
-          show: "fail"
+      - name: Upload failure summary
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        if: success() || failure()
+        with:
+          name: failure-summary
+          path: failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json
--- a/go.mod
+++ b/go.mod
@ -218,7 +218,7 @@ require (
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/ory-am/dockertest.v3 v3.3.4
-	gotest.tools/gotestsum v1.9.0
+	gotest.tools/gotestsum v1.10.0
 	honnef.co/go/tools v0.4.3
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 	layeh.com/radius v0.0.0-20190322222518-890bc1058917
--- a/go.sum
+++ b/go.sum
@ -3825,8 +3825,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
-gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A=
-gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w=
+gotest.tools/gotestsum v1.10.0 h1:lVO4uQJoxdsJb7jgmr1fg8QW7zGQ/tuqvsq5fHKyoHQ=
+gotest.tools/gotestsum v1.10.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=