backport of commit 702c52148988fc6907b8ee6457accd1536a2c25f (#21781)

Co-authored-by: Mike Palmiotto <mike.palmiotto@hashicorp.com>

.github/scripts/generate-test-package-lists.sh

@@ -1,298 +0,0 @@
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: MPL-2.0
-
-# This script is meant to be sourced into the shell running in a Github
-# workflow.
-
-# This script is a temporary measure until we implement a dynamic test-splitting
-# solution. It distributes the entire set of test packages into 16 sublists,
-# which should roughly take an equal amount of time to complete.
-
-set -e
-
-test_packages=()
-
-base="github.com/hashicorp/vault"
-
-# Total time: 526
-test_packages[1]+=" $base/api"
-test_packages[1]+=" $base/command"
-test_packages[1]+=" $base/sdk/helper/keysutil"
-
-# Total time: 1160
-test_packages[2]+=" $base/sdk/helper/ocsp"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[2]+=" $base/vault/external_tests/replication-perf"
-fi
-
-# Total time: 1009
-test_packages[3]+=" $base/builtin/credential/approle"
-test_packages[3]+=" $base/command/agentproxyshared/sink/file"
-test_packages[3]+=" $base/command/agent/template"
-test_packages[3]+=" $base/helper/random"
-test_packages[3]+=" $base/helper/storagepacker"
-test_packages[3]+=" $base/sdk/helper/certutil"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[3]+=" $base/vault/external_tests/entropy"
-fi
-test_packages[3]+=" $base/vault/external_tests/raft"
-
-# Total time: 830
-test_packages[4]+=" $base/builtin/plugin"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[4]+=" $base/enthelpers/fsm"
-fi
-test_packages[4]+=" $base/http"
-test_packages[4]+=" $base/sdk/helper/pluginutil"
-test_packages[4]+=" $base/serviceregistration/kubernetes"
-test_packages[4]+=" $base/tools/codechecker/pkg/godoctests"
-test_packages[4]+=" $base/tools/codechecker/pkg/gonilnilfunctions"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[4]+=" $base/vault/external_tests/apilock"
-    test_packages[4]+=" $base/vault/external_tests/filteredpaths"
-    test_packages[4]+=" $base/vault/external_tests/perfstandby"
-    test_packages[4]+=" $base/vault/external_tests/replication-dr"
-fi
-
-
-# Total time: 258
-test_packages[5]+=" $base/builtin/credential/aws"
-test_packages[5]+=" $base/builtin/credential/cert"
-test_packages[5]+=" $base/builtin/logical/aws"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[5]+=" $base/enthelpers/logshipper"
-    test_packages[5]+=" $base/enthelpers/merkle"
-fi
-test_packages[5]+=" $base/helper/hostutil"
-test_packages[5]+=" $base/helper/pgpkeys"
-test_packages[5]+=" $base/sdk/physical/inmem"
-test_packages[5]+=" $base/vault/activity"
-test_packages[5]+=" $base/vault/diagnose"
-test_packages[5]+=" $base/vault/external_tests/pprof"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[5]+=" $base/vault/external_tests/resolver"
-fi
-test_packages[5]+=" $base/vault/external_tests/response"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[5]+=" $base/vault/external_tests/seal"
-fi
-test_packages[5]+=" $base/vault/external_tests/sealmigration"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[5]+=" $base/vault/external_tests/transform"
-fi
-
-# Total time: 588
-test_packages[6]+=" $base"
-test_packages[6]+=" $base/audit"
-test_packages[6]+=" $base/builtin/audit/file"
-test_packages[6]+=" $base/builtin/credential/github"
-test_packages[6]+=" $base/builtin/credential/okta"
-test_packages[6]+=" $base/builtin/logical/database/dbplugin"
-test_packages[6]+=" $base/command/agentproxyshared/auth/cert"
-test_packages[6]+=" $base/command/agentproxyshared/auth/jwt"
-test_packages[6]+=" $base/command/agentproxyshared/auth/kerberos"
-test_packages[6]+=" $base/command/agentproxyshared/auth/kubernetes"
-test_packages[6]+=" $base/command/agentproxyshared/auth/token-file"
-test_packages[6]+=" $base/command/agentproxyshared"
-test_packages[6]+=" $base/command/agentproxyshared/cache"
-test_packages[6]+=" $base/command/agentproxyshared/cache/cacheboltdb"
-test_packages[6]+=" $base/command/agentproxyshared/cache/cachememdb"
-test_packages[6]+=" $base/command/agentproxyshared/cache/keymanager"
-test_packages[6]+=" $base/command/agent/config"
-test_packages[6]+=" $base/command/agent/exec"
-test_packages[6]+=" $base/command/proxy/config"
-test_packages[6]+=" $base/command/config"
-test_packages[6]+=" $base/command/token"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[6]+=" $base/enthelpers/namespace"
-    test_packages[6]+=" $base/enthelpers/replicatedpaths"
-    test_packages[6]+=" $base/enthelpers/sealrewrap"
-fi
-test_packages[6]+=" $base/helper/builtinplugins"
-test_packages[6]+=" $base/helper/dhutil"
-test_packages[6]+=" $base/helper/fairshare"
-test_packages[6]+=" $base/helper/flag-kv"
-test_packages[6]+=" $base/helper/flag-slice"
-test_packages[6]+=" $base/helper/forwarding"
-test_packages[6]+=" $base/helper/logging"
-test_packages[6]+=" $base/helper/metricsutil"
-test_packages[6]+=" $base/helper/namespace"
-test_packages[6]+=" $base/helper/osutil"
-test_packages[6]+=" $base/helper/parseip"
-test_packages[6]+=" $base/helper/policies"
-test_packages[6]+=" $base/helper/testhelpers/logical"
-test_packages[6]+=" $base/helper/timeutil"
-test_packages[6]+=" $base/helper/useragent"
-test_packages[6]+=" $base/helper/versions"
-test_packages[6]+=" $base/internalshared/configutil"
-test_packages[6]+=" $base/internalshared/listenerutil"
-test_packages[6]+=" $base/physical/alicloudoss"
-test_packages[6]+=" $base/physical/gcs"
-test_packages[6]+=" $base/physical/manta"
-test_packages[6]+=" $base/physical/mssql"
-test_packages[6]+=" $base/physical/oci"
-test_packages[6]+=" $base/physical/s3"
-test_packages[6]+=" $base/physical/spanner"
-test_packages[6]+=" $base/physical/swift"
-test_packages[6]+=" $base/physical/zookeeper"
-test_packages[6]+=" $base/plugins/database/hana"
-test_packages[6]+=" $base/plugins/database/redshift"
-test_packages[6]+=" $base/sdk/database/dbplugin/v5"
-test_packages[6]+=" $base/sdk/database/helper/credsutil"
-test_packages[6]+=" $base/sdk/helper/authmetadata"
-test_packages[6]+=" $base/sdk/helper/compressutil"
-test_packages[6]+=" $base/sdk/helper/cryptoutil"
-test_packages[6]+=" $base/sdk/helper/identitytpl"
-test_packages[6]+=" $base/sdk/helper/kdf"
-test_packages[6]+=" $base/sdk/helper/locksutil"
-test_packages[6]+=" $base/sdk/helper/pathmanager"
-test_packages[6]+=" $base/sdk/helper/roottoken"
-test_packages[6]+=" $base/sdk/helper/testhelpers"
-test_packages[6]+=" $base/sdk/helper/testhelpers/schema"
-test_packages[6]+=" $base/sdk/helper/xor"
-test_packages[6]+=" $base/sdk/physical/file"
-test_packages[6]+=" $base/sdk/plugin/pb"
-test_packages[6]+=" $base/serviceregistration/kubernetes/client"
-test_packages[6]+=" $base/shamir"
-test_packages[6]+=" $base/vault/cluster"
-test_packages[6]+=" $base/vault/eventbus"
-test_packages[6]+=" $base/vault/external_tests/api"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[6]+=" $base/vault/external_tests/consistencyheaders"
-fi
-test_packages[6]+=" $base/vault/external_tests/expiration"
-test_packages[6]+=" $base/vault/external_tests/hcp_link"
-test_packages[6]+=" $base/vault/external_tests/kv"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[6]+=" $base/vault/external_tests/plugins"
-fi
-test_packages[6]+=" $base/vault/external_tests/quotas"
-test_packages[6]+=" $base/vault/seal"
-
-# Total time: 389
-test_packages[7]+=" $base/builtin/credential/userpass"
-test_packages[7]+=" $base/builtin/logical/pki"
-test_packages[7]+=" $base/builtin/logical/transit"
-test_packages[7]+=" $base/command/agent"
-test_packages[7]+=" $base/helper/monitor"
-test_packages[7]+=" $base/sdk/database/helper/connutil"
-test_packages[7]+=" $base/sdk/database/helper/dbutil"
-test_packages[7]+=" $base/sdk/helper/cidrutil"
-test_packages[7]+=" $base/sdk/helper/custommetadata"
-test_packages[7]+=" $base/sdk/helper/jsonutil"
-test_packages[7]+=" $base/sdk/helper/ldaputil"
-test_packages[7]+=" $base/sdk/helper/logging"
-test_packages[7]+=" $base/sdk/helper/policyutil"
-test_packages[7]+=" $base/sdk/helper/salt"
-test_packages[7]+=" $base/sdk/helper/template"
-test_packages[7]+=" $base/sdk/helper/useragent"
-test_packages[7]+=" $base/sdk/logical"
-test_packages[7]+=" $base/sdk/plugin/mock"
-test_packages[7]+=" $base/sdk/queue"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[7]+=" $base/vault/autosnapshots"
-    test_packages[7]+=" $base/vault/external_tests/activity"
-fi
-test_packages[7]+=" $base/vault/external_tests/approle"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[7]+=" $base/vault/external_tests/kmip"
-fi
-test_packages[7]+=" $base/vault/external_tests/mfa"
-test_packages[7]+=" $base/vault/external_tests/misc"
-test_packages[7]+=" $base/vault/quotas"
-
-# Total time: 779
-test_packages[8]+=" $base/builtin/credential/aws/pkcs7"
-test_packages[8]+=" $base/builtin/logical/totp"
-test_packages[8]+=" $base/command/agentproxyshared/auth"
-test_packages[8]+=" $base/physical/raft"
-test_packages[8]+=" $base/sdk/framework"
-test_packages[8]+=" $base/sdk/plugin"
-test_packages[8]+=" $base/vault"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[8]+=" $base/vault/external_tests/barrier"
-    test_packages[8]+=" $base/vault/external_tests/cubbyholes"
-fi
-test_packages[8]+=" $base/vault/external_tests/metrics"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[8]+=" $base/vault/external_tests/replication"
-fi
-test_packages[8]+=" $base/vault/external_tests/router"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[8]+=" $base/vault/external_tests/system"
-    test_packages[8]+=" $base/vault/managed_key"
-fi
-
-# Total time: 310
-test_packages[9]+=" $base/vault/hcp_link/capabilities/api_capability"
-test_packages[9]+=" $base/vault/external_tests/plugin"
-
-# Total time: 925
-test_packages[10]+=" $base/builtin/credential/ldap"
-test_packages[10]+=" $base/builtin/logical/database"
-test_packages[10]+=" $base/physical/etcd"
-test_packages[10]+=" $base/physical/postgresql"
-
-# Total time: 851
-test_packages[11]+=" $base/builtin/logical/rabbitmq"
-test_packages[11]+=" $base/physical/dynamodb"
-test_packages[11]+=" $base/plugins/database/influxdb"
-test_packages[11]+=" $base/vault/external_tests/identity"
-test_packages[11]+=" $base/vault/external_tests/token"
-
-# Total time: 340
-test_packages[12]+=" $base/builtin/logical/consul"
-test_packages[12]+=" $base/physical/couchdb"
-test_packages[12]+=" $base/plugins/database/mongodb"
-test_packages[12]+=" $base/plugins/database/mssql"
-test_packages[12]+=" $base/plugins/database/mysql"
-
-# Total time: 704
-test_packages[13]+=" $base/builtin/logical/pkiext"
-test_packages[13]+=" $base/command/server"
-test_packages[13]+=" $base/physical/aerospike"
-test_packages[13]+=" $base/physical/cockroachdb"
-test_packages[13]+=" $base/plugins/database/postgresql"
-test_packages[13]+=" $base/plugins/database/postgresql/scram"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[13]+=" $base/vault/external_tests/filteredpathsext"
-fi
-test_packages[13]+=" $base/vault/external_tests/policy"
-
-# Total time: 374
-test_packages[14]+=" $base/builtin/credential/radius"
-test_packages[14]+=" $base/builtin/logical/ssh"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[14]+=" $base/enthelpers/wal"
-fi
-test_packages[14]+=" $base/physical/azure"
-test_packages[14]+=" $base/serviceregistration/consul"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[14]+=" $base/vault/external_tests/quotas-docker"
-fi
-test_packages[14]+=" $base/vault/external_tests/raftha"
-
-# Total time: 362
-test_packages[15]+=" $base/builtin/logical/nomad"
-test_packages[15]+=" $base/physical/mysql"
-test_packages[15]+=" $base/plugins/database/cassandra"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[15]+=" $base/vault/external_tests/namespaces"
-    test_packages[15]+=" $base/vault/external_tests/census"
-fi
-test_packages[15]+=" $base/vault/external_tests/sealmigrationext"
-
-# Total time: 635
-test_packages[16]+=" $base/physical/cassandra"
-test_packages[16]+=" $base/physical/consul"
-if [ "${ENTERPRISE:+x}" == "x" ] ; then
-    test_packages[16]+=" $base/vault/external_tests/autosnapshots"
-    test_packages[16]+=" $base/vault/external_tests/replicationext"
-    test_packages[16]+=" $base/vault/external_tests/sealext"
-fi
-
-for i in $(cd $(git rev-parse --show-toplevel) && go list -test -json ./... |
-  jq -r '.ForTest | select(.!=null) | select(.|test("_binary$"))');
-do
-  test_packages[17]+=" $i"
-done

.github/scripts/test-generate-test-package-lists.sh

@@ -1,70 +0,0 @@
-#!/bin/bash
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: MPL-2.0
-
-set -e${DEBUG+x}o pipefail
-
-#
-# This script is run to make sure that every package returned by 
-# go list -test ./... (when run from the repo root, api/, and sdk/ directory)
-# appear in the test_packages array defined in the sibling file
-# generate-test-package-lists.sh
-#
-# This script is executed as part of the ci workflow triggered by pull_requests
-# events. In the event that the job that runs this script fails, examine the
-# output of the 'test' step in that job to obtain the list of test packages that
-# are missing in the test_packages array or that should be removed from it.
-#
-
-dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-source generate-test-package-lists.sh
-
-get_module_packages() {
-    for d in "../.." "../../sdk" "../../api"; do
-        cd "$dir/$d"
-        go list -test -json ./... | jq -r '.ForTest | select(.!=null)' |
-          grep -v vault/integ | grep '^github.com/hashicorp/'
-    done |sort -u
-}
-
-count_test_packages() {
-    count=0
-    for test_package in "${test_packages[@]}" ; do
-        count=$((${count}+$(wc -w <<< "${test_package}")))
-    done
-
-    echo $count
-}
-
-contains() {
-    target=$1; shift
-    for i; do
-        if [[ "$i" == "$target" ]]; then
-            return 0
-        fi
-    done
-    return 1
-}
-
-all_packages=( $(get_module_packages) )
-
-for package in "${all_packages[@]}" ; do
-    if ! contains $package ${test_packages[@]}; then
-        echo "Error: package ${package} is not present in test_packages"
-        exit 1
-    fi
-done
-
-test_package_count=$(count_test_packages)
-if (( ${#all_packages[@]} != $test_package_count )) ; then
-    echo "Error: there are currently ${#all_packages[@]} packages in the repository but $test_package_count packages in test_packages"
-
-    unused_packages="${test_packages[@]} "
-    for ap in ${all_packages[@]} ; do
-        unused_packages="$(echo "$unused_packages" | sed -r "s~$ap ~ ~" )"
-    done
-
-    echo "Packages in test_packages that aren't used: ${unused_packages// /}"
-    exit 1
-fi

.github/workflows/test-go.yml

@@ -51,12 +51,11 @@ on:
 env: ${{ fromJSON(inputs.env-vars) }}
 
 jobs:
-  test-generate-test-package-list:
+  test-matrix:
-    runs-on: ${{ fromJSON(inputs.runs-on) }}
-    name: Verify Test Package Distribution
     permissions:
       id-token: write  # Note: this permission is explicitly required for Vault auth
       contents: read
+    runs-on: ${{ fromJSON(inputs.runs-on) }}
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - uses: ./.github/actions/set-up-go
@@ -76,7 +75,15 @@ jobs:
           caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
           token: ${{ steps.vault-auth.outputs.token }}
           secrets: |
+            kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
             kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
+            kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI;
+            kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2;
+            kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS;
+            kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL;
+            kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID;
+            kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET;
+            kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID;
       - id: setup-git-private
         name: Setup Git configuration (private)
         if: github.repository == 'hashicorp/vault-enterprise'
@@ -87,43 +94,66 @@ jobs:
         if: github.repository != 'hashicorp/vault-enterprise'
         run: |
           git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
-      - id: test
+      - run: go install gotest.tools/gotestsum@v1.9.0
-        working-directory: .github/scripts
+
+      - run: mkdir -p test-results/go-test
+
+      # We use a unique "read-" prefix to guarantee that we're not scribbling on
+      # the aggregated test data in the event of test failure. This key is
+      # unique for every test run and just used to restore the previous
+      # aggregated data. We persist all test data after a successful run and
+      # store that in the go-test-reports- cache.
+      - id: restore-from-cache
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          path: test-results/go-test
+          key: read-go-test-reports-${{ github.run_number }}
+          restore-keys: go-test-reports-
+      - name: List cached results
+        id: list-cached-results
+        run: ls -lhR test-results/go-test
+      - name: Build matrix excluding binary and integration tests
+        id: build-non-binary
         env:
           GOPRIVATE: github.com/hashicorp/*
         run: |
-          ENTERPRISE=${{ inputs.enterprise }} ./test-generate-test-package-lists.sh
+          (
-  runner-indexes:
+            go list ./... | grep -v "_binary" | grep -v "vault/integ" | gotestsum tool ci-matrix --debug \
-    runs-on: ${{ fromJSON(inputs.runs-on) }}
+              --partitions 16 \
-    name: Generate runner indexes
+              --timing-files 'test-results/go-test/*.json' > matrix.json
-    #
+          )
-    # This job generates a JSON Array of integers ranging from 1 to 16.
+      - name: Capture list of binary tests
-    # That array is used in the matrix section of the test-go job below.
+        id: list-binary-tests
-    #
-    outputs:
-      runner-indexes: ${{ steps.generate-index-list.outputs.indexes }}
-    steps:
-      - id: generate-index-list
         run: |
-          INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})"
+          LIST="$(go list ./... | grep "_binary" | xargs)"
-          INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")"
+          echo "list=$LIST" >> "$GITHUB_OUTPUT"
-          echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}"
+      - name: Build complete matrix
+        id: build
+        run: |
+            set -exo pipefail
+            export BINARY_TESTS="${{ steps.list-binary-tests.outputs.list }}"
+            (
+              echo -n "matrix="
+              jq -c --arg BINARY "${BINARY_TESTS}" \
+                '.include += [{
+                  "id": 16,
+                  "estimatedRuntime": "N/A",
+                  "packages": $BINARY,
+                  "description": "partition 16 - binary test packages"
+              }]' matrix.json
+            ) >> "$GITHUB_OUTPUT"
+    outputs:
+      matrix: ${{ steps.build.outputs.matrix }}
+
   test-go:
+    needs: test-matrix
     permissions:
       id-token: write  # Note: this permission is explicitly required for Vault auth
       contents: read
-    name: "${{ matrix.runner-index }}"
-    needs:
-      - runner-indexes
     runs-on: ${{ fromJSON(inputs.runs-on) }}
     strategy:
       fail-fast: false
-      matrix:
+      matrix: ${{ fromJSON(needs.test-matrix.outputs.matrix) }}
-        #
-        # Initialize the runner-index key with the JSON array of integers
-        # generated above.
-        #
-        runner-index: ${{ fromJSON(needs.runner-indexes.outputs.runner-indexes) }}
     env:
       GOPRIVATE: github.com/hashicorp/*
       TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
@@ -164,13 +194,8 @@ jobs:
         if: github.repository != 'hashicorp/vault-enterprise'
         run: |
           git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
-      - id: go-mod-download
-        if: matrix.runner-index > 16
-        env:
-          GOPRIVATE: github.com/hashicorp/*
-        run: time go mod download -x
       - id: build
-        if: matrix.runner-index > 16
+        if: contains(matrix.packages, '_binary')
         env:
           GOPRIVATE: github.com/hashicorp/*
         run: time make ci-bootstrap dev
@@ -182,19 +207,9 @@ jobs:
         run: |
           set -exo pipefail
 
-          #
-          # This script creates a Bash array with 16 elements each
-          # containing a space delimited list of package names. The
-          # array element corresponding to this instance's
-          # matrix.runner-index value.
-          #
-          ENTERPRISE=${{ inputs.enterprise }} source .github/scripts/generate-test-package-lists.sh
-
           # Build the dynamically generated source files.
           make prep
 
-          mkdir -p test-results/go-test
-
           # We don't want VAULT_LICENSE set when running Go tests, because that's
           # not what developers have in their environments and it could break some
           # tests; it would be like setting VAULT_TOKEN.  However some non-Go
@@ -226,16 +241,15 @@ jobs:
           # shellcheck disable=SC2086 # can't quote package list
           GOARCH=${{ inputs.go-arch }} \
             go run gotest.tools/gotestsum --format=short-verbose \
-              --junitfile test-results/go-test/results-${{ matrix.runner-index }}.xml \
+              --junitfile test-results/go-test/results-${{ matrix.id }}.xml \
-              --jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \
+              --jsonfile test-results/go-test/results-${{ matrix.id }}.json \
-              --jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \
+              --jsonfile-timing-events failure-summary-${{ matrix.id }}${{inputs.name}}.json \
               -- \
               -tags "${{ inputs.go-tags }}" \
               -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
               -parallel=${{ inputs.go-test-parallelism }} \
               ${{ inputs.extra-flags }} \
-                \
+              ${{ matrix.packages }}
-              ${test_packages[${{ matrix.runner-index }}]}
       - name: Prepare datadog-ci
         if: github.repository == 'hashicorp/vault' && (success() || failure())
         continue-on-error: true
@@ -250,17 +264,35 @@ jobs:
           if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then
             export DATADOG_API_KEY=${{ secrets.DATADOG_API_KEY }}
           fi
-          datadog-ci junit upload --service "$GITHUB_REPOSITORY" test-results/go-test/results-${{ matrix.runner-index }}.xml
+          datadog-ci junit upload --service "$GITHUB_REPOSITORY" test-results/go-test/results-${{ matrix.id }}.xml
         if: success() || failure()
       - name: Archive test results
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: test-results${{ inputs.name }}
-          path: test-results/
+          path: test-results/go-test
         if: success() || failure()
       - name: Upload failure summary
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         if: success() || failure()
         with:
           name: failure-summary
-          path: failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json
+          path: failure-summary-${{ matrix.id }}${{inputs.name}}.json
+
+  test-collect-reports:
+    needs: test-go
+    runs-on: ${{ fromJSON(inputs.runs-on) }}
+    steps:
+      - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          path: test-results/go-test
+          key: go-test-reports-${{ github.run_number }}
+          restore-keys: go-test-reports-
+      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: test-results
+          path: test-results/go-test
+      - run: |
+          ls -lhR test-results/go-test
+          find test-results/go-test -mindepth 1 -mtime +3 -delete
+          ls -lhR test-results/go-test