From 9528a3bf1f054c4aa0c828db0eeb08d99a1264d7 Mon Sep 17 00:00:00 2001
From: Dhia Ayachi <dhia@hashicorp.com>
Date: Mon, 20 Sep 2021 16:04:59 -0400
Subject: [PATCH] Rerun go-test and go-test-32bits tests up to 3 times before
 failing (#11059)

* Rerun go-test and go-test-32bits tests up to 3 times before failing

* do not run rerun fail report in oss

* set back rerun fail report but print to stdin instead of comment
---
 .circleci/config.yml                    | 20 ++++++++++++++++----
 .circleci/scripts/rerun-fails-report.sh | 21 +++++++++++++++++++++
 2 files changed, 37 insertions(+), 4 deletions(-)
 create mode 100755 .circleci/scripts/rerun-fails-report.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e12accdb2..54798f322 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -220,7 +220,6 @@ jobs:
       - run:
           name: go test
           command: |
-            mkdir -p $TEST_RESULTS_DIR
             mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
             PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./... | circleci tests split --split-by=timings --timings-type=classname)
             echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
@@ -228,10 +227,14 @@ jobs:
             gotestsum \
               --format=short-verbose \
               --jsonfile /tmp/jsonfile/go-test-${CIRCLE_NODE_INDEX}.log \
+              --debug \
+              --rerun-fails=3 \
+              --rerun-fails-max-failures=40 \
+              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
+              --packages="$PACKAGE_NAMES" \
               --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
               -tags="$GOTAGS" -p 2 \
-              -cover -coverprofile=coverage.txt \
-              $PACKAGE_NAMES
+              -cover -coverprofile=coverage.txt
 
       - store_test_results:
           path: *TEST_RESULTS_DIR
@@ -239,6 +242,10 @@ jobs:
           path: *TEST_RESULTS_DIR
       - store_artifacts:
           path: /tmp/jsonfile
+      - run: &rerun-fails-report
+          name: "Re-run fails report"
+          command: |
+            .circleci/scripts/rerun-fails-report.sh /tmp/gotestsum-rerun-fails
       - run: *notify-slack-failure
 
   go-test-race:
@@ -301,11 +308,16 @@ jobs:
           command: |
             mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile
             go env
+            PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./...)
             gotestsum \
               --jsonfile /tmp/jsonfile/go-test-32bit.log \
+              --rerun-fails=3 \
+              --rerun-fails-max-failures=40 \
+              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
+              --packages="$PACKAGE_NAMES" \
               --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
               -tags="$GOTAGS" -p 2 \
-              -short ./...
+              -short
 
       - store_test_results:
           path: *TEST_RESULTS_DIR
diff --git a/.circleci/scripts/rerun-fails-report.sh b/.circleci/scripts/rerun-fails-report.sh
new file mode 100755
index 000000000..959f0708e
--- /dev/null
+++ b/.circleci/scripts/rerun-fails-report.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Add a comment on the github PR if there were any rerun tests.
+#
+set -eu -o pipefail
+
+report_filename="${1?report filename is required}"
+if [ ! -s "$report_filename" ]; then
+    echo "gotestsum rerun report file is empty or missing"
+    exit 0
+fi
+
+function report {
+    echo ":repeat: gotestsum re-ran some tests in $CIRCLE_BUILD_URL"
+    echo
+    echo '```'
+    cat "$report_filename"
+    echo '```'
+}
+
+report
\ No newline at end of file