e2e: tweak failure messages

Tweak the error messages for the flakiest tests, so that on test failure, we get
more output

e2e/connect/multi_service.go

@@ -1,6 +1,7 @@
 package connect
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
@@ -9,6 +10,7 @@ import (
 	"github.com/hashicorp/nomad/e2e/framework"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/jobspec"
+	"github.com/hashicorp/nomad/testutil"
 	"github.com/kr/pretty"
 	"github.com/stretchr/testify/require"
 )
@@ -45,9 +47,9 @@ EVAL:
 	case "complete":
 		// Ok!
 	case "failed", "canceled", "blocked":
-		t.Fatalf("eval %s\n%s\n", eval.Status, pretty.Sprint(eval))
+		require.Failf(t, "expected complete status", "eval %s\n%s", eval.Status, pretty.Sprint(eval))
 	default:
-		t.Fatalf("unknown eval status: %s\n%s\n", eval.Status, pretty.Sprint(eval))
+		require.Failf(t, "expected complete status", "unknown eval status: %s\n%s", eval.Status, pretty.Sprint(eval))
 	}
 
 	// Assert there were 0 placement failures
@@ -85,7 +87,7 @@ EVAL:
 	allocIDs := make(map[string]bool, 1)
 	for _, a := range allocs {
 		if a.ClientStatus != "running" || a.DesiredStatus != "run" {
-			t.Fatalf("alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus)
+			require.Failf(t, "expected running status", "alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus)
 		}
 		allocIDs[a.ID] = true
 	}
@@ -94,7 +96,9 @@ EVAL:
 	agentapi := tc.Consul().Agent()
 
 	failing := map[string]*consulapi.AgentCheck{}
-	require.Eventually(t, func() bool {
+	testutil.WaitForResultRetries(60, func() (bool, error) {
+		defer time.Sleep(time.Second)
+
 		checks, err := agentapi.Checks()
 		require.NoError(t, err)
 
@@ -123,12 +127,14 @@ EVAL:
 		}
 
 		if len(failing) == 0 {
-			return true
+			return true, nil
 		}
 
 		t.Logf("still %d checks not passing", len(failing))
-		return false
-	}, time.Minute, time.Second)
+		return false, fmt.Errorf("checks are not passing %v %v", len(failing), pretty.Sprint(failing))
+	}, func(e error) {
+		require.NoError(t, err)
+	})
 
 	require.Len(t, failing, 0, pretty.Sprint(failing))
 }

e2e/consulacls/manage.go

@@ -62,7 +62,7 @@ func (m *tfManager) Enable(t *testing.T) string {
 
 	response, err := exec.CommandContext(ctx,
 		"consulacls/consul-acls-manage.sh", "enable").CombinedOutput()
-	require.NoError(t, err)
+	require.NoError(t, err, "consul-acls-manage.sh failed: %v", string(response))
 	fmt.Println(string(response))
 
 	// Read the Consul ACL master token that was generated (or if the token

e2e/e2eutil/allocs.go

@@ -9,17 +9,23 @@ import (
 
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/testutil"
+	"github.com/kr/pretty"
 )
 
 // WaitForAllocStatusExpected polls 'nomad job status' and exactly compares
 // the status of all allocations (including any previous versions) against the
 // expected list.
 func WaitForAllocStatusExpected(jobID, ns string, expected []string) error {
-	return WaitForAllocStatusComparison(
+	err := WaitForAllocStatusComparison(
 		func() ([]string, error) { return AllocStatuses(jobID, ns) },
 		func(got []string) bool { return reflect.DeepEqual(got, expected) },
 		nil,
 	)
+	if err != nil {
+		allocs, _ := AllocsForJob(jobID, ns)
+		err = fmt.Errorf("%v\nallocs: %v", err, pretty.Sprint(allocs))
+	}
+	return err
 }
 
 // WaitForAllocStatusComparison is a convenience wrapper that polls the query
@@ -208,7 +214,7 @@ func AllocExec(allocID, taskID, execCmd, ns string, wc *WaitConfig) (string, err
 		got, err = Command(cmd[0], cmd[1:]...)
 		return err == nil, err
 	}, func(e error) {
-		err = fmt.Errorf("exec failed: '%s': %v", strings.Join(cmd, " "), e)
+		err = fmt.Errorf("exec failed: '%s': %v\nGot: %v", strings.Join(cmd, " "), e, got)
 	})
 	return got, err
 }

e2e/volumes/volumes.go

@@ -71,7 +71,7 @@ func (tc *VolumesTest) TestVolumeMounts(f *framework.F) {
 
 	out, err := e2e.AllocExec(allocID, "docker_task", cmdToExec, ns, nil)
 	f.NoError(err, "could not exec into task: docker_task")
-	f.Equal(out, allocID+"\n", "alloc data is missing from docker_task")
+	f.Equal(allocID+"\n", out, "alloc data is missing from docker_task")
 
 	out, err = e2e.AllocExec(allocID, "exec_task", cmdToExec, ns, nil)
 	f.NoError(err, "could not exec into task: exec_task")