Merge pull request #9868 from hashicorp/e2e-tests-20200121

Deflake some e2e tests

e2e/e2eutil/allocs.go

@@ -208,7 +208,7 @@ func AllocExec(allocID, taskID, execCmd, ns string, wc *WaitConfig) (string, err
 		got, err = Command(cmd[0], cmd[1:]...)
 		return err == nil, err
 	}, func(e error) {
-		err = fmt.Errorf("exec failed: '%s'", strings.Join(cmd, " "))
+		err = fmt.Errorf("exec failed: '%s': %v", strings.Join(cmd, " "), e)
 	})
 	return got, err
 }

e2e/e2eutil/cli.go

@@ -10,8 +10,12 @@ import (
 // Command sends a command line argument to Nomad and returns the unbuffered
 // stdout as a string (or, if there's an error, the stderr)
 func Command(cmd string, args ...string) (string, error) {
-	out, err := exec.Command(cmd, args...).CombinedOutput()
+	bytes, err := exec.Command(cmd, args...).CombinedOutput()
-	return string(out), err
+	out := string(bytes)
+	if err != nil {
+		return out, fmt.Errorf("command %v %v failed: %v\nOutput: %v", cmd, args, err, out)
+	}
+	return out, err
 }
 
 // GetField returns the value of an output field (ex. the "Submit Date" field

e2e/events/events.go

@@ -218,7 +218,10 @@ func (tc *EventsTest) TestStartIndex(f *framework.F) {
 			select {
 			case <-ctx.Done():
 				return
-			case event := <-streamCh:
+			case event, ok := <-streamCh:
+				if !ok {
+					return
+				}
 				if event.IsHeartbeat() {
 					continue
 				}

e2e/namespaces/namespaces.go

@@ -3,6 +3,7 @@ package namespaces
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	e2e "github.com/hashicorp/nomad/e2e/e2eutil"
 	"github.com/hashicorp/nomad/e2e/framework"
@@ -88,61 +89,69 @@ func (tc *NamespacesE2ETest) TestNamespacesFiltering(f *framework.F) {
 	jobDefault := run("namespaces/input/namespace_default.nomad", "")
 
 	// exercise 'nomad job status' filtering
+	parse := func(out string) []map[string]string {
+		rows, err := e2e.ParseColumns(out)
+		f.NoError(err, "failed to parse job status output: %v", out)
+
+		result := make([]map[string]string, 0, len(rows))
+		for _, row := range rows {
+			jobID := row["Job ID"]
+			if jobID == "" {
+				jobID = row["ID"]
+			}
+			if strings.HasPrefix(jobID, "test-namespace-") {
+				result = append(result, row)
+			}
+		}
+		return result
+	}
 
 	out, err := e2e.Command("nomad", "job", "status", "-namespace", "NamespaceA")
 	f.NoError(err, "'nomad job status -namespace NamespaceA' failed")
-	rows, err := e2e.ParseColumns(out)
+	rows := parse(out)
-	f.NoError(err, "could not parse job status output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobA, rows[0]["ID"])
 
 	out, err = e2e.Command("nomad", "job", "status", "-namespace", "NamespaceB")
 	f.NoError(err, "'nomad job status -namespace NamespaceB' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse job status output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobB, rows[0]["ID"])
 
 	out, err = e2e.Command("nomad", "job", "status", "-namespace", "*")
 	f.NoError(err, "'nomad job status -namespace *' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse job status output")
 	f.Equal(3, len(rows))
 
 	out, err = e2e.Command("nomad", "job", "status")
 	f.NoError(err, "'nomad job status' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse job status output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobDefault, rows[0]["ID"])
 
 	// exercise 'nomad status' filtering
 
 	out, err = e2e.Command("nomad", "status", "-namespace", "NamespaceA")
 	f.NoError(err, "'nomad job status -namespace NamespaceA' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse status output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobA, rows[0]["ID"])
 
 	out, err = e2e.Command("nomad", "status", "-namespace", "NamespaceB")
 	f.NoError(err, "'nomad job status -namespace NamespaceB' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse status output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobB, rows[0]["ID"])
 
 	out, err = e2e.Command("nomad", "status", "-namespace", "*")
 	f.NoError(err, "'nomad job status -namespace *' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse status output")
 	f.Equal(3, len(rows))
 
 	out, err = e2e.Command("nomad", "status")
 	f.NoError(err, "'nomad status' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse status output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobDefault, rows[0]["ID"])
 
 	// exercise 'nomad deployment list' filtering
@@ -150,23 +159,20 @@ func (tc *NamespacesE2ETest) TestNamespacesFiltering(f *framework.F) {
 
 	out, err = e2e.Command("nomad", "deployment", "list", "-namespace", "NamespaceA")
 	f.NoError(err, "'nomad job status -namespace NamespaceA' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse deployment list output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobA, rows[0]["Job ID"])
 
 	out, err = e2e.Command("nomad", "deployment", "list", "-namespace", "NamespaceB")
 	f.NoError(err, "'nomad job status -namespace NamespaceB' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse deployment list output")
 	f.Equal(len(rows), 1)
 	f.Equal(jobB, rows[0]["Job ID"])
 
 	out, err = e2e.Command("nomad", "deployment", "list")
 	f.NoError(err, "'nomad deployment list' failed")
-	rows, err = e2e.ParseColumns(out)
+	rows = parse(out)
-	f.NoError(err, "could not parse deployment list output")
+	f.Len(rows, 1)
-	f.Equal(1, len(rows))
 	f.Equal(jobDefault, rows[0]["Job ID"])
 
 	out, err = e2e.Command("nomad", "job", "stop", jobA)

e2e/volumes/volumes.go

@@ -3,12 +3,14 @@ package volumes
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/hashicorp/nomad/api"
 	e2e "github.com/hashicorp/nomad/e2e/e2eutil"
 	"github.com/hashicorp/nomad/e2e/framework"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/jobspec"
+	"github.com/hashicorp/nomad/testutil"
 )
 
 const ns = ""
@@ -94,8 +96,22 @@ func (tc *VolumesTest) TestVolumeMounts(f *framework.F) {
 	_, _, err = tc.Nomad().Jobs().Register(job, nil)
 	f.NoError(err, "could not register updated job")
 
-	allocs, err = e2e.AllocsForJob(jobID, ns)
+	testutil.WaitForResultRetries(5000, func() (bool, error) {
-	f.NoError(err, "could not get allocs for job")
+		time.Sleep(time.Millisecond * 100)
+		allocs, err = e2e.AllocsForJob(jobID, ns)
+		if err != nil {
+			return false, err
+		}
+		if len(allocs) < 2 {
+			return false, fmt.Errorf("no new allocation for %v: %v", jobID, allocs)
+		}
+
+		return true, nil
+	}, func(e error) {
+		f.NoError(e, "failed to get new alloc")
+
+	})
+
 	newAllocID := allocs[0]["ID"]
 
 	newCmdToExec := fmt.Sprintf("cat /tmp/foo/%s", newAllocID)