diff --git a/e2e/e2eutil/allocs.go b/e2e/e2eutil/allocs.go index 625732f44..fcf6f6057 100644 --- a/e2e/e2eutil/allocs.go +++ b/e2e/e2eutil/allocs.go @@ -208,7 +208,7 @@ func AllocExec(allocID, taskID, execCmd, ns string, wc *WaitConfig) (string, err got, err = Command(cmd[0], cmd[1:]...) return err == nil, err }, func(e error) { - err = fmt.Errorf("exec failed: '%s'", strings.Join(cmd, " ")) + err = fmt.Errorf("exec failed: '%s': %v", strings.Join(cmd, " "), e) }) return got, err } diff --git a/e2e/e2eutil/cli.go b/e2e/e2eutil/cli.go index be2aebf6d..9d5293fd9 100644 --- a/e2e/e2eutil/cli.go +++ b/e2e/e2eutil/cli.go @@ -10,8 +10,12 @@ import ( // Command sends a command line argument to Nomad and returns the unbuffered // stdout as a string (or, if there's an error, the stderr) func Command(cmd string, args ...string) (string, error) { - out, err := exec.Command(cmd, args...).CombinedOutput() - return string(out), err + bytes, err := exec.Command(cmd, args...).CombinedOutput() + out := string(bytes) + if err != nil { + return out, fmt.Errorf("command %v %v failed: %v\nOutput: %v", cmd, args, err, out) + } + return out, err } // GetField returns the value of an output field (ex. the "Submit Date" field diff --git a/e2e/events/events.go b/e2e/events/events.go index 2a545676b..28c0614a1 100644 --- a/e2e/events/events.go +++ b/e2e/events/events.go @@ -218,7 +218,10 @@ func (tc *EventsTest) TestStartIndex(f *framework.F) { select { case <-ctx.Done(): return - case event := <-streamCh: + case event, ok := <-streamCh: + if !ok { + return + } if event.IsHeartbeat() { continue } diff --git a/e2e/namespaces/namespaces.go b/e2e/namespaces/namespaces.go index a484bcc7f..67a81d913 100644 --- a/e2e/namespaces/namespaces.go +++ b/e2e/namespaces/namespaces.go @@ -3,6 +3,7 @@ package namespaces import ( "fmt" "os" + "strings" e2e "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" @@ -88,61 +89,69 @@ func (tc *NamespacesE2ETest) TestNamespacesFiltering(f *framework.F) { jobDefault := run("namespaces/input/namespace_default.nomad", "") // exercise 'nomad job status' filtering + parse := func(out string) []map[string]string { + rows, err := e2e.ParseColumns(out) + f.NoError(err, "failed to parse job status output: %v", out) + + result := make([]map[string]string, 0, len(rows)) + for _, row := range rows { + jobID := row["Job ID"] + if jobID == "" { + jobID = row["ID"] + } + if strings.HasPrefix(jobID, "test-namespace-") { + result = append(result, row) + } + } + return result + } out, err := e2e.Command("nomad", "job", "status", "-namespace", "NamespaceA") f.NoError(err, "'nomad job status -namespace NamespaceA' failed") - rows, err := e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") - f.Equal(1, len(rows)) + rows := parse(out) + f.Len(rows, 1) f.Equal(jobA, rows[0]["ID"]) out, err = e2e.Command("nomad", "job", "status", "-namespace", "NamespaceB") f.NoError(err, "'nomad job status -namespace NamespaceB' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobB, rows[0]["ID"]) out, err = e2e.Command("nomad", "job", "status", "-namespace", "*") f.NoError(err, "'nomad job status -namespace *' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") + rows = parse(out) f.Equal(3, len(rows)) out, err = e2e.Command("nomad", "job", "status") f.NoError(err, "'nomad job status' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobDefault, rows[0]["ID"]) // exercise 'nomad status' filtering out, err = e2e.Command("nomad", "status", "-namespace", "NamespaceA") f.NoError(err, "'nomad job status -namespace NamespaceA' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobA, rows[0]["ID"]) out, err = e2e.Command("nomad", "status", "-namespace", "NamespaceB") f.NoError(err, "'nomad job status -namespace NamespaceB' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobB, rows[0]["ID"]) out, err = e2e.Command("nomad", "status", "-namespace", "*") f.NoError(err, "'nomad job status -namespace *' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") + rows = parse(out) f.Equal(3, len(rows)) out, err = e2e.Command("nomad", "status") f.NoError(err, "'nomad status' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobDefault, rows[0]["ID"]) // exercise 'nomad deployment list' filtering @@ -150,23 +159,20 @@ func (tc *NamespacesE2ETest) TestNamespacesFiltering(f *framework.F) { out, err = e2e.Command("nomad", "deployment", "list", "-namespace", "NamespaceA") f.NoError(err, "'nomad job status -namespace NamespaceA' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse deployment list output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobA, rows[0]["Job ID"]) out, err = e2e.Command("nomad", "deployment", "list", "-namespace", "NamespaceB") f.NoError(err, "'nomad job status -namespace NamespaceB' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse deployment list output") + rows = parse(out) f.Equal(len(rows), 1) f.Equal(jobB, rows[0]["Job ID"]) out, err = e2e.Command("nomad", "deployment", "list") f.NoError(err, "'nomad deployment list' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse deployment list output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobDefault, rows[0]["Job ID"]) out, err = e2e.Command("nomad", "job", "stop", jobA) diff --git a/e2e/volumes/volumes.go b/e2e/volumes/volumes.go index 0ad9596af..0a38aa548 100644 --- a/e2e/volumes/volumes.go +++ b/e2e/volumes/volumes.go @@ -3,12 +3,14 @@ package volumes import ( "fmt" "os" + "time" "github.com/hashicorp/nomad/api" e2e "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/jobspec" + "github.com/hashicorp/nomad/testutil" ) const ns = "" @@ -94,8 +96,22 @@ func (tc *VolumesTest) TestVolumeMounts(f *framework.F) { _, _, err = tc.Nomad().Jobs().Register(job, nil) f.NoError(err, "could not register updated job") - allocs, err = e2e.AllocsForJob(jobID, ns) - f.NoError(err, "could not get allocs for job") + testutil.WaitForResultRetries(5000, func() (bool, error) { + time.Sleep(time.Millisecond * 100) + allocs, err = e2e.AllocsForJob(jobID, ns) + if err != nil { + return false, err + } + if len(allocs) < 2 { + return false, fmt.Errorf("no new allocation for %v: %v", jobID, allocs) + } + + return true, nil + }, func(e error) { + f.NoError(e, "failed to get new alloc") + + }) + newAllocID := allocs[0]["ID"] newCmdToExec := fmt.Sprintf("cat /tmp/foo/%s", newAllocID)