From 2e8bcac2610023425c351b7850de3285e85c2697 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Jan 2021 12:41:07 -0500 Subject: [PATCH 1/4] e2e: deflake events Handle streamCh channel being closed. --- e2e/events/events.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/events/events.go b/e2e/events/events.go index 2a545676b..28c0614a1 100644 --- a/e2e/events/events.go +++ b/e2e/events/events.go @@ -218,7 +218,10 @@ func (tc *EventsTest) TestStartIndex(f *framework.F) { select { case <-ctx.Done(): return - case event := <-streamCh: + case event, ok := <-streamCh: + if !ok { + return + } if event.IsHeartbeat() { continue } From 95b7fc80b8f1a49f5a9bf273e2fa5ef5df4e19cc Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Jan 2021 10:24:23 -0500 Subject: [PATCH 2/4] e2e deflake namespaces: only check namespace jobs Deflake namespace e2e test by only asserting on jobs related to the namespace tests. During our e2e tests, some left over jobs (e.g. prometheus) are left running while being shutdown and cause the test to fail. --- e2e/e2eutil/allocs.go | 2 +- e2e/namespaces/namespaces.go | 66 ++++++++++++++++++++---------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/e2e/e2eutil/allocs.go b/e2e/e2eutil/allocs.go index 625732f44..fcf6f6057 100644 --- a/e2e/e2eutil/allocs.go +++ b/e2e/e2eutil/allocs.go @@ -208,7 +208,7 @@ func AllocExec(allocID, taskID, execCmd, ns string, wc *WaitConfig) (string, err got, err = Command(cmd[0], cmd[1:]...) return err == nil, err }, func(e error) { - err = fmt.Errorf("exec failed: '%s'", strings.Join(cmd, " ")) + err = fmt.Errorf("exec failed: '%s': %v", strings.Join(cmd, " "), e) }) return got, err } diff --git a/e2e/namespaces/namespaces.go b/e2e/namespaces/namespaces.go index a484bcc7f..67a81d913 100644 --- a/e2e/namespaces/namespaces.go +++ b/e2e/namespaces/namespaces.go @@ -3,6 +3,7 @@ package namespaces import ( "fmt" "os" + "strings" e2e "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" @@ -88,61 +89,69 @@ func (tc *NamespacesE2ETest) TestNamespacesFiltering(f *framework.F) { jobDefault := run("namespaces/input/namespace_default.nomad", "") // exercise 'nomad job status' filtering + parse := func(out string) []map[string]string { + rows, err := e2e.ParseColumns(out) + f.NoError(err, "failed to parse job status output: %v", out) + + result := make([]map[string]string, 0, len(rows)) + for _, row := range rows { + jobID := row["Job ID"] + if jobID == "" { + jobID = row["ID"] + } + if strings.HasPrefix(jobID, "test-namespace-") { + result = append(result, row) + } + } + return result + } out, err := e2e.Command("nomad", "job", "status", "-namespace", "NamespaceA") f.NoError(err, "'nomad job status -namespace NamespaceA' failed") - rows, err := e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") - f.Equal(1, len(rows)) + rows := parse(out) + f.Len(rows, 1) f.Equal(jobA, rows[0]["ID"]) out, err = e2e.Command("nomad", "job", "status", "-namespace", "NamespaceB") f.NoError(err, "'nomad job status -namespace NamespaceB' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobB, rows[0]["ID"]) out, err = e2e.Command("nomad", "job", "status", "-namespace", "*") f.NoError(err, "'nomad job status -namespace *' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") + rows = parse(out) f.Equal(3, len(rows)) out, err = e2e.Command("nomad", "job", "status") f.NoError(err, "'nomad job status' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse job status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobDefault, rows[0]["ID"]) // exercise 'nomad status' filtering out, err = e2e.Command("nomad", "status", "-namespace", "NamespaceA") f.NoError(err, "'nomad job status -namespace NamespaceA' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobA, rows[0]["ID"]) out, err = e2e.Command("nomad", "status", "-namespace", "NamespaceB") f.NoError(err, "'nomad job status -namespace NamespaceB' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobB, rows[0]["ID"]) out, err = e2e.Command("nomad", "status", "-namespace", "*") f.NoError(err, "'nomad job status -namespace *' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") + rows = parse(out) f.Equal(3, len(rows)) out, err = e2e.Command("nomad", "status") f.NoError(err, "'nomad status' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse status output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobDefault, rows[0]["ID"]) // exercise 'nomad deployment list' filtering @@ -150,23 +159,20 @@ func (tc *NamespacesE2ETest) TestNamespacesFiltering(f *framework.F) { out, err = e2e.Command("nomad", "deployment", "list", "-namespace", "NamespaceA") f.NoError(err, "'nomad job status -namespace NamespaceA' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse deployment list output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobA, rows[0]["Job ID"]) out, err = e2e.Command("nomad", "deployment", "list", "-namespace", "NamespaceB") f.NoError(err, "'nomad job status -namespace NamespaceB' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse deployment list output") + rows = parse(out) f.Equal(len(rows), 1) f.Equal(jobB, rows[0]["Job ID"]) out, err = e2e.Command("nomad", "deployment", "list") f.NoError(err, "'nomad deployment list' failed") - rows, err = e2e.ParseColumns(out) - f.NoError(err, "could not parse deployment list output") - f.Equal(1, len(rows)) + rows = parse(out) + f.Len(rows, 1) f.Equal(jobDefault, rows[0]["Job ID"]) out, err = e2e.Command("nomad", "job", "stop", jobA) From 923725bf3d77923e50ce34261546a87133e7bfed Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Jan 2021 15:59:17 -0500 Subject: [PATCH 3/4] e2e: deflake TestVolumeMounts After submitting an update, the test ought to wait until the new allocations are placed. Previously, we'd use the original to-be-stopped allocations and the test fails when attempting to exec. --- e2e/volumes/volumes.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/e2e/volumes/volumes.go b/e2e/volumes/volumes.go index 0ad9596af..0a38aa548 100644 --- a/e2e/volumes/volumes.go +++ b/e2e/volumes/volumes.go @@ -3,12 +3,14 @@ package volumes import ( "fmt" "os" + "time" "github.com/hashicorp/nomad/api" e2e "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/jobspec" + "github.com/hashicorp/nomad/testutil" ) const ns = "" @@ -94,8 +96,22 @@ func (tc *VolumesTest) TestVolumeMounts(f *framework.F) { _, _, err = tc.Nomad().Jobs().Register(job, nil) f.NoError(err, "could not register updated job") - allocs, err = e2e.AllocsForJob(jobID, ns) - f.NoError(err, "could not get allocs for job") + testutil.WaitForResultRetries(5000, func() (bool, error) { + time.Sleep(time.Millisecond * 100) + allocs, err = e2e.AllocsForJob(jobID, ns) + if err != nil { + return false, err + } + if len(allocs) < 2 { + return false, fmt.Errorf("no new allocation for %v: %v", jobID, allocs) + } + + return true, nil + }, func(e error) { + f.NoError(e, "failed to get new alloc") + + }) + newAllocID := allocs[0]["ID"] newCmdToExec := fmt.Sprintf("cat /tmp/foo/%s", newAllocID) From 9dcdafe4cfd1169bfb66eb147799a8969476f057 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Jan 2021 09:28:54 -0500 Subject: [PATCH 4/4] e2e: show command output on failure When a command fails, it's nice to have the full output, as it contains diagnostic information. The status code isn't sufficient for debugging. --- e2e/e2eutil/cli.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/e2eutil/cli.go b/e2e/e2eutil/cli.go index be2aebf6d..9d5293fd9 100644 --- a/e2e/e2eutil/cli.go +++ b/e2e/e2eutil/cli.go @@ -10,8 +10,12 @@ import ( // Command sends a command line argument to Nomad and returns the unbuffered // stdout as a string (or, if there's an error, the stderr) func Command(cmd string, args ...string) (string, error) { - out, err := exec.Command(cmd, args...).CombinedOutput() - return string(out), err + bytes, err := exec.Command(cmd, args...).CombinedOutput() + out := string(bytes) + if err != nil { + return out, fmt.Errorf("command %v %v failed: %v\nOutput: %v", cmd, args, err, out) + } + return out, err } // GetField returns the value of an output field (ex. the "Submit Date" field