From ee522ab587cd5efc797ac39697fd0a179afa3cff Mon Sep 17 00:00:00 2001
From: Jasmine Dahilig <jasmine@dahilig.com>
Date: Tue, 18 Aug 2020 10:49:50 -0700
Subject: [PATCH] task lifecycle: e2e tests

---
 e2e/e2e_test.go                    |   1 +
 e2e/lifecycle/inputs/batch.nomad   | 127 +++++++++++++++++++++++
 e2e/lifecycle/inputs/service.nomad | 160 +++++++++++++++++++++++++++++
 e2e/lifecycle/lifecycle.go         | 115 +++++++++++++++++++++
 4 files changed, 403 insertions(+)
 create mode 100644 e2e/lifecycle/inputs/batch.nomad
 create mode 100644 e2e/lifecycle/inputs/service.nomad
 create mode 100644 e2e/lifecycle/lifecycle.go

diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go
index 0b0f024cf..f8d4032d3 100644
--- a/e2e/e2e_test.go
+++ b/e2e/e2e_test.go
@@ -17,6 +17,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/deployment"
 	_ "github.com/hashicorp/nomad/e2e/example"
 	_ "github.com/hashicorp/nomad/e2e/hostvolumes"
+	_ "github.com/hashicorp/nomad/e2e/lifecycle"
 	_ "github.com/hashicorp/nomad/e2e/metrics"
 	_ "github.com/hashicorp/nomad/e2e/nomad09upgrade"
 	_ "github.com/hashicorp/nomad/e2e/nomadexec"
diff --git a/e2e/lifecycle/inputs/batch.nomad b/e2e/lifecycle/inputs/batch.nomad
new file mode 100644
index 000000000..744bcf7b6
--- /dev/null
+++ b/e2e/lifecycle/inputs/batch.nomad
@@ -0,0 +1,127 @@
+# lifecycle hook test job for batch jobs. touches, removes, and tests
+# for the existence of files to assert the order of running tasks.
+# all tasks should exit 0 and the alloc dir should contain the following
+# files: ./init-ran, ./main-ran, ./poststart-run
+
+job "batch-lifecycle" {
+
+  datacenters = ["dc1"]
+
+  type = "batch"
+
+  group "test" {
+
+    task "init" {
+
+      lifecycle {
+        hook = "prestart"
+      }
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/prestart.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+sleep 1
+touch ${NOMAD_ALLOC_DIR}/init-ran
+touch ${NOMAD_ALLOC_DIR}/init-running
+sleep 5
+if [ -f ${NOMAD_ALLOC_DIR}/main ]; then exit 7; fi
+if [ -f ${NOMAD_ALLOC_DIR}/poststart-running ]; then exit 8; fi
+rm ${NOMAD_ALLOC_DIR}/init-running
+EOT
+
+        destination = "local/prestart.sh"
+
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+    task "main" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/main.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+sleep 1
+touch ${NOMAD_ALLOC_DIR}/main-running
+touch ${NOMAD_ALLOC_DIR}/main-started
+# NEED TO HANG AROUND TO GIVE POSTSTART TIME TO GET STARTED
+sleep 10
+if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 9; fi
+if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 10; fi
+
+if [ ! -f ${NOMAD_ALLOC_DIR}/poststart-started ]; then exit 11; fi
+
+touch ${NOMAD_ALLOC_DIR}/main-ran
+rm ${NOMAD_ALLOC_DIR}/main-running
+EOT
+
+        destination = "local/main.sh"
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+
+    task "poststart" {
+
+      lifecycle {
+        hook = "poststart"
+      }
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/poststart.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+sleep 1
+touch ${NOMAD_ALLOC_DIR}/poststart-ran
+touch ${NOMAD_ALLOC_DIR}/poststart-running
+touch ${NOMAD_ALLOC_DIR}/poststart-started
+sleep 10
+# THIS IS WHERE THE ACTUAL TESTING HAPPENS
+# IF init-ran doesn't exist, then the init task hasn't run yet, so fail
+if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 12; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/main-started ]; then exit 15; fi
+if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 14; fi
+rm ${NOMAD_ALLOC_DIR}/poststart-running
+EOT
+
+        destination = "local/poststart.sh"
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+  }
+}
diff --git a/e2e/lifecycle/inputs/service.nomad b/e2e/lifecycle/inputs/service.nomad
new file mode 100644
index 000000000..ce4929721
--- /dev/null
+++ b/e2e/lifecycle/inputs/service.nomad
@@ -0,0 +1,160 @@
+# lifecycle hook test job for service jobs. touches, removes, and tests
+# for the existence of files to assert the order of running tasks.
+# after stopping, the alloc dir should contain the following files:
+# files: ./init-ran, ./sidecar-ran, ./main-ran, ./poststart-run but not
+# the ./main-running, ./sidecar-running, or ./poststart-running files
+
+job "service-lifecycle" {
+
+  datacenters = ["dc1"]
+
+  type = "batch"
+
+  group "test" {
+
+    task "init" {
+
+      lifecycle {
+        hook = "prestart"
+      }
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/prestart.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+sleep 1
+touch ${NOMAD_ALLOC_DIR}/init-ran
+touch ${NOMAD_ALLOC_DIR}/init-running
+if [ -f ${NOMAD_ALLOC_DIR}/main ]; then exit 7; fi
+if [ -f ${NOMAD_ALLOC_DIR}/poststart ]; then exit 8; fi
+rm ${NOMAD_ALLOC_DIR}/init-running
+EOT
+
+        destination = "local/prestart.sh"
+
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+    task "sidecar" {
+
+      lifecycle {
+        hook    = "prestart"
+        sidecar = true
+      }
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/sidecar.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+touch ${NOMAD_ALLOC_DIR}/sidecar-ran
+touch ${NOMAD_ALLOC_DIR}/sidecar-running
+sleep 5
+if [ ! -f ${NOMAD_ALLOC_DIR}/main-running ]; then exit 9; fi
+if [ -f ${NOMAD_ALLOC_DIR}/poststart-running ]; then exit 10; fi
+sleep 300
+EOT
+
+        destination = "local/sidecar.sh"
+
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+    task "main" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/main.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+touch ${NOMAD_ALLOC_DIR}/main-ran
+touch ${NOMAD_ALLOC_DIR}/main-running
+touch ${NOMAD_ALLOC_DIR}/main-started
+# NEED TO HANG AROUND TO GIVE POSTSTART TIME TO GET STARTED
+sleep 10
+if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 11; fi
+if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 12; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-ran ]; then exit 13; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-running ]; then exit 14; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/poststart-started ]; then exit 15; fi
+sleep 300
+EOT
+
+        destination = "local/main.sh"
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+
+    task "poststart" {
+
+      lifecycle {
+        hook = "poststart"
+      }
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["local/poststart.sh"]
+      }
+
+      template {
+        data = <<EOT
+#!/bin/sh
+touch ${NOMAD_ALLOC_DIR}/poststart-ran
+touch ${NOMAD_ALLOC_DIR}/poststart-running
+touch ${NOMAD_ALLOC_DIR}/poststart-started
+if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 16; fi
+if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 17; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-ran ]; then exit 18; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-running ]; then exit 19; fi
+if [ ! -f ${NOMAD_ALLOC_DIR}/main-started ]; then exit 20; fi
+rm ${NOMAD_ALLOC_DIR}/poststart-running
+EOT
+
+        destination = "local/poststart.sh"
+      }
+
+      resources {
+        cpu    = 64
+        memory = 64
+      }
+    }
+
+  }
+}
diff --git a/e2e/lifecycle/lifecycle.go b/e2e/lifecycle/lifecycle.go
new file mode 100644
index 000000000..63b3502e4
--- /dev/null
+++ b/e2e/lifecycle/lifecycle.go
@@ -0,0 +1,115 @@
+package lifecycle
+
+import (
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/framework"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/stretchr/testify/require"
+)
+
+type LifecycleE2ETest struct {
+	framework.TC
+	jobIDs []string
+}
+
+func init() {
+	framework.AddSuites(&framework.TestSuite{
+		Component: "Lifecycle",
+		// YOU COULD RUN THIS LOCALLY BC DIS FLAG
+		CanRunLocal: true,
+		Cases:       []framework.TestCase{new(LifecycleE2ETest)},
+	})
+}
+
+// Ensure cluster has leader and at least 1 client node
+// in a ready state before running tests
+func (tc *LifecycleE2ETest) BeforeAll(f *framework.F) {
+	e2eutil.WaitForLeader(f.T(), tc.Nomad())
+	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
+}
+
+// TestBatchJob runs a batch job with prestart and poststop hooks
+func (tc *LifecycleE2ETest) TestBatchJob(f *framework.F) {
+	t := f.T()
+	require := require.New(t)
+	nomadClient := tc.Nomad()
+	uuid := uuid.Generate()
+	jobID := "lifecycle-" + uuid[0:8]
+	tc.jobIDs = append(tc.jobIDs, jobID)
+
+	allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "lifecycle/inputs/batch.nomad", jobID, "")
+	require.Equal(1, len(allocs))
+	allocID := allocs[0].ID
+
+	// wait for the job to stop and assert we stopped successfully, not failed
+	e2eutil.WaitForAllocStopped(t, nomadClient, allocID)
+	alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
+	require.NoError(err)
+	require.Equal(structs.AllocClientStatusComplete, alloc.ClientStatus)
+
+	// assert the files were written as expected
+	afi, _, err := nomadClient.AllocFS().List(alloc, "alloc", nil)
+	require.NoError(err)
+	expected := map[string]bool{
+		"init-ran": true, "main-ran": true, "poststart-ran": true,
+		"init-running": false, "main-running": false, "poststart-running": false}
+	got := checkFiles(expected, afi)
+	require.Equal(expected, got)
+}
+
+// TODO: cleanup == poststop
+//       q: what is a good example for a poststart?
+//       a: notify(-slack)
+
+// TestServiceJob runs a service job with prestart and poststop hooks
+func (tc *LifecycleE2ETest) TestServiceJob(f *framework.F) {
+	t := f.T()
+	require := require.New(t)
+	nomadClient := tc.Nomad()
+	uuid := uuid.Generate()
+	jobID := "lifecycle-" + uuid[0:8]
+	tc.jobIDs = append(tc.jobIDs, jobID)
+
+	allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "lifecycle/inputs/service.nomad", jobID, "")
+	require.Equal(1, len(allocs))
+	allocID := allocs[0].ID
+
+	e2eutil.WaitForAllocRunning(t, nomadClient, allocID)
+
+	alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
+	require.NoError(err)
+
+	// stop the job
+	_, _, err = nomadClient.Jobs().Deregister(jobID, false, nil)
+	require.NoError(err)
+	e2eutil.WaitForAllocStopped(t, nomadClient, allocID)
+
+	// assert the files were written as expected
+	afi, _, err := nomadClient.AllocFS().List(alloc, "alloc", nil)
+	require.NoError(err)
+	expected := map[string]bool{
+		"init-ran": true, "sidecar-ran": true, "main-ran": true, "poststart-ran": true,
+		"poststart-started": true, "main-started": true,
+		"init-running": false, "sidecar-running": false,
+		"main-running": false, "poststart-running": false}
+	got := checkFiles(expected, afi)
+	require.Equal(expected, got)
+}
+
+// checkFiles returns a map of whether the expected files were found
+// in the file info response
+func checkFiles(expected map[string]bool, got []*api.AllocFileInfo) map[string]bool {
+	results := map[string]bool{}
+	for expect := range expected {
+		results[expect] = false
+	}
+	for _, file := range got {
+		// there will be files unrelated to the test, so ignore those
+		if _, ok := results[file.Name]; ok {
+			results[file.Name] = true
+		}
+	}
+	return results
+}