e2e: add e2e tests for nomad service disco checks

This PR adds 2 e2e tests for ensuring nomad service discovery checks
get created and produce status results as expected.

e2e/e2eutil/allocs.go

@@ -240,6 +240,13 @@ func AllocLogs(allocID string, logStream LogStream) (string, error) {
 	return Command(cmd[0], cmd[1:]...)
 }
 
+// AllocChecks returns the CLI output from 'nomad alloc checks' on the given
+// alloc ID.
+func AllocChecks(allocID string) (string, error) {
+	cmd := []string{"nomad", "alloc", "checks", allocID}
+	return Command(cmd[0], cmd[1:]...)
+}
+
 func AllocTaskLogs(allocID, task string, logStream LogStream) (string, error) {
 	cmd := []string{"nomad", "alloc", "logs"}
 	if logStream == LogsStdErr {

e2e/servicediscovery/input/checks_happy.nomad

@@ -0,0 +1,37 @@
+job "checks_happy" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+    network {
+      mode = "host"
+      port "http" {}
+    }
+
+    service {
+      provider = "nomad"
+      name     = "http-server"
+      port     = "http"
+      check {
+        name     = "http-server-check"
+        type     = "http"
+        path     = "/"
+        interval = "2s"
+        timeout  = "1s"
+      }
+    }
+
+    task "python-http" {
+      driver = "raw_exec"
+      config {
+        command = "python3"
+        args    = ["-m", "http.server", "${NOMAD_PORT_http}"]
+      }
+    }
+  }
+}

e2e/servicediscovery/input/checks_sad.nomad

@@ -0,0 +1,38 @@
+job "checks_sad" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+    network {
+      mode = "host"
+      port "http" {}
+    }
+
+    service {
+      provider = "nomad"
+      name     = "http-server"
+      port     = "http"
+      check {
+        name     = "http-server-check"
+        type     = "http"
+        path     = "/"
+        method   = "POST" # not allowed by http.server
+        interval = "2s"
+        timeout  = "1s"
+      }
+    }
+
+    task "python-http" {
+      driver = "raw_exec"
+      config {
+        command = "python3"
+        args    = ["-m", "http.server", "${NOMAD_PORT_http}"]
+      }
+    }
+  }
+}

e2e/servicediscovery/nomad_checks_test.go

@@ -0,0 +1,83 @@
+package servicediscovery
+
+import (
+	"context"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/shoenig/test/must"
+)
+
+func testChecksHappy(t *testing.T) {
+	nomadClient := e2eutil.NomadClient(t)
+
+	// Generate our unique job ID which will be used for this test.
+	jobID := "nsd-check-happy-" + uuid.Short()
+	jobIDs := []string{jobID}
+
+	// Defer a cleanup function to remove the job. This will trigger if the
+	// test fails, unless the cancel function is called.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	defer e2eutil.CleanupJobsAndGCWithContext(t, ctx, &jobIDs)
+
+	// Register the happy checks job.
+	allocStubs := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, jobChecksHappy, jobID, "")
+	must.Len(t, 1, allocStubs)
+
+	// wait for the alloc to be running
+	e2eutil.WaitForAllocRunning(t, nomadClient, allocStubs[0].ID)
+
+	// get the output of 'nomad alloc checks'
+	output, err := e2eutil.AllocChecks(allocStubs[0].ID)
+	must.NoError(t, err)
+
+	// assert the output contains success
+	statusRe := regexp.MustCompile(`Status\s+=\s+success`)
+	must.RegexMatch(t, statusRe, output)
+
+	// assert the output contains 200 status code
+	statusCodeRe := regexp.MustCompile(`StatusCode\s+=\s+200`)
+	must.RegexMatch(t, statusCodeRe, output)
+
+	// assert output contains nomad's success string
+	must.StrContains(t, output, `nomad: http ok`)
+}
+
+func testChecksSad(t *testing.T) {
+	nomadClient := e2eutil.NomadClient(t)
+
+	// Generate our unique job ID which will be used for this test.
+	jobID := "nsd-check-sad-" + uuid.Short()
+	jobIDs := []string{jobID}
+
+	// Defer a cleanup function to remove the job. This will trigger if the
+	// test fails, unless the cancel function is called.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	defer e2eutil.CleanupJobsAndGCWithContext(t, ctx, &jobIDs)
+
+	// Register the sad checks job.
+	allocStubs := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, jobChecksSad, jobID, "")
+	must.Len(t, 1, allocStubs)
+
+	// wait for the alloc to be running
+	e2eutil.WaitForAllocRunning(t, nomadClient, allocStubs[0].ID)
+
+	// get the output of 'nomad alloc checks'
+	output, err := e2eutil.AllocChecks(allocStubs[0].ID)
+	must.NoError(t, err)
+
+	// assert the output contains failure
+	statusRe := regexp.MustCompile(`Status\s+=\s+failure`)
+	must.RegexMatch(t, statusRe, output)
+
+	// assert the output contains 501 status code
+	statusCodeRe := regexp.MustCompile(`StatusCode\s+=\s+501`)
+	must.RegexMatch(t, statusCodeRe, output)
+
+	// assert output contains error output from python http.server
+	must.StrContains(t, output, `<p>Error code explanation: HTTPStatus.NOT_IMPLEMENTED - Server does not support this operation.</p>`)
+}

e2e/servicediscovery/service_discovery_test.go

@@ -20,6 +20,8 @@ const (
 	jobMultiProvider    = "./input/multi_provider.nomad"
 	jobSimpleLBReplicas = "./input/simple_lb_replicas.nomad"
 	jobSimpleLBClients  = "./input/simple_lb_clients.nomad"
+	jobChecksHappy      = "./input/checks_happy.nomad"
+	jobChecksSad        = "./input/checks_sad.nomad"
 )
 
 const (
@@ -41,6 +43,8 @@ func TestServiceDiscovery(t *testing.T) {
 	t.Run("TestServiceDiscovery_MultiProvider", testMultiProvider)
 	t.Run("TestServiceDiscovery_UpdateProvider", testUpdateProvider)
 	t.Run("TestServiceDiscovery_SimpleLoadBalancing", testSimpleLoadBalancing)
+	t.Run("TestServiceDiscovery_ChecksHappy", testChecksHappy)
+	t.Run("TestServiceDiscovery_ChecksSad", testChecksSad)
 }
 
 // testMultiProvider tests service discovery where multi providers are used

e2e/servicediscovery/simple_lb_test.go

@@ -15,8 +15,8 @@ import (
 func testSimpleLoadBalancing(t *testing.T) {
 	nomadClient := e2eutil.NomadClient(t)
 
-	// Generate our job ID which will be used for the entire test.
-	jobID := "nsd-simple-lb-replicas" + uuid.Short()
+	// Generate our unique job ID which will be used for this test.
+	jobID := "nsd-simple-lb-replicas-" + uuid.Short()
 	jobIDs := []string{jobID}
 
 	// Defer a cleanup function to remove the job. This will trigger if the