From f7d2a74ddf1d5a010a26ffeb0e049af76b35d724 Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Wed, 28 Jun 2017 11:35:43 -0700
Subject: [PATCH 1/2] System jobs should be running until stopped

Prior to this commit they would be marked as dead if they had no
currently running allocations -- even though they would spring back to
life (running) if the cluster state changed such that a new eval+alloc
was created.
---
 nomad/state/state_store.go      | 16 ++++++++++++++
 nomad/state/state_store_test.go | 37 +++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go
index 91785decf..97bc0c5b5 100644
--- a/nomad/state/state_store.go
+++ b/nomad/state/state_store.go
@@ -2124,6 +2124,22 @@ func (s *StateStore) getJobStatus(txn *memdb.Txn, job *structs.Job, evalDelete b
 		}
 	}
 
+	// system jobs are running until explicitly stopped (which is handled elsewhere)
+	if job.Type == structs.JobTypeSystem {
+		if job.Stop {
+			return structs.JobStatusDead, nil
+		}
+
+		if hasEval {
+			// At least one completed eval
+			return structs.JobStatusRunning, nil
+		}
+
+		// Pending until at least one eval has completed
+		return structs.JobStatusPending, nil
+
+	}
+
 	// The job is dead if all the allocations and evals are terminal or if there
 	// are no evals because of garbage collection.
 	if evalDelete || hasEval || hasAlloc {
diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go
index 2151a4a42..54aa73e64 100644
--- a/nomad/state/state_store_test.go
+++ b/nomad/state/state_store_test.go
@@ -4464,6 +4464,43 @@ func TestStateStore_SetJobStatus_PendingEval(t *testing.T) {
 	}
 }
 
+// TestStateStore_SetJobStatus_SystemJob asserts that system jobs are still
+// considered running until explicitly stopped.
+func TestStateStore_SetJobStatus_SystemJob(t *testing.T) {
+	state := testStateStore(t)
+	job := mock.SystemJob()
+
+	// Create a mock eval that is pending.
+	eval := mock.Eval()
+	eval.JobID = job.ID
+	eval.Type = job.Type
+	eval.Status = structs.EvalStatusComplete
+	if err := state.UpsertEvals(1000, []*structs.Evaluation{eval}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	txn := state.db.Txn(false)
+	status, err := state.getJobStatus(txn, job, true)
+	if err != nil {
+		t.Fatalf("getJobStatus() failed: %v", err)
+	}
+
+	if expected := structs.JobStatusRunning; status != expected {
+		t.Fatalf("getJobStatus() returned %v; expected %v", status, expected)
+	}
+
+	// Stop the job
+	job.Stop = true
+	status, err = state.getJobStatus(txn, job, true)
+	if err != nil {
+		t.Fatalf("getJobStatus() failed: %v", err)
+	}
+
+	if expected := structs.JobStatusDead; status != expected {
+		t.Fatalf("getJobStatus() returned %v; expected %v", status, expected)
+	}
+}
+
 func TestStateJobSummary_UpdateJobCount(t *testing.T) {
 	state := testStateStore(t)
 	alloc := mock.Alloc()

From 8d3e13ab8ad6b63358b0c4355bcfa8e35603a4c3 Mon Sep 17 00:00:00 2001
From: Michael Schurter <schmichael@hashicorp.com>
Date: Mon, 3 Jul 2017 13:39:39 -0700
Subject: [PATCH 2/2] System jobs without evals are running too

---
 nomad/state/state_store.go      | 8 +-------
 nomad/state/state_store_test.go | 1 +
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go
index 97bc0c5b5..91b459844 100644
--- a/nomad/state/state_store.go
+++ b/nomad/state/state_store.go
@@ -2130,14 +2130,8 @@ func (s *StateStore) getJobStatus(txn *memdb.Txn, job *structs.Job, evalDelete b
 			return structs.JobStatusDead, nil
 		}
 
-		if hasEval {
-			// At least one completed eval
-			return structs.JobStatusRunning, nil
-		}
-
 		// Pending until at least one eval has completed
-		return structs.JobStatusPending, nil
-
+		return structs.JobStatusRunning, nil
 	}
 
 	// The job is dead if all the allocations and evals are terminal or if there
diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go
index 54aa73e64..1783f1cdb 100644
--- a/nomad/state/state_store_test.go
+++ b/nomad/state/state_store_test.go
@@ -1635,6 +1635,7 @@ func TestStateStore_JobsByScheduler(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		job := mock.SystemJob()
+		job.Status = structs.JobStatusRunning
 		sysJobs = append(sysJobs, job)
 
 		err := state.UpsertJob(2000+uint64(i), job)