luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

GC-ing dead batch jobs

This commit is contained in:
Diptanu Choudhury 2016-06-22 11:40:27 -07:00
parent 40c9171c0b
commit 0fe8746692
2 changed files with 83 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nomad/core_sched.go
View File

@ -231,9 +231,8 @@ func (c *CoreScheduler) gcEval(eval *structs.Evaluation, thresholdIndex uint64)
return false, nil, err
}
// If the job has been deregistered, we want to garbage collect the
// allocations and evaluations.
if job != nil {
// We don't want to gc anything related to a job which is not dead
if job != nil && job.Status != structs.JobStatusDead {
return false, nil, nil
}
}

86
nomad/core_sched_test.go
View File

@ -187,7 +187,7 @@ func TestCoreScheduler_EvalGC_Batch_NoAllocs(t *testing.T) {
}
}
func TestCoreScheduler_EvalGC_Batch_Allocs_WithJob(t *testing.T) {
func TestCoreScheduler_EvalGC_Batch_Allocs_WithRunningJob(t *testing.T) {
s1 := testServer(t, nil)
defer s1.Shutdown()
testutil.WaitForLeader(t, s1.RPC)
@ -201,21 +201,27 @@ func TestCoreScheduler_EvalGC_Batch_Allocs_WithJob(t *testing.T) {
t.Fatalf("err: %v", err)
}
// Insert "dead" eval
// Insert "complete" eval
eval := mock.Eval()
eval.Type = structs.JobTypeBatch
eval.Status = structs.EvalStatusFailed
eval.Status = structs.EvalStatusComplete
eval.JobID = job.ID
if err := state.UpsertEvals(1001, []*structs.Evaluation{eval}); err != nil {
t.Fatalf("err: %v", err)
}
// Insert "dead" alloc
// Insert "failed" alloc
alloc := mock.Alloc()
alloc.EvalID = eval.ID
alloc.JobID = job.ID
alloc.DesiredStatus = structs.AllocDesiredStatusFailed
err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
// Insert "running" alloc
alloc2 := mock.Alloc()
alloc2.EvalID = eval.ID
alloc2.JobID = job.ID
alloc2.DesiredStatus = structs.AllocDesiredStatusRun
err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc2})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -256,6 +262,76 @@ func TestCoreScheduler_EvalGC_Batch_Allocs_WithJob(t *testing.T) {
}
}
func TestCoreScheduler_EvalGC_Batch_Allocs_WithJobInTerminalJob(t *testing.T) {
s1 := testServer(t, nil)
defer s1.Shutdown()
testutil.WaitForLeader(t, s1.RPC)
// Insert job.
state := s1.fsm.State()
job := mock.Job()
job.Type = structs.JobTypeBatch
err := state.UpsertJob(1000, job)
if err != nil {
t.Fatalf("err: %v", err)
}
// Insert "complete" eval
eval := mock.Eval()
eval.Type = structs.JobTypeBatch
eval.Status = structs.EvalStatusComplete
eval.JobID = job.ID
if err := state.UpsertEvals(1001, []*structs.Evaluation{eval}); err != nil {
t.Fatalf("err: %v", err)
}
// Insert "complete" alloc
alloc := mock.Alloc()
alloc.EvalID = eval.ID
alloc.JobID = job.ID
alloc.DesiredStatus = structs.AllocDesiredStatusRun
alloc.ClientStatus = structs.AllocClientStatusComplete
err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
if err != nil {
t.Fatalf("err: %v", err)
}
// Update the time tables to make this work
tt := s1.fsm.TimeTable()
tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
// Create a core scheduler
snap, err := state.Snapshot()
if err != nil {
t.Fatalf("err: %v", err)
}
core := NewCoreScheduler(s1, snap)
// Attempt the GC
gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000)
err = core.Process(gc)
if err != nil {
t.Fatalf("err: %v", err)
}
// Shouldn't be gone because there are associated allocs.
out, err := state.EvalByID(eval.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
if out != nil {
t.Fatalf("expected eval: %v to be GC-ed", out)
}
outA, err := state.AllocByID(alloc.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
if outA != nil {
t.Fatalf("expected alloc: %v to be GC-ed", outA)
}
}
func TestCoreScheduler_EvalGC_Batch_Allocs_NoJob(t *testing.T) {
s1 := testServer(t, nil)
defer s1.Shutdown()