From 6688a3f76ced1a7d6d46b65abdc3c08eeee0720a Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 27 Jun 2017 10:31:32 -0700 Subject: [PATCH] FSM Tests --- nomad/fsm.go | 3 - nomad/fsm_test.go | 484 +++++++++++++++++++++++++++++++-------- nomad/structs/structs.go | 2 +- 3 files changed, 392 insertions(+), 97 deletions(-) diff --git a/nomad/fsm.go b/nomad/fsm.go index a157d7320..56151d024 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -571,7 +571,6 @@ func (n *nomadFSM) applyPlanResults(buf []byte, index uint64) interface{} { return nil } -// TODO test // applyDeploymentStatusUpdate is used to update the status of an existing // deployment func (n *nomadFSM) applyDeploymentStatusUpdate(buf []byte, index uint64) interface{} { @@ -593,7 +592,6 @@ func (n *nomadFSM) applyDeploymentStatusUpdate(buf []byte, index uint64) interfa return nil } -// TODO test // applyDeploymentPromotion is used to promote canaries in a deployment func (n *nomadFSM) applyDeploymentPromotion(buf []byte, index uint64) interface{} { defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_deployment_promotion"}, time.Now()) @@ -614,7 +612,6 @@ func (n *nomadFSM) applyDeploymentPromotion(buf []byte, index uint64) interface{ return nil } -// TODO test // applyDeploymentAllocHealth is used to set the health of allocations as part // of a deployment func (n *nomadFSM) applyDeploymentAllocHealth(buf []byte, index uint64) interface{} { diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 2eaf4d079..b706f8135 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -9,6 +9,7 @@ import ( "time" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" @@ -1047,6 +1048,396 @@ func TestFSM_DeregisterVaultAccessor(t *testing.T) { } } +func TestFSM_ApplyPlanResults(t *testing.T) { + fsm := testFSM(t) + + // Create the request and create a deployment + alloc := mock.Alloc() + job := alloc.Job + alloc.Job = nil + + d := mock.Deployment() + d.JobID = job.ID + d.JobModifyIndex = job.ModifyIndex + d.JobVersion = job.Version + + alloc.DeploymentID = d.ID + + fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) + req := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + Job: job, + Alloc: []*structs.Allocation{alloc}, + }, + CreatedDeployment: d, + } + buf, err := structs.Encode(structs.ApplyPlanResultsRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp := fsm.Apply(makeLog(buf)) + if resp != nil { + t.Fatalf("resp: %v", resp) + } + + // Verify the allocation is registered + ws := memdb.NewWatchSet() + out, err := fsm.State().AllocByID(ws, alloc.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + alloc.CreateIndex = out.CreateIndex + alloc.ModifyIndex = out.ModifyIndex + alloc.AllocModifyIndex = out.AllocModifyIndex + + // Job should be re-attached + alloc.Job = job + if !reflect.DeepEqual(alloc, out) { + t.Fatalf("bad: %#v %#v", alloc, out) + } + + dout, err := fsm.State().DeploymentByID(ws, d.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if tg, ok := dout.TaskGroups[alloc.TaskGroup]; !ok || tg.PlacedAllocs != 1 { + t.Fatalf("err: %v %v", tg, err) + } + + // Ensure that the original job is used + evictAlloc := alloc.Copy() + job = mock.Job() + job.Priority = 123 + + evictAlloc.Job = nil + evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict + req2 := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + Job: job, + Alloc: []*structs.Allocation{evictAlloc}, + }, + } + buf, err = structs.Encode(structs.ApplyPlanResultsRequestType, req2) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp = fsm.Apply(makeLog(buf)) + if resp != nil { + t.Fatalf("resp: %v", resp) + } + + // Verify we are evicted + out, err = fsm.State().AllocByID(ws, alloc.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if out.DesiredStatus != structs.AllocDesiredStatusEvict { + t.Fatalf("alloc found!") + } + if out.Job == nil || out.Job.Priority == 123 { + t.Fatalf("bad job") + } +} + +func TestFSM_DeploymentStatusUpdate(t *testing.T) { + fsm := testFSM(t) + fsm.evalBroker.SetEnabled(true) + state := fsm.State() + + // Upsert a deployment + d := mock.Deployment() + if err := state.UpsertDeployment(1, d, false); err != nil { + t.Fatalf("bad: %v", err) + } + + // Create a request to update the deployment, create an eval and job + e := mock.Eval() + j := mock.Job() + status, desc := structs.DeploymentStatusFailed, "foo" + req := &structs.DeploymentStatusUpdateRequest{ + DeploymentUpdate: &structs.DeploymentStatusUpdate{ + DeploymentID: d.ID, + Status: status, + StatusDescription: desc, + }, + Job: j, + Eval: e, + } + buf, err := structs.Encode(structs.DeploymentStatusUpdateRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := fsm.Apply(makeLog(buf)) + if resp != nil { + t.Fatalf("resp: %v", resp) + } + + // Check that the status was updated properly + ws := memdb.NewWatchSet() + dout, err := state.DeploymentByID(ws, d.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if dout.Status != status || dout.StatusDescription != desc { + t.Fatalf("bad: %#v", dout) + } + + // Check that the evaluation was created + eout, _ := state.EvalByID(ws, e.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if eout == nil { + t.Fatalf("bad: %#v", eout) + } + + // Check that the job was created + jout, _ := state.JobByID(ws, j.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if jout == nil { + t.Fatalf("bad: %#v", jout) + } + + // Assert the eval was enqueued + stats := fsm.evalBroker.Stats() + if stats.TotalReady != 1 { + t.Fatalf("bad: %#v %#v", stats, e) + } +} + +func TestFSM_DeploymentPromotion(t *testing.T) { + fsm := testFSM(t) + fsm.evalBroker.SetEnabled(true) + state := fsm.State() + + // Create a job with two task groups + j := mock.Job() + tg1 := j.TaskGroups[0] + tg2 := tg1.Copy() + tg2.Name = "foo" + j.TaskGroups = append(j.TaskGroups, tg2) + if err := state.UpsertJob(1, j); err != nil { + t.Fatalf("bad: %v", err) + } + + // Create a deployment + d := mock.Deployment() + d.JobID = j.ID + d.TaskGroups = map[string]*structs.DeploymentState{ + "web": &structs.DeploymentState{ + DesiredTotal: 10, + DesiredCanaries: 1, + }, + "foo": &structs.DeploymentState{ + DesiredTotal: 10, + DesiredCanaries: 1, + }, + } + if err := state.UpsertDeployment(2, d, false); err != nil { + t.Fatalf("bad: %v", err) + } + + // Create a set of allocations + c1 := mock.Alloc() + c1.JobID = j.ID + c1.DeploymentID = d.ID + c1.Canary = true + c1.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + } + c2 := mock.Alloc() + c2.JobID = j.ID + c2.DeploymentID = d.ID + c2.Canary = true + c2.TaskGroup = tg2.Name + c2.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + } + + if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil { + t.Fatalf("err: %v", err) + } + + // Create an eval + e := mock.Eval() + + // Promote the canaries + req := &structs.ApplyDeploymentPromoteRequest{ + DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ + DeploymentID: d.ID, + All: true, + }, + Eval: e, + } + buf, err := structs.Encode(structs.DeploymentPromoteRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := fsm.Apply(makeLog(buf)) + if resp != nil { + t.Fatalf("resp: %v", resp) + } + + // Check that the status per task group was updated properly + ws := memdb.NewWatchSet() + dout, err := state.DeploymentByID(ws, d.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if len(dout.TaskGroups) != 2 { + t.Fatalf("bad: %#v", dout.TaskGroups) + } + for tg, state := range dout.TaskGroups { + if !state.Promoted { + t.Fatalf("bad: group %q not promoted %#v", tg, state) + } + } + + // Check that the allocs were promoted + out1, err := state.AllocByID(ws, c1.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + out2, err := state.AllocByID(ws, c2.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + + for _, alloc := range []*structs.Allocation{out1, out2} { + if alloc.DeploymentStatus == nil { + t.Fatalf("bad: alloc %q has nil deployment status", alloc.ID) + } + if !alloc.DeploymentStatus.Promoted { + t.Fatalf("bad: alloc %q not promoted", alloc.ID) + } + } + + // Check that the evaluation was created + eout, _ := state.EvalByID(ws, e.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if eout == nil { + t.Fatalf("bad: %#v", eout) + } + + // Assert the eval was enqueued + stats := fsm.evalBroker.Stats() + if stats.TotalReady != 1 { + t.Fatalf("bad: %#v %#v", stats, e) + } +} + +func TestFSM_DeploymentAllocHealth(t *testing.T) { + fsm := testFSM(t) + fsm.evalBroker.SetEnabled(true) + state := fsm.State() + + // Insert a deployment + d := mock.Deployment() + if err := state.UpsertDeployment(1, d, false); err != nil { + t.Fatalf("bad: %v", err) + } + + // Insert two allocations + a1 := mock.Alloc() + a1.DeploymentID = d.ID + a2 := mock.Alloc() + a2.DeploymentID = d.ID + if err := state.UpsertAllocs(2, []*structs.Allocation{a1, a2}); err != nil { + t.Fatalf("bad: %v", err) + } + + // Create a job to roll back to + j := mock.Job() + + // Create an eval that should be upserted + e := mock.Eval() + + // Create a status update for the deployment + status, desc := structs.DeploymentStatusFailed, "foo" + u := &structs.DeploymentStatusUpdate{ + DeploymentID: d.ID, + Status: status, + StatusDescription: desc, + } + + // Set health against the deployment + req := &structs.ApplyDeploymentAllocHealthRequest{ + DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ + DeploymentID: d.ID, + HealthyAllocationIDs: []string{a1.ID}, + UnhealthyAllocationIDs: []string{a2.ID}, + }, + Job: j, + Eval: e, + DeploymentUpdate: u, + } + buf, err := structs.Encode(structs.DeploymentAllocHealthRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := fsm.Apply(makeLog(buf)) + if resp != nil { + t.Fatalf("resp: %v", resp) + } + + // Check that the status was updated properly + ws := memdb.NewWatchSet() + dout, err := state.DeploymentByID(ws, d.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if dout.Status != status || dout.StatusDescription != desc { + t.Fatalf("bad: %#v", dout) + } + + // Check that the evaluation was created + eout, _ := state.EvalByID(ws, e.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if eout == nil { + t.Fatalf("bad: %#v", eout) + } + + // Check that the job was created + jout, _ := state.JobByID(ws, j.ID) + if err != nil { + t.Fatalf("bad: %v", err) + } + if jout == nil { + t.Fatalf("bad: %#v", jout) + } + + // Check the status of the allocs + out1, err := state.AllocByID(ws, a1.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + out2, err := state.AllocByID(ws, a2.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + + if !out1.DeploymentStatus.IsHealthy() { + t.Fatalf("bad: alloc %q not healthy", out1.ID) + } + if !out2.DeploymentStatus.IsUnhealthy() { + t.Fatalf("bad: alloc %q not unhealthy", out2.ID) + } + + // Assert the eval was enqueued + stats := fsm.evalBroker.Stats() + if stats.TotalReady != 1 { + t.Fatalf("bad: %#v %#v", stats, e) + } +} + func testSnapshotRestore(t *testing.T, fsm *nomadFSM) *nomadFSM { // Snapshot snap, err := fsm.Snapshot() @@ -1484,96 +1875,3 @@ func TestFSM_ReconcileSummaries(t *testing.T) { t.Fatalf("expected: %#v, actual: %#v", &expected, out2) } } - -func TestFSM_ApplyPlanResults(t *testing.T) { - fsm := testFSM(t) - - // Create the request and create a deployment - alloc := mock.Alloc() - job := alloc.Job - alloc.Job = nil - - d := mock.Deployment() - d.JobID = job.ID - d.JobModifyIndex = job.ModifyIndex - d.JobVersion = job.Version - - alloc.DeploymentID = d.ID - - fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) - req := structs.ApplyPlanResultsRequest{ - AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: job, - Alloc: []*structs.Allocation{alloc}, - }, - CreatedDeployment: d, - } - buf, err := structs.Encode(structs.ApplyPlanResultsRequestType, req) - if err != nil { - t.Fatalf("err: %v", err) - } - - resp := fsm.Apply(makeLog(buf)) - if resp != nil { - t.Fatalf("resp: %v", resp) - } - - // Verify the allocation is registered - ws := memdb.NewWatchSet() - out, err := fsm.State().AllocByID(ws, alloc.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - alloc.CreateIndex = out.CreateIndex - alloc.ModifyIndex = out.ModifyIndex - alloc.AllocModifyIndex = out.AllocModifyIndex - - // Job should be re-attached - alloc.Job = job - if !reflect.DeepEqual(alloc, out) { - t.Fatalf("bad: %#v %#v", alloc, out) - } - - dout, err := fsm.State().DeploymentByID(ws, d.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if tg, ok := dout.TaskGroups[alloc.TaskGroup]; !ok || tg.PlacedAllocs != 1 { - t.Fatalf("err: %v %v", tg, err) - } - - // Ensure that the original job is used - evictAlloc := alloc.Copy() - job = mock.Job() - job.Priority = 123 - - evictAlloc.Job = nil - evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict - req2 := structs.ApplyPlanResultsRequest{ - AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: job, - Alloc: []*structs.Allocation{evictAlloc}, - }, - } - buf, err = structs.Encode(structs.ApplyPlanResultsRequestType, req2) - if err != nil { - t.Fatalf("err: %v", err) - } - - resp = fsm.Apply(makeLog(buf)) - if resp != nil { - t.Fatalf("resp: %v", resp) - } - - // Verify we are evicted - out, err = fsm.State().AllocByID(ws, alloc.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out.DesiredStatus != structs.AllocDesiredStatusEvict { - t.Fatalf("alloc found!") - } - if out.Job == nil || out.Job.Priority == 123 { - t.Fatalf("bad job") - } -} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 81d42cc8f..fdde1e2c4 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -56,7 +56,7 @@ const ( ApplyPlanResultsRequestType DeploymentStatusUpdateRequestType DeploymentPromoteRequestType - DeploymentAllocHealtRequestType + DeploymentAllocHealthRequestType ) const (