nomad: thread through failed allocations

nomad/plan_apply.go

@@ -52,7 +52,7 @@ func (s *Server) planApply() {
 		}
 
 		// Apply the plan if there is anything to do
-		if len(result.NodeEvict) != 0 || len(result.NodeAllocation) != 0 {
+		if len(result.NodeEvict) != 0 || len(result.NodeAllocation) != 0 || len(result.FailedAllocs) != 0 {
 			allocIndex, err := s.applyPlan(result)
 			if err != nil {
 				s.logger.Printf("[ERR] nomad: failed to apply plan: %v", err)
@@ -77,6 +77,7 @@ func (s *Server) applyPlan(result *structs.PlanResult) (uint64, error) {
 	for _, allocList := range result.NodeAllocation {
 		req.Alloc = append(req.Alloc, allocList...)
 	}
+	req.Alloc = append(req.Alloc, result.FailedAllocs...)
 
 	_, index, err := s.raftApply(structs.AllocUpdateRequestType, &req)
 	return index, err
@@ -92,6 +93,7 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR
 	result := &structs.PlanResult{
 		NodeEvict:      make(map[string][]string),
 		NodeAllocation: make(map[string][]*structs.Allocation),
+		FailedAllocs:   plan.FailedAllocs,
 	}
 
 	// Check each allocation to see if it should be allowed

nomad/plan_apply_test.go

@@ -1,6 +1,7 @@
 package nomad
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/hashicorp/nomad/nomad/mock"
@@ -36,6 +37,7 @@ func TestPlanApply_applyPlan(t *testing.T) {
 
 	// Register alloc
 	alloc := mock.Alloc()
+	allocFail := mock.Alloc()
 	plan := &structs.PlanResult{
 		NodeEvict: map[string][]string{
 			node.ID: []string{},
@@ -43,6 +45,7 @@ func TestPlanApply_applyPlan(t *testing.T) {
 		NodeAllocation: map[string][]*structs.Allocation{
 			node.ID: []*structs.Allocation{alloc},
 		},
+		FailedAllocs: []*structs.Allocation{allocFail},
 	}
 
 	// Apply the plan
@@ -63,6 +66,15 @@ func TestPlanApply_applyPlan(t *testing.T) {
 		t.Fatalf("missing alloc")
 	}
 
+	// Lookup the allocation
+	out, err = s1.fsm.State().GetAllocByID(allocFail.ID)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if out == nil {
+		t.Fatalf("missing alloc")
+	}
+
 	// Evict alloc, Register alloc2
 	alloc2 := mock.Alloc()
 	plan = &structs.PlanResult{
@@ -109,10 +121,12 @@ func TestPlanApply_EvalPlan_Simple(t *testing.T) {
 	snap, _ := state.Snapshot()
 
 	alloc := mock.Alloc()
+	allocFail := mock.Alloc()
 	plan := &structs.Plan{
 		NodeAllocation: map[string][]*structs.Allocation{
 			node.ID: []*structs.Allocation{alloc},
 		},
+		FailedAllocs: []*structs.Allocation{allocFail},
 	}
 
 	result, err := evaluatePlan(snap, plan)
@@ -122,6 +136,9 @@ func TestPlanApply_EvalPlan_Simple(t *testing.T) {
 	if result == nil {
 		t.Fatalf("missing result")
 	}
+	if !reflect.DeepEqual(result.FailedAllocs, plan.FailedAllocs) {
+		t.Fatalf("missing failed allocs")
+	}
 }
 
 func TestPlanApply_EvalPlan_Partial(t *testing.T) {

nomad/structs/structs.go

@@ -883,6 +883,11 @@ type PlanResult struct {
 	// NodeAllocation contains all the allocations that were committed.
 	NodeAllocation map[string][]*Allocation
 
+	// FailedAllocs are allocations that could not be made,
+	// but are persisted so that the user can use the feedback
+	// to determine the cause.
+	FailedAllocs []*Allocation
+
 	// RefreshIndex is the index the worker should refresh state up to.
 	// This allows all evictions and allocations to be materialized.
 	// If any allocations were rejected due to stale data (node state,