629 lines
14 KiB
Go
629 lines
14 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
require "github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestFeasibleRankIterator(t *testing.T) {
|
|
_, ctx := testContext(t)
|
|
var nodes []*structs.Node
|
|
for i := 0; i < 10; i++ {
|
|
nodes = append(nodes, mock.Node())
|
|
}
|
|
static := NewStaticIterator(ctx, nodes)
|
|
|
|
feasible := NewFeasibleRankIterator(ctx, static)
|
|
|
|
out := collectRanked(feasible)
|
|
if len(out) != len(nodes) {
|
|
t.Fatalf("bad: %v", out)
|
|
}
|
|
}
|
|
|
|
func TestBinPackIterator_NoExistingAlloc(t *testing.T) {
|
|
_, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
Reserved: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
// Overloaded
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
Reserved: &structs.Resources{
|
|
CPU: 512,
|
|
MemoryMB: 512,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
// 50% fit
|
|
Resources: &structs.Resources{
|
|
CPU: 4096,
|
|
MemoryMB: 4096,
|
|
},
|
|
Reserved: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
taskGroup := &structs.TaskGroup{
|
|
EphemeralDisk: &structs.EphemeralDisk{},
|
|
Tasks: []*structs.Task{
|
|
{
|
|
Name: "web",
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
binp := NewBinPackIterator(ctx, static, false, 0)
|
|
binp.SetTaskGroup(taskGroup)
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, binp)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
if len(out) != 2 {
|
|
t.Fatalf("Bad: %v", out)
|
|
}
|
|
if out[0] != nodes[0] || out[1] != nodes[2] {
|
|
t.Fatalf("Bad: %v", out)
|
|
}
|
|
|
|
if out[0].FinalScore != 1.0 {
|
|
t.Fatalf("Bad Score: %v", out[0].FinalScore)
|
|
}
|
|
if out[1].FinalScore < 0.75 || out[1].FinalScore > 0.95 {
|
|
t.Fatalf("Bad Score: %v", out[1].FinalScore)
|
|
}
|
|
}
|
|
|
|
func TestBinPackIterator_PlannedAlloc(t *testing.T) {
|
|
_, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
ID: uuid.Generate(),
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
ID: uuid.Generate(),
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
// Add a planned alloc to node1 that fills it
|
|
plan := ctx.Plan()
|
|
plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{
|
|
{
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
}
|
|
|
|
// Add a planned alloc to node2 that half fills it
|
|
plan.NodeAllocation[nodes[1].Node.ID] = []*structs.Allocation{
|
|
{
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
}
|
|
|
|
taskGroup := &structs.TaskGroup{
|
|
EphemeralDisk: &structs.EphemeralDisk{},
|
|
Tasks: []*structs.Task{
|
|
{
|
|
Name: "web",
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
binp := NewBinPackIterator(ctx, static, false, 0)
|
|
binp.SetTaskGroup(taskGroup)
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, binp)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
if len(out) != 1 {
|
|
t.Fatalf("Bad: %#v", out)
|
|
}
|
|
if out[0] != nodes[1] {
|
|
t.Fatalf("Bad Score: %v", out)
|
|
}
|
|
|
|
if out[0].FinalScore != 1.0 {
|
|
t.Fatalf("Bad Score: %v", out[0].FinalScore)
|
|
}
|
|
}
|
|
|
|
func TestBinPackIterator_ExistingAlloc(t *testing.T) {
|
|
state, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
ID: uuid.Generate(),
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
ID: uuid.Generate(),
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
// Add existing allocations
|
|
j1, j2 := mock.Job(), mock.Job()
|
|
alloc1 := &structs.Allocation{
|
|
Namespace: structs.DefaultNamespace,
|
|
ID: uuid.Generate(),
|
|
EvalID: uuid.Generate(),
|
|
NodeID: nodes[0].Node.ID,
|
|
JobID: j1.ID,
|
|
Job: j1,
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
DesiredStatus: structs.AllocDesiredStatusRun,
|
|
ClientStatus: structs.AllocClientStatusPending,
|
|
TaskGroup: "web",
|
|
}
|
|
alloc2 := &structs.Allocation{
|
|
Namespace: structs.DefaultNamespace,
|
|
ID: uuid.Generate(),
|
|
EvalID: uuid.Generate(),
|
|
NodeID: nodes[1].Node.ID,
|
|
JobID: j2.ID,
|
|
Job: j2,
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
DesiredStatus: structs.AllocDesiredStatusRun,
|
|
ClientStatus: structs.AllocClientStatusPending,
|
|
TaskGroup: "web",
|
|
}
|
|
noErr(t, state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)))
|
|
noErr(t, state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)))
|
|
noErr(t, state.UpsertAllocs(1000, []*structs.Allocation{alloc1, alloc2}))
|
|
|
|
taskGroup := &structs.TaskGroup{
|
|
EphemeralDisk: &structs.EphemeralDisk{},
|
|
Tasks: []*structs.Task{
|
|
{
|
|
Name: "web",
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
binp := NewBinPackIterator(ctx, static, false, 0)
|
|
binp.SetTaskGroup(taskGroup)
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, binp)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
if len(out) != 1 {
|
|
t.Fatalf("Bad: %#v", out)
|
|
}
|
|
if out[0] != nodes[1] {
|
|
t.Fatalf("Bad: %v", out)
|
|
}
|
|
if out[0].FinalScore != 1.0 {
|
|
t.Fatalf("Bad Score: %v", out[0].FinalScore)
|
|
}
|
|
}
|
|
|
|
func TestBinPackIterator_ExistingAlloc_PlannedEvict(t *testing.T) {
|
|
state, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
ID: uuid.Generate(),
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
// Perfect fit
|
|
ID: uuid.Generate(),
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
// Add existing allocations
|
|
j1, j2 := mock.Job(), mock.Job()
|
|
alloc1 := &structs.Allocation{
|
|
Namespace: structs.DefaultNamespace,
|
|
ID: uuid.Generate(),
|
|
EvalID: uuid.Generate(),
|
|
NodeID: nodes[0].Node.ID,
|
|
JobID: j1.ID,
|
|
Job: j1,
|
|
Resources: &structs.Resources{
|
|
CPU: 2048,
|
|
MemoryMB: 2048,
|
|
},
|
|
DesiredStatus: structs.AllocDesiredStatusRun,
|
|
ClientStatus: structs.AllocClientStatusPending,
|
|
TaskGroup: "web",
|
|
}
|
|
alloc2 := &structs.Allocation{
|
|
Namespace: structs.DefaultNamespace,
|
|
ID: uuid.Generate(),
|
|
EvalID: uuid.Generate(),
|
|
NodeID: nodes[1].Node.ID,
|
|
JobID: j2.ID,
|
|
Job: j2,
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
DesiredStatus: structs.AllocDesiredStatusRun,
|
|
ClientStatus: structs.AllocClientStatusPending,
|
|
TaskGroup: "web",
|
|
}
|
|
noErr(t, state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)))
|
|
noErr(t, state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)))
|
|
noErr(t, state.UpsertAllocs(1000, []*structs.Allocation{alloc1, alloc2}))
|
|
|
|
// Add a planned eviction to alloc1
|
|
plan := ctx.Plan()
|
|
plan.NodeUpdate[nodes[0].Node.ID] = []*structs.Allocation{alloc1}
|
|
|
|
taskGroup := &structs.TaskGroup{
|
|
EphemeralDisk: &structs.EphemeralDisk{},
|
|
Tasks: []*structs.Task{
|
|
{
|
|
Name: "web",
|
|
Resources: &structs.Resources{
|
|
CPU: 1024,
|
|
MemoryMB: 1024,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
binp := NewBinPackIterator(ctx, static, false, 0)
|
|
binp.SetTaskGroup(taskGroup)
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, binp)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
if len(out) != 2 {
|
|
t.Fatalf("Bad: %#v", out)
|
|
}
|
|
if out[0] != nodes[0] || out[1] != nodes[1] {
|
|
t.Fatalf("Bad: %v", out)
|
|
}
|
|
if out[0].FinalScore < 0.50 || out[0].FinalScore > 0.95 {
|
|
t.Fatalf("Bad Score: %v", out[0].FinalScore)
|
|
}
|
|
if out[1].FinalScore != 1 {
|
|
t.Fatalf("Bad Score: %v", out[1].FinalScore)
|
|
}
|
|
}
|
|
|
|
func TestJobAntiAffinity_PlannedAlloc(t *testing.T) {
|
|
_, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: &structs.Node{
|
|
ID: uuid.Generate(),
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
ID: uuid.Generate(),
|
|
},
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
job := mock.Job()
|
|
job.ID = "foo"
|
|
tg := job.TaskGroups[0]
|
|
tg.Count = 4
|
|
|
|
// Add a planned alloc to node1 that fills it
|
|
plan := ctx.Plan()
|
|
plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{
|
|
{
|
|
ID: uuid.Generate(),
|
|
JobID: "foo",
|
|
TaskGroup: tg.Name,
|
|
},
|
|
{
|
|
ID: uuid.Generate(),
|
|
JobID: "foo",
|
|
TaskGroup: tg.Name,
|
|
},
|
|
}
|
|
|
|
// Add a planned alloc to node2 that half fills it
|
|
plan.NodeAllocation[nodes[1].Node.ID] = []*structs.Allocation{
|
|
{
|
|
JobID: "bar",
|
|
},
|
|
}
|
|
|
|
jobAntiAff := NewJobAntiAffinityIterator(ctx, static, "foo")
|
|
jobAntiAff.SetJob(job)
|
|
jobAntiAff.SetTaskGroup(tg)
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, jobAntiAff)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
if len(out) != 2 {
|
|
t.Fatalf("Bad: %#v", out)
|
|
}
|
|
if out[0] != nodes[0] {
|
|
t.Fatalf("Bad: %v", out)
|
|
}
|
|
// Score should be -(#collissions+1/desired_count) => -(3/4)
|
|
if out[0].FinalScore != -0.75 {
|
|
t.Fatalf("Bad Score: %#v", out[0].FinalScore)
|
|
}
|
|
|
|
if out[1] != nodes[1] {
|
|
t.Fatalf("Bad: %v", out)
|
|
}
|
|
if out[1].FinalScore != 0.0 {
|
|
t.Fatalf("Bad Score: %v", out[1].FinalScore)
|
|
}
|
|
}
|
|
|
|
func collectRanked(iter RankIterator) (out []*RankedNode) {
|
|
for {
|
|
next := iter.Next()
|
|
if next == nil {
|
|
break
|
|
}
|
|
out = append(out, next)
|
|
}
|
|
return
|
|
}
|
|
|
|
func TestNodeAntiAffinity_PenaltyNodes(t *testing.T) {
|
|
_, ctx := testContext(t)
|
|
node1 := &structs.Node{
|
|
ID: uuid.Generate(),
|
|
}
|
|
node2 := &structs.Node{
|
|
ID: uuid.Generate(),
|
|
}
|
|
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: node1,
|
|
},
|
|
{
|
|
Node: node2,
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
nodeAntiAffIter := NewNodeReschedulingPenaltyIterator(ctx, static)
|
|
nodeAntiAffIter.SetPenaltyNodes(map[string]struct{}{node1.ID: {}})
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, nodeAntiAffIter)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
|
|
require := require.New(t)
|
|
require.Equal(2, len(out))
|
|
require.Equal(node1.ID, out[0].Node.ID)
|
|
require.Equal(-1.0, out[0].FinalScore)
|
|
|
|
require.Equal(node2.ID, out[1].Node.ID)
|
|
require.Equal(0.0, out[1].FinalScore)
|
|
|
|
}
|
|
|
|
func TestScoreNormalizationIterator(t *testing.T) {
|
|
// Test normalized scores when there is more than one scorer
|
|
_, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{
|
|
Node: &structs.Node{
|
|
ID: uuid.Generate(),
|
|
},
|
|
},
|
|
{
|
|
Node: &structs.Node{
|
|
ID: uuid.Generate(),
|
|
},
|
|
},
|
|
}
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
job := mock.Job()
|
|
job.ID = "foo"
|
|
tg := job.TaskGroups[0]
|
|
tg.Count = 4
|
|
|
|
// Add a planned alloc to node1 that fills it
|
|
plan := ctx.Plan()
|
|
plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{
|
|
{
|
|
ID: uuid.Generate(),
|
|
JobID: "foo",
|
|
TaskGroup: tg.Name,
|
|
},
|
|
{
|
|
ID: uuid.Generate(),
|
|
JobID: "foo",
|
|
TaskGroup: tg.Name,
|
|
},
|
|
}
|
|
|
|
// Add a planned alloc to node2 that half fills it
|
|
plan.NodeAllocation[nodes[1].Node.ID] = []*structs.Allocation{
|
|
{
|
|
JobID: "bar",
|
|
},
|
|
}
|
|
|
|
jobAntiAff := NewJobAntiAffinityIterator(ctx, static, "foo")
|
|
jobAntiAff.SetJob(job)
|
|
jobAntiAff.SetTaskGroup(tg)
|
|
|
|
nodeReschedulePenaltyIter := NewNodeReschedulingPenaltyIterator(ctx, jobAntiAff)
|
|
nodeReschedulePenaltyIter.SetPenaltyNodes(map[string]struct{}{nodes[0].Node.ID: {}})
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, nodeReschedulePenaltyIter)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
require := require.New(t)
|
|
|
|
require.Equal(2, len(out))
|
|
require.Equal(out[0], nodes[0])
|
|
// Score should be averaged between both scorers
|
|
// -0.75 from job anti affinity and -1 from node rescheduling penalty
|
|
require.Equal(-0.875, out[0].FinalScore)
|
|
require.Equal(out[1], nodes[1])
|
|
require.Equal(out[1].FinalScore, 0.0)
|
|
}
|
|
|
|
func TestNodeAffinityIterator(t *testing.T) {
|
|
_, ctx := testContext(t)
|
|
nodes := []*RankedNode{
|
|
{Node: mock.Node()},
|
|
{Node: mock.Node()},
|
|
{Node: mock.Node()},
|
|
{Node: mock.Node()},
|
|
}
|
|
|
|
nodes[0].Node.Attributes["kernel.version"] = "4.9"
|
|
nodes[1].Node.Datacenter = "dc2"
|
|
nodes[2].Node.Datacenter = "dc2"
|
|
nodes[2].Node.NodeClass = "large"
|
|
|
|
affinities := []*structs.Affinity{
|
|
{
|
|
Operand: "=",
|
|
LTarget: "${node.datacenter}",
|
|
RTarget: "dc1",
|
|
Weight: 200,
|
|
},
|
|
{
|
|
Operand: "=",
|
|
LTarget: "${node.datacenter}",
|
|
RTarget: "dc2",
|
|
Weight: -100,
|
|
},
|
|
{
|
|
Operand: "version",
|
|
LTarget: "${attr.kernel.version}",
|
|
RTarget: ">4.0",
|
|
Weight: 50,
|
|
},
|
|
{
|
|
Operand: "is",
|
|
LTarget: "${node.class}",
|
|
RTarget: "large",
|
|
Weight: 50,
|
|
},
|
|
}
|
|
|
|
static := NewStaticRankIterator(ctx, nodes)
|
|
|
|
job := mock.Job()
|
|
job.ID = "foo"
|
|
tg := job.TaskGroups[0]
|
|
tg.Affinities = affinities
|
|
|
|
nodeAffinity := NewNodeAffinityIterator(ctx, static)
|
|
nodeAffinity.SetTaskGroup(tg)
|
|
|
|
scoreNorm := NewScoreNormalizationIterator(ctx, nodeAffinity)
|
|
|
|
out := collectRanked(scoreNorm)
|
|
expectedScores := make(map[string]float64)
|
|
// Total weight = 400
|
|
// Node 0 matches two affinities(dc and kernel version), total weight =250
|
|
expectedScores[nodes[0].Node.ID] = 0.625
|
|
|
|
// Node 1 matches an anti affinity, weight = -100
|
|
expectedScores[nodes[1].Node.ID] = -0.25
|
|
|
|
// Node 2 matches one affinity(node class) with weight 50
|
|
expectedScores[nodes[2].Node.ID] = -0.125
|
|
|
|
// Node 3 matches one affinity (dc) with weight = 200
|
|
expectedScores[nodes[3].Node.ID] = 0.5
|
|
|
|
require := require.New(t)
|
|
for _, n := range out {
|
|
require.Equal(expectedScores[n.Node.ID], n.FinalScore)
|
|
}
|
|
|
|
}
|