2015-08-12 01:27:54 +00:00
|
|
|
package scheduler
|
|
|
|
|
2015-08-13 20:08:15 +00:00
|
|
|
import (
|
|
|
|
"fmt"
|
2018-07-16 13:47:18 +00:00
|
|
|
"math"
|
|
|
|
|
2015-08-13 20:08:15 +00:00
|
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
|
|
)
|
2015-08-12 01:27:54 +00:00
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
const (
|
|
|
|
// binPackingMaxFitScore is the maximum possible bin packing fitness score.
|
|
|
|
// This is used to normalize bin packing score to a value between 0 and 1
|
|
|
|
binPackingMaxFitScore = 18.0
|
|
|
|
)
|
|
|
|
|
2015-08-12 01:27:54 +00:00
|
|
|
// Rank is used to provide a score and various ranking metadata
|
|
|
|
// along with a node when iterating. This state can be modified as
|
|
|
|
// various rank methods are applied.
|
|
|
|
type RankedNode struct {
|
2016-08-27 03:08:03 +00:00
|
|
|
Node *structs.Node
|
2018-07-16 13:47:18 +00:00
|
|
|
FinalScore float64
|
|
|
|
Scores []float64
|
2018-10-02 20:36:04 +00:00
|
|
|
TaskResources map[string]*structs.AllocatedTaskResources
|
2015-08-16 17:28:58 +00:00
|
|
|
|
|
|
|
// Allocs is used to cache the proposed allocations on the
|
|
|
|
// node. This can be shared between iterators that require it.
|
|
|
|
Proposed []*structs.Allocation
|
2018-09-21 21:05:00 +00:00
|
|
|
|
|
|
|
// PreemptedAllocs is used by the BinpackIterator to identify allocs
|
|
|
|
// that should be preempted in order to make the placement
|
|
|
|
PreemptedAllocs []*structs.Allocation
|
2015-08-12 01:27:54 +00:00
|
|
|
}
|
|
|
|
|
2015-08-13 20:08:15 +00:00
|
|
|
func (r *RankedNode) GoString() string {
|
2018-07-16 13:47:18 +00:00
|
|
|
return fmt.Sprintf("<Node: %s Score: %0.3f>", r.Node.ID, r.FinalScore)
|
2015-08-13 20:08:15 +00:00
|
|
|
}
|
|
|
|
|
2015-09-07 23:19:21 +00:00
|
|
|
func (r *RankedNode) ProposedAllocs(ctx Context) ([]*structs.Allocation, error) {
|
|
|
|
if r.Proposed != nil {
|
|
|
|
return r.Proposed, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
p, err := ctx.ProposedAllocs(r.Node.ID)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
r.Proposed = p
|
|
|
|
return p, nil
|
|
|
|
}
|
|
|
|
|
2015-09-13 21:31:32 +00:00
|
|
|
func (r *RankedNode) SetTaskResources(task *structs.Task,
|
2018-10-02 20:36:04 +00:00
|
|
|
resource *structs.AllocatedTaskResources) {
|
2015-09-13 21:31:32 +00:00
|
|
|
if r.TaskResources == nil {
|
2018-10-02 20:36:04 +00:00
|
|
|
r.TaskResources = make(map[string]*structs.AllocatedTaskResources)
|
2015-09-13 21:31:32 +00:00
|
|
|
}
|
|
|
|
r.TaskResources[task.Name] = resource
|
|
|
|
}
|
|
|
|
|
2015-08-12 01:27:54 +00:00
|
|
|
// RankFeasibleIterator is used to iteratively yield nodes along
|
|
|
|
// with ranking metadata. The iterators may manage some state for
|
|
|
|
// performance optimizations.
|
|
|
|
type RankIterator interface {
|
2015-08-13 22:01:02 +00:00
|
|
|
// Next yields a ranked option or nil if exhausted
|
2015-08-12 01:27:54 +00:00
|
|
|
Next() *RankedNode
|
2015-08-13 22:01:02 +00:00
|
|
|
|
|
|
|
// Reset is invoked when an allocation has been placed
|
|
|
|
// to reset any stale state.
|
|
|
|
Reset()
|
2015-08-12 01:27:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// FeasibleRankIterator is used to consume from a FeasibleIterator
|
|
|
|
// and return an unranked node with base ranking.
|
|
|
|
type FeasibleRankIterator struct {
|
2015-08-12 01:30:45 +00:00
|
|
|
ctx Context
|
2015-08-12 01:27:54 +00:00
|
|
|
source FeasibleIterator
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewFeasibleRankIterator is used to return a new FeasibleRankIterator
|
|
|
|
// from a FeasibleIterator source.
|
|
|
|
func NewFeasibleRankIterator(ctx Context, source FeasibleIterator) *FeasibleRankIterator {
|
|
|
|
iter := &FeasibleRankIterator{
|
2015-08-12 01:30:45 +00:00
|
|
|
ctx: ctx,
|
2015-08-12 01:27:54 +00:00
|
|
|
source: source,
|
|
|
|
}
|
|
|
|
return iter
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *FeasibleRankIterator) Next() *RankedNode {
|
|
|
|
option := iter.source.Next()
|
2015-08-13 17:13:11 +00:00
|
|
|
if option == nil {
|
|
|
|
return nil
|
|
|
|
}
|
2015-08-12 01:27:54 +00:00
|
|
|
ranked := &RankedNode{
|
|
|
|
Node: option,
|
|
|
|
}
|
|
|
|
return ranked
|
|
|
|
}
|
|
|
|
|
2015-08-13 22:01:02 +00:00
|
|
|
func (iter *FeasibleRankIterator) Reset() {
|
|
|
|
iter.source.Reset()
|
|
|
|
}
|
|
|
|
|
2015-08-12 01:30:45 +00:00
|
|
|
// StaticRankIterator is a RankIterator that returns a static set of results.
|
|
|
|
// This is largely only useful for testing.
|
|
|
|
type StaticRankIterator struct {
|
|
|
|
ctx Context
|
|
|
|
nodes []*RankedNode
|
|
|
|
offset int
|
2015-08-13 22:01:02 +00:00
|
|
|
seen int
|
2015-08-12 01:30:45 +00:00
|
|
|
}
|
|
|
|
|
2015-08-13 17:05:54 +00:00
|
|
|
// NewStaticRankIterator returns a new static rank iterator over the given nodes
|
|
|
|
func NewStaticRankIterator(ctx Context, nodes []*RankedNode) *StaticRankIterator {
|
|
|
|
iter := &StaticRankIterator{
|
|
|
|
ctx: ctx,
|
|
|
|
nodes: nodes,
|
|
|
|
}
|
|
|
|
return iter
|
|
|
|
}
|
|
|
|
|
2015-08-12 01:30:45 +00:00
|
|
|
func (iter *StaticRankIterator) Next() *RankedNode {
|
|
|
|
// Check if exhausted
|
2015-08-13 22:01:02 +00:00
|
|
|
n := len(iter.nodes)
|
|
|
|
if iter.offset == n || iter.seen == n {
|
|
|
|
if iter.seen != n {
|
|
|
|
iter.offset = 0
|
|
|
|
} else {
|
|
|
|
return nil
|
|
|
|
}
|
2015-08-12 01:30:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return the next offset
|
|
|
|
offset := iter.offset
|
|
|
|
iter.offset += 1
|
2015-08-13 22:01:02 +00:00
|
|
|
iter.seen += 1
|
2015-08-12 01:30:45 +00:00
|
|
|
return iter.nodes[offset]
|
|
|
|
}
|
|
|
|
|
2015-08-13 22:01:02 +00:00
|
|
|
func (iter *StaticRankIterator) Reset() {
|
|
|
|
iter.seen = 0
|
|
|
|
}
|
|
|
|
|
2015-08-12 01:27:54 +00:00
|
|
|
// BinPackIterator is a RankIterator that scores potential options
|
|
|
|
// based on a bin-packing algorithm.
|
|
|
|
type BinPackIterator struct {
|
2016-08-25 17:27:19 +00:00
|
|
|
ctx Context
|
|
|
|
source RankIterator
|
|
|
|
evict bool
|
|
|
|
priority int
|
|
|
|
taskGroup *structs.TaskGroup
|
2015-08-12 01:27:54 +00:00
|
|
|
}
|
|
|
|
|
2015-09-13 21:31:32 +00:00
|
|
|
// NewBinPackIterator returns a BinPackIterator which tries to fit tasks
|
|
|
|
// potentially evicting other tasks based on a given priority.
|
|
|
|
func NewBinPackIterator(ctx Context, source RankIterator, evict bool, priority int) *BinPackIterator {
|
2015-08-12 01:27:54 +00:00
|
|
|
iter := &BinPackIterator{
|
2015-09-13 21:31:32 +00:00
|
|
|
ctx: ctx,
|
|
|
|
source: source,
|
|
|
|
evict: evict,
|
|
|
|
priority: priority,
|
2015-08-12 01:27:54 +00:00
|
|
|
}
|
|
|
|
return iter
|
|
|
|
}
|
|
|
|
|
2015-08-14 00:48:26 +00:00
|
|
|
func (iter *BinPackIterator) SetPriority(p int) {
|
|
|
|
iter.priority = p
|
|
|
|
}
|
|
|
|
|
2016-08-25 17:27:19 +00:00
|
|
|
func (iter *BinPackIterator) SetTaskGroup(taskGroup *structs.TaskGroup) {
|
|
|
|
iter.taskGroup = taskGroup
|
2015-09-13 21:31:32 +00:00
|
|
|
}
|
|
|
|
|
2015-08-12 01:27:54 +00:00
|
|
|
func (iter *BinPackIterator) Next() *RankedNode {
|
2015-09-13 21:31:32 +00:00
|
|
|
OUTER:
|
2015-08-12 01:27:54 +00:00
|
|
|
for {
|
2015-08-13 18:54:59 +00:00
|
|
|
// Get the next potential option
|
2015-08-12 01:27:54 +00:00
|
|
|
option := iter.source.Next()
|
|
|
|
if option == nil {
|
|
|
|
return nil
|
|
|
|
}
|
2015-08-13 18:54:59 +00:00
|
|
|
|
2015-08-16 17:28:58 +00:00
|
|
|
// Get the proposed allocations
|
2015-09-07 23:19:21 +00:00
|
|
|
proposed, err := option.ProposedAllocs(iter.ctx)
|
|
|
|
if err != nil {
|
2018-09-15 23:23:13 +00:00
|
|
|
iter.ctx.Logger().Named("binpack").Error("failed retrieving proposed allocations", "error", err)
|
2015-09-07 23:19:21 +00:00
|
|
|
continue
|
2015-08-13 18:54:59 +00:00
|
|
|
}
|
2015-08-13 18:28:02 +00:00
|
|
|
|
2015-09-13 21:31:32 +00:00
|
|
|
// Index the existing network usage
|
2015-09-13 21:37:09 +00:00
|
|
|
netIdx := structs.NewNetworkIndex()
|
2015-09-13 21:31:32 +00:00
|
|
|
netIdx.SetNode(option.Node)
|
|
|
|
netIdx.AddAllocs(proposed)
|
|
|
|
|
2018-10-15 22:15:46 +00:00
|
|
|
// Create a device allocator
|
|
|
|
devAllocator := newDeviceAllocator(iter.ctx, option.Node)
|
|
|
|
devAllocator.AddAllocs(proposed)
|
|
|
|
|
2018-10-17 18:04:54 +00:00
|
|
|
// Track the affinities of the devices
|
2018-11-08 23:01:58 +00:00
|
|
|
totalDeviceAffinityWeight := 0.0
|
|
|
|
sumMatchingAffinities := 0.0
|
2018-10-17 18:04:54 +00:00
|
|
|
|
2015-09-13 21:31:32 +00:00
|
|
|
// Assign the resources for each task
|
2018-10-02 20:36:04 +00:00
|
|
|
total := &structs.AllocatedResources{
|
|
|
|
Tasks: make(map[string]*structs.AllocatedTaskResources,
|
|
|
|
len(iter.taskGroup.Tasks)),
|
|
|
|
Shared: structs.AllocatedSharedResources{
|
2018-10-16 22:34:32 +00:00
|
|
|
DiskMB: int64(iter.taskGroup.EphemeralDisk.SizeMB),
|
2018-10-02 20:36:04 +00:00
|
|
|
},
|
2016-08-25 17:27:19 +00:00
|
|
|
}
|
2018-09-21 21:05:00 +00:00
|
|
|
|
|
|
|
var allocsToPreempt []*structs.Allocation
|
|
|
|
|
2018-10-29 17:34:03 +00:00
|
|
|
// Initialize preemptor with node
|
2018-11-15 03:23:38 +00:00
|
|
|
preemptor := NewPreemptor(iter.priority, iter.ctx)
|
2018-10-29 17:34:03 +00:00
|
|
|
preemptor.SetNode(option.Node)
|
|
|
|
|
2018-09-21 21:05:00 +00:00
|
|
|
// Count the number of existing preemptions
|
|
|
|
allPreemptions := iter.ctx.Plan().NodePreemptions
|
|
|
|
var currentPreemptions []*structs.Allocation
|
|
|
|
for _, allocs := range allPreemptions {
|
|
|
|
currentPreemptions = append(currentPreemptions, allocs...)
|
|
|
|
}
|
2018-10-29 17:34:03 +00:00
|
|
|
preemptor.SetPreemptions(currentPreemptions)
|
2018-09-21 21:05:00 +00:00
|
|
|
|
2016-08-25 17:27:19 +00:00
|
|
|
for _, task := range iter.taskGroup.Tasks {
|
2018-10-02 20:36:04 +00:00
|
|
|
// Allocate the resources
|
|
|
|
taskResources := &structs.AllocatedTaskResources{
|
|
|
|
Cpu: structs.AllocatedCpuResources{
|
2018-10-16 22:34:32 +00:00
|
|
|
CpuShares: int64(task.Resources.CPU),
|
2018-10-02 20:36:04 +00:00
|
|
|
},
|
|
|
|
Memory: structs.AllocatedMemoryResources{
|
2018-10-16 22:34:32 +00:00
|
|
|
MemoryMB: int64(task.Resources.MemoryMB),
|
2018-10-02 20:36:04 +00:00
|
|
|
},
|
|
|
|
}
|
2015-09-13 21:31:32 +00:00
|
|
|
|
|
|
|
// Check if we need a network resource
|
2018-10-02 20:36:04 +00:00
|
|
|
if len(task.Resources.Networks) > 0 {
|
|
|
|
ask := task.Resources.Networks[0].Copy()
|
2015-09-13 23:41:32 +00:00
|
|
|
offer, err := netIdx.AssignNetwork(ask)
|
2015-09-13 21:31:32 +00:00
|
|
|
if offer == nil {
|
2018-09-21 21:05:00 +00:00
|
|
|
// If eviction is not enabled, mark this node as exhausted and continue
|
|
|
|
if !iter.evict {
|
|
|
|
iter.ctx.Metrics().ExhaustedNode(option.Node,
|
|
|
|
fmt.Sprintf("network: %s", err))
|
|
|
|
netIdx.Release()
|
|
|
|
continue OUTER
|
|
|
|
}
|
|
|
|
|
|
|
|
// Look for preemptible allocations to satisfy the network resource for this task
|
2018-10-29 17:34:03 +00:00
|
|
|
preemptor.SetCandidates(proposed)
|
|
|
|
|
2018-10-30 14:14:56 +00:00
|
|
|
netPreemptions := preemptor.PreemptForNetwork(ask, netIdx)
|
2018-10-18 04:49:37 +00:00
|
|
|
if netPreemptions == nil {
|
2018-11-02 22:59:53 +00:00
|
|
|
iter.ctx.Logger().Named("binpack").Error(fmt.Sprintf("unable to meet network resource %v after preemption", ask))
|
2018-09-21 21:05:00 +00:00
|
|
|
netIdx.Release()
|
|
|
|
continue OUTER
|
|
|
|
}
|
2018-10-18 04:49:37 +00:00
|
|
|
allocsToPreempt = append(allocsToPreempt, netPreemptions...)
|
2018-09-21 21:05:00 +00:00
|
|
|
|
|
|
|
// First subtract out preempted allocations
|
2018-10-18 04:49:37 +00:00
|
|
|
proposed = structs.RemoveAllocs(proposed, netPreemptions)
|
2018-09-21 21:05:00 +00:00
|
|
|
|
|
|
|
// Reset the network index and try the offer again
|
2016-02-20 20:18:22 +00:00
|
|
|
netIdx.Release()
|
2018-09-21 21:05:00 +00:00
|
|
|
netIdx = structs.NewNetworkIndex()
|
|
|
|
netIdx.SetNode(option.Node)
|
|
|
|
netIdx.AddAllocs(proposed)
|
|
|
|
|
|
|
|
offer, err = netIdx.AssignNetwork(ask)
|
|
|
|
if offer == nil {
|
2018-11-02 22:59:53 +00:00
|
|
|
iter.ctx.Logger().Named("binpack").Error(fmt.Sprintf("unexpected error, unable to create offer after preempting:%v", err))
|
2018-09-21 21:05:00 +00:00
|
|
|
netIdx.Release()
|
|
|
|
continue OUTER
|
|
|
|
}
|
2015-09-13 21:31:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Reserve this to prevent another task from colliding
|
|
|
|
netIdx.AddReserved(offer)
|
|
|
|
|
|
|
|
// Update the network ask to the offer
|
|
|
|
taskResources.Networks = []*structs.NetworkResource{offer}
|
|
|
|
}
|
|
|
|
|
2018-10-15 22:15:46 +00:00
|
|
|
// Check if we need to assign devices
|
|
|
|
for _, req := range task.Resources.Devices {
|
2018-11-08 23:01:58 +00:00
|
|
|
offer, sumAffinities, err := devAllocator.AssignDevice(req)
|
2018-10-15 22:15:46 +00:00
|
|
|
if offer == nil {
|
2018-11-15 03:23:38 +00:00
|
|
|
// If eviction is not enabled, mark this node as exhausted and continue
|
|
|
|
if !iter.evict {
|
|
|
|
iter.ctx.Metrics().ExhaustedNode(option.Node, fmt.Sprintf("devices: %s", err))
|
|
|
|
continue OUTER
|
|
|
|
}
|
|
|
|
|
|
|
|
// Attempt preemption
|
|
|
|
preemptor.SetCandidates(proposed)
|
|
|
|
devicePreemptions := preemptor.PreemptForDevice(req, devAllocator)
|
|
|
|
|
|
|
|
if devicePreemptions == nil {
|
|
|
|
iter.ctx.Logger().Named("binpack").Error(fmt.Sprintf("unable to meet device resource need after attempting preemption:%v", req))
|
|
|
|
netIdx.Release()
|
|
|
|
continue OUTER
|
|
|
|
}
|
|
|
|
allocsToPreempt = append(allocsToPreempt, devicePreemptions...)
|
|
|
|
|
|
|
|
// First subtract out preempted allocations
|
|
|
|
proposed = structs.RemoveAllocs(proposed, allocsToPreempt)
|
|
|
|
|
|
|
|
// Reset the device allocator with new set of proposed allocs
|
|
|
|
devAllocator := newDeviceAllocator(iter.ctx, option.Node)
|
|
|
|
devAllocator.AddAllocs(proposed)
|
|
|
|
|
|
|
|
// Try offer again
|
|
|
|
offer, sumAffinities, err = devAllocator.AssignDevice(req)
|
|
|
|
if offer == nil {
|
|
|
|
iter.ctx.Logger().Named("binpack").Error(fmt.Sprintf("unexpected error, unable to create device offer after preempting:%v", err))
|
|
|
|
continue OUTER
|
|
|
|
}
|
2018-10-15 22:15:46 +00:00
|
|
|
}
|
2018-10-17 18:04:54 +00:00
|
|
|
|
|
|
|
// Store the resource
|
2018-10-15 22:15:46 +00:00
|
|
|
devAllocator.AddReserved(offer)
|
|
|
|
taskResources.Devices = append(taskResources.Devices, offer)
|
2018-10-17 18:04:54 +00:00
|
|
|
|
|
|
|
// Add the scores
|
|
|
|
if len(req.Affinities) != 0 {
|
2018-11-08 23:01:58 +00:00
|
|
|
for _, a := range req.Affinities {
|
|
|
|
totalDeviceAffinityWeight += math.Abs(a.Weight)
|
|
|
|
}
|
|
|
|
sumMatchingAffinities += sumAffinities
|
2018-10-17 18:04:54 +00:00
|
|
|
}
|
2018-10-15 22:15:46 +00:00
|
|
|
}
|
|
|
|
|
2015-09-13 21:31:32 +00:00
|
|
|
// Store the task resource
|
|
|
|
option.SetTaskResources(task, taskResources)
|
|
|
|
|
|
|
|
// Accumulate the total resource requirement
|
2018-10-02 20:36:04 +00:00
|
|
|
total.Tasks[task.Name] = taskResources
|
2015-09-13 21:31:32 +00:00
|
|
|
}
|
|
|
|
|
2018-10-18 04:49:37 +00:00
|
|
|
// Store current set of running allocs before adding resources for the task group
|
|
|
|
current := proposed
|
|
|
|
|
2015-08-13 18:54:59 +00:00
|
|
|
// Add the resources we are trying to fit
|
2018-10-02 20:36:04 +00:00
|
|
|
proposed = append(proposed, &structs.Allocation{AllocatedResources: total})
|
2015-08-13 18:28:02 +00:00
|
|
|
|
2018-10-17 22:03:38 +00:00
|
|
|
// Check if these allocations fit, if they do not, simply skip this node
|
|
|
|
fit, dim, util, _ := structs.AllocsFit(option.Node, proposed, netIdx, false)
|
2016-02-20 20:18:22 +00:00
|
|
|
netIdx.Release()
|
2015-08-13 18:54:59 +00:00
|
|
|
if !fit {
|
2018-09-21 21:05:00 +00:00
|
|
|
// Skip the node if evictions are not enabled
|
|
|
|
if !iter.evict {
|
|
|
|
iter.ctx.Metrics().ExhaustedNode(option.Node, dim)
|
|
|
|
continue
|
|
|
|
}
|
2018-10-18 05:07:56 +00:00
|
|
|
|
2018-09-21 21:05:00 +00:00
|
|
|
// If eviction is enabled and the node doesn't fit the alloc, check if
|
|
|
|
// any allocs can be preempted
|
2018-10-29 17:34:03 +00:00
|
|
|
|
|
|
|
// Initialize preemptor with candidate set
|
|
|
|
preemptor.SetCandidates(current)
|
2015-08-13 18:54:59 +00:00
|
|
|
|
2018-10-30 14:14:56 +00:00
|
|
|
preemptedAllocs := preemptor.PreemptForTaskGroup(total)
|
2018-10-29 17:34:03 +00:00
|
|
|
allocsToPreempt = append(allocsToPreempt, preemptedAllocs...)
|
2018-10-18 04:49:37 +00:00
|
|
|
|
2018-09-21 21:05:00 +00:00
|
|
|
// If we were unable to find preempted allocs to meet these requirements
|
|
|
|
// mark as exhausted and continue
|
2018-10-29 17:34:03 +00:00
|
|
|
if len(preemptedAllocs) == 0 {
|
2018-09-21 21:05:00 +00:00
|
|
|
iter.ctx.Metrics().ExhaustedNode(option.Node, dim)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(allocsToPreempt) > 0 {
|
|
|
|
option.PreemptedAllocs = allocsToPreempt
|
2015-08-13 18:54:59 +00:00
|
|
|
}
|
2015-08-13 19:02:42 +00:00
|
|
|
|
2015-08-13 18:54:59 +00:00
|
|
|
// Score the fit normally otherwise
|
2015-08-16 17:28:58 +00:00
|
|
|
fitness := structs.ScoreFit(option.Node, util)
|
2018-07-20 14:33:47 +00:00
|
|
|
normalizedFit := fitness / binPackingMaxFitScore
|
2018-07-16 13:47:18 +00:00
|
|
|
option.Scores = append(option.Scores, normalizedFit)
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "binpack", normalizedFit)
|
2018-10-17 18:04:54 +00:00
|
|
|
|
|
|
|
// Score the device affinity
|
2018-11-08 23:01:58 +00:00
|
|
|
if totalDeviceAffinityWeight != 0 {
|
|
|
|
sumMatchingAffinities /= totalDeviceAffinityWeight
|
|
|
|
option.Scores = append(option.Scores, sumMatchingAffinities)
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "devices", sumMatchingAffinities)
|
2018-10-17 18:04:54 +00:00
|
|
|
}
|
|
|
|
|
2015-08-13 18:54:59 +00:00
|
|
|
return option
|
2015-08-13 18:28:02 +00:00
|
|
|
}
|
|
|
|
}
|
2015-08-13 22:01:02 +00:00
|
|
|
|
|
|
|
func (iter *BinPackIterator) Reset() {
|
|
|
|
iter.source.Reset()
|
|
|
|
}
|
2015-08-16 17:32:25 +00:00
|
|
|
|
|
|
|
// JobAntiAffinityIterator is used to apply an anti-affinity to allocating
|
|
|
|
// along side other allocations from this job. This is used to help distribute
|
|
|
|
// load across the cluster.
|
|
|
|
type JobAntiAffinityIterator struct {
|
2018-07-16 13:47:18 +00:00
|
|
|
ctx Context
|
|
|
|
source RankIterator
|
|
|
|
jobID string
|
|
|
|
taskGroup string
|
|
|
|
desiredCount int
|
2015-08-16 17:32:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewJobAntiAffinityIterator is used to create a JobAntiAffinityIterator that
|
|
|
|
// applies the given penalty for co-placement with allocs from this job.
|
2018-07-16 13:47:18 +00:00
|
|
|
func NewJobAntiAffinityIterator(ctx Context, source RankIterator, jobID string) *JobAntiAffinityIterator {
|
2015-08-16 17:32:25 +00:00
|
|
|
iter := &JobAntiAffinityIterator{
|
2018-07-16 13:47:18 +00:00
|
|
|
ctx: ctx,
|
|
|
|
source: source,
|
|
|
|
jobID: jobID,
|
2015-08-16 17:32:25 +00:00
|
|
|
}
|
|
|
|
return iter
|
|
|
|
}
|
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
func (iter *JobAntiAffinityIterator) SetJob(job *structs.Job) {
|
|
|
|
iter.jobID = job.ID
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *JobAntiAffinityIterator) SetTaskGroup(tg *structs.TaskGroup) {
|
|
|
|
iter.taskGroup = tg.Name
|
|
|
|
iter.desiredCount = tg.Count
|
2015-08-16 17:32:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *JobAntiAffinityIterator) Next() *RankedNode {
|
|
|
|
for {
|
|
|
|
option := iter.source.Next()
|
|
|
|
if option == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the proposed allocations
|
2015-09-07 23:19:21 +00:00
|
|
|
proposed, err := option.ProposedAllocs(iter.ctx)
|
|
|
|
if err != nil {
|
2018-09-15 23:23:13 +00:00
|
|
|
iter.ctx.Logger().Named("job_anti_affinity").Error("failed retrieving proposed allocations", "error", err)
|
2015-09-07 23:19:21 +00:00
|
|
|
continue
|
2015-08-16 17:32:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Determine the number of collisions
|
|
|
|
collisions := 0
|
|
|
|
for _, alloc := range proposed {
|
2018-07-16 13:47:18 +00:00
|
|
|
if alloc.JobID == iter.jobID && alloc.TaskGroup == iter.taskGroup {
|
2015-08-16 17:32:25 +00:00
|
|
|
collisions += 1
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
// Calculate the penalty based on number of collisions
|
|
|
|
// TODO(preetha): Figure out if batch jobs need a different scoring penalty where collisions matter less
|
2015-08-16 17:32:25 +00:00
|
|
|
if collisions > 0 {
|
2018-07-24 15:49:50 +00:00
|
|
|
scorePenalty := -1 * float64(collisions+1) / float64(iter.desiredCount)
|
2018-07-16 13:47:18 +00:00
|
|
|
option.Scores = append(option.Scores, scorePenalty)
|
2015-08-16 17:32:25 +00:00
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "job-anti-affinity", scorePenalty)
|
2018-10-18 02:06:24 +00:00
|
|
|
} else {
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "job-anti-affinity", 0)
|
2015-08-16 17:32:25 +00:00
|
|
|
}
|
|
|
|
return option
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *JobAntiAffinityIterator) Reset() {
|
|
|
|
iter.source.Reset()
|
|
|
|
}
|
2018-01-14 22:47:21 +00:00
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
// NodeReschedulingPenaltyIterator is used to apply a penalty to
|
2018-01-14 22:47:21 +00:00
|
|
|
// a node that had a previous failed allocation for the same job.
|
|
|
|
// This is used when attempting to reschedule a failed alloc
|
2018-07-16 13:47:18 +00:00
|
|
|
type NodeReschedulingPenaltyIterator struct {
|
2018-01-14 22:47:21 +00:00
|
|
|
ctx Context
|
|
|
|
source RankIterator
|
|
|
|
penaltyNodes map[string]struct{}
|
|
|
|
}
|
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
// NewNodeReschedulingPenaltyIterator is used to create a NodeReschedulingPenaltyIterator that
|
|
|
|
// applies the given scoring penalty for placement onto nodes in penaltyNodes
|
|
|
|
func NewNodeReschedulingPenaltyIterator(ctx Context, source RankIterator) *NodeReschedulingPenaltyIterator {
|
|
|
|
iter := &NodeReschedulingPenaltyIterator{
|
|
|
|
ctx: ctx,
|
|
|
|
source: source,
|
2018-01-14 22:47:21 +00:00
|
|
|
}
|
|
|
|
return iter
|
|
|
|
}
|
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
func (iter *NodeReschedulingPenaltyIterator) SetPenaltyNodes(penaltyNodes map[string]struct{}) {
|
2018-01-14 22:47:21 +00:00
|
|
|
iter.penaltyNodes = penaltyNodes
|
|
|
|
}
|
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
func (iter *NodeReschedulingPenaltyIterator) Next() *RankedNode {
|
2018-01-14 22:47:21 +00:00
|
|
|
for {
|
|
|
|
option := iter.source.Next()
|
|
|
|
if option == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
_, ok := iter.penaltyNodes[option.Node.ID]
|
|
|
|
if ok {
|
2018-07-16 13:47:18 +00:00
|
|
|
option.Scores = append(option.Scores, -1)
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "node-reschedule-penalty", -1)
|
2018-10-18 02:06:24 +00:00
|
|
|
} else {
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "node-reschedule-penalty", 0)
|
2018-01-14 22:47:21 +00:00
|
|
|
}
|
|
|
|
return option
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-16 13:47:18 +00:00
|
|
|
func (iter *NodeReschedulingPenaltyIterator) Reset() {
|
2018-01-19 14:41:53 +00:00
|
|
|
iter.penaltyNodes = make(map[string]struct{})
|
2018-01-14 22:47:21 +00:00
|
|
|
iter.source.Reset()
|
|
|
|
}
|
2018-07-16 13:47:18 +00:00
|
|
|
|
2018-07-18 20:25:45 +00:00
|
|
|
// NodeAffinityIterator is used to resolve any affinity rules in the job or task group,
|
|
|
|
// and apply a weighted score to nodes if they match.
|
2018-07-16 13:47:18 +00:00
|
|
|
type NodeAffinityIterator struct {
|
|
|
|
ctx Context
|
|
|
|
source RankIterator
|
|
|
|
jobAffinities []*structs.Affinity
|
|
|
|
affinities []*structs.Affinity
|
|
|
|
}
|
|
|
|
|
2018-07-18 20:25:45 +00:00
|
|
|
// NewNodeAffinityIterator is used to create a NodeAffinityIterator that
|
|
|
|
// applies a weighted score according to whether nodes match any
|
|
|
|
// affinities in the job or task group.
|
2018-07-16 13:47:18 +00:00
|
|
|
func NewNodeAffinityIterator(ctx Context, source RankIterator) *NodeAffinityIterator {
|
|
|
|
return &NodeAffinityIterator{
|
|
|
|
ctx: ctx,
|
|
|
|
source: source,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *NodeAffinityIterator) SetJob(job *structs.Job) {
|
2018-07-18 20:25:45 +00:00
|
|
|
iter.jobAffinities = job.Affinities
|
2018-07-16 13:47:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *NodeAffinityIterator) SetTaskGroup(tg *structs.TaskGroup) {
|
|
|
|
// Merge job affinities
|
|
|
|
if iter.jobAffinities != nil {
|
|
|
|
iter.affinities = append(iter.affinities, iter.jobAffinities...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Merge task group affinities and task affinities
|
|
|
|
if tg.Affinities != nil {
|
|
|
|
iter.affinities = append(iter.affinities, tg.Affinities...)
|
|
|
|
}
|
|
|
|
for _, task := range tg.Tasks {
|
|
|
|
if task.Affinities != nil {
|
|
|
|
iter.affinities = append(iter.affinities, task.Affinities...)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *NodeAffinityIterator) Reset() {
|
|
|
|
iter.source.Reset()
|
|
|
|
// This method is called between each task group, so only reset the merged list
|
|
|
|
iter.affinities = nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *NodeAffinityIterator) hasAffinities() bool {
|
|
|
|
return len(iter.affinities) > 0
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *NodeAffinityIterator) Next() *RankedNode {
|
|
|
|
option := iter.source.Next()
|
|
|
|
if option == nil {
|
|
|
|
return nil
|
|
|
|
}
|
2018-07-18 20:25:45 +00:00
|
|
|
if !iter.hasAffinities() {
|
2018-10-18 02:06:24 +00:00
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "node-affinity", 0)
|
2018-07-16 13:47:18 +00:00
|
|
|
return option
|
|
|
|
}
|
|
|
|
// TODO(preetha): we should calculate normalized weights once and reuse it here
|
|
|
|
sumWeight := 0.0
|
|
|
|
for _, affinity := range iter.affinities {
|
|
|
|
sumWeight += math.Abs(affinity.Weight)
|
|
|
|
}
|
|
|
|
|
|
|
|
totalAffinityScore := 0.0
|
|
|
|
for _, affinity := range iter.affinities {
|
|
|
|
if matchesAffinity(iter.ctx, affinity, option.Node) {
|
2018-07-24 15:49:50 +00:00
|
|
|
totalAffinityScore += affinity.Weight
|
2018-07-16 13:47:18 +00:00
|
|
|
}
|
|
|
|
}
|
2018-07-24 15:49:50 +00:00
|
|
|
normScore := totalAffinityScore / sumWeight
|
2018-07-16 13:47:18 +00:00
|
|
|
if totalAffinityScore != 0.0 {
|
2018-07-24 15:49:50 +00:00
|
|
|
option.Scores = append(option.Scores, normScore)
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "node-affinity", normScore)
|
2018-07-16 13:47:18 +00:00
|
|
|
}
|
|
|
|
return option
|
|
|
|
}
|
|
|
|
|
|
|
|
func matchesAffinity(ctx Context, affinity *structs.Affinity, option *structs.Node) bool {
|
2018-07-18 20:25:45 +00:00
|
|
|
//TODO(preetha): Add a step here that filters based on computed node class for potential speedup
|
2018-07-16 13:47:18 +00:00
|
|
|
// Resolve the targets
|
|
|
|
lVal, ok := resolveTarget(affinity.LTarget, option)
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
rVal, ok := resolveTarget(affinity.RTarget, option)
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if satisfied
|
|
|
|
return checkAffinity(ctx, affinity.Operand, lVal, rVal)
|
|
|
|
}
|
|
|
|
|
2018-07-18 20:25:45 +00:00
|
|
|
// ScoreNormalizationIterator is used to combine scores from various prior
|
|
|
|
// iterators and combine them into one final score. The current implementation
|
|
|
|
// averages the scores together.
|
2018-07-16 13:47:18 +00:00
|
|
|
type ScoreNormalizationIterator struct {
|
|
|
|
ctx Context
|
|
|
|
source RankIterator
|
|
|
|
}
|
|
|
|
|
2018-07-18 20:25:45 +00:00
|
|
|
// NewScoreNormalizationIterator is used to create a ScoreNormalizationIterator that
|
|
|
|
// averages scores from various iterators into a final score.
|
2018-07-16 13:47:18 +00:00
|
|
|
func NewScoreNormalizationIterator(ctx Context, source RankIterator) *ScoreNormalizationIterator {
|
|
|
|
return &ScoreNormalizationIterator{
|
|
|
|
ctx: ctx,
|
|
|
|
source: source}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *ScoreNormalizationIterator) Reset() {
|
|
|
|
iter.source.Reset()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (iter *ScoreNormalizationIterator) Next() *RankedNode {
|
|
|
|
option := iter.source.Next()
|
2018-07-18 20:25:45 +00:00
|
|
|
if option == nil || len(option.Scores) == 0 {
|
|
|
|
return option
|
2018-07-16 13:47:18 +00:00
|
|
|
}
|
|
|
|
numScorers := len(option.Scores)
|
2018-07-18 20:25:45 +00:00
|
|
|
sum := 0.0
|
|
|
|
for _, score := range option.Scores {
|
|
|
|
sum += score
|
2018-07-16 13:47:18 +00:00
|
|
|
}
|
2018-07-18 20:25:45 +00:00
|
|
|
option.FinalScore = sum / float64(numScorers)
|
2018-07-16 13:47:18 +00:00
|
|
|
//TODO(preetha): Turn map in allocmetrics into a heap of topK scores
|
|
|
|
iter.ctx.Metrics().ScoreNode(option.Node, "normalized-score", option.FinalScore)
|
|
|
|
return option
|
|
|
|
}
|