145 lines
4.5 KiB
Go
145 lines
4.5 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package spread
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/nomad/api"
|
|
"github.com/hashicorp/nomad/e2e/e2eutil"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/shoenig/test/must"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
const (
|
|
evenJobFilePath = "./input/even_spread.nomad"
|
|
multipleJobFilePath = "./input/multiple_spread.nomad"
|
|
)
|
|
|
|
func TestSpread(t *testing.T) {
|
|
|
|
nomadClient := e2eutil.NomadClient(t)
|
|
e2eutil.WaitForLeader(t, nomadClient)
|
|
e2eutil.WaitForNodesReady(t, nomadClient, 4)
|
|
|
|
// Run our test cases.
|
|
t.Run("TestSpread_Even", testSpreadEven)
|
|
t.Run("TestSpread_Multiple", testSpreadMultiple)
|
|
}
|
|
|
|
func testSpreadEven(t *testing.T) {
|
|
|
|
nomadClient := e2eutil.NomadClient(t)
|
|
|
|
// Generate a job ID and register the test job.
|
|
jobID := "spread-" + uuid.Short()
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, evenJobFilePath, jobID, "")
|
|
|
|
// Ensure the test cleans its own job and allocations fully, so it does not
|
|
// impact other spread tests.
|
|
t.Cleanup(func() { cleanupJob(t, nomadClient, jobID, allocs) })
|
|
|
|
dcToAllocs := make(map[string]int)
|
|
|
|
for _, allocStub := range allocs {
|
|
alloc, _, err := nomadClient.Allocations().Info(allocStub.ID, nil)
|
|
must.NoError(t, err)
|
|
must.Greater(t, 0, len(alloc.Metrics.ScoreMetaData))
|
|
node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
|
|
must.NoError(t, err)
|
|
dcToAllocs[node.Datacenter]++
|
|
}
|
|
|
|
must.Eq(t, map[string]int{"dc1": 3, "dc2": 3}, dcToAllocs)
|
|
}
|
|
|
|
func testSpreadMultiple(t *testing.T) {
|
|
|
|
nomadClient := e2eutil.NomadClient(t)
|
|
|
|
// Generate a job ID and register the test job.
|
|
jobID := "spread-" + uuid.Short()
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, multipleJobFilePath, jobID, "")
|
|
|
|
// Ensure the test cleans its own job and allocations fully, so it does not
|
|
// impact other spread tests.
|
|
t.Cleanup(func() { cleanupJob(t, nomadClient, jobID, allocs) })
|
|
|
|
// Verify spread score and alloc distribution
|
|
dcToAllocs := make(map[string]int)
|
|
rackToAllocs := make(map[string]int)
|
|
allocMetrics := make(map[string]*api.AllocationMetric)
|
|
|
|
for _, allocStub := range allocs {
|
|
alloc, _, err := nomadClient.Allocations().Info(allocStub.ID, nil)
|
|
must.NoError(t, err)
|
|
must.Greater(t, 0, len(alloc.Metrics.ScoreMetaData))
|
|
allocMetrics[allocStub.ID] = alloc.Metrics
|
|
|
|
node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
|
|
must.NoError(t, err)
|
|
dcToAllocs[node.Datacenter]++
|
|
|
|
if rack := node.Meta["rack"]; rack != "" {
|
|
rackToAllocs[rack]++
|
|
}
|
|
}
|
|
|
|
failureReport := report(allocMetrics)
|
|
|
|
must.Eq(t, map[string]int{"dc1": 5, "dc2": 5}, dcToAllocs, failureReport)
|
|
must.Eq(t, map[string]int{"r1": 7, "r2": 3}, rackToAllocs, failureReport)
|
|
}
|
|
|
|
func cleanupJob(t *testing.T, nomadClient *api.Client, jobID string, allocs []*api.AllocationListStub) {
|
|
|
|
_, _, err := nomadClient.Jobs().Deregister(jobID, true, nil)
|
|
assert.NoError(t, err)
|
|
|
|
// Ensure that all allocations have been removed from state. This is an
|
|
// important aspect of the cleaning required which allows the spread
|
|
// test to run successfully.
|
|
assert.Eventually(t, func() bool {
|
|
|
|
// Run the garbage collector to remove all terminal allocations.
|
|
must.NoError(t, nomadClient.System().GarbageCollect())
|
|
|
|
for _, allocStub := range allocs {
|
|
_, _, err := nomadClient.Allocations().Info(allocStub.ID, nil)
|
|
if err == nil {
|
|
return false
|
|
} else {
|
|
if !strings.Contains(err.Error(), "alloc not found") {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}, 10*time.Second, 200*time.Millisecond)
|
|
}
|
|
|
|
func report(metrics map[string]*api.AllocationMetric) must.Setting {
|
|
var s strings.Builder
|
|
for allocID, m := range metrics {
|
|
s.WriteString("Alloc ID: " + allocID + "\n")
|
|
s.WriteString(fmt.Sprintf(" NodesEvaluated: %d\n", m.NodesEvaluated))
|
|
s.WriteString(fmt.Sprintf(" NodesAvailable: %#v\n", m.NodesAvailable))
|
|
s.WriteString(fmt.Sprintf(" ClassFiltered: %#v\n", m.ClassFiltered))
|
|
s.WriteString(fmt.Sprintf(" ConstraintFiltered: %#v\n", m.ConstraintFiltered))
|
|
s.WriteString(fmt.Sprintf(" NodesExhausted: %d\n", m.NodesExhausted))
|
|
s.WriteString(fmt.Sprintf(" ClassExhausted: %#v\n", m.ClassExhausted))
|
|
s.WriteString(fmt.Sprintf(" DimensionExhausted: %#v\n", m.DimensionExhausted))
|
|
s.WriteString(fmt.Sprintf(" QuotaExhausted: %#v\n", m.QuotaExhausted))
|
|
for _, nodeMeta := range m.ScoreMetaData {
|
|
s.WriteString(fmt.Sprintf(" NodeID: %s, NormScore: %f, Scores: %#v\n",
|
|
nodeMeta.NodeID, nodeMeta.NormScore, nodeMeta.Scores))
|
|
}
|
|
}
|
|
return must.Sprint(s.String())
|
|
}
|