135 lines
4.0 KiB
Go
135 lines
4.0 KiB
Go
package spread
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/hashicorp/nomad/api"
|
|
"github.com/hashicorp/nomad/e2e/e2eutil"
|
|
"github.com/hashicorp/nomad/e2e/framework"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
)
|
|
|
|
type SpreadTest struct {
|
|
framework.TC
|
|
jobIds []string
|
|
}
|
|
|
|
func init() {
|
|
framework.AddSuites(&framework.TestSuite{
|
|
Component: "Spread",
|
|
CanRunLocal: true,
|
|
Cases: []framework.TestCase{
|
|
new(SpreadTest),
|
|
},
|
|
})
|
|
}
|
|
|
|
func (tc *SpreadTest) BeforeAll(f *framework.F) {
|
|
// Ensure cluster has leader before running tests
|
|
e2eutil.WaitForLeader(f.T(), tc.Nomad())
|
|
e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 4)
|
|
}
|
|
|
|
func (tc *SpreadTest) TestEvenSpread(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
uuid := uuid.Generate()
|
|
jobId := "spread" + uuid[0:8]
|
|
tc.jobIds = append(tc.jobIds, jobId)
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "spread/input/even_spread.nomad", jobId, "")
|
|
|
|
jobAllocs := nomadClient.Allocations()
|
|
dcToAllocs := make(map[string]int)
|
|
require := require.New(f.T())
|
|
// Verify spread score and alloc distribution
|
|
for _, allocStub := range allocs {
|
|
alloc, _, err := jobAllocs.Info(allocStub.ID, nil)
|
|
require.Nil(err)
|
|
require.NotEmpty(alloc.Metrics.ScoreMetaData)
|
|
|
|
node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
|
|
require.Nil(err)
|
|
dcToAllocs[node.Datacenter]++
|
|
}
|
|
|
|
expectedDcToAllocs := make(map[string]int)
|
|
expectedDcToAllocs["dc1"] = 3
|
|
expectedDcToAllocs["dc2"] = 3
|
|
require.Equal(expectedDcToAllocs, dcToAllocs)
|
|
}
|
|
|
|
func (tc *SpreadTest) TestMultipleSpreads(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
uuid := uuid.Generate()
|
|
jobId := "spread" + uuid[0:8]
|
|
tc.jobIds = append(tc.jobIds, jobId)
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "spread/input/multiple_spread.nomad", jobId, "")
|
|
|
|
jobAllocs := nomadClient.Allocations()
|
|
dcToAllocs := make(map[string]int)
|
|
rackToAllocs := make(map[string]int)
|
|
allocMetrics := make(map[string]*api.AllocationMetric)
|
|
|
|
require := require.New(f.T())
|
|
// Verify spread score and alloc distribution
|
|
for _, allocStub := range allocs {
|
|
alloc, _, err := jobAllocs.Info(allocStub.ID, nil)
|
|
|
|
require.Nil(err)
|
|
require.NotEmpty(alloc.Metrics.ScoreMetaData)
|
|
allocMetrics[allocStub.ID] = alloc.Metrics
|
|
|
|
node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
|
|
require.Nil(err)
|
|
dcToAllocs[node.Datacenter]++
|
|
rack := node.Meta["rack"]
|
|
if rack != "" {
|
|
rackToAllocs[rack]++
|
|
}
|
|
}
|
|
|
|
expectedDcToAllocs := make(map[string]int)
|
|
expectedDcToAllocs["dc1"] = 5
|
|
expectedDcToAllocs["dc2"] = 5
|
|
require.Equal(expectedDcToAllocs, dcToAllocs, report(allocMetrics))
|
|
|
|
expectedRackToAllocs := make(map[string]int)
|
|
expectedRackToAllocs["r1"] = 7
|
|
expectedRackToAllocs["r2"] = 3
|
|
require.Equal(expectedRackToAllocs, rackToAllocs, report(allocMetrics))
|
|
|
|
}
|
|
|
|
func report(metrics map[string]*api.AllocationMetric) string {
|
|
var s strings.Builder
|
|
for allocID, m := range metrics {
|
|
s.WriteString("Alloc ID: " + allocID + "\n")
|
|
s.WriteString(fmt.Sprintf(" NodesEvaluated: %d\n", m.NodesEvaluated))
|
|
s.WriteString(fmt.Sprintf(" NodesAvailable: %#v\n", m.NodesAvailable))
|
|
s.WriteString(fmt.Sprintf(" ClassFiltered: %#v\n", m.ClassFiltered))
|
|
s.WriteString(fmt.Sprintf(" ConstraintFiltered: %#v\n", m.ConstraintFiltered))
|
|
s.WriteString(fmt.Sprintf(" NodesExhausted: %d\n", m.NodesExhausted))
|
|
s.WriteString(fmt.Sprintf(" ClassExhausted: %#v\n", m.ClassExhausted))
|
|
s.WriteString(fmt.Sprintf(" DimensionExhausted: %#v\n", m.DimensionExhausted))
|
|
s.WriteString(fmt.Sprintf(" QuotaExhausted: %#v\n", m.QuotaExhausted))
|
|
for _, nodeMeta := range m.ScoreMetaData {
|
|
s.WriteString(fmt.Sprintf(" NodeID: %s, NormScore: %f, Scores: %#v\n",
|
|
nodeMeta.NodeID, nodeMeta.NormScore, nodeMeta.Scores))
|
|
}
|
|
}
|
|
return s.String()
|
|
}
|
|
|
|
func (tc *SpreadTest) AfterEach(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobs := nomadClient.Jobs()
|
|
// Stop all jobs in test
|
|
for _, id := range tc.jobIds {
|
|
jobs.Deregister(id, true, nil)
|
|
}
|
|
// Garbage collect
|
|
nomadClient.System().GarbageCollect()
|
|
}
|