From 29d2f8172a2ca0114d43941d211999a0ab10bded Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Tue, 18 Dec 2018 18:39:45 -0600 Subject: [PATCH] Affinities e2e tests --- e2e/affinities/affinities.go | 218 ++++++++++++++++++++++++++++++++ e2e/affinities/input/aff1.nomad | 22 ++++ e2e/affinities/input/aff2.nomad | 27 ++++ e2e/affinities/input/aff3.nomad | 27 ++++ e2e/e2e_test.go | 1 + 5 files changed, 295 insertions(+) create mode 100644 e2e/affinities/affinities.go create mode 100644 e2e/affinities/input/aff1.nomad create mode 100644 e2e/affinities/input/aff2.nomad create mode 100644 e2e/affinities/input/aff3.nomad diff --git a/e2e/affinities/affinities.go b/e2e/affinities/affinities.go new file mode 100644 index 000000000..75f43f3bb --- /dev/null +++ b/e2e/affinities/affinities.go @@ -0,0 +1,218 @@ +package affinities + +import ( + "time" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/framework" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/jobspec" + "github.com/stretchr/testify/require" + + . "github.com/onsi/gomega" +) + +type BasicAffinityTest struct { + framework.TC + jobIds []string +} + +func init() { + framework.AddSuites(&framework.TestSuite{ + Component: "Affinity", + CanRunLocal: true, + Cases: []framework.TestCase{ + new(BasicAffinityTest), + }, + }) +} + +func (tc *BasicAffinityTest) TestSingleAffinities(f *framework.F) { + nomadClient := tc.Nomad() + + // Parse job + job, err := jobspec.ParseFile("affinities/input/aff1.nomad") + require := require.New(f.T()) + require.Nil(err) + uuid := uuid.Generate() + jobId := helper.StringToPtr("aff" + uuid[0:8]) + job.ID = jobId + + tc.jobIds = append(tc.jobIds, *jobId) + + // Register job + jobs := nomadClient.Jobs() + resp, _, err := jobs.Register(job, nil) + require.Nil(err) + require.NotEmpty(resp.EvalID) + + g := NewGomegaWithT(f.T()) + + // Wrap in retry to wait until placement + g.Eventually(func() []*api.AllocationListStub { + // Look for allocations + allocs, _, _ := jobs.Allocations(*job.ID, false, nil) + return allocs + }, 5*time.Second, time.Second).ShouldNot(BeEmpty()) + + jobAllocs := nomadClient.Allocations() + + allocs, _, _ := jobs.Allocations(*job.ID, false, nil) + + // Verify affinity score metadata + for _, allocStub := range allocs { + alloc, _, err := jobAllocs.Info(allocStub.ID, nil) + require.Nil(err) + require.NotEmpty(alloc.Metrics.ScoreMetaData) + // Expect node affinity score to be 1.0 (dc1 nodes) if found + // TODO(preetha) make this more realistic + for _, sm := range alloc.Metrics.ScoreMetaData { + score, ok := sm.Scores["node-affinity"] + if ok { + require.Equal(1.0, score) + } + } + } + +} + +func (tc *BasicAffinityTest) TestMultipleAffinities(f *framework.F) { + nomadClient := tc.Nomad() + + // Parse job + job, err := jobspec.ParseFile("affinities/input/aff2.nomad") + require := require.New(f.T()) + require.Nil(err) + uuid := uuid.Generate() + jobId := helper.StringToPtr("aff" + uuid[0:8]) + job.ID = jobId + + tc.jobIds = append(tc.jobIds, *jobId) + + // Register job + jobs := nomadClient.Jobs() + resp, _, err := jobs.Register(job, nil) + require.Nil(err) + require.NotEmpty(resp.EvalID) + + g := NewGomegaWithT(f.T()) + + // Wrap in retry to wait until placement + g.Eventually(func() []*api.AllocationListStub { + // Look for allocations + allocs, _, _ := jobs.Allocations(*job.ID, false, nil) + return allocs + }, 5*time.Second, time.Second).ShouldNot(BeEmpty()) + + jobAllocs := nomadClient.Allocations() + + allocs, _, _ := jobs.Allocations(*job.ID, false, nil) + + // Verify affinity score metadata + for _, allocStub := range allocs { + alloc, _, err := jobAllocs.Info(allocStub.ID, nil) + require.Nil(err) + require.NotEmpty(alloc.Metrics.ScoreMetaData) + + node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil) + require.Nil(err) + + dcMatch := node.Datacenter == "dc1" + rackMatch := node.Meta != nil && node.Meta["rack"] == "r1" + + // Figure out expected node affinity score based on whether both affinities match or just one does + expectedNodeAffinityScore := 0.0 + if dcMatch && rackMatch { + expectedNodeAffinityScore = 1.0 + } else if dcMatch || rackMatch { + expectedNodeAffinityScore = 0.5 + } + + nodeScore := 0.0 + // Find the node's score for this alloc + for _, sm := range alloc.Metrics.ScoreMetaData { + score, ok := sm.Scores["node-affinity"] + if ok && sm.NodeID == alloc.NodeID { + nodeScore = score + } + } + require.Equal(nodeScore, expectedNodeAffinityScore) + } +} + +func (tc *BasicAffinityTest) TestAntiAffinities(f *framework.F) { + nomadClient := tc.Nomad() + + // Parse job + job, err := jobspec.ParseFile("affinities/input/aff3.nomad") + require := require.New(f.T()) + require.Nil(err) + uuid := uuid.Generate() + jobId := helper.StringToPtr("aff" + uuid[0:8]) + job.ID = jobId + + tc.jobIds = append(tc.jobIds, *jobId) + + // Register job + jobs := nomadClient.Jobs() + resp, _, err := jobs.Register(job, nil) + require.Nil(err) + require.NotEmpty(resp.EvalID) + + g := NewGomegaWithT(f.T()) + + // Wrap in retry to wait until placement + g.Eventually(func() []*api.AllocationListStub { + // Look for allocations + allocs, _, _ := jobs.Allocations(*job.ID, false, nil) + return allocs + }, 5*time.Second, time.Second).ShouldNot(BeEmpty()) + + jobAllocs := nomadClient.Allocations() + + allocs, _, _ := jobs.Allocations(*job.ID, false, nil) + + // Verify affinity score metadata + for _, allocStub := range allocs { + alloc, _, err := jobAllocs.Info(allocStub.ID, nil) + require.Nil(err) + require.NotEmpty(alloc.Metrics.ScoreMetaData) + + node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil) + require.Nil(err) + + dcMatch := node.Datacenter == "dc1" + rackMatch := node.Meta != nil && node.Meta["rack"] == "r1" + + // Figure out expected node affinity score based on whether both affinities match or just one does + expectedAntiAffinityScore := 0.0 + if dcMatch && rackMatch { + expectedAntiAffinityScore = -1.0 + } else if dcMatch || rackMatch { + expectedAntiAffinityScore = -0.5 + } + + nodeScore := 0.0 + + // Find the node's score for this alloc + for _, sm := range alloc.Metrics.ScoreMetaData { + score, ok := sm.Scores["node-affinity"] + if ok && sm.NodeID == alloc.NodeID { + nodeScore = score + } + } + require.Equal(nodeScore, expectedAntiAffinityScore) + } +} + +func (tc *BasicAffinityTest) AfterEach(f *framework.F) { + nomadClient := tc.Nomad() + jobs := nomadClient.Jobs() + // Stop all jobs in test + for _, id := range tc.jobIds { + jobs.Deregister(id, true, nil) + } + // Garbage collect + nomadClient.System().GarbageCollect() +} diff --git a/e2e/affinities/input/aff1.nomad b/e2e/affinities/input/aff1.nomad new file mode 100644 index 000000000..758f0dbb2 --- /dev/null +++ b/e2e/affinities/input/aff1.nomad @@ -0,0 +1,22 @@ +job "test1" { + datacenters = ["dc1", "dc2"] + type = "service" + + group "test1" { + count = 5 + affinity { + attribute ="${node.datacenter}" + operator = "=" + value = "dc1" + weight = 100 + } + task "test" { + driver = "raw_exec" + + config { + command = "bash" + args = ["-c", "sleep 15000"] + } + } + } +} \ No newline at end of file diff --git a/e2e/affinities/input/aff2.nomad b/e2e/affinities/input/aff2.nomad new file mode 100644 index 000000000..fbc04334a --- /dev/null +++ b/e2e/affinities/input/aff2.nomad @@ -0,0 +1,27 @@ +job "test1" { + datacenters = ["dc1", "dc2"] + type = "service" + affinity { + attribute ="${meta.rack}" + operator = "=" + value = "r1" + weight = 100 + } + group "test1" { + count = 4 + affinity { + attribute ="${node.datacenter}" + operator = "=" + value = "dc1" + weight = 100 + } + task "test" { + driver = "raw_exec" + + config { + command = "bash" + args = ["-c", "sleep 15000"] + } + } + } +} \ No newline at end of file diff --git a/e2e/affinities/input/aff3.nomad b/e2e/affinities/input/aff3.nomad new file mode 100644 index 000000000..6a005f871 --- /dev/null +++ b/e2e/affinities/input/aff3.nomad @@ -0,0 +1,27 @@ +job "test1" { + datacenters = ["dc1", "dc2"] + type = "service" + affinity { + attribute ="${meta.rack}" + operator = "=" + value = "r1" + weight = -50 + } + group "test1" { + count = 4 + affinity { + attribute ="${node.datacenter}" + operator = "=" + value = "dc1" + weight = -50 + } + task "test" { + driver = "raw_exec" + + config { + command = "bash" + args = ["-c", "sleep 15000"] + } + } + } +} \ No newline at end of file diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index c2a3896ac..5686cab01 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -3,6 +3,7 @@ package e2e import ( "testing" + _ "github.com/hashicorp/nomad/e2e/affinities" _ "github.com/hashicorp/nomad/e2e/example" )