package consul import ( "fmt" "os" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/require" ) const ( consulJobBasic = "consul/input/consul_example.nomad" consulJobCanaryTags = "consul/input/canary_tags.nomad" ) type ConsulE2ETest struct { framework.TC jobIds []string } func init() { framework.AddSuites(&framework.TestSuite{ Component: "Consul", CanRunLocal: true, Consul: true, Cases: []framework.TestCase{ new(ConsulE2ETest), new(ScriptChecksE2ETest), }, }) } func (tc *ConsulE2ETest) BeforeAll(f *framework.F) { e2eutil.WaitForLeader(f.T(), tc.Nomad()) e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) } func (tc *ConsulE2ETest) AfterEach(f *framework.F) { if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { return } for _, id := range tc.jobIds { tc.Nomad().Jobs().Deregister(id, true, nil) } tc.jobIds = []string{} tc.Nomad().System().GarbageCollect() } // TestConsulRegistration asserts that a job registers services with tags in Consul. func (tc *ConsulE2ETest) TestConsulRegistration(f *framework.F) { t := f.T() nomadClient := tc.Nomad() catalog := tc.Consul().Catalog() jobId := "consul" + uuid.Generate()[0:8] tc.jobIds = append(tc.jobIds, jobId) allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, consulJobBasic, jobId, "") require.Equal(t, 3, len(allocs)) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) expectedTags := []string{ "cache", "global", } // Assert services get registered testutil.WaitForResult(func() (bool, error) { services, _, err := catalog.Service("consul-example", "", nil) if err != nil { return false, fmt.Errorf("error contacting Consul: %v", err) } if expected := 3; len(services) != expected { return false, fmt.Errorf("expected %d services but found %d", expected, len(services)) } for _, s := range services { // If we've made it this far the tags should *always* match require.True(t, helper.CompareSliceSetString(expectedTags, s.ServiceTags)) } return true, nil }, func(err error) { t.Fatalf("error waiting for services to be registered: %v", err) }) // Stop the job e2eutil.WaitForJobStopped(t, nomadClient, jobId) // Verify that services were deregistered in Consul testutil.WaitForResult(func() (bool, error) { s, _, err := catalog.Service("consul-example", "", nil) if err != nil { return false, err } return len(s) == 0, fmt.Errorf("expected 0 services but found: %v", s) }, func(err error) { t.Fatalf("error waiting for services to be deregistered: %v", err) }) } // TestCanaryInplaceUpgrades verifies setting and unsetting canary tags func (tc *ConsulE2ETest) TestCanaryInplaceUpgrades(f *framework.F) { t := f.T() nomadClient := tc.Nomad() consulClient := tc.Consul() jobId := "consul" + uuid.Generate()[0:8] tc.jobIds = append(tc.jobIds, jobId) allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, consulJobCanaryTags, jobId, "") require.Equal(t, 2, len(allocs)) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) e2eutil.WaitForAllocsRunning(t, nomadClient, allocIDs) // Start a deployment job, _, err := nomadClient.Jobs().Info(jobId, nil) require.NoError(t, err) job.Meta = map[string]string{"version": "2"} resp, _, err := nomadClient.Jobs().Register(job, nil) require.NoError(t, err) require.NotEmpty(t, resp.EvalID) // Eventually have a canary var activeDeploy *api.Deployment testutil.WaitForResult(func() (bool, error) { deploys, _, err := nomadClient.Jobs().Deployments(jobId, false, nil) if err != nil { return false, err } if expected := 2; len(deploys) != expected { return false, fmt.Errorf("expected 2 deploys but found %v", deploys) } for _, d := range deploys { if d.Status == structs.DeploymentStatusRunning { activeDeploy = d break } } if activeDeploy == nil { return false, fmt.Errorf("no running deployments: %v", deploys) } if expected := 1; len(activeDeploy.TaskGroups["consul_canary_test"].PlacedCanaries) != expected { return false, fmt.Errorf("expected %d placed canaries but found %#v", expected, activeDeploy.TaskGroups["consul_canary_test"]) } return true, nil }, func(err error) { t.Fatalf("error while waiting for deploys: %v", err) }) allocID := activeDeploy.TaskGroups["consul_canary_test"].PlacedCanaries[0] testutil.WaitForResult(func() (bool, error) { alloc, _, err := nomadClient.Allocations().Info(allocID, nil) if err != nil { return false, err } if alloc.DeploymentStatus == nil { return false, fmt.Errorf("canary alloc %s has no deployment status", allocID) } if alloc.DeploymentStatus.Healthy == nil { return false, fmt.Errorf("canary alloc %s has no deployment health: %#v", allocID, alloc.DeploymentStatus) } return *alloc.DeploymentStatus.Healthy, fmt.Errorf("expected healthy canary but found: %#v", alloc.DeploymentStatus) }, func(err error) { t.Fatalf("error waiting for canary to be healthy: %v", err) }) // Check Consul for canary tags testutil.WaitForResult(func() (bool, error) { consulServices, _, err := consulClient.Catalog().Service("canarytest", "", nil) if err != nil { return false, err } for _, s := range consulServices { if helper.CompareSliceSetString([]string{"canary", "foo"}, s.ServiceTags) { return true, nil } } return false, fmt.Errorf(`could not find service tags {"canary", "foo"}: %#v`, consulServices) }, func(err error) { t.Fatalf("error waiting for canary tags: %v", err) }) // Promote canary { resp, _, err := nomadClient.Deployments().PromoteAll(activeDeploy.ID, nil) require.NoError(t, err) require.NotEmpty(t, resp.EvalID) } // Eventually canary is promoted testutil.WaitForResult(func() (bool, error) { alloc, _, err := nomadClient.Allocations().Info(allocID, nil) if err != nil { return false, err } return !alloc.DeploymentStatus.Canary, fmt.Errorf("still a canary") }, func(err error) { t.Fatalf("error waiting for canary to be promoted: %v", err) }) // Verify that no instances have canary tags expected := []string{"foo", "bar"} testutil.WaitForResult(func() (bool, error) { consulServices, _, err := consulClient.Catalog().Service("canarytest", "", nil) if err != nil { return false, err } for _, s := range consulServices { if !helper.CompareSliceSetString(expected, s.ServiceTags) { return false, fmt.Errorf("expected %#v Consul tags but found %#v", expected, s.ServiceTags) } } return true, nil }, func(err error) { t.Fatalf("error waiting for non-canary tags: %v", err) }) }