//go:build !ent // +build !ent // Nomad OSS ignores Consul Namespace configuration in jobs, these e2e tests // verify everything still works and is registered into the "default" namespace, // since e2e always uses Consul Enterprise. With Consul OSS, there are no namespaces. // and these tests will not work. package consul import ( "sort" capi "github.com/hashicorp/consul/api" "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" "github.com/stretchr/testify/require" ) func (tc *ConsulNamespacesE2ETest) TestConsulRegisterGroupServices(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-group-services" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, "") require.Len(f.T(), allocations, 3) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace // Verify our services were registered into "default" e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1) // Verify our services are all healthy e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing") // Verify our services were NOT registered into specified consul namespaces e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0) e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0) e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0) e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) // Verify that services were de-registered in Consul e2eutil.RequireConsulDeregistered(r, c, namespace, "b1") e2eutil.RequireConsulDeregistered(r, c, namespace, "b2") e2eutil.RequireConsulDeregistered(r, c, namespace, "c1") e2eutil.RequireConsulDeregistered(r, c, namespace, "c2") e2eutil.RequireConsulDeregistered(r, c, namespace, "z1") e2eutil.RequireConsulDeregistered(r, c, namespace, "z2") } func (tc *ConsulNamespacesE2ETest) TestConsulRegisterTaskServices(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-task-services" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, "") require.Len(f.T(), allocations, 3) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace // Verify our services were registered into "default" e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1) // Verify our services are all healthy e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing") e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing") // Verify our services were NOT registered into specified consul namespaces e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0) e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0) e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0) e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0) e2eutil.RequireConsulRegistered(r, c, "cherry", "z1", 0) e2eutil.RequireConsulRegistered(r, c, "cherry", "z2", 0) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) // Verify that services were de-registered from Consul e2eutil.RequireConsulDeregistered(r, c, namespace, "b1") e2eutil.RequireConsulDeregistered(r, c, namespace, "b2") e2eutil.RequireConsulDeregistered(r, c, namespace, "c1") e2eutil.RequireConsulDeregistered(r, c, namespace, "b2") e2eutil.RequireConsulDeregistered(r, c, namespace, "z1") e2eutil.RequireConsulDeregistered(r, c, namespace, "z2") } func (tc *ConsulNamespacesE2ETest) TestConsulTemplateKV(f *framework.F) { t := f.T() nomadClient := tc.Nomad() jobID := "cns-template-kv" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs to complete allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, "") require.Len(t, allocations, 2) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs) // Sort allocs by name sort.Sort(e2eutil.AllocsByName(allocations)) // Check template read from default namespace even if namespace set textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut) require.NoError(t, err) require.Equal(t, "value: ns_default", textB) // Check template read from default namespace if no namespace set textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut) require.NoError(t, err) require.Equal(t, "value: ns_default", textZ) // Stop the job e2eutil.WaitForJobStopped(t, nomadClient, jobID) } func (tc *ConsulNamespacesE2ETest) TestConsulConnectSidecars(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-connect-sidecars" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, "") require.Len(f.T(), allocations, 4) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace // Verify services with cns set were registered into "default" e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-sidecar-proxy", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-sidecar-proxy", 1) // Verify services without cns set were registered into "default" e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z-sidecar-proxy", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z-sidecar-proxy", 1) // Verify our services were NOT registered into specified consul namespaces e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0) e2eutil.RequireConsulRegistered(r, c, "apple", "count-api-sidecar-proxy", 0) e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0) e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashb0ard-sidecar-proxy", 0) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) // Verify that services were de-registered from Consul e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-sidecar-proxy") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-sidecar-proxy") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z-sidecar-proxy") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z") e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z-sidecar-proxy") } func (tc *ConsulNamespacesE2ETest) TestConsulConnectIngressGateway(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-connect-ingress" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, "") require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace // Verify services with cns set were registered into "default" e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api", 1) // Verify services without cns set were registered into "default" e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service-z", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api-z", 1) // Verify services with cns set were NOT registered into specified consul namespaces e2eutil.RequireConsulRegistered(r, c, "apple", "my-ingress-service", 0) e2eutil.RequireConsulRegistered(r, c, "apple", "uuid-api", 0) // Read the config entry of gateway with cns set, checking it exists in "default' namespace ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service") require.Equal(f.T(), namespace, ce.GetNamespace()) // Read the config entry of gateway without cns set, checking it exists in "default' namespace ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z") require.Equal(f.T(), namespace, ceZ.GetNamespace()) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) // Remove the config entries e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service") e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z") } func (tc *ConsulNamespacesE2ETest) TestConsulConnectTerminatingGateway(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-connect-terminating" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, "") require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace // Verify services with cns set were registered into "default" Consul namespace e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1) // Verify services without cns set were registered into "default" Consul namespace e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway-z", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1) e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1) // Verify services with cns set were NOT registered into specified consul namespaces e2eutil.RequireConsulRegistered(r, c, "apple", "api-gateway", 0) e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0) e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0) // Read the config entry of gateway with cns set, checking it exists in "default' namespace ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway") require.Equal(f.T(), namespace, ce.GetNamespace()) // Read the config entry of gateway without cns set, checking it exists in "default' namespace ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z") require.Equal(f.T(), namespace, ceZ.GetNamespace()) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) // Remove the config entries e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway") e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z") } func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksTask(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-script-checks-task" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, "") require.Len(f.T(), allocations, 2) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace sort.Sort(e2eutil.AllocsByName(allocations)) allocsWithSetNamespace := allocations[0:1] allocsWithNoNamespace := allocations[1:2] // Verify checks were registered into "default" Consul namespace e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing) e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning) e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical) // Check in warning state becomes healthy after check passes for the service // with specified Consul namespace // // (ensures UpdateTTL is respecting namespace) _, _, err := exec(nomadClient, allocsWithSetNamespace, []string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"}) r.NoError(err) e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing) // Verify checks were registered into "default" Consul namespace when no // namespace was specified. e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing) e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning) e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical) // Check in warning state becomes healthy after check passes for the service // with specified Consul namespace // // (ensures UpdateTTL is respecting namespace) _, _, errZ := exec(nomadClient, allocsWithNoNamespace, []string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"}) r.NoError(errZ) e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) } func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksGroup(f *framework.F) { nomadClient := tc.Nomad() jobID := "cns-script-checks-group" tc.jobIDs = append(tc.jobIDs, jobID) // Run job and wait for allocs allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, "") require.Len(f.T(), allocations, 2) allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) r := f.Assertions c := tc.Consul() namespace := consulNamespace sort.Sort(e2eutil.AllocsByName(allocations)) allocsWithSetNamespace := allocations[0:1] allocsWithNoNamespace := allocations[1:2] // Verify checks were registered into "default" Consul namespace e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing) e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning) e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical) // Check in warning state becomes healthy after check passes for the service // with specified Consul namespace // // (ensures UpdateTTL is respecting namespace) _, _, err := exec(nomadClient, allocsWithSetNamespace, []string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"}) r.NoError(err) e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing) // Verify checks were registered into "default" Consul namespace when no // namespace was specified. e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing) e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning) e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical) // Check in warning state becomes healthy after check passes for the service // with specified Consul namespace // // (ensures UpdateTTL is respecting namespace) _, _, errZ := exec(nomadClient, allocsWithNoNamespace, []string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"}) r.NoError(errZ) e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing) // Stop the job e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) }