7f1191111d
This PR adds a set of tests to the Consul test suite for testing Nomad OSS's behavior around setting Consul Namespace on groups, which is to ignore the setting (as Consul Namespaces are currently an Enterprise feature). Tests are generally a reduced facsimile of existing tests, modified to check behavior of when group.consul.namespace is set and not set. Verification is oriented around what happens in Consul; the in-depth functional correctness of these features is left to the original tests. Nomad ENT will get its own version of these tests in `namespaces_ent.go`.
387 lines
17 KiB
Go
387 lines
17 KiB
Go
// +build !ent
|
|
|
|
// Nomad OSS ignores Consul Namespace configuration in jobs, these e2e tests
|
|
// verify everything still works and is registered into the "default" namespace,
|
|
// since e2e always uses Consul Enterprise. With Consul OSS, there are no namespaces.
|
|
// and these tests will not work.
|
|
|
|
package consul
|
|
|
|
import (
|
|
"sort"
|
|
|
|
capi "github.com/hashicorp/consul/api"
|
|
"github.com/hashicorp/nomad/e2e/e2eutil"
|
|
"github.com/hashicorp/nomad/e2e/framework"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulRegisterGroupServices(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-group-services"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, "")
|
|
require.Len(f.T(), allocations, 3)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify our services were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1)
|
|
|
|
// Verify our services are all healthy
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing")
|
|
|
|
// Verify our services were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Verify that services were de-registered in Consul
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "c1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "c2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z2")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulRegisterTaskServices(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-task-services"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, "")
|
|
require.Len(f.T(), allocations, 3)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify our services were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1)
|
|
|
|
// Verify our services are all healthy
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing")
|
|
|
|
// Verify our services were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "z1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "z2", 0)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Verify that services were de-registered from Consul
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "c1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z2")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulTemplateKV(f *framework.F) {
|
|
t := f.T()
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-template-kv"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs to complete
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, "")
|
|
require.Len(t, allocations, 2)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
// Sort allocs by name
|
|
sort.Sort(e2eutil.AllocsByName(allocations))
|
|
|
|
// Check template read from default namespace even if namespace set
|
|
textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "value: ns_default", textB)
|
|
|
|
// Check template read from default namespace if no namespace set
|
|
textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "value: ns_default", textZ)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(t, nomadClient, jobID)
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulConnectSidecars(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-connect-sidecars"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, "")
|
|
require.Len(f.T(), allocations, 4)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify services with cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-sidecar-proxy", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-sidecar-proxy", 1)
|
|
|
|
// Verify services without cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z-sidecar-proxy", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z-sidecar-proxy", 1)
|
|
|
|
// Verify our services were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-api-sidecar-proxy", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashb0ard-sidecar-proxy", 0)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Verify that services were de-registered from Consul
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-sidecar-proxy")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-sidecar-proxy")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z-sidecar-proxy")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z-sidecar-proxy")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulConnectIngressGateway(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-connect-ingress"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, "")
|
|
require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify services with cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api", 1)
|
|
|
|
// Verify services without cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api-z", 1)
|
|
|
|
// Verify services with cns set were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "my-ingress-service", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "uuid-api", 0)
|
|
|
|
// Read the config entry of gateway with cns set, checking it exists in "default' namespace
|
|
ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service")
|
|
require.Equal(f.T(), namespace, ce.GetNamespace())
|
|
|
|
// Read the config entry of gateway without cns set, checking it exists in "default' namespace
|
|
ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z")
|
|
require.Equal(f.T(), namespace, ceZ.GetNamespace())
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Remove the config entries
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service")
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulConnectTerminatingGateway(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-connect-terminating"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, "")
|
|
require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify services with cns set were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1)
|
|
|
|
// Verify services without cns set were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1)
|
|
|
|
// Verify services with cns set were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "api-gateway", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0)
|
|
|
|
// Read the config entry of gateway with cns set, checking it exists in "default' namespace
|
|
ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway")
|
|
require.Equal(f.T(), namespace, ce.GetNamespace())
|
|
|
|
// Read the config entry of gateway without cns set, checking it exists in "default' namespace
|
|
ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z")
|
|
require.Equal(f.T(), namespace, ceZ.GetNamespace())
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Remove the config entries
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway")
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksTask(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-script-checks-task"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, "")
|
|
require.Len(f.T(), allocations, 2)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
sort.Sort(e2eutil.AllocsByName(allocations))
|
|
allocsWithSetNamespace := allocations[0:1]
|
|
allocsWithNoNamespace := allocations[1:2]
|
|
|
|
// Verify checks were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, err := exec(nomadClient, allocsWithSetNamespace,
|
|
[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"})
|
|
r.NoError(err)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing)
|
|
|
|
// Verify checks were registered into "default" Consul namespace when no
|
|
// namespace was specified.
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
|
|
[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"})
|
|
r.NoError(errZ)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksGroup(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-script-checks-group"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, "")
|
|
require.Len(f.T(), allocations, 2)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
sort.Sort(e2eutil.AllocsByName(allocations))
|
|
allocsWithSetNamespace := allocations[0:1]
|
|
allocsWithNoNamespace := allocations[1:2]
|
|
|
|
// Verify checks were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, err := exec(nomadClient, allocsWithSetNamespace,
|
|
[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"})
|
|
r.NoError(err)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing)
|
|
|
|
// Verify checks were registered into "default" Consul namespace when no
|
|
// namespace was specified.
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
|
|
[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"})
|
|
r.NoError(errZ)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
}
|