388 lines
17 KiB
Go
388 lines
17 KiB
Go
//go:build !ent
|
|
// +build !ent
|
|
|
|
// Nomad OSS ignores Consul Namespace configuration in jobs, these e2e tests
|
|
// verify everything still works and is registered into the "default" namespace,
|
|
// since e2e always uses Consul Enterprise. With Consul OSS, there are no namespaces.
|
|
// and these tests will not work.
|
|
|
|
package consul
|
|
|
|
import (
|
|
"sort"
|
|
|
|
capi "github.com/hashicorp/consul/api"
|
|
"github.com/hashicorp/nomad/e2e/e2eutil"
|
|
"github.com/hashicorp/nomad/e2e/framework"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulRegisterGroupServices(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-group-services"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, "")
|
|
require.Len(f.T(), allocations, 3)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify our services were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1)
|
|
|
|
// Verify our services are all healthy
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing")
|
|
|
|
// Verify our services were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Verify that services were de-registered in Consul
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "c1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "c2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z2")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulRegisterTaskServices(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-task-services"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, "")
|
|
require.Len(f.T(), allocations, 3)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify our services were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1)
|
|
|
|
// Verify our services are all healthy
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing")
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing")
|
|
|
|
// Verify our services were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "z1", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "cherry", "z2", 0)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Verify that services were de-registered from Consul
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "c1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z1")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "z2")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulTemplateKV(f *framework.F) {
|
|
t := f.T()
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-template-kv"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs to complete
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, "")
|
|
require.Len(t, allocations, 2)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
// Sort allocs by name
|
|
sort.Sort(e2eutil.AllocsByName(allocations))
|
|
|
|
// Check template read from default namespace even if namespace set
|
|
textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "value: ns_default", textB)
|
|
|
|
// Check template read from default namespace if no namespace set
|
|
textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "value: ns_default", textZ)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(t, nomadClient, jobID)
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulConnectSidecars(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-connect-sidecars"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, "")
|
|
require.Len(f.T(), allocations, 4)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify services with cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-sidecar-proxy", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-sidecar-proxy", 1)
|
|
|
|
// Verify services without cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z-sidecar-proxy", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z-sidecar-proxy", 1)
|
|
|
|
// Verify our services were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-api-sidecar-proxy", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashb0ard-sidecar-proxy", 0)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Verify that services were de-registered from Consul
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-sidecar-proxy")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-sidecar-proxy")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z-sidecar-proxy")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z")
|
|
e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z-sidecar-proxy")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulConnectIngressGateway(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-connect-ingress"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, "")
|
|
require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify services with cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api", 1)
|
|
|
|
// Verify services without cns set were registered into "default"
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api-z", 1)
|
|
|
|
// Verify services with cns set were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "my-ingress-service", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "uuid-api", 0)
|
|
|
|
// Read the config entry of gateway with cns set, checking it exists in "default' namespace
|
|
ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service")
|
|
require.Equal(f.T(), namespace, ce.GetNamespace())
|
|
|
|
// Read the config entry of gateway without cns set, checking it exists in "default' namespace
|
|
ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z")
|
|
require.Equal(f.T(), namespace, ceZ.GetNamespace())
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Remove the config entries
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service")
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulConnectTerminatingGateway(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-connect-terminating"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, "")
|
|
require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
// Verify services with cns set were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1)
|
|
|
|
// Verify services without cns set were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1)
|
|
e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1)
|
|
|
|
// Verify services with cns set were NOT registered into specified consul namespaces
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "api-gateway", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0)
|
|
e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0)
|
|
|
|
// Read the config entry of gateway with cns set, checking it exists in "default' namespace
|
|
ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway")
|
|
require.Equal(f.T(), namespace, ce.GetNamespace())
|
|
|
|
// Read the config entry of gateway without cns set, checking it exists in "default' namespace
|
|
ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z")
|
|
require.Equal(f.T(), namespace, ceZ.GetNamespace())
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
|
|
// Remove the config entries
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway")
|
|
e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z")
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksTask(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-script-checks-task"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, "")
|
|
require.Len(f.T(), allocations, 2)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
sort.Sort(e2eutil.AllocsByName(allocations))
|
|
allocsWithSetNamespace := allocations[0:1]
|
|
allocsWithNoNamespace := allocations[1:2]
|
|
|
|
// Verify checks were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, err := exec(nomadClient, allocsWithSetNamespace,
|
|
[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"})
|
|
r.NoError(err)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing)
|
|
|
|
// Verify checks were registered into "default" Consul namespace when no
|
|
// namespace was specified.
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
|
|
[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"})
|
|
r.NoError(errZ)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
}
|
|
|
|
func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksGroup(f *framework.F) {
|
|
nomadClient := tc.Nomad()
|
|
jobID := "cns-script-checks-group"
|
|
tc.jobIDs = append(tc.jobIDs, jobID)
|
|
|
|
// Run job and wait for allocs
|
|
allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, "")
|
|
require.Len(f.T(), allocations, 2)
|
|
allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
|
|
e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
|
|
|
|
r := f.Assertions
|
|
c := tc.Consul()
|
|
namespace := consulNamespace
|
|
|
|
sort.Sort(e2eutil.AllocsByName(allocations))
|
|
allocsWithSetNamespace := allocations[0:1]
|
|
allocsWithNoNamespace := allocations[1:2]
|
|
|
|
// Verify checks were registered into "default" Consul namespace
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, err := exec(nomadClient, allocsWithSetNamespace,
|
|
[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"})
|
|
r.NoError(err)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing)
|
|
|
|
// Verify checks were registered into "default" Consul namespace when no
|
|
// namespace was specified.
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical)
|
|
|
|
// Check in warning state becomes healthy after check passes for the service
|
|
// with specified Consul namespace
|
|
//
|
|
// (ensures UpdateTTL is respecting namespace)
|
|
_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
|
|
[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"})
|
|
r.NoError(errZ)
|
|
e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing)
|
|
|
|
// Stop the job
|
|
e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
|
|
}
|