c8260c3940
There are bits of logic in callers of RemoveWorkload on group/task cleanup hooks which call RemoveWorkload with the "Canary" version of the workload, in case the alloc is marked as a Canary. This logic triggers an extra sync with Consul, and also doesn't do the intended behavior - for which no special casing is necessary anyway. When the workload is marked for removal, all associated services and checks will be removed regardless of the Canary status, because the service and check IDs do not incorporate the canary-ness in the first place. The only place where canary-ness matters is when updating a workload, where we need to compute the hash of the services and checks to determine whether they have been modified, the Canary flag of which is a part of that. Fixes #10842
305 lines
9.3 KiB
Go
305 lines
9.3 KiB
Go
package allocrunner
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"testing"
|
|
"time"
|
|
|
|
consulapi "github.com/hashicorp/consul/api"
|
|
ctestutil "github.com/hashicorp/consul/sdk/testutil"
|
|
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
|
|
"github.com/hashicorp/nomad/client/consul"
|
|
"github.com/hashicorp/nomad/client/taskenv"
|
|
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
|
|
"github.com/hashicorp/nomad/helper"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
var _ interfaces.RunnerPrerunHook = (*groupServiceHook)(nil)
|
|
var _ interfaces.RunnerUpdateHook = (*groupServiceHook)(nil)
|
|
var _ interfaces.RunnerPostrunHook = (*groupServiceHook)(nil)
|
|
var _ interfaces.RunnerPreKillHook = (*groupServiceHook)(nil)
|
|
var _ interfaces.RunnerTaskRestartHook = (*groupServiceHook)(nil)
|
|
|
|
// TestGroupServiceHook_NoGroupServices asserts calling group service hooks
|
|
// without group services does not error.
|
|
func TestGroupServiceHook_NoGroupServices(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.Job.TaskGroups[0].Services = []*structs.Service{{
|
|
Name: "foo",
|
|
PortLabel: "9999",
|
|
}}
|
|
logger := testlog.HCLogger(t)
|
|
consulClient := consul.NewMockConsulServiceClient(t, logger)
|
|
|
|
h := newGroupServiceHook(groupServiceHookConfig{
|
|
alloc: alloc,
|
|
consul: consulClient,
|
|
restarter: agentconsul.NoopRestarter(),
|
|
taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region),
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, h.Prerun())
|
|
|
|
req := &interfaces.RunnerUpdateRequest{Alloc: alloc}
|
|
require.NoError(t, h.Update(req))
|
|
|
|
require.NoError(t, h.Postrun())
|
|
|
|
require.NoError(t, h.PreTaskRestart())
|
|
|
|
ops := consulClient.GetOps()
|
|
require.Len(t, ops, 5)
|
|
require.Equal(t, "add", ops[0].Op) // Prerun
|
|
require.Equal(t, "update", ops[1].Op) // Update
|
|
require.Equal(t, "remove", ops[2].Op) // Postrun
|
|
require.Equal(t, "remove", ops[3].Op) // Restart -> preKill
|
|
require.Equal(t, "add", ops[4].Op) // Restart -> preRun
|
|
}
|
|
|
|
// TestGroupServiceHook_ShutdownDelayUpdate asserts calling group service hooks
|
|
// update updates the hooks delay value.
|
|
func TestGroupServiceHook_ShutdownDelayUpdate(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.Job.TaskGroups[0].ShutdownDelay = helper.TimeToPtr(10 * time.Second)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
consulClient := consul.NewMockConsulServiceClient(t, logger)
|
|
|
|
h := newGroupServiceHook(groupServiceHookConfig{
|
|
alloc: alloc,
|
|
consul: consulClient,
|
|
restarter: agentconsul.NoopRestarter(),
|
|
taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region),
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, h.Prerun())
|
|
|
|
// Incease shutdown Delay
|
|
alloc.Job.TaskGroups[0].ShutdownDelay = helper.TimeToPtr(15 * time.Second)
|
|
req := &interfaces.RunnerUpdateRequest{Alloc: alloc}
|
|
require.NoError(t, h.Update(req))
|
|
|
|
// Assert that update updated the delay value
|
|
require.Equal(t, h.delay, 15*time.Second)
|
|
|
|
// Remove shutdown delay
|
|
alloc.Job.TaskGroups[0].ShutdownDelay = nil
|
|
req = &interfaces.RunnerUpdateRequest{Alloc: alloc}
|
|
require.NoError(t, h.Update(req))
|
|
|
|
// Assert that update updated the delay value
|
|
require.Equal(t, h.delay, 0*time.Second)
|
|
}
|
|
|
|
// TestGroupServiceHook_GroupServices asserts group service hooks with group
|
|
// services does not error.
|
|
func TestGroupServiceHook_GroupServices(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
alloc := mock.ConnectAlloc()
|
|
logger := testlog.HCLogger(t)
|
|
consulClient := consul.NewMockConsulServiceClient(t, logger)
|
|
|
|
h := newGroupServiceHook(groupServiceHookConfig{
|
|
alloc: alloc,
|
|
consul: consulClient,
|
|
restarter: agentconsul.NoopRestarter(),
|
|
taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region),
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, h.Prerun())
|
|
|
|
req := &interfaces.RunnerUpdateRequest{Alloc: alloc}
|
|
require.NoError(t, h.Update(req))
|
|
|
|
require.NoError(t, h.Postrun())
|
|
|
|
require.NoError(t, h.PreTaskRestart())
|
|
|
|
ops := consulClient.GetOps()
|
|
require.Len(t, ops, 5)
|
|
require.Equal(t, "add", ops[0].Op) // Prerun
|
|
require.Equal(t, "update", ops[1].Op) // Update
|
|
require.Equal(t, "remove", ops[2].Op) // Postrun
|
|
require.Equal(t, "remove", ops[3].Op) // Restart -> preKill
|
|
require.Equal(t, "add", ops[4].Op) // Restart -> preRun
|
|
}
|
|
|
|
// TestGroupServiceHook_Error asserts group service hooks with group
|
|
// services but no group network is handled gracefully.
|
|
func TestGroupServiceHook_NoNetwork(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.Job.TaskGroups[0].Networks = []*structs.NetworkResource{}
|
|
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "testconnect",
|
|
PortLabel: "9999",
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
logger := testlog.HCLogger(t)
|
|
|
|
consulClient := consul.NewMockConsulServiceClient(t, logger)
|
|
|
|
h := newGroupServiceHook(groupServiceHookConfig{
|
|
alloc: alloc,
|
|
consul: consulClient,
|
|
restarter: agentconsul.NoopRestarter(),
|
|
taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region),
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, h.Prerun())
|
|
|
|
req := &interfaces.RunnerUpdateRequest{Alloc: alloc}
|
|
require.NoError(t, h.Update(req))
|
|
|
|
require.NoError(t, h.Postrun())
|
|
|
|
require.NoError(t, h.PreTaskRestart())
|
|
|
|
ops := consulClient.GetOps()
|
|
require.Len(t, ops, 5)
|
|
require.Equal(t, "add", ops[0].Op) // Prerun
|
|
require.Equal(t, "update", ops[1].Op) // Update
|
|
require.Equal(t, "remove", ops[2].Op) // Postrun
|
|
require.Equal(t, "remove", ops[3].Op) // Restart -> preKill
|
|
require.Equal(t, "add", ops[4].Op) // Restart -> preRun
|
|
}
|
|
|
|
func TestGroupServiceHook_getWorkloadServices(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
alloc := mock.Alloc()
|
|
alloc.Job.TaskGroups[0].Networks = []*structs.NetworkResource{}
|
|
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "testconnect",
|
|
PortLabel: "9999",
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{},
|
|
},
|
|
},
|
|
}
|
|
logger := testlog.HCLogger(t)
|
|
|
|
consulClient := consul.NewMockConsulServiceClient(t, logger)
|
|
|
|
h := newGroupServiceHook(groupServiceHookConfig{
|
|
alloc: alloc,
|
|
consul: consulClient,
|
|
restarter: agentconsul.NoopRestarter(),
|
|
taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region),
|
|
logger: logger,
|
|
})
|
|
|
|
services := h.getWorkloadServices()
|
|
require.Len(t, services.Services, 1)
|
|
}
|
|
|
|
// TestGroupServiceHook_Update08Alloc asserts that adding group services to a previously
|
|
// 0.8 alloc works.
|
|
//
|
|
// COMPAT(0.11) Only valid for upgrades from 0.8.
|
|
func TestGroupServiceHook_Update08Alloc(t *testing.T) {
|
|
// Create an embedded Consul server
|
|
testconsul, err := ctestutil.NewTestServerConfigT(t, func(c *ctestutil.TestServerConfig) {
|
|
// If -v wasn't specified squelch consul logging
|
|
if !testing.Verbose() {
|
|
c.Stdout = ioutil.Discard
|
|
c.Stderr = ioutil.Discard
|
|
}
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("error starting test consul server: %v", err)
|
|
}
|
|
defer testconsul.Stop()
|
|
|
|
consulConfig := consulapi.DefaultConfig()
|
|
consulConfig.Address = testconsul.HTTPAddr
|
|
consulClient, err := consulapi.NewClient(consulConfig)
|
|
require.NoError(t, err)
|
|
namespacesClient := agentconsul.NewNamespacesClient(consulClient.Namespaces(), consulClient.Agent())
|
|
|
|
serviceClient := agentconsul.NewServiceClient(consulClient.Agent(), namespacesClient, testlog.HCLogger(t), true)
|
|
|
|
// Lower periodicInterval to ensure periodic syncing doesn't improperly
|
|
// remove Connect services.
|
|
//const interval = 50 * time.Millisecond
|
|
//serviceClient.periodicInterval = interval
|
|
|
|
// Disable deregistration probation to test syncing
|
|
//serviceClient.deregisterProbationExpiry = time.Time{}
|
|
|
|
go serviceClient.Run()
|
|
defer serviceClient.Shutdown()
|
|
|
|
// Create new 0.10-style alloc
|
|
alloc := mock.Alloc()
|
|
alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
|
|
{
|
|
Mode: "bridge",
|
|
IP: "10.0.0.1",
|
|
DynamicPorts: []structs.Port{
|
|
{
|
|
Label: "connect-proxy-testconnect",
|
|
Value: 9999,
|
|
To: 9998,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
|
|
tg.Services = []*structs.Service{
|
|
{
|
|
Name: "testconnect",
|
|
PortLabel: "9999",
|
|
Connect: &structs.ConsulConnect{
|
|
SidecarService: &structs.ConsulSidecarService{
|
|
Proxy: &structs.ConsulProxy{
|
|
LocalServicePort: 9000,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Create old 0.8-style alloc from new alloc
|
|
oldAlloc := alloc.Copy()
|
|
oldAlloc.AllocatedResources = nil
|
|
oldAlloc.Job.LookupTaskGroup(alloc.TaskGroup).Services = nil
|
|
|
|
// Create the group service hook
|
|
h := newGroupServiceHook(groupServiceHookConfig{
|
|
alloc: oldAlloc,
|
|
consul: serviceClient,
|
|
restarter: agentconsul.NoopRestarter(),
|
|
taskEnvBuilder: taskenv.NewBuilder(mock.Node(), oldAlloc, nil, oldAlloc.Job.Region),
|
|
logger: testlog.HCLogger(t),
|
|
})
|
|
|
|
require.NoError(t, h.Prerun())
|
|
require.NoError(t, h.Update(&interfaces.RunnerUpdateRequest{Alloc: alloc}))
|
|
|
|
// Assert the group and sidecar services are registered
|
|
require.Eventually(t, func() bool {
|
|
services, err := consulClient.Agent().Services()
|
|
require.NoError(t, err)
|
|
return len(services) == 2
|
|
}, 3*time.Second, 100*time.Millisecond)
|
|
|
|
}
|