2018-10-04 23:22:01 +00:00
|
|
|
package allocrunner
|
2018-06-22 00:35:07 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"time"
|
|
|
|
|
2018-08-23 19:03:17 +00:00
|
|
|
multierror "github.com/hashicorp/go-multierror"
|
2018-10-04 23:22:01 +00:00
|
|
|
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
|
2019-06-14 03:05:57 +00:00
|
|
|
clientconfig "github.com/hashicorp/nomad/client/config"
|
2020-02-11 16:39:16 +00:00
|
|
|
cstructs "github.com/hashicorp/nomad/client/structs"
|
2019-11-18 18:04:01 +00:00
|
|
|
"github.com/hashicorp/nomad/client/taskenv"
|
2018-08-30 21:33:50 +00:00
|
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
2018-06-22 00:35:07 +00:00
|
|
|
)
|
|
|
|
|
2020-02-11 16:39:16 +00:00
|
|
|
type hookResourceSetter interface {
|
|
|
|
GetAllocHookResources() *cstructs.AllocHookResources
|
|
|
|
SetAllocHookResources(*cstructs.AllocHookResources)
|
|
|
|
}
|
|
|
|
|
|
|
|
type allocHookResourceSetter struct {
|
|
|
|
ar *allocRunner
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *allocHookResourceSetter) GetAllocHookResources() *cstructs.AllocHookResources {
|
|
|
|
a.ar.hookStateMu.RLock()
|
|
|
|
defer a.ar.hookStateMu.RUnlock()
|
|
|
|
|
|
|
|
return a.ar.hookState
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *allocHookResourceSetter) SetAllocHookResources(res *cstructs.AllocHookResources) {
|
|
|
|
a.ar.hookStateMu.Lock()
|
|
|
|
defer a.ar.hookStateMu.Unlock()
|
|
|
|
|
|
|
|
a.ar.hookState = res
|
|
|
|
|
|
|
|
// Propagate to all of the TRs within the lock to ensure consistent state.
|
|
|
|
// TODO: Refactor so TR's pull state from AR?
|
|
|
|
for _, tr := range a.ar.tasks {
|
|
|
|
tr.SetAllocHookResources(res)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-11 00:07:59 +00:00
|
|
|
// allocHealthSetter is a shim to allow the alloc health watcher hook to set
|
|
|
|
// and clear the alloc health without full access to the alloc runner state
|
|
|
|
type allocHealthSetter struct {
|
|
|
|
ar *allocRunner
|
|
|
|
}
|
|
|
|
|
2018-12-07 01:24:43 +00:00
|
|
|
// HasHealth returns true if a deployment status is already set.
|
|
|
|
func (a *allocHealthSetter) HasHealth() bool {
|
|
|
|
a.ar.stateLock.Lock()
|
|
|
|
defer a.ar.stateLock.Unlock()
|
|
|
|
return a.ar.state.DeploymentStatus.HasHealth()
|
|
|
|
}
|
|
|
|
|
2018-09-11 00:07:59 +00:00
|
|
|
// ClearHealth allows the health watcher hook to clear the alloc's deployment
|
|
|
|
// health if the deployment id changes. It does not update the server as the
|
|
|
|
// status is only cleared when already receiving an update from the server.
|
|
|
|
//
|
|
|
|
// Only for use by health hook.
|
|
|
|
func (a *allocHealthSetter) ClearHealth() {
|
|
|
|
a.ar.stateLock.Lock()
|
|
|
|
a.ar.state.ClearDeploymentStatus()
|
2018-12-07 01:24:43 +00:00
|
|
|
a.ar.persistDeploymentStatus(nil)
|
2018-09-11 00:07:59 +00:00
|
|
|
a.ar.stateLock.Unlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetHealth allows the health watcher hook to set the alloc's
|
|
|
|
// deployment/migration health and emit task events.
|
|
|
|
//
|
|
|
|
// Only for use by health hook.
|
|
|
|
func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) {
|
|
|
|
// Updating alloc deployment state is tricky because it may be nil, but
|
|
|
|
// if it's not then we need to maintain the values of Canary and
|
|
|
|
// ModifyIndex as they're only mutated by the server.
|
|
|
|
a.ar.stateLock.Lock()
|
|
|
|
a.ar.state.SetDeploymentStatus(time.Now(), healthy)
|
2018-12-07 01:24:43 +00:00
|
|
|
a.ar.persistDeploymentStatus(a.ar.state.DeploymentStatus)
|
2019-05-21 13:55:05 +00:00
|
|
|
terminalDesiredState := a.ar.Alloc().ServerTerminalStatus()
|
2018-09-11 00:07:59 +00:00
|
|
|
a.ar.stateLock.Unlock()
|
|
|
|
|
|
|
|
// If deployment is unhealthy emit task events explaining why
|
2019-01-22 22:43:32 +00:00
|
|
|
if !healthy && isDeploy && !terminalDesiredState {
|
2018-09-11 00:07:59 +00:00
|
|
|
for task, event := range trackerTaskEvents {
|
|
|
|
if tr, ok := a.ar.tasks[task]; ok {
|
2018-09-11 03:24:53 +00:00
|
|
|
// Append but don't emit event since the server
|
|
|
|
// will be updated below
|
|
|
|
tr.AppendEvent(event)
|
2018-09-11 00:07:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Gather the state of the other tasks
|
|
|
|
states := make(map[string]*structs.TaskState, len(a.ar.tasks))
|
|
|
|
for name, tr := range a.ar.tasks {
|
|
|
|
states[name] = tr.TaskState()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Build the client allocation
|
|
|
|
calloc := a.ar.clientAlloc(states)
|
|
|
|
|
|
|
|
// Update the server
|
|
|
|
a.ar.stateUpdater.AllocStateUpdated(calloc)
|
|
|
|
|
|
|
|
// Broadcast client alloc to listeners
|
|
|
|
a.ar.allocBroadcaster.Send(calloc)
|
|
|
|
}
|
|
|
|
|
2020-04-20 13:28:19 +00:00
|
|
|
// initRunnerHooks initializes the runners hooks.
|
2019-06-14 03:05:57 +00:00
|
|
|
func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error {
|
2018-06-22 00:35:07 +00:00
|
|
|
hookLogger := ar.logger.Named("runner_hook")
|
|
|
|
|
2018-09-11 00:07:59 +00:00
|
|
|
// create health setting shim
|
|
|
|
hs := &allocHealthSetter{ar}
|
|
|
|
|
2019-05-21 04:11:48 +00:00
|
|
|
// create network isolation setting shim
|
|
|
|
ns := &allocNetworkIsolationSetter{ar: ar}
|
|
|
|
|
2020-02-11 16:39:16 +00:00
|
|
|
// create hook resource setting shim
|
|
|
|
hrs := &allocHookResourceSetter{ar: ar}
|
2020-02-17 13:18:57 +00:00
|
|
|
hrs.SetAllocHookResources(&cstructs.AllocHookResources{})
|
2020-02-11 16:39:16 +00:00
|
|
|
|
2019-05-21 04:11:48 +00:00
|
|
|
// build the network manager
|
|
|
|
nm, err := newNetworkManager(ar.Alloc(), ar.driverManager)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to configure network manager: %v", err)
|
|
|
|
}
|
|
|
|
|
2019-06-14 03:05:57 +00:00
|
|
|
// create network configurator
|
2019-09-04 20:33:25 +00:00
|
|
|
nc, err := newNetworkConfigurator(hookLogger, ar.Alloc(), config)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to initialize network configurator: %v", err)
|
|
|
|
}
|
2019-06-14 03:05:57 +00:00
|
|
|
|
2021-09-16 06:13:09 +00:00
|
|
|
// Create a new taskenv.Builder which is used and mutated by networkHook.
|
|
|
|
envBuilder := taskenv.NewBuilder(
|
|
|
|
config.Node, ar.Alloc(), nil, config.Region).SetAllocDir(ar.allocDir.AllocDir)
|
|
|
|
|
|
|
|
// Create a taskenv.TaskEnv which is used for read only purposes by the
|
|
|
|
// newNetworkHook.
|
|
|
|
builtTaskEnv := envBuilder.Build()
|
|
|
|
|
2018-06-22 00:35:07 +00:00
|
|
|
// Create the alloc directory hook. This is run first to ensure the
|
2018-10-05 02:36:40 +00:00
|
|
|
// directory path exists for other hooks.
|
2019-09-03 15:43:38 +00:00
|
|
|
alloc := ar.Alloc()
|
2019-05-21 04:11:48 +00:00
|
|
|
ar.runnerHooks = []interfaces.RunnerHook{
|
2018-08-31 00:13:00 +00:00
|
|
|
newAllocDirHook(hookLogger, ar.allocDir),
|
2021-04-08 05:04:47 +00:00
|
|
|
newCgroupHook(ar.Alloc(), ar.cpusetManager),
|
2018-12-06 11:15:59 +00:00
|
|
|
newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
|
|
|
|
newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
|
2022-06-07 14:18:19 +00:00
|
|
|
newAllocHealthWatcherHook(hookLogger, alloc, hs, ar.Listener(), ar.consulClient, ar.checkStore),
|
2021-09-16 06:13:09 +00:00
|
|
|
newNetworkHook(hookLogger, ns, alloc, nm, nc, ar, builtTaskEnv),
|
2019-11-18 18:04:01 +00:00
|
|
|
newGroupServiceHook(groupServiceHookConfig{
|
2022-06-07 14:18:19 +00:00
|
|
|
alloc: alloc,
|
2022-09-22 14:38:21 +00:00
|
|
|
providerNamespace: alloc.ServiceProviderNamespace(),
|
2022-06-07 14:18:19 +00:00
|
|
|
serviceRegWrapper: ar.serviceRegWrapper,
|
|
|
|
restarter: ar,
|
|
|
|
taskEnvBuilder: envBuilder,
|
|
|
|
networkStatus: ar,
|
|
|
|
logger: hookLogger,
|
|
|
|
shutdownDelayCtx: ar.shutdownDelayCtx,
|
2019-11-18 18:04:01 +00:00
|
|
|
}),
|
2022-11-21 15:19:09 +00:00
|
|
|
newConsulGRPCSocketHook(hookLogger, alloc, ar.allocDir, config.ConsulConfig, config.Node.Attributes),
|
2020-07-13 21:53:10 +00:00
|
|
|
newConsulHTTPSocketHook(hookLogger, alloc, ar.allocDir, config.ConsulConfig),
|
2022-01-07 20:23:47 +00:00
|
|
|
newCSIHook(alloc, hookLogger, ar.csiManager, ar.rpcClient, ar, hrs, ar.clientConfig.Node.SecretID),
|
2022-06-07 14:18:19 +00:00
|
|
|
newChecksHook(hookLogger, alloc, ar.checkStore, ar),
|
2019-05-21 04:11:48 +00:00
|
|
|
}
|
2019-05-08 14:30:50 +00:00
|
|
|
|
2019-05-14 01:26:39 +00:00
|
|
|
return nil
|
2018-06-22 00:35:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// prerun is used to run the runners prerun hooks.
|
|
|
|
func (ar *allocRunner) prerun() error {
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start := time.Now()
|
|
|
|
ar.logger.Trace("running pre-run hooks", "start", start)
|
|
|
|
defer func() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start))
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
pre, ok := hook.(interfaces.RunnerPrerunHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := pre.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
2018-08-23 19:03:17 +00:00
|
|
|
ar.logger.Trace("running pre-run hook", "name", name, "start", start)
|
2018-06-22 00:35:07 +00:00
|
|
|
}
|
|
|
|
|
2019-03-12 22:03:54 +00:00
|
|
|
if err := pre.Prerun(); err != nil {
|
2018-09-11 00:39:34 +00:00
|
|
|
return fmt.Errorf("pre-run hook %q failed: %v", name, err)
|
2018-06-22 00:35:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
2019-02-21 23:37:22 +00:00
|
|
|
ar.logger.Trace("finished pre-run hook", "name", name, "end", end, "duration", end.Sub(start))
|
2018-06-22 00:35:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-30 21:33:50 +00:00
|
|
|
// update runs the alloc runner update hooks. Update hooks are run
|
|
|
|
// asynchronously with all other alloc runner operations.
|
|
|
|
func (ar *allocRunner) update(update *structs.Allocation) error {
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start := time.Now()
|
|
|
|
ar.logger.Trace("running update hooks", "start", start)
|
|
|
|
defer func() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start))
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
req := &interfaces.RunnerUpdateRequest{
|
|
|
|
Alloc: update,
|
|
|
|
}
|
|
|
|
|
2018-09-11 00:39:34 +00:00
|
|
|
var merr multierror.Error
|
2018-08-30 21:33:50 +00:00
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
h, ok := hook.(interfaces.RunnerUpdateHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := h.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
2019-12-19 21:44:04 +00:00
|
|
|
ar.logger.Trace("running update hook", "name", name, "start", start)
|
2018-08-30 21:33:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := h.Update(req); err != nil {
|
2018-09-11 00:39:34 +00:00
|
|
|
merr.Errors = append(merr.Errors, fmt.Errorf("update hook %q failed: %v", name, err))
|
2018-08-30 21:33:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-11 00:39:34 +00:00
|
|
|
return merr.ErrorOrNil()
|
2018-08-30 21:33:50 +00:00
|
|
|
}
|
|
|
|
|
2018-06-22 00:35:07 +00:00
|
|
|
// postrun is used to run the runners postrun hooks.
|
|
|
|
func (ar *allocRunner) postrun() error {
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start := time.Now()
|
|
|
|
ar.logger.Trace("running post-run hooks", "start", start)
|
|
|
|
defer func() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start))
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
post, ok := hook.(interfaces.RunnerPostrunHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := post.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
|
|
|
ar.logger.Trace("running post-run hook", "name", name, "start", start)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := post.Postrun(); err != nil {
|
|
|
|
return fmt.Errorf("hook %q failed: %v", name, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-23 19:03:17 +00:00
|
|
|
// destroy is used to run the runners destroy hooks. All hooks are run and
|
|
|
|
// errors are returned as a multierror.
|
|
|
|
func (ar *allocRunner) destroy() error {
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start := time.Now()
|
|
|
|
ar.logger.Trace("running destroy hooks", "start", start)
|
|
|
|
defer func() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start))
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
var merr multierror.Error
|
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
h, ok := hook.(interfaces.RunnerDestroyHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := h.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
|
|
|
ar.logger.Trace("running destroy hook", "name", name, "start", start)
|
|
|
|
}
|
2018-06-22 00:35:07 +00:00
|
|
|
|
2018-08-23 19:03:17 +00:00
|
|
|
if err := h.Destroy(); err != nil {
|
|
|
|
merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err))
|
|
|
|
}
|
2018-06-22 00:35:07 +00:00
|
|
|
|
2018-08-23 19:03:17 +00:00
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start))
|
|
|
|
}
|
|
|
|
}
|
2018-06-22 00:35:07 +00:00
|
|
|
|
2019-01-18 23:09:38 +00:00
|
|
|
return merr.ErrorOrNil()
|
2018-08-23 19:03:17 +00:00
|
|
|
}
|
2018-11-14 18:29:07 +00:00
|
|
|
|
2019-11-18 16:16:25 +00:00
|
|
|
func (ar *allocRunner) preKillHooks() {
|
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
pre, ok := hook.(interfaces.RunnerPreKillHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := pre.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
|
|
|
ar.logger.Trace("running alloc pre shutdown hook", "name", name, "start", start)
|
|
|
|
}
|
|
|
|
|
|
|
|
pre.PreKill()
|
|
|
|
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished alloc pre shutdown hook", "name", name, "end", end, "duration", end.Sub(start))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-14 18:29:07 +00:00
|
|
|
// shutdownHooks calls graceful shutdown hooks for when the agent is exiting.
|
|
|
|
func (ar *allocRunner) shutdownHooks() {
|
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
sh, ok := hook.(interfaces.ShutdownHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := sh.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
|
|
|
ar.logger.Trace("running shutdown hook", "name", name, "start", start)
|
|
|
|
}
|
|
|
|
|
|
|
|
sh.Shutdown()
|
|
|
|
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished shutdown hooks", "name", name, "end", end, "duration", end.Sub(start))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-01-21 16:36:00 +00:00
|
|
|
|
|
|
|
func (ar *allocRunner) taskRestartHooks() {
|
|
|
|
for _, hook := range ar.runnerHooks {
|
|
|
|
re, ok := hook.(interfaces.RunnerTaskRestartHook)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
name := re.Name()
|
|
|
|
var start time.Time
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
start = time.Now()
|
|
|
|
ar.logger.Trace("running alloc task restart hook",
|
|
|
|
"name", name, "start", start)
|
|
|
|
}
|
|
|
|
|
|
|
|
re.PreTaskRestart()
|
|
|
|
|
|
|
|
if ar.logger.IsTrace() {
|
|
|
|
end := time.Now()
|
|
|
|
ar.logger.Trace("finished alloc task restart hook",
|
|
|
|
"name", name, "end", end, "duration", end.Sub(start))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|