2018-07-12 23:15:33 +00:00
|
|
|
package taskrunner
|
|
|
|
|
2018-07-16 21:37:27 +00:00
|
|
|
import (
|
|
|
|
"context"
|
2018-07-12 23:15:33 +00:00
|
|
|
|
2018-07-16 21:37:27 +00:00
|
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
2018-10-04 19:08:20 +00:00
|
|
|
"github.com/hashicorp/nomad/plugins/drivers"
|
2018-07-16 21:37:27 +00:00
|
|
|
)
|
|
|
|
|
2018-09-20 22:44:27 +00:00
|
|
|
// Restart a task. Returns immediately if no task is running. Blocks until
|
|
|
|
// existing task exits or passed-in context is canceled.
|
2018-07-16 21:37:27 +00:00
|
|
|
func (tr *TaskRunner) Restart(ctx context.Context, event *structs.TaskEvent, failure bool) error {
|
|
|
|
// Grab the handle
|
2018-10-04 19:08:20 +00:00
|
|
|
handle := tr.getDriverHandle()
|
2018-11-05 20:20:45 +00:00
|
|
|
|
2018-07-16 21:37:27 +00:00
|
|
|
// Check it is running
|
|
|
|
if handle == nil {
|
|
|
|
return ErrTaskNotRunning
|
|
|
|
}
|
|
|
|
|
2018-09-11 03:24:53 +00:00
|
|
|
// Emit the event since it may take a long time to kill
|
|
|
|
tr.EmitEvent(event)
|
2018-07-16 21:37:27 +00:00
|
|
|
|
2018-11-05 20:20:45 +00:00
|
|
|
// Run the hooks prior to restarting the task
|
|
|
|
tr.killing()
|
|
|
|
|
2018-07-16 21:37:27 +00:00
|
|
|
// Tell the restart tracker that a restart triggered the exit
|
|
|
|
tr.restartTracker.SetRestartTriggered(failure)
|
|
|
|
|
|
|
|
// Kill the task using an exponential backoff in-case of failures.
|
2018-11-05 20:20:45 +00:00
|
|
|
if err := tr.killTask(handle); err != nil {
|
2018-07-16 21:37:27 +00:00
|
|
|
// We couldn't successfully destroy the resource created.
|
|
|
|
tr.logger.Error("failed to kill task. Resources may have been leaked", "error", err)
|
|
|
|
}
|
|
|
|
|
2018-09-20 22:44:27 +00:00
|
|
|
// Drain the wait channel or wait for the request context to be canceled
|
2018-10-04 19:08:20 +00:00
|
|
|
waitCh, err := handle.WaitCh(ctx)
|
|
|
|
if err != nil {
|
2018-10-11 02:11:45 +00:00
|
|
|
return err
|
2018-10-04 19:08:20 +00:00
|
|
|
}
|
2018-10-11 02:11:45 +00:00
|
|
|
|
2018-11-05 20:22:45 +00:00
|
|
|
select {
|
|
|
|
case <-waitCh:
|
|
|
|
case <-ctx.Done():
|
|
|
|
}
|
2018-07-12 23:15:33 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-10-04 19:08:20 +00:00
|
|
|
func (tr *TaskRunner) Signal(event *structs.TaskEvent, s string) error {
|
2018-07-16 21:37:27 +00:00
|
|
|
// Grab the handle
|
2018-10-04 19:08:20 +00:00
|
|
|
handle := tr.getDriverHandle()
|
2018-07-16 21:37:27 +00:00
|
|
|
|
|
|
|
// Check it is running
|
|
|
|
if handle == nil {
|
|
|
|
return ErrTaskNotRunning
|
|
|
|
}
|
|
|
|
|
|
|
|
// Emit the event
|
|
|
|
tr.EmitEvent(event)
|
|
|
|
|
|
|
|
// Send the signal
|
|
|
|
return handle.Signal(s)
|
|
|
|
}
|
|
|
|
|
2018-08-23 19:03:17 +00:00
|
|
|
// Kill a task. Blocks until task exits or context is canceled. State is set to
|
|
|
|
// dead.
|
2018-07-16 21:37:27 +00:00
|
|
|
func (tr *TaskRunner) Kill(ctx context.Context, event *structs.TaskEvent) error {
|
2018-09-20 22:44:27 +00:00
|
|
|
// Cancel the task runner to break out of restart delay or the main run
|
|
|
|
// loop.
|
2018-10-18 20:39:02 +00:00
|
|
|
tr.killCtxCancel()
|
2018-09-20 22:44:27 +00:00
|
|
|
|
2018-07-16 21:37:27 +00:00
|
|
|
// Grab the handle
|
2018-10-04 19:08:20 +00:00
|
|
|
handle := tr.getDriverHandle()
|
2018-07-16 21:37:27 +00:00
|
|
|
|
2018-10-04 19:08:20 +00:00
|
|
|
// Check it is running
|
2018-07-16 21:37:27 +00:00
|
|
|
if handle == nil {
|
|
|
|
return ErrTaskNotRunning
|
|
|
|
}
|
|
|
|
|
2018-09-11 03:24:53 +00:00
|
|
|
// Emit the event since it may take a long time to kill
|
|
|
|
tr.EmitEvent(event)
|
2018-07-16 21:37:27 +00:00
|
|
|
|
|
|
|
// Run the hooks prior to killing the task
|
2018-11-05 20:20:45 +00:00
|
|
|
tr.killing()
|
2018-07-16 21:37:27 +00:00
|
|
|
|
2018-11-05 20:20:45 +00:00
|
|
|
// Tell the restart tracker that the task has been killed so it doesn't
|
|
|
|
// attempt to restart it.
|
2018-07-16 21:37:27 +00:00
|
|
|
tr.restartTracker.SetKilled()
|
|
|
|
|
|
|
|
// Kill the task using an exponential backoff in-case of failures.
|
2018-11-05 20:20:45 +00:00
|
|
|
killErr := tr.killTask(handle)
|
|
|
|
if killErr != nil {
|
2018-07-16 21:37:27 +00:00
|
|
|
// We couldn't successfully destroy the resource created.
|
2018-11-05 20:20:45 +00:00
|
|
|
tr.logger.Error("failed to kill task. Resources may have been leaked", "error", killErr)
|
2018-07-16 21:37:27 +00:00
|
|
|
}
|
|
|
|
|
2018-09-20 22:44:27 +00:00
|
|
|
// Block until task has exited.
|
2018-10-04 19:08:20 +00:00
|
|
|
waitCh, err := handle.WaitCh(ctx)
|
|
|
|
|
2018-10-10 03:01:20 +00:00
|
|
|
// The error should be nil or TaskNotFound, if it's something else then a
|
2018-10-16 23:42:19 +00:00
|
|
|
// failure in the driver or transport layer occurred
|
2018-10-11 00:08:57 +00:00
|
|
|
if err != nil {
|
|
|
|
if err == drivers.ErrTaskNotFound {
|
|
|
|
return nil
|
|
|
|
}
|
2018-10-04 19:08:20 +00:00
|
|
|
tr.logger.Error("failed to wait on task. Resources may have been leaked", "error", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-11-05 20:22:45 +00:00
|
|
|
select {
|
|
|
|
case <-waitCh:
|
|
|
|
case <-ctx.Done():
|
|
|
|
}
|
2018-07-16 21:37:27 +00:00
|
|
|
|
2018-11-05 20:20:45 +00:00
|
|
|
if killErr != nil {
|
|
|
|
return killErr
|
2018-10-11 00:08:57 +00:00
|
|
|
} else if err := ctx.Err(); err != nil {
|
|
|
|
return err
|
2018-07-16 21:37:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2018-07-12 23:15:33 +00:00
|
|
|
}
|