From 29591a7c2eabf42c50a23e8d0315ce2db76ede41 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Fri, 16 Nov 2018 14:17:23 -0500 Subject: [PATCH] task_runner: emit event on task exit with exit result details --- client/allocrunner/taskrunner/task_runner.go | 18 ++++++++++++++++++ nomad/structs/structs.go | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 7a61fdd9b..48a63d3ee 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -369,6 +369,9 @@ MAIN: select { case result = <-resultCh: // WaitCh returned a result + if result != nil { + tr.handleTaskExitResult(result) + } case <-tr.ctx.Done(): // TaskRunner was told to exit immediately return @@ -410,6 +413,7 @@ MAIN: event := structs.NewTaskEvent(structs.TaskTerminated). SetExitCode(result.ExitCode). SetSignal(result.Signal). + SetOOMKilled(result.OOMKilled). SetExitMessage(result.Err) tr.UpdateState(structs.TaskStateDead, event) } @@ -422,6 +426,20 @@ MAIN: tr.logger.Debug("task run loop exiting") } +func (tr *TaskRunner) handleTaskExitResult(result *drivers.ExitResult) { + event := structs.NewTaskEvent(structs.TaskTerminated). + SetExitCode(result.ExitCode). + SetSignal(result.Signal). + SetOOMKilled(result.OOMKilled). + SetExitMessage(result.Err) + + tr.EmitEvent(event) + + if !tr.clientConfig.DisableTaggedMetrics { + metrics.IncrCounterWithLabels([]string{"client", "allocs", "oom_killed"}, 1, tr.baseLabels) + } +} + // handleUpdates runs update hooks when triggerUpdateCh is ticked and exits // when Run has returned. Should only be run in a goroutine from Run. func (tr *TaskRunner) handleUpdates() { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ff2db8dee..751e6b36b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -6196,6 +6196,11 @@ func (e *TaskEvent) SetDriverMessage(m string) *TaskEvent { return e } +func (e *TaskEvent) SetOOMKilled(oom bool) *TaskEvent { + e.Details["oom_killed"] = strconv.FormatBool(oom) + return e +} + // TaskArtifact is an artifact to download before running the task. type TaskArtifact struct { // GetterSource is the source to download an artifact using go-getter