open-nomad/api/tasks.go

989 lines
26 KiB
Go
Raw Normal View History

2015-09-09 20:02:39 +00:00
package api
import (
"fmt"
"path"
"path/filepath"
2017-02-13 23:18:17 +00:00
"strings"
"time"
)
const (
// RestartPolicyModeDelay causes an artificial delay till the next interval is
// reached when the specified attempts have been reached in the interval.
RestartPolicyModeDelay = "delay"
2017-02-06 19:48:28 +00:00
// RestartPolicyModeFail causes a job to fail if the specified number of
// attempts are reached within an interval.
RestartPolicyModeFail = "fail"
)
2016-05-27 22:24:22 +00:00
// MemoryStats holds memory usage related stats
2016-04-29 20:03:02 +00:00
type MemoryStats struct {
RSS uint64
Cache uint64
Swap uint64
Usage uint64
2016-04-29 20:03:02 +00:00
MaxUsage uint64
KernelUsage uint64
KernelMaxUsage uint64
2016-06-10 02:45:41 +00:00
Measured []string
2016-04-29 20:03:02 +00:00
}
2016-05-27 22:24:22 +00:00
// CpuStats holds cpu usage related stats
2016-05-21 07:49:17 +00:00
type CpuStats struct {
2016-05-19 21:06:19 +00:00
SystemMode float64
UserMode float64
TotalTicks float64
2016-04-29 20:03:02 +00:00
ThrottledPeriods uint64
ThrottledTime uint64
2016-05-19 21:06:19 +00:00
Percent float64
2016-06-10 02:45:41 +00:00
Measured []string
2016-04-29 20:03:02 +00:00
}
2016-05-27 22:24:22 +00:00
// ResourceUsage holds information related to cpu and memory stats
type ResourceUsage struct {
2016-04-29 20:03:02 +00:00
MemoryStats *MemoryStats
2016-05-21 07:49:17 +00:00
CpuStats *CpuStats
DeviceStats []*DeviceGroupStats
}
2016-05-27 22:24:22 +00:00
// TaskResourceUsage holds aggregated resource usage of all processes in a Task
// and the resource usage of the individual pids
type TaskResourceUsage struct {
ResourceUsage *ResourceUsage
2016-05-27 22:24:22 +00:00
Timestamp int64
Pids map[string]*ResourceUsage
2016-04-29 20:03:02 +00:00
}
// AllocResourceUsage holds the aggregated task resource usage of the
// allocation.
type AllocResourceUsage struct {
ResourceUsage *ResourceUsage
Tasks map[string]*TaskResourceUsage
Timestamp int64
}
// RestartPolicy defines how the Nomad client restarts
// tasks in a taskgroup when they fail
type RestartPolicy struct {
2017-02-13 23:18:17 +00:00
Interval *time.Duration
Attempts *int
Delay *time.Duration
Mode *string
}
func (r *RestartPolicy) Merge(rp *RestartPolicy) {
if rp.Interval != nil {
r.Interval = rp.Interval
}
if rp.Attempts != nil {
r.Attempts = rp.Attempts
}
if rp.Delay != nil {
r.Delay = rp.Delay
}
if rp.Mode != nil {
r.Mode = rp.Mode
}
2015-10-31 04:28:56 +00:00
}
// Reschedule configures how Tasks are rescheduled when they crash or fail.
type ReschedulePolicy struct {
// Attempts limits the number of rescheduling attempts that can occur in an interval.
Attempts *int `mapstructure:"attempts"`
// Interval is a duration in which we can limit the number of reschedule attempts.
Interval *time.Duration `mapstructure:"interval"`
// Delay is a minimum duration to wait between reschedule attempts.
// The delay function determines how much subsequent reschedule attempts are delayed by.
Delay *time.Duration `mapstructure:"delay"`
// DelayFunction determines how the delay progressively changes on subsequent reschedule
2018-03-26 19:45:09 +00:00
// attempts. Valid values are "exponential", "constant", and "fibonacci".
DelayFunction *string `mapstructure:"delay_function"`
2018-03-13 15:06:26 +00:00
// MaxDelay is an upper bound on the delay.
MaxDelay *time.Duration `mapstructure:"max_delay"`
// Unlimited allows rescheduling attempts until they succeed
Unlimited *bool `mapstructure:"unlimited"`
}
func (r *ReschedulePolicy) Merge(rp *ReschedulePolicy) {
if rp == nil {
return
}
if rp.Interval != nil {
r.Interval = rp.Interval
}
if rp.Attempts != nil {
r.Attempts = rp.Attempts
}
if rp.Delay != nil {
r.Delay = rp.Delay
}
if rp.DelayFunction != nil {
r.DelayFunction = rp.DelayFunction
}
2018-03-13 15:06:26 +00:00
if rp.MaxDelay != nil {
r.MaxDelay = rp.MaxDelay
}
if rp.Unlimited != nil {
r.Unlimited = rp.Unlimited
}
}
func (r *ReschedulePolicy) Canonicalize(jobType string) {
dp := NewDefaultReschedulePolicy(jobType)
if r.Interval == nil {
r.Interval = dp.Interval
}
if r.Attempts == nil {
r.Attempts = dp.Attempts
}
if r.Delay == nil {
r.Delay = dp.Delay
}
if r.DelayFunction == nil {
r.DelayFunction = dp.DelayFunction
}
if r.MaxDelay == nil {
r.MaxDelay = dp.MaxDelay
}
if r.Unlimited == nil {
r.Unlimited = dp.Unlimited
}
}
2018-07-16 13:30:58 +00:00
// Affinity is used to serialize task group affinities
type Affinity struct {
LTarget string // Left-hand target
RTarget string // Right-hand target
Operand string // Constraint operand (<=, <, =, !=, >, >=), set_contains_all, set_contains_any
Weight *int8 // Weight applied to nodes that match the affinity. Can be negative
2018-07-16 13:30:58 +00:00
}
func NewAffinity(LTarget string, Operand string, RTarget string, Weight int8) *Affinity {
2018-07-16 13:30:58 +00:00
return &Affinity{
LTarget: LTarget,
RTarget: RTarget,
Operand: Operand,
Weight: int8ToPtr(Weight),
}
}
func (a *Affinity) Canonicalize() {
if a.Weight == nil {
a.Weight = int8ToPtr(50)
2018-07-16 13:30:58 +00:00
}
}
func NewDefaultReschedulePolicy(jobType string) *ReschedulePolicy {
var dp *ReschedulePolicy
switch jobType {
case "service":
// This needs to be in sync with DefaultServiceJobReschedulePolicy
// in nomad/structs/structs.go
dp = &ReschedulePolicy{
Delay: timeToPtr(30 * time.Second),
DelayFunction: stringToPtr("exponential"),
MaxDelay: timeToPtr(1 * time.Hour),
Unlimited: boolToPtr(true),
Attempts: intToPtr(0),
Interval: timeToPtr(0),
}
case "batch":
// This needs to be in sync with DefaultBatchJobReschedulePolicy
// in nomad/structs/structs.go
dp = &ReschedulePolicy{
Attempts: intToPtr(1),
Interval: timeToPtr(24 * time.Hour),
Delay: timeToPtr(5 * time.Second),
DelayFunction: stringToPtr("constant"),
MaxDelay: timeToPtr(0),
Unlimited: boolToPtr(false),
}
case "system":
dp = &ReschedulePolicy{
Attempts: intToPtr(0),
Interval: timeToPtr(0),
Delay: timeToPtr(0),
DelayFunction: stringToPtr(""),
MaxDelay: timeToPtr(0),
Unlimited: boolToPtr(false),
}
default:
// GH-7203: it is possible an unknown job type is passed to this
// function and we need to ensure a non-nil object is returned so that
// the canonicalization runs without panicking.
dp = &ReschedulePolicy{
Attempts: intToPtr(0),
Interval: timeToPtr(0),
Delay: timeToPtr(0),
DelayFunction: stringToPtr(""),
MaxDelay: timeToPtr(0),
Unlimited: boolToPtr(false),
}
}
return dp
}
func (r *ReschedulePolicy) Copy() *ReschedulePolicy {
if r == nil {
return nil
}
nrp := new(ReschedulePolicy)
*nrp = *r
return nrp
}
func (p *ReschedulePolicy) String() string {
if p == nil {
return ""
}
if *p.Unlimited {
return fmt.Sprintf("unlimited with %v delay, max_delay = %v", *p.DelayFunction, *p.MaxDelay)
}
2018-03-21 14:15:29 +00:00
return fmt.Sprintf("%v in %v with %v delay, max_delay = %v", *p.Attempts, *p.Interval, *p.DelayFunction, *p.MaxDelay)
}
// Spread is used to serialize task group allocation spread preferences
type Spread struct {
Attribute string
Weight *int8
SpreadTarget []*SpreadTarget
}
// SpreadTarget is used to serialize target allocation spread percentages
type SpreadTarget struct {
Value string
Percent uint8
}
func NewSpreadTarget(value string, percent uint8) *SpreadTarget {
return &SpreadTarget{
Value: value,
Percent: percent,
}
}
func NewSpread(attribute string, weight int8, spreadTargets []*SpreadTarget) *Spread {
return &Spread{
Attribute: attribute,
Weight: int8ToPtr(weight),
SpreadTarget: spreadTargets,
}
}
func (s *Spread) Canonicalize() {
if s.Weight == nil {
s.Weight = int8ToPtr(50)
}
}
// EphemeralDisk is an ephemeral disk object
type EphemeralDisk struct {
2017-02-06 19:48:28 +00:00
Sticky *bool
Migrate *bool
SizeMB *int `mapstructure:"size"`
}
func DefaultEphemeralDisk() *EphemeralDisk {
return &EphemeralDisk{
Sticky: boolToPtr(false),
Migrate: boolToPtr(false),
SizeMB: intToPtr(300),
2017-02-06 19:48:28 +00:00
}
}
func (e *EphemeralDisk) Canonicalize() {
if e.Sticky == nil {
e.Sticky = boolToPtr(false)
2017-02-06 19:48:28 +00:00
}
if e.Migrate == nil {
e.Migrate = boolToPtr(false)
2017-02-06 19:48:28 +00:00
}
if e.SizeMB == nil {
e.SizeMB = intToPtr(300)
2017-02-06 19:48:28 +00:00
}
}
// MigrateStrategy describes how allocations for a task group should be
// migrated between nodes (eg when draining).
type MigrateStrategy struct {
MaxParallel *int `mapstructure:"max_parallel"`
HealthCheck *string `mapstructure:"health_check"`
MinHealthyTime *time.Duration `mapstructure:"min_healthy_time"`
HealthyDeadline *time.Duration `mapstructure:"healthy_deadline"`
}
func DefaultMigrateStrategy() *MigrateStrategy {
return &MigrateStrategy{
MaxParallel: intToPtr(1),
HealthCheck: stringToPtr("checks"),
MinHealthyTime: timeToPtr(10 * time.Second),
HealthyDeadline: timeToPtr(5 * time.Minute),
}
}
func (m *MigrateStrategy) Canonicalize() {
if m == nil {
return
}
defaults := DefaultMigrateStrategy()
if m.MaxParallel == nil {
m.MaxParallel = defaults.MaxParallel
}
if m.HealthCheck == nil {
m.HealthCheck = defaults.HealthCheck
}
if m.MinHealthyTime == nil {
m.MinHealthyTime = defaults.MinHealthyTime
}
if m.HealthyDeadline == nil {
m.HealthyDeadline = defaults.HealthyDeadline
}
}
2018-03-01 19:21:32 +00:00
func (m *MigrateStrategy) Merge(o *MigrateStrategy) {
if o.MaxParallel != nil {
m.MaxParallel = o.MaxParallel
}
if o.HealthCheck != nil {
m.HealthCheck = o.HealthCheck
}
if o.MinHealthyTime != nil {
m.MinHealthyTime = o.MinHealthyTime
}
if o.HealthyDeadline != nil {
m.HealthyDeadline = o.HealthyDeadline
}
}
func (m *MigrateStrategy) Copy() *MigrateStrategy {
if m == nil {
return nil
}
nm := new(MigrateStrategy)
*nm = *m
return nm
}
2019-08-12 14:22:27 +00:00
// VolumeRequest is a representation of a storage volume that a TaskGroup wishes to use.
type VolumeRequest struct {
csi: add mount_options to volumes and volume requests (#7398) Add mount_options to both the volume definition on registration and to the volume block in the group where the volume is requested. If both are specified, the options provided in the request replace the options defined in the volume. They get passed to the NodePublishVolume, which causes the node plugin to actually mount the volume on the host. Individual tasks just mount bind into the host mounted volume (unchanged behavior). An operator can mount the same volume with different options by specifying it twice in the group context. closes #7007 * nomad/structs/volumes: add MountOptions to volume request * jobspec/test-fixtures/basic.hcl: add mount_options to volume block * jobspec/parse_test: add expected MountOptions * api/tasks: add mount_options * jobspec/parse_group: use hcl decode not mapstructure, mount_options * client/allocrunner/csi_hook: pass MountOptions through client/allocrunner/csi_hook: add a VolumeMountOptions client/allocrunner/csi_hook: drop Options client/allocrunner/csi_hook: use the structs options * client/pluginmanager/csimanager/interface: UsageOptions.MountOptions * client/pluginmanager/csimanager/volume: pass MountOptions in capabilities * plugins/csi/plugin: remove todo 7007 comment * nomad/structs/csi: MountOptions * api/csi: add options to the api for parsing, match structs * plugins/csi/plugin: move VolumeMountOptions to structs * api/csi: use specific type for mount_options * client/allocrunner/csi_hook: merge MountOptions here * rename CSIOptions to CSIMountOptions * client/allocrunner/csi_hook * client/pluginmanager/csimanager/volume * nomad/structs/csi * plugins/csi/fake/client: add PrevVolumeCapability * plugins/csi/plugin * client/pluginmanager/csimanager/volume_test: remove debugging * client/pluginmanager/csimanager/volume: fix odd merging logic * api: rename CSIOptions -> CSIMountOptions * nomad/csi_endpoint: remove a 7007 comment * command/alloc_status: show mount options in the volume list * nomad/structs/csi: include MountOptions in the volume stub * api/csi: add MountOptions to stub * command/volume_status_csi: clean up csiVolMountOption, add it * command/alloc_status: csiVolMountOption lives in volume_csi_status * command/node_status: display mount flags * nomad/structs/volumes: npe * plugins/csi/plugin: npe in ToCSIRepresentation * jobspec/parse_test: expand volume parse test cases * command/agent/job_endpoint: ApiTgToStructsTG needs MountOptions * command/volume_status_csi: copy paste error * jobspec/test-fixtures/basic: hclfmt * command/volume_status_csi: clean up csiVolMountOption
2020-03-23 17:55:26 +00:00
Name string
Type string
Source string
ReadOnly bool `hcl:"read_only"`
MountOptions *CSIMountOptions `hcl:"mount_options"`
ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
}
const (
VolumeMountPropagationPrivate = "private"
VolumeMountPropagationHostToTask = "host-to-task"
VolumeMountPropagationBidirectional = "bidirectional"
)
2019-08-12 14:22:27 +00:00
// VolumeMount represents the relationship between a destination path in a task
// and the task group volume that should be mounted there.
type VolumeMount struct {
Volume *string
Destination *string
ReadOnly *bool `mapstructure:"read_only"`
PropagationMode *string `mapstructure:"propagation_mode"`
}
func (vm *VolumeMount) Canonicalize() {
if vm.PropagationMode == nil {
vm.PropagationMode = stringToPtr(VolumeMountPropagationPrivate)
}
if vm.ReadOnly == nil {
vm.ReadOnly = boolToPtr(false)
}
}
2015-09-09 20:02:39 +00:00
// TaskGroup is the unit of scheduling.
type TaskGroup struct {
Name *string
Count *int
Constraints []*Constraint
2018-07-16 13:30:58 +00:00
Affinities []*Affinity
Tasks []*Task
Spreads []*Spread
2019-08-12 14:22:27 +00:00
Volumes map[string]*VolumeRequest
RestartPolicy *RestartPolicy
ReschedulePolicy *ReschedulePolicy
EphemeralDisk *EphemeralDisk
Update *UpdateStrategy
Migrate *MigrateStrategy
Networks []*NetworkResource
Meta map[string]string
Services []*Service
ShutdownDelay *time.Duration `mapstructure:"shutdown_delay"`
Scaling *ScalingPolicy
2015-09-09 20:02:39 +00:00
}
// NewTaskGroup creates a new TaskGroup.
2015-09-10 00:59:18 +00:00
func NewTaskGroup(name string, count int) *TaskGroup {
2015-09-09 20:02:39 +00:00
return &TaskGroup{
Name: stringToPtr(name),
Count: intToPtr(count),
2017-02-06 19:48:28 +00:00
}
}
// Canonicalize sets defaults and merges settings that should be inherited from the job
func (g *TaskGroup) Canonicalize(job *Job) {
2017-02-06 19:48:28 +00:00
if g.Name == nil {
g.Name = stringToPtr("")
2017-02-06 19:48:28 +00:00
}
if g.Count == nil {
if g.Scaling != nil && g.Scaling.Min != nil {
g.Count = intToPtr(int(*g.Scaling.Min))
} else {
g.Count = intToPtr(1)
}
2017-02-06 19:48:28 +00:00
}
if g.Scaling != nil {
g.Scaling.Canonicalize(*g.Count)
}
2017-02-06 19:48:28 +00:00
if g.EphemeralDisk == nil {
g.EphemeralDisk = DefaultEphemeralDisk()
} else {
g.EphemeralDisk.Canonicalize()
}
2017-02-13 23:18:17 +00:00
// Merge the update policy from the job
if ju, tu := job.Update != nil, g.Update != nil; ju && tu {
// Merge the jobs and task groups definition of the update strategy
jc := job.Update.Copy()
jc.Merge(g.Update)
g.Update = jc
} else if ju && !job.Update.Empty() {
// Inherit the jobs as long as it is non-empty.
jc := job.Update.Copy()
g.Update = jc
}
if g.Update != nil {
g.Update.Canonicalize()
}
// Merge the reschedule policy from the job
if jr, tr := job.Reschedule != nil, g.ReschedulePolicy != nil; jr && tr {
jobReschedule := job.Reschedule.Copy()
jobReschedule.Merge(g.ReschedulePolicy)
g.ReschedulePolicy = jobReschedule
2018-01-23 01:58:23 +00:00
} else if jr {
jobReschedule := job.Reschedule.Copy()
g.ReschedulePolicy = jobReschedule
}
// Only use default reschedule policy for non system jobs
if g.ReschedulePolicy == nil && *job.Type != "system" {
g.ReschedulePolicy = NewDefaultReschedulePolicy(*job.Type)
}
if g.ReschedulePolicy != nil {
g.ReschedulePolicy.Canonicalize(*job.Type)
}
2018-03-01 19:21:32 +00:00
// Merge the migrate strategy from the job
if jm, tm := job.Migrate != nil, g.Migrate != nil; jm && tm {
jobMigrate := job.Migrate.Copy()
jobMigrate.Merge(g.Migrate)
g.Migrate = jobMigrate
} else if jm {
jobMigrate := job.Migrate.Copy()
g.Migrate = jobMigrate
}
// Merge with default reschedule policy
if g.Migrate == nil && *job.Type == "service" {
g.Migrate = &MigrateStrategy{}
}
if g.Migrate != nil {
g.Migrate.Canonicalize()
2018-03-01 19:21:32 +00:00
}
2017-02-13 23:18:17 +00:00
var defaultRestartPolicy *RestartPolicy
switch *job.Type {
2017-02-13 23:18:17 +00:00
case "service", "system":
2020-03-07 02:52:58 +00:00
defaultRestartPolicy = defaultServiceJobRestartPolicy()
2017-02-13 23:18:17 +00:00
default:
2020-03-07 02:52:58 +00:00
defaultRestartPolicy = defaultBatchJobRestartPolicy()
2017-02-13 23:18:17 +00:00
}
if g.RestartPolicy != nil {
defaultRestartPolicy.Merge(g.RestartPolicy)
2015-09-09 20:02:39 +00:00
}
2017-02-13 23:18:17 +00:00
g.RestartPolicy = defaultRestartPolicy
2020-03-07 02:52:58 +00:00
for _, t := range g.Tasks {
t.Canonicalize(g, job)
}
for _, spread := range g.Spreads {
spread.Canonicalize()
}
for _, a := range g.Affinities {
a.Canonicalize()
}
for _, n := range g.Networks {
n.Canonicalize()
}
for _, s := range g.Services {
s.Canonicalize(nil, g, job)
}
2015-09-09 20:02:39 +00:00
}
2020-03-07 02:52:58 +00:00
// These needs to be in sync with DefaultServiceJobRestartPolicy in
// in nomad/structs/structs.go
func defaultServiceJobRestartPolicy() *RestartPolicy {
return &RestartPolicy{
Delay: timeToPtr(15 * time.Second),
Attempts: intToPtr(2),
Interval: timeToPtr(30 * time.Minute),
Mode: stringToPtr(RestartPolicyModeFail),
}
}
// These needs to be in sync with DefaultBatchJobRestartPolicy in
// in nomad/structs/structs.go
func defaultBatchJobRestartPolicy() *RestartPolicy {
return &RestartPolicy{
Delay: timeToPtr(15 * time.Second),
Attempts: intToPtr(3),
Interval: timeToPtr(24 * time.Hour),
Mode: stringToPtr(RestartPolicyModeFail),
}
}
2015-09-09 20:02:39 +00:00
// Constrain is used to add a constraint to a task group.
func (g *TaskGroup) Constrain(c *Constraint) *TaskGroup {
g.Constraints = append(g.Constraints, c)
return g
}
// AddMeta is used to add a meta k/v pair to a task group
func (g *TaskGroup) SetMeta(key, val string) *TaskGroup {
if g.Meta == nil {
g.Meta = make(map[string]string)
}
g.Meta[key] = val
return g
}
// AddTask is used to add a new task to a task group.
func (g *TaskGroup) AddTask(t *Task) *TaskGroup {
g.Tasks = append(g.Tasks, t)
return g
}
2018-07-16 13:30:58 +00:00
// AddAffinity is used to add a new affinity to a task group.
func (g *TaskGroup) AddAffinity(a *Affinity) *TaskGroup {
g.Affinities = append(g.Affinities, a)
return g
}
// RequireDisk adds a ephemeral disk to the task group
func (g *TaskGroup) RequireDisk(disk *EphemeralDisk) *TaskGroup {
g.EphemeralDisk = disk
2016-08-26 03:10:25 +00:00
return g
}
// AddSpread is used to add a new spread preference to a task group.
func (g *TaskGroup) AddSpread(s *Spread) *TaskGroup {
g.Spreads = append(g.Spreads, s)
return g
}
// LogConfig provides configuration for log rotation
type LogConfig struct {
MaxFiles *int `mapstructure:"max_files"`
MaxFileSizeMB *int `mapstructure:"max_file_size"`
2017-02-06 19:48:28 +00:00
}
func DefaultLogConfig() *LogConfig {
return &LogConfig{
MaxFiles: intToPtr(10),
MaxFileSizeMB: intToPtr(10),
2017-02-06 19:48:28 +00:00
}
}
func (l *LogConfig) Canonicalize() {
if l.MaxFiles == nil {
l.MaxFiles = intToPtr(10)
2017-02-06 19:48:28 +00:00
}
if l.MaxFileSizeMB == nil {
l.MaxFileSizeMB = intToPtr(10)
2017-02-06 19:48:28 +00:00
}
}
// DispatchPayloadConfig configures how a task gets its input from a job dispatch
type DispatchPayloadConfig struct {
2016-12-14 20:50:08 +00:00
File string
2016-11-23 22:56:50 +00:00
}
const (
TaskLifecycleHookPrestart = "prestart"
)
2019-10-11 17:10:45 +00:00
type TaskLifecycle struct {
Hook string `mapstructure:"hook"`
Sidecar bool `mapstructure:"sidecar"`
2019-12-12 18:59:38 +00:00
}
// Determine if lifecycle has user-input values
func (l *TaskLifecycle) Empty() bool {
return l == nil || (l.Hook == "")
2019-12-12 18:59:38 +00:00
}
2015-09-09 20:02:39 +00:00
// Task is a single process in a task group.
type Task struct {
Name string
Driver string
User string
2019-10-11 17:10:45 +00:00
Lifecycle *TaskLifecycle
Config map[string]interface{}
Constraints []*Constraint
2018-07-16 13:30:58 +00:00
Affinities []*Affinity
Env map[string]string
2017-03-01 23:30:01 +00:00
Services []*Service
Resources *Resources
2020-03-07 02:52:58 +00:00
RestartPolicy *RestartPolicy
Meta map[string]string
KillTimeout *time.Duration `mapstructure:"kill_timeout"`
LogConfig *LogConfig `mapstructure:"logs"`
Artifacts []*TaskArtifact
Vault *Vault
Templates []*Template
DispatchPayload *DispatchPayloadConfig
VolumeMounts []*VolumeMount
CSI Plugin Registration (#6555) This changeset implements the initial registration and fingerprinting of CSI Plugins as part of #5378. At a high level, it introduces the following: * A `csi_plugin` stanza as part of a Nomad task configuration, to allow a task to expose that it is a plugin. * A new task runner hook: `csi_plugin_supervisor`. This hook does two things. When the `csi_plugin` stanza is detected, it will automatically configure the plugin task to receive bidirectional mounts to the CSI intermediary directory. At runtime, it will then perform an initial heartbeat of the plugin and handle submitting it to the new `dynamicplugins.Registry` for further use by the client, and then run a lightweight heartbeat loop that will emit task events when health changes. * The `dynamicplugins.Registry` for handling plugins that run as Nomad tasks, in contrast to the existing catalog that requires `go-plugin` type plugins and to know the plugin configuration in advance. * The `csimanager` which fingerprints CSI plugins, in a similar way to `drivermanager` and `devicemanager`. It currently only fingerprints the NodeID from the plugin, and assumes that all plugins are monolithic. Missing features * We do not use the live updates of the `dynamicplugin` registry in the `csimanager` yet. * We do not deregister the plugins from the client when they shutdown yet, they just become indefinitely marked as unhealthy. This is deliberate until we figure out how we should manage deploying new versions of plugins/transitioning them.
2019-10-22 13:20:26 +00:00
CSIPluginConfig *TaskCSIPluginConfig `mapstructure:"csi_plugin" json:"csi_plugin,omitempty"`
2017-02-21 00:36:41 +00:00
Leader bool
ShutdownDelay time.Duration `mapstructure:"shutdown_delay"`
2017-12-06 21:23:24 +00:00
KillSignal string `mapstructure:"kill_signal"`
Kind string
}
func (t *Task) Canonicalize(tg *TaskGroup, job *Job) {
if t.Resources == nil {
2017-11-10 01:09:37 +00:00
t.Resources = &Resources{}
}
2017-11-10 01:09:37 +00:00
t.Resources.Canonicalize()
if t.KillTimeout == nil {
t.KillTimeout = timeToPtr(5 * time.Second)
}
2017-02-06 19:48:28 +00:00
if t.LogConfig == nil {
t.LogConfig = DefaultLogConfig()
} else {
t.LogConfig.Canonicalize()
}
for _, artifact := range t.Artifacts {
artifact.Canonicalize()
}
if t.Vault != nil {
t.Vault.Canonicalize()
}
2017-02-06 19:48:28 +00:00
for _, tmpl := range t.Templates {
tmpl.Canonicalize()
}
2017-03-01 23:30:01 +00:00
for _, s := range t.Services {
s.Canonicalize(t, tg, job)
}
for _, a := range t.Affinities {
a.Canonicalize()
}
for _, vm := range t.VolumeMounts {
vm.Canonicalize()
}
if t.Lifecycle.Empty() {
t.Lifecycle = nil
2019-12-12 18:59:38 +00:00
}
CSI Plugin Registration (#6555) This changeset implements the initial registration and fingerprinting of CSI Plugins as part of #5378. At a high level, it introduces the following: * A `csi_plugin` stanza as part of a Nomad task configuration, to allow a task to expose that it is a plugin. * A new task runner hook: `csi_plugin_supervisor`. This hook does two things. When the `csi_plugin` stanza is detected, it will automatically configure the plugin task to receive bidirectional mounts to the CSI intermediary directory. At runtime, it will then perform an initial heartbeat of the plugin and handle submitting it to the new `dynamicplugins.Registry` for further use by the client, and then run a lightweight heartbeat loop that will emit task events when health changes. * The `dynamicplugins.Registry` for handling plugins that run as Nomad tasks, in contrast to the existing catalog that requires `go-plugin` type plugins and to know the plugin configuration in advance. * The `csimanager` which fingerprints CSI plugins, in a similar way to `drivermanager` and `devicemanager`. It currently only fingerprints the NodeID from the plugin, and assumes that all plugins are monolithic. Missing features * We do not use the live updates of the `dynamicplugin` registry in the `csimanager` yet. * We do not deregister the plugins from the client when they shutdown yet, they just become indefinitely marked as unhealthy. This is deliberate until we figure out how we should manage deploying new versions of plugins/transitioning them.
2019-10-22 13:20:26 +00:00
if t.CSIPluginConfig != nil {
t.CSIPluginConfig.Canonicalize()
}
2020-03-07 02:52:58 +00:00
if t.RestartPolicy == nil {
t.RestartPolicy = tg.RestartPolicy
} else {
tgrp := &RestartPolicy{}
*tgrp = *tg.RestartPolicy
tgrp.Merge(t.RestartPolicy)
t.RestartPolicy = tgrp
}
2017-02-06 19:48:28 +00:00
}
2016-03-16 03:21:52 +00:00
// TaskArtifact is used to download artifacts before running a task.
type TaskArtifact struct {
GetterSource *string `mapstructure:"source"`
GetterOptions map[string]string `mapstructure:"options"`
GetterMode *string `mapstructure:"mode"`
RelativeDest *string `mapstructure:"destination"`
2017-02-06 19:48:28 +00:00
}
func (a *TaskArtifact) Canonicalize() {
if a.GetterMode == nil {
a.GetterMode = stringToPtr("any")
}
if a.GetterSource == nil {
// Shouldn't be possible, but we don't want to panic
a.GetterSource = stringToPtr("")
}
2017-02-06 19:48:28 +00:00
if a.RelativeDest == nil {
switch *a.GetterMode {
case "file":
// File mode should default to local/filename
dest := *a.GetterSource
dest = path.Base(dest)
dest = filepath.Join("local", dest)
a.RelativeDest = &dest
default:
// Default to a directory
a.RelativeDest = stringToPtr("local/")
}
2017-02-06 19:48:28 +00:00
}
2015-09-09 20:02:39 +00:00
}
2016-09-23 22:39:52 +00:00
type Template struct {
SourcePath *string `mapstructure:"source"`
DestPath *string `mapstructure:"destination"`
EmbeddedTmpl *string `mapstructure:"data"`
ChangeMode *string `mapstructure:"change_mode"`
ChangeSignal *string `mapstructure:"change_signal"`
Splay *time.Duration `mapstructure:"splay"`
Perms *string `mapstructure:"perms"`
LeftDelim *string `mapstructure:"left_delimiter"`
RightDelim *string `mapstructure:"right_delimiter"`
Envvars *bool `mapstructure:"env"`
VaultGrace *time.Duration `mapstructure:"vault_grace"`
2017-02-06 19:48:28 +00:00
}
func (tmpl *Template) Canonicalize() {
if tmpl.SourcePath == nil {
tmpl.SourcePath = stringToPtr("")
2017-02-06 19:48:28 +00:00
}
if tmpl.DestPath == nil {
tmpl.DestPath = stringToPtr("")
2017-02-06 19:48:28 +00:00
}
if tmpl.EmbeddedTmpl == nil {
tmpl.EmbeddedTmpl = stringToPtr("")
}
if tmpl.ChangeMode == nil {
tmpl.ChangeMode = stringToPtr("restart")
2017-02-06 19:48:28 +00:00
}
if tmpl.ChangeSignal == nil {
if *tmpl.ChangeMode == "signal" {
tmpl.ChangeSignal = stringToPtr("SIGHUP")
} else {
tmpl.ChangeSignal = stringToPtr("")
}
} else {
2017-02-13 23:18:17 +00:00
sig := *tmpl.ChangeSignal
tmpl.ChangeSignal = stringToPtr(strings.ToUpper(sig))
2017-02-13 23:18:17 +00:00
}
if tmpl.Splay == nil {
tmpl.Splay = timeToPtr(5 * time.Second)
}
if tmpl.Perms == nil {
tmpl.Perms = stringToPtr("0644")
}
2017-02-21 00:43:28 +00:00
if tmpl.LeftDelim == nil {
tmpl.LeftDelim = stringToPtr("{{")
2017-02-21 00:43:28 +00:00
}
if tmpl.RightDelim == nil {
tmpl.RightDelim = stringToPtr("}}")
2017-02-21 00:43:28 +00:00
}
if tmpl.Envvars == nil {
tmpl.Envvars = boolToPtr(false)
}
//COMPAT(0.12) VaultGrace is deprecated and unused as of Vault 0.5
if tmpl.VaultGrace == nil {
tmpl.VaultGrace = timeToPtr(0)
}
2016-09-23 22:39:52 +00:00
}
2016-08-17 04:32:25 +00:00
type Vault struct {
Policies []string
2017-02-06 19:48:28 +00:00
Env *bool
ChangeMode *string `mapstructure:"change_mode"`
ChangeSignal *string `mapstructure:"change_signal"`
2017-02-06 19:48:28 +00:00
}
func (v *Vault) Canonicalize() {
if v.Env == nil {
v.Env = boolToPtr(true)
2017-02-06 19:48:28 +00:00
}
if v.ChangeMode == nil {
v.ChangeMode = stringToPtr("restart")
2017-02-06 19:48:28 +00:00
}
if v.ChangeSignal == nil {
v.ChangeSignal = stringToPtr("SIGHUP")
2017-02-06 19:48:28 +00:00
}
2016-08-17 04:32:25 +00:00
}
2015-09-09 20:02:39 +00:00
// NewTask creates and initializes a new Task.
func NewTask(name, driver string) *Task {
return &Task{
Name: name,
Driver: driver,
}
}
// Configure is used to configure a single k/v pair on
// the task.
2016-08-22 16:35:25 +00:00
func (t *Task) SetConfig(key string, val interface{}) *Task {
2015-09-09 20:02:39 +00:00
if t.Config == nil {
2015-11-14 04:51:30 +00:00
t.Config = make(map[string]interface{})
2015-09-09 20:02:39 +00:00
}
t.Config[key] = val
return t
}
// SetMeta is used to add metadata k/v pairs to the task.
func (t *Task) SetMeta(key, val string) *Task {
if t.Meta == nil {
t.Meta = make(map[string]string)
}
t.Meta[key] = val
return t
}
// Require is used to add resource requirements to a task.
func (t *Task) Require(r *Resources) *Task {
t.Resources = r
return t
}
2015-09-10 00:29:43 +00:00
// Constraint adds a new constraints to a single task.
func (t *Task) Constrain(c *Constraint) *Task {
t.Constraints = append(t.Constraints, c)
return t
}
2015-11-12 23:28:22 +00:00
2018-07-16 13:30:58 +00:00
// AddAffinity adds a new affinity to a single task.
func (t *Task) AddAffinity(a *Affinity) *Task {
t.Affinities = append(t.Affinities, a)
return t
}
2016-02-10 21:36:47 +00:00
// SetLogConfig sets a log config to a task
func (t *Task) SetLogConfig(l *LogConfig) *Task {
t.LogConfig = l
return t
}
2015-11-12 23:28:22 +00:00
// TaskState tracks the current state of a task and events that caused state
2016-05-15 16:41:34 +00:00
// transitions.
2015-11-12 23:28:22 +00:00
type TaskState struct {
State string
Failed bool
Restarts uint64
LastRestart time.Time
StartedAt time.Time
FinishedAt time.Time
Events []*TaskEvent
2015-11-12 23:28:22 +00:00
}
const (
2017-02-10 01:40:13 +00:00
TaskSetup = "Task Setup"
2016-10-10 21:49:37 +00:00
TaskSetupFailure = "Setup Failure"
TaskDriverFailure = "Driver Failure"
TaskDriverMessage = "Driver"
TaskReceived = "Received"
TaskFailedValidation = "Failed Validation"
TaskStarted = "Started"
TaskTerminated = "Terminated"
TaskKilling = "Killing"
TaskKilled = "Killed"
TaskRestarting = "Restarting"
TaskNotRestarting = "Not Restarting"
TaskDownloadingArtifacts = "Downloading Artifacts"
TaskArtifactDownloadFailed = "Failed Artifact Download"
2017-02-11 01:55:19 +00:00
TaskSiblingFailed = "Sibling Task Failed"
2016-10-05 20:41:29 +00:00
TaskSignaling = "Signaling"
2016-10-05 22:11:09 +00:00
TaskRestartSignal = "Restart Signaled"
2017-02-11 01:55:19 +00:00
TaskLeaderDead = "Leader Task Dead"
2017-07-07 06:04:42 +00:00
TaskBuildingTaskDir = "Building Task Directory"
2015-11-12 23:28:22 +00:00
)
// TaskEvent is an event that effects the state of a task and contains meta-data
// appropriate to the events type.
type TaskEvent struct {
2017-11-03 17:48:55 +00:00
Type string
Time int64
DisplayMessage string
Details map[string]string
// DEPRECATION NOTICE: The following fields are all deprecated. see TaskEvent struct in structs.go for details.
FailsTask bool
RestartReason string
SetupError string
DriverError string
DriverMessage string
ExitCode int
Signal int
Message string
KillReason string
KillTimeout time.Duration
KillError string
StartDelay int64
DownloadError string
ValidationError string
DiskLimit int64
DiskSize int64
FailedSibling string
VaultError string
TaskSignalReason string
TaskSignal string
2017-08-08 04:26:04 +00:00
GenericSource string
2015-11-12 23:28:22 +00:00
}
CSI Plugin Registration (#6555) This changeset implements the initial registration and fingerprinting of CSI Plugins as part of #5378. At a high level, it introduces the following: * A `csi_plugin` stanza as part of a Nomad task configuration, to allow a task to expose that it is a plugin. * A new task runner hook: `csi_plugin_supervisor`. This hook does two things. When the `csi_plugin` stanza is detected, it will automatically configure the plugin task to receive bidirectional mounts to the CSI intermediary directory. At runtime, it will then perform an initial heartbeat of the plugin and handle submitting it to the new `dynamicplugins.Registry` for further use by the client, and then run a lightweight heartbeat loop that will emit task events when health changes. * The `dynamicplugins.Registry` for handling plugins that run as Nomad tasks, in contrast to the existing catalog that requires `go-plugin` type plugins and to know the plugin configuration in advance. * The `csimanager` which fingerprints CSI plugins, in a similar way to `drivermanager` and `devicemanager`. It currently only fingerprints the NodeID from the plugin, and assumes that all plugins are monolithic. Missing features * We do not use the live updates of the `dynamicplugin` registry in the `csimanager` yet. * We do not deregister the plugins from the client when they shutdown yet, they just become indefinitely marked as unhealthy. This is deliberate until we figure out how we should manage deploying new versions of plugins/transitioning them.
2019-10-22 13:20:26 +00:00
// CSIPluginType is an enum string that encapsulates the valid options for a
// CSIPlugin stanza's Type. These modes will allow the plugin to be used in
// different ways by the client.
type CSIPluginType string
const (
// CSIPluginTypeNode indicates that Nomad should only use the plugin for
// performing Node RPCs against the provided plugin.
CSIPluginTypeNode CSIPluginType = "node"
// CSIPluginTypeController indicates that Nomad should only use the plugin for
// performing Controller RPCs against the provided plugin.
CSIPluginTypeController CSIPluginType = "controller"
// CSIPluginTypeMonolith indicates that Nomad can use the provided plugin for
// both controller and node rpcs.
CSIPluginTypeMonolith CSIPluginType = "monolith"
)
// TaskCSIPluginConfig contains the data that is required to setup a task as a
// CSI plugin. This will be used by the csi_plugin_supervisor_hook to configure
// mounts for the plugin and initiate the connection to the plugin catalog.
type TaskCSIPluginConfig struct {
// ID is the identifier of the plugin.
// Ideally this should be the FQDN of the plugin.
ID string `mapstructure:"id"`
// CSIPluginType instructs Nomad on how to handle processing a plugin
Type CSIPluginType `mapstructure:"type"`
// MountDir is the destination that nomad should mount in its CSI
// directory for the plugin. It will then expect a file called CSISocketName
// to be created by the plugin, and will provide references into
// "MountDir/CSIIntermediaryDirname/VolumeName/AllocID for mounts.
//
// Default is /csi.
MountDir string `mapstructure:"mount_dir"`
}
func (t *TaskCSIPluginConfig) Canonicalize() {
if t.MountDir == "" {
t.MountDir = "/csi"
}
}