diff --git a/api/jobs_test.go b/api/jobs_test.go index dc2dd351f..41fb41140 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -136,8 +136,12 @@ func TestJobs_Canonicalize(t *testing.T) { Mode: helper.StringToPtr("fail"), }, ReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(2), - Interval: helper.TimeToPtr(1 * time.Hour), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr("exponential"), + Delay: helper.TimeToPtr(30 * time.Second), + DelayCeiling: helper.TimeToPtr(1 * time.Hour), + Unlimited: helper.BoolToPtr(true), }, Tasks: []*Task{ { @@ -202,8 +206,12 @@ func TestJobs_Canonicalize(t *testing.T) { Mode: helper.StringToPtr("fail"), }, ReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(2), - Interval: helper.TimeToPtr(1 * time.Hour), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr("exponential"), + Delay: helper.TimeToPtr(30 * time.Second), + DelayCeiling: helper.TimeToPtr(1 * time.Hour), + Unlimited: helper.BoolToPtr(true), }, Tasks: []*Task{ { @@ -335,8 +343,12 @@ func TestJobs_Canonicalize(t *testing.T) { Mode: helper.StringToPtr("delay"), }, ReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(2), - Interval: helper.TimeToPtr(1 * time.Hour), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr("exponential"), + Delay: helper.TimeToPtr(30 * time.Second), + DelayCeiling: helper.TimeToPtr(1 * time.Hour), + Unlimited: helper.BoolToPtr(true), }, EphemeralDisk: &EphemeralDisk{ Sticky: helper.BoolToPtr(false), @@ -550,8 +562,12 @@ func TestJobs_Canonicalize(t *testing.T) { Mode: helper.StringToPtr("fail"), }, ReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(2), - Interval: helper.TimeToPtr(1 * time.Hour), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr("exponential"), + Delay: helper.TimeToPtr(30 * time.Second), + DelayCeiling: helper.TimeToPtr(1 * time.Hour), + Unlimited: helper.BoolToPtr(true), }, Update: &UpdateStrategy{ Stagger: helper.TimeToPtr(2 * time.Second), @@ -586,8 +602,12 @@ func TestJobs_Canonicalize(t *testing.T) { Mode: helper.StringToPtr("fail"), }, ReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(2), - Interval: helper.TimeToPtr(1 * time.Hour), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr("exponential"), + Delay: helper.TimeToPtr(30 * time.Second), + DelayCeiling: helper.TimeToPtr(1 * time.Hour), + Unlimited: helper.BoolToPtr(true), }, Update: &UpdateStrategy{ Stagger: helper.TimeToPtr(1 * time.Second), diff --git a/api/search_test.go b/api/search_test.go index e62d99667..892394c39 100644 --- a/api/search_test.go +++ b/api/search_test.go @@ -4,11 +4,11 @@ import ( "testing" "github.com/hashicorp/nomad/api/contexts" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSearch_List(t *testing.T) { - assert := assert.New(t) + require := require.New(t) t.Parallel() c, s := makeClient(t, nil, nil) @@ -16,16 +16,17 @@ func TestSearch_List(t *testing.T) { job := testJob() _, _, err := c.Jobs().Register(job, nil) - assert.Nil(err) + require.Nil(err) id := *job.ID prefix := id[:len(id)-2] resp, qm, err := c.Search().PrefixSearch(prefix, contexts.Jobs, nil) - assert.Nil(err) - assert.NotNil(qm) + require.Nil(err) + require.NotNil(qm) + require.NotNil(qm) jobMatches := resp.Matches[contexts.Jobs] - assert.Equal(1, len(jobMatches)) - assert.Equal(id, jobMatches[0]) + require.Equal(1, len(jobMatches)) + require.Equal(id, jobMatches[0]) } diff --git a/api/tasks.go b/api/tasks.go index 69a8022f9..ef123fa06 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -86,6 +86,20 @@ type ReschedulePolicy struct { // Interval is a duration in which we can limit the number of reschedule attempts. Interval *time.Duration `mapstructure:"interval"` + + // Delay is a minimum duration to wait between reschedule attempts. + // The delay function determines how much subsequent reschedule attempts are delayed by. + Delay *time.Duration `mapstructure:"delay"` + + // DelayFunction determines how the delay progressively changes on subsequent reschedule + // attempts. Valid values are "exponential", "linear", and "fibonacci". + DelayFunction *string `mapstructure:"delay_function"` + + // DelayCeiling is an upper bound on the delay. + DelayCeiling *time.Duration `mapstructure:"delay_ceiling"` + + // Unlimited allows rescheduling attempts until they succeed + Unlimited *bool `mapstructure:"unlimited"` } func (r *ReschedulePolicy) Merge(rp *ReschedulePolicy) { @@ -95,6 +109,18 @@ func (r *ReschedulePolicy) Merge(rp *ReschedulePolicy) { if rp.Attempts != nil { r.Attempts = rp.Attempts } + if rp.Delay != nil { + r.Delay = rp.Delay + } + if rp.DelayFunction != nil { + r.DelayFunction = rp.DelayFunction + } + if rp.DelayCeiling != nil { + r.DelayCeiling = rp.DelayCeiling + } + if rp.Unlimited != nil { + r.Unlimited = rp.Unlimited + } } func (r *ReschedulePolicy) Copy() *ReschedulePolicy { @@ -316,13 +342,21 @@ func (g *TaskGroup) Canonicalize(job *Job) { switch *job.Type { case "service": defaultReschedulePolicy = &ReschedulePolicy{ - Attempts: helper.IntToPtr(structs.DefaultServiceJobReschedulePolicy.Attempts), - Interval: helper.TimeToPtr(structs.DefaultServiceJobReschedulePolicy.Interval), + Attempts: helper.IntToPtr(structs.DefaultServiceJobReschedulePolicy.Attempts), + Interval: helper.TimeToPtr(structs.DefaultServiceJobReschedulePolicy.Interval), + Delay: helper.TimeToPtr(structs.DefaultServiceJobReschedulePolicy.Delay), + DelayFunction: helper.StringToPtr(structs.DefaultServiceJobReschedulePolicy.DelayFunction), + DelayCeiling: helper.TimeToPtr(structs.DefaultServiceJobReschedulePolicy.DelayCeiling), + Unlimited: helper.BoolToPtr(structs.DefaultServiceJobReschedulePolicy.Unlimited), } case "batch": defaultReschedulePolicy = &ReschedulePolicy{ - Attempts: helper.IntToPtr(structs.DefaultBatchJobReschedulePolicy.Attempts), - Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Attempts: helper.IntToPtr(structs.DefaultBatchJobReschedulePolicy.Attempts), + Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Delay: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Delay), + DelayFunction: helper.StringToPtr(structs.DefaultBatchJobReschedulePolicy.DelayFunction), + DelayCeiling: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.DelayCeiling), + Unlimited: helper.BoolToPtr(structs.DefaultBatchJobReschedulePolicy.Unlimited), } default: defaultReschedulePolicy = &ReschedulePolicy{ diff --git a/api/tasks_test.go b/api/tasks_test.go index 37c47d514..284b6e8a4 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -284,70 +284,115 @@ func TestTaskGroup_Canonicalize_ReschedulePolicy(t *testing.T) { jobReschedulePolicy: nil, taskReschedulePolicy: nil, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(structs.DefaultBatchJobReschedulePolicy.Attempts), - Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Attempts: helper.IntToPtr(structs.DefaultBatchJobReschedulePolicy.Attempts), + Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Delay: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Delay), + DelayFunction: helper.StringToPtr(structs.DefaultBatchJobReschedulePolicy.DelayFunction), + DelayCeiling: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.DelayCeiling), + Unlimited: helper.BoolToPtr(structs.DefaultBatchJobReschedulePolicy.Unlimited), }, }, { desc: "Empty job reschedule policy", jobReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(0), - Interval: helper.TimeToPtr(0), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + Delay: helper.TimeToPtr(0), + DelayCeiling: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr(""), + Unlimited: helper.BoolToPtr(false), }, taskReschedulePolicy: nil, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(0), - Interval: helper.TimeToPtr(0), + Attempts: helper.IntToPtr(0), + Interval: helper.TimeToPtr(0), + Delay: helper.TimeToPtr(0), + DelayCeiling: helper.TimeToPtr(0), + DelayFunction: helper.StringToPtr(""), + Unlimited: helper.BoolToPtr(false), }, }, { desc: "Inherit from job", jobReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(1), - Interval: helper.TimeToPtr(20 * time.Second), + Attempts: helper.IntToPtr(1), + Interval: helper.TimeToPtr(20 * time.Second), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(10 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, taskReschedulePolicy: nil, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(1), - Interval: helper.TimeToPtr(20 * time.Second), + Attempts: helper.IntToPtr(1), + Interval: helper.TimeToPtr(20 * time.Second), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(10 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, }, { desc: "Set in task", jobReschedulePolicy: nil, taskReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(5), - Interval: helper.TimeToPtr(2 * time.Minute), + Attempts: helper.IntToPtr(5), + Interval: helper.TimeToPtr(2 * time.Minute), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(10 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(5), - Interval: helper.TimeToPtr(2 * time.Minute), + Attempts: helper.IntToPtr(5), + Interval: helper.TimeToPtr(2 * time.Minute), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(10 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, }, { desc: "Merge from job", jobReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(1), + Attempts: helper.IntToPtr(1), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(10 * time.Minute), }, taskReschedulePolicy: &ReschedulePolicy{ - Interval: helper.TimeToPtr(5 * time.Minute), + Interval: helper.TimeToPtr(5 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(1), - Interval: helper.TimeToPtr(5 * time.Minute), + Attempts: helper.IntToPtr(1), + Interval: helper.TimeToPtr(5 * time.Minute), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(10 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, }, { desc: "Override from group", jobReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(1), + Attempts: helper.IntToPtr(1), + DelayCeiling: helper.TimeToPtr(10 * time.Second), }, taskReschedulePolicy: &ReschedulePolicy{ - Attempts: helper.IntToPtr(5), + Attempts: helper.IntToPtr(5), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(20 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(5), - Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Attempts: helper.IntToPtr(5), + Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Delay: helper.TimeToPtr(20 * time.Second), + DelayCeiling: helper.TimeToPtr(20 * time.Minute), + DelayFunction: helper.StringToPtr("linear"), + Unlimited: helper.BoolToPtr(false), }, }, { @@ -357,8 +402,12 @@ func TestTaskGroup_Canonicalize_ReschedulePolicy(t *testing.T) { }, taskReschedulePolicy: nil, expected: &ReschedulePolicy{ - Attempts: helper.IntToPtr(1), - Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Attempts: helper.IntToPtr(1), + Interval: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Interval), + Delay: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.Delay), + DelayFunction: helper.StringToPtr(structs.DefaultBatchJobReschedulePolicy.DelayFunction), + DelayCeiling: helper.TimeToPtr(structs.DefaultBatchJobReschedulePolicy.DelayCeiling), + Unlimited: helper.BoolToPtr(structs.DefaultBatchJobReschedulePolicy.Unlimited), }, }, } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index c661e4b0b..a059343ab 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -639,8 +639,12 @@ func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) { } tg.ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: *taskGroup.ReschedulePolicy.Attempts, - Interval: *taskGroup.ReschedulePolicy.Interval, + Attempts: *taskGroup.ReschedulePolicy.Attempts, + Interval: *taskGroup.ReschedulePolicy.Interval, + Delay: *taskGroup.ReschedulePolicy.Delay, + DelayFunction: *taskGroup.ReschedulePolicy.DelayFunction, + DelayCeiling: *taskGroup.ReschedulePolicy.DelayCeiling, + Unlimited: *taskGroup.ReschedulePolicy.Unlimited, } tg.EphemeralDisk = &structs.EphemeralDisk{ diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 2cf2e6fa8..ccc58276a 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -1172,8 +1172,12 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Mode: helper.StringToPtr("delay"), }, ReschedulePolicy: &api.ReschedulePolicy{ - Interval: helper.TimeToPtr(12 * time.Hour), - Attempts: helper.IntToPtr(5), + Interval: helper.TimeToPtr(12 * time.Hour), + Attempts: helper.IntToPtr(5), + DelayFunction: helper.StringToPtr("linear"), + Delay: helper.TimeToPtr(30 * time.Second), + Unlimited: helper.BoolToPtr(true), + DelayCeiling: helper.TimeToPtr(20 * time.Minute), }, EphemeralDisk: &api.EphemeralDisk{ SizeMB: helper.IntToPtr(100), @@ -1384,8 +1388,12 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Mode: "delay", }, ReschedulePolicy: &structs.ReschedulePolicy{ - Interval: 12 * time.Hour, - Attempts: 5, + Interval: 12 * time.Hour, + Attempts: 5, + DelayFunction: "linear", + Delay: 30 * time.Second, + Unlimited: true, + DelayCeiling: 20 * time.Minute, }, EphemeralDisk: &structs.EphemeralDisk{ SizeMB: 100, diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ea9612714..41deedf20 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -2773,8 +2773,7 @@ type ReschedulePolicy struct { // DelayCeiling is an upper bound on the delay. DelayCeiling time.Duration - // Unlimited allows infinite rescheduling attempts. Only allowed when delay is set between reschedule - // attempts. + // Unlimited allows rescheduling attempts until they succeed Unlimited bool }