luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Backport of feature: Add new field render_templates on restart block into release/1.6.x (#18094) 

This pull request was automerged via backport-assistant
This commit is contained in:
hc-github-team-nomad-core 2023-07-28 13:54:00 -05:00 committed by GitHub
parent bebed09677
commit 2ed92e0c6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 314 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/18054.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
jobspec: Add new parameter `render_templates` for `restart` block to allow explicit re-render of templates on task restart. The default value is `false` and is fully backward compatible
```

14
api/jobs_test.go
View File

@ -324,6 +324,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -409,6 +410,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(3),
Interval: pointerOf(24 * time.Hour),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(1),
@ -499,6 +501,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -667,6 +670,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(10),
Delay: pointerOf(25 * time.Second),
Mode: pointerOf("delay"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -711,6 +715,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(20),
Delay: pointerOf(25 * time.Second),
Mode: pointerOf("delay"),
RenderTemplates: pointerOf(false),
},
Resources: &Resources{
CPU: pointerOf(500),
@ -835,7 +840,6 @@ func TestJobs_Canonicalize(t *testing.T) {
},
},
},
{
name: "update_merge",
input: &Job{
@ -932,6 +936,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -979,6 +984,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -1016,7 +1022,6 @@ func TestJobs_Canonicalize(t *testing.T) {
},
},
},
{
name: "restart_merge",
input: &Job{
@ -1038,6 +1043,7 @@ func TestJobs_Canonicalize(t *testing.T) {
RestartPolicy: &RestartPolicy{
Attempts: pointerOf(5),
Delay: pointerOf(1 * time.Second),
RenderTemplates: pointerOf(true),
},
},
},
@ -1109,6 +1115,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -1144,6 +1151,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Delay: pointerOf(1 * time.Second),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(true),
},
},
},
@ -1161,6 +1169,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
@ -1196,6 +1205,7 @@ func TestJobs_Canonicalize(t *testing.T) {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
},
},

6
api/tasks.go
View File

@ -92,6 +92,7 @@ type RestartPolicy struct {
Attempts *int `hcl:"attempts,optional"`
Delay *time.Duration `hcl:"delay,optional"`
Mode *string `hcl:"mode,optional"`
RenderTemplates *bool `mapstructure:"render_templates" hcl:"render_templates,optional"`
}
func (r *RestartPolicy) Merge(rp *RestartPolicy) {
@ -107,6 +108,9 @@ func (r *RestartPolicy) Merge(rp *RestartPolicy) {
if rp.Mode != nil {
r.Mode = rp.Mode
}
if rp.RenderTemplates != nil {
r.RenderTemplates = rp.RenderTemplates
}
}
// Reschedule configures how Tasks are rescheduled when they crash or fail.
@ -584,6 +588,7 @@ func defaultServiceJobRestartPolicy() *RestartPolicy {
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf(RestartPolicyModeFail),
RenderTemplates: pointerOf(false),
}
}
@ -595,6 +600,7 @@ func defaultBatchJobRestartPolicy() *RestartPolicy {
Attempts: pointerOf(3),
Interval: pointerOf(24 * time.Hour),
Mode: pointerOf(RestartPolicyModeFail),
RenderTemplates: pointerOf(false),
}
}

1
client/allocrunner/taskrunner/task_runner_hooks.go
View File

@ -119,6 +119,7 @@ func (tr *TaskRunner) initHooks() {
envBuilder: tr.envBuilder,
consulNamespace: consulNamespace,
nomadNamespace: tr.alloc.Job.Namespace,
renderOnTaskRestart: task.RestartPolicy.RenderTemplates,
}))
}

8
client/allocrunner/taskrunner/template_hook.go
View File

@ -45,6 +45,9 @@ type templateHookConfig struct {
// nomadNamespace is the job's Nomad namespace
nomadNamespace string
// renderOnTaskRestart is flag to explicitly render templates on task restart
renderOnTaskRestart bool
}
type templateHook struct {
@ -97,8 +100,13 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
// If we have already run prerun before exit early.
if h.templateManager != nil {
if !h.config.renderOnTaskRestart {
return nil
}
h.logger.Info("re-rendering templates on task restart")
h.templateManager.Stop()
h.templateManager = nil
}
// Store the current Vault token and the task directory
h.taskDir = req.TaskDir.Dir

2
command/agent/job_endpoint.go
View File

@ -1062,6 +1062,7 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta
Interval: *taskGroup.RestartPolicy.Interval,
Delay: *taskGroup.RestartPolicy.Delay,
Mode: *taskGroup.RestartPolicy.Mode,
RenderTemplates: *taskGroup.RestartPolicy.RenderTemplates,
}
if taskGroup.ShutdownDelay != nil {
@ -1213,6 +1214,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
Interval: *apiTask.RestartPolicy.Interval,
Delay: *apiTask.RestartPolicy.Delay,
Mode: *apiTask.RestartPolicy.Mode,
RenderTemplates: *apiTask.RestartPolicy.RenderTemplates,
}
}

7
command/agent/job_endpoint_test.go
View File

@ -2522,6 +2522,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: pointer.Of(5),
Delay: pointer.Of(10 * time.Second),
Mode: pointer.Of("delay"),
RenderTemplates: pointer.Of(false),
},
ReschedulePolicy: &api.ReschedulePolicy{
Interval: pointer.Of(12 * time.Hour),
@ -2665,6 +2666,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: pointer.Of(10),
Delay: pointer.Of(20 * time.Second),
Mode: pointer.Of("delay"),
RenderTemplates: pointer.Of(false),
},
Services: []*api.Service{
{
@ -2931,6 +2933,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: 5,
Delay: 10 * time.Second,
Mode: "delay",
RenderTemplates: false,
},
Spreads: []*structs.Spread{
{
@ -3079,6 +3082,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: 10,
Delay: 20 * time.Second,
Mode: "delay",
RenderTemplates: false,
},
Services: []*structs.Service{
{
@ -3287,6 +3291,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: pointer.Of(5),
Delay: pointer.Of(10 * time.Second),
Mode: pointer.Of("delay"),
RenderTemplates: pointer.Of(false),
},
EphemeralDisk: &api.EphemeralDisk{
SizeMB: pointer.Of(100),
@ -3408,6 +3413,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: 5,
Delay: 10 * time.Second,
Mode: "delay",
RenderTemplates: false,
},
EphemeralDisk: &structs.EphemeralDisk{
SizeMB: 100,
@ -3466,6 +3472,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Attempts: 5,
Delay: 10 * time.Second,
Mode: "delay",
RenderTemplates: false,
},
Meta: map[string]string{
"lol": "code",

1
command/agent/testingutils_test.go
View File

@ -39,6 +39,7 @@ func MockJob() *api.Job {
Interval: pointer.Of(10 * time.Minute),
Delay: pointer.Of(1 * time.Minute),
Mode: pointer.Of("delay"),
RenderTemplates: pointer.Of(false),
},
Networks: []*api.NetworkResource{
{

2
command/helpers_test.go
View File

@ -227,6 +227,7 @@ const (
attempts = 10
mode = "delay"
interval = "15s"
render_templates = false
}
}
}`
@ -246,6 +247,7 @@ var (
Attempts: pointer.Of(10),
Interval: pointer.Of(15 * time.Second),
Mode: pointer.Of("delay"),
RenderTemplates: pointer.Of(false),
},
Tasks: []*api.Task{

1
jobspec/parse_group.go
View File

@ -321,6 +321,7 @@ func parseRestartPolicy(final **api.RestartPolicy, list *ast.ObjectList) error {
"interval",
"delay",
"mode",
"render_templates",
}
if err := checkHCLKeys(obj.Val, valid); err != nil {
return err

1
jobspec/parse_test.go
View File

@ -197,6 +197,7 @@ func TestParse(t *testing.T) {
Attempts: intToPtr(5),
Delay: timeToPtr(15 * time.Second),
Mode: stringToPtr("delay"),
RenderTemplates: boolToPtr(false),
},
Spreads: []*api.Spread{
{

1
jobspec/test-fixtures/basic.hcl
View File

@ -106,6 +106,7 @@ job "binstore-storagelocker" {
interval = "10m"
delay = "15s"
mode = "delay"
render_templates = false
}
reschedule {

18
jobspec2/parse_test.go
View File

@ -1086,3 +1086,21 @@ func TestErrMissingKey(t *testing.T) {
require.NotNil(t, tmpl.ErrMissingKey)
require.True(t, *tmpl.ErrMissingKey)
}
func TestRestartRenderTemplates(t *testing.T) {
ci.Parallel(t)
hclBytes, err := os.ReadFile("test-fixtures/restart-render-templates.hcl")
require.NoError(t, err)
job, err := ParseWithConfig(&ParseConfig{
Path: "test-fixtures/restart-render-templates.hcl",
Body: hclBytes,
AllowFS: false,
})
require.NoError(t, err)
tg := job.TaskGroups[0]
require.NotNil(t, tg.RestartPolicy)
require.True(t, *tg.RestartPolicy.RenderTemplates)
require.Nil(t, tg.Tasks[0].RestartPolicy)
require.False(t, *tg.Tasks[1].RestartPolicy.RenderTemplates)
}

17
jobspec2/test-fixtures/restart-render-templates.hcl Normal file
View File

@ -0,0 +1,17 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
job "example" {
group "group" {
restart {
render_templates = true
}
task "foo" {
}
task "bar" {
restart {
render_templates = false
}
}
}
}

1
nomad/mock/job.go
View File

@ -48,6 +48,7 @@ func Job() *structs.Job {
Interval: 10 * time.Minute,
Delay: 1 * time.Minute,
Mode: structs.RestartPolicyModeDelay,
RenderTemplates: false,
},
ReschedulePolicy: &structs.ReschedulePolicy{
Attempts: 2,

27
nomad/structs/diff_test.go
View File

@ -1861,6 +1861,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "",
New: "fail",
},
{
Type: DiffTypeAdded,
Name: "RenderTemplates",
Old: "",
New: "false",
},
},
},
},
@ -1908,6 +1914,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "fail",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "RenderTemplates",
Old: "false",
New: "",
},
},
},
},
@ -1929,6 +1941,7 @@ func TestTaskGroupDiff(t *testing.T) {
Interval: 2 * time.Second,
Delay: 2 * time.Second,
Mode: "delay",
RenderTemplates: true,
},
},
Expected: &TaskGroupDiff{
@ -1962,6 +1975,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "fail",
New: "delay",
},
{
Type: DiffTypeEdited,
Name: "RenderTemplates",
Old: "false",
New: "true",
},
},
},
},
@ -1976,6 +1995,7 @@ func TestTaskGroupDiff(t *testing.T) {
Interval: 1 * time.Second,
Delay: 1 * time.Second,
Mode: "fail",
RenderTemplates: false,
},
},
New: &TaskGroup{
@ -1984,6 +2004,7 @@ func TestTaskGroupDiff(t *testing.T) {
Interval: 2 * time.Second,
Delay: 1 * time.Second,
Mode: "fail",
RenderTemplates: true,
},
},
Expected: &TaskGroupDiff{
@ -2017,6 +2038,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "fail",
New: "fail",
},
{
Type: DiffTypeEdited,
Name: "RenderTemplates",
Old: "false",
New: "true",
},
},
},
},

5
nomad/structs/structs.go
View File

@ -5869,12 +5869,14 @@ var (
Attempts: 2,
Interval: 30 * time.Minute,
Mode: RestartPolicyModeFail,
RenderTemplates: false,
}
DefaultBatchJobRestartPolicy = RestartPolicy{
Delay: 15 * time.Second,
Attempts: 3,
Interval: 24 * time.Hour,
Mode: RestartPolicyModeFail,
RenderTemplates: false,
}
)
@ -6207,6 +6209,9 @@ type RestartPolicy struct {
// Mode controls what happens when the task restarts more than attempt times
// in an interval.
Mode string
// RenderTemplates is flag to explicitly render all templates on task restart
RenderTemplates bool
}
func (r *RestartPolicy) Copy() *RestartPolicy {

1
nomad/structs/structs_test.go
View File

@ -4310,6 +4310,7 @@ func TestRestartPolicy_Validate(t *testing.T) {
Mode: RestartPolicyModeFail,
Attempts: 0,
Interval: 5 * time.Second,
RenderTemplates: true,
}
if err := p.Validate(); err != nil {
t.Fatalf("err: %v", err)

11
scheduler/util.go
View File

@ -258,6 +258,12 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison {
return difference("volume request", a.Volumes, b.Volumes)
}
// Check if restart.render_templates is updated
// this requires a destructive update for template hook to receive the new config
if a.RestartPolicy.RenderTemplates != b.RestartPolicy.RenderTemplates {
return difference("group restart render_templates", a.RestartPolicy.RenderTemplates, b.RestartPolicy.RenderTemplates)
}
// Check each task
for _, at := range a.Tasks {
bt := b.LookupTask(at.Name)
@ -319,6 +325,11 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison {
if at.LogConfig.Disabled != bt.LogConfig.Disabled {
return difference("task log disabled", at.LogConfig.Disabled, bt.LogConfig.Disabled)
}
// Check if restart.render_templates is updated
if at.RestartPolicy.RenderTemplates != bt.RestartPolicy.RenderTemplates {
return difference("task restart render_templates", at.RestartPolicy.RenderTemplates, bt.RestartPolicy.RenderTemplates)
}
}
// none of the fields that trigger a destructive update were modified,

16
scheduler/util_test.go
View File

@ -1407,3 +1407,19 @@ func TestUtil_UpdateNonTerminalAllocsToLost(t *testing.T) {
expected = []string{}
require.True(t, reflect.DeepEqual(allocsLost, expected), "actual: %v, expected: %v", allocsLost, expected)
}
func TestTaskGroupUpdated_Restart(t *testing.T) {
ci.Parallel(t)
j1 := mock.Job()
name := j1.TaskGroups[0].Name
j2 := j1.Copy()
j3 := j1.Copy()
must.False(t, tasksUpdated(j1, j2, name).modified)
j2.TaskGroups[0].RestartPolicy.RenderTemplates = true
must.True(t, tasksUpdated(j1, j2, name).modified)
j3.TaskGroups[0].Tasks[0].RestartPolicy.RenderTemplates = true
must.True(t, tasksUpdated(j1, j3, name).modified)
}

9
website/content/docs/job-specification/restart.mdx
View File

@ -40,6 +40,7 @@ restart {
attempts = 2
delay = "15s"
mode = "fail"
render_templates = true
}
```
@ -59,6 +60,7 @@ restart {
attempts = 5
delay = "15s"
mode = "fail"
render_templates = true
}
```
@ -86,6 +88,11 @@ level, so that the Connect sidecar can inherit the default `restart`.
than `attempts` times in an interval. For a detailed explanation of these
values and their behavior, please see the [mode values section](#mode-values).
- `render_templates` `(bool: false)` - Specifies whether to re-render all
templates when a task is restarted. If set to `true`, all templates will be re-rendered
when the task restarts. This can be useful for re-fetching Vault secrets, even if the
lease on the existing secrets has not yet expired.
### `restart` Parameter Defaults
The values for many of the `restart` parameters vary by job type. Here are the
@ -99,6 +106,7 @@ defaults by job type:
delay = "15s"
interval = "24h"
mode = "fail"
render_templates = false
}
```
@ -110,6 +118,7 @@ defaults by job type:
attempts = 2
delay = "15s"
mode = "fail"
render_templates = false
}
```