Backport of feature: Add new field render_templates on restart block into release/1.6.x (#18094)

This pull request was automerged via backport-assistant

.changelog/18054.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+jobspec: Add new parameter `render_templates` for `restart` block to allow explicit re-render of templates on task restart. The default value is `false` and is fully backward compatible
+```

api/jobs_test.go

@@ -324,6 +324,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(2),
 							Interval:        pointerOf(30 * time.Minute),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -409,6 +410,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(3),
 							Interval:        pointerOf(24 * time.Hour),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(1),
@@ -499,6 +501,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(2),
 							Interval:        pointerOf(30 * time.Minute),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -667,6 +670,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(10),
 							Delay:           pointerOf(25 * time.Second),
 							Mode:            pointerOf("delay"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -711,6 +715,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 									Attempts:        pointerOf(20),
 									Delay:           pointerOf(25 * time.Second),
 									Mode:            pointerOf("delay"),
+									RenderTemplates: pointerOf(false),
 								},
 								Resources: &Resources{
 									CPU:      pointerOf(500),
@@ -835,7 +840,6 @@ func TestJobs_Canonicalize(t *testing.T) {
 				},
 			},
 		},
-
 		{
 			name: "update_merge",
 			input: &Job{
@@ -932,6 +936,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(2),
 							Interval:        pointerOf(30 * time.Minute),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -979,6 +984,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(2),
 							Interval:        pointerOf(30 * time.Minute),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -1016,7 +1022,6 @@ func TestJobs_Canonicalize(t *testing.T) {
 				},
 			},
 		},
-
 		{
 			name: "restart_merge",
 			input: &Job{
@@ -1038,6 +1043,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 								RestartPolicy: &RestartPolicy{
 									Attempts:        pointerOf(5),
 									Delay:           pointerOf(1 * time.Second),
+									RenderTemplates: pointerOf(true),
 								},
 							},
 						},
@@ -1109,6 +1115,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(2),
 							Interval:        pointerOf(30 * time.Minute),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -1144,6 +1151,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 									Delay:           pointerOf(1 * time.Second),
 									Interval:        pointerOf(30 * time.Minute),
 									Mode:            pointerOf("fail"),
+									RenderTemplates: pointerOf(true),
 								},
 							},
 						},
@@ -1161,6 +1169,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Attempts:        pointerOf(2),
 							Interval:        pointerOf(30 * time.Minute),
 							Mode:            pointerOf("fail"),
+							RenderTemplates: pointerOf(false),
 						},
 						ReschedulePolicy: &ReschedulePolicy{
 							Attempts:      pointerOf(0),
@@ -1196,6 +1205,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 									Attempts:        pointerOf(2),
 									Interval:        pointerOf(30 * time.Minute),
 									Mode:            pointerOf("fail"),
+									RenderTemplates: pointerOf(false),
 								},
 							},
 						},

api/tasks.go

@@ -92,6 +92,7 @@ type RestartPolicy struct {
 	Attempts        *int           `hcl:"attempts,optional"`
 	Delay           *time.Duration `hcl:"delay,optional"`
 	Mode            *string        `hcl:"mode,optional"`
+	RenderTemplates *bool          `mapstructure:"render_templates" hcl:"render_templates,optional"`
 }
 
 func (r *RestartPolicy) Merge(rp *RestartPolicy) {
@@ -107,6 +108,9 @@ func (r *RestartPolicy) Merge(rp *RestartPolicy) {
 	if rp.Mode != nil {
 		r.Mode = rp.Mode
 	}
+	if rp.RenderTemplates != nil {
+		r.RenderTemplates = rp.RenderTemplates
+	}
 }
 
 // Reschedule configures how Tasks are rescheduled  when they crash or fail.
@@ -584,6 +588,7 @@ func defaultServiceJobRestartPolicy() *RestartPolicy {
 		Attempts:        pointerOf(2),
 		Interval:        pointerOf(30 * time.Minute),
 		Mode:            pointerOf(RestartPolicyModeFail),
+		RenderTemplates: pointerOf(false),
 	}
 }
 
@@ -595,6 +600,7 @@ func defaultBatchJobRestartPolicy() *RestartPolicy {
 		Attempts:        pointerOf(3),
 		Interval:        pointerOf(24 * time.Hour),
 		Mode:            pointerOf(RestartPolicyModeFail),
+		RenderTemplates: pointerOf(false),
 	}
 }
 

client/allocrunner/taskrunner/task_runner_hooks.go

@@ -119,6 +119,7 @@ func (tr *TaskRunner) initHooks() {
 			envBuilder:          tr.envBuilder,
 			consulNamespace:     consulNamespace,
 			nomadNamespace:      tr.alloc.Job.Namespace,
+			renderOnTaskRestart: task.RestartPolicy.RenderTemplates,
 		}))
 	}
 

client/allocrunner/taskrunner/template_hook.go

@@ -45,6 +45,9 @@ type templateHookConfig struct {
 
 	// nomadNamespace is the job's Nomad namespace
 	nomadNamespace string
+
+	// renderOnTaskRestart is flag to explicitly render templates on task restart
+	renderOnTaskRestart bool
 }
 
 type templateHook struct {
@@ -97,8 +100,13 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
 
 	// If we have already run prerun before exit early.
 	if h.templateManager != nil {
+		if !h.config.renderOnTaskRestart {
 			return nil
 		}
+		h.logger.Info("re-rendering templates on task restart")
+		h.templateManager.Stop()
+		h.templateManager = nil
+	}
 
 	// Store the current Vault token and the task directory
 	h.taskDir = req.TaskDir.Dir

command/agent/job_endpoint.go

@@ -1062,6 +1062,7 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta
 		Interval:        *taskGroup.RestartPolicy.Interval,
 		Delay:           *taskGroup.RestartPolicy.Delay,
 		Mode:            *taskGroup.RestartPolicy.Mode,
+		RenderTemplates: *taskGroup.RestartPolicy.RenderTemplates,
 	}
 
 	if taskGroup.ShutdownDelay != nil {
@@ -1213,6 +1214,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
 			Interval:        *apiTask.RestartPolicy.Interval,
 			Delay:           *apiTask.RestartPolicy.Delay,
 			Mode:            *apiTask.RestartPolicy.Mode,
+			RenderTemplates: *apiTask.RestartPolicy.RenderTemplates,
 		}
 	}
 

command/agent/job_endpoint_test.go

@@ -2522,6 +2522,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 					Attempts:        pointer.Of(5),
 					Delay:           pointer.Of(10 * time.Second),
 					Mode:            pointer.Of("delay"),
+					RenderTemplates: pointer.Of(false),
 				},
 				ReschedulePolicy: &api.ReschedulePolicy{
 					Interval:      pointer.Of(12 * time.Hour),
@@ -2665,6 +2666,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 							Attempts:        pointer.Of(10),
 							Delay:           pointer.Of(20 * time.Second),
 							Mode:            pointer.Of("delay"),
+							RenderTemplates: pointer.Of(false),
 						},
 						Services: []*api.Service{
 							{
@@ -2931,6 +2933,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 					Attempts:        5,
 					Delay:           10 * time.Second,
 					Mode:            "delay",
+					RenderTemplates: false,
 				},
 				Spreads: []*structs.Spread{
 					{
@@ -3079,6 +3082,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 							Attempts:        10,
 							Delay:           20 * time.Second,
 							Mode:            "delay",
+							RenderTemplates: false,
 						},
 						Services: []*structs.Service{
 							{
@@ -3287,6 +3291,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 					Attempts:        pointer.Of(5),
 					Delay:           pointer.Of(10 * time.Second),
 					Mode:            pointer.Of("delay"),
+					RenderTemplates: pointer.Of(false),
 				},
 				EphemeralDisk: &api.EphemeralDisk{
 					SizeMB:  pointer.Of(100),
@@ -3408,6 +3413,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 					Attempts:        5,
 					Delay:           10 * time.Second,
 					Mode:            "delay",
+					RenderTemplates: false,
 				},
 				EphemeralDisk: &structs.EphemeralDisk{
 					SizeMB:  100,
@@ -3466,6 +3472,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 							Attempts:        5,
 							Delay:           10 * time.Second,
 							Mode:            "delay",
+							RenderTemplates: false,
 						},
 						Meta: map[string]string{
 							"lol": "code",

command/agent/testingutils_test.go

@@ -39,6 +39,7 @@ func MockJob() *api.Job {
 					Interval:        pointer.Of(10 * time.Minute),
 					Delay:           pointer.Of(1 * time.Minute),
 					Mode:            pointer.Of("delay"),
+					RenderTemplates: pointer.Of(false),
 				},
 				Networks: []*api.NetworkResource{
 					{

command/helpers_test.go

@@ -227,6 +227,7 @@ const (
       attempts         = 10
       mode             = "delay"
       interval         = "15s"
+      render_templates = false
     }
   }
 }`
@@ -246,6 +247,7 @@ var (
 					Attempts:        pointer.Of(10),
 					Interval:        pointer.Of(15 * time.Second),
 					Mode:            pointer.Of("delay"),
+					RenderTemplates: pointer.Of(false),
 				},
 
 				Tasks: []*api.Task{

jobspec/parse_group.go

@@ -321,6 +321,7 @@ func parseRestartPolicy(final **api.RestartPolicy, list *ast.ObjectList) error {
 		"interval",
 		"delay",
 		"mode",
+		"render_templates",
 	}
 	if err := checkHCLKeys(obj.Val, valid); err != nil {
 		return err

jobspec/parse_test.go

@@ -197,6 +197,7 @@ func TestParse(t *testing.T) {
 							Attempts:        intToPtr(5),
 							Delay:           timeToPtr(15 * time.Second),
 							Mode:            stringToPtr("delay"),
+							RenderTemplates: boolToPtr(false),
 						},
 						Spreads: []*api.Spread{
 							{

jobspec/test-fixtures/basic.hcl

@@ -106,6 +106,7 @@ job "binstore-storagelocker" {
       interval         = "10m"
       delay            = "15s"
       mode             = "delay"
+      render_templates = false
     }
 
     reschedule {

jobspec2/parse_test.go

@@ -1086,3 +1086,21 @@ func TestErrMissingKey(t *testing.T) {
 	require.NotNil(t, tmpl.ErrMissingKey)
 	require.True(t, *tmpl.ErrMissingKey)
 }
+
+func TestRestartRenderTemplates(t *testing.T) {
+	ci.Parallel(t)
+	hclBytes, err := os.ReadFile("test-fixtures/restart-render-templates.hcl")
+	require.NoError(t, err)
+	job, err := ParseWithConfig(&ParseConfig{
+		Path:    "test-fixtures/restart-render-templates.hcl",
+		Body:    hclBytes,
+		AllowFS: false,
+	})
+	require.NoError(t, err)
+	tg := job.TaskGroups[0]
+	require.NotNil(t, tg.RestartPolicy)
+	require.True(t, *tg.RestartPolicy.RenderTemplates)
+
+	require.Nil(t, tg.Tasks[0].RestartPolicy)
+	require.False(t, *tg.Tasks[1].RestartPolicy.RenderTemplates)
+}

jobspec2/test-fixtures/restart-render-templates.hcl

@@ -0,0 +1,17 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+job "example" {
+  group "group" {
+    restart {
+      render_templates = true
+    }
+    task "foo" {
+    }
+    task "bar" {
+      restart {
+        render_templates = false
+      }
+    }
+  }
+}

nomad/mock/job.go

@@ -48,6 +48,7 @@ func Job() *structs.Job {
 					Interval:        10 * time.Minute,
 					Delay:           1 * time.Minute,
 					Mode:            structs.RestartPolicyModeDelay,
+					RenderTemplates: false,
 				},
 				ReschedulePolicy: &structs.ReschedulePolicy{
 					Attempts:      2,

nomad/structs/diff_test.go

@@ -1861,6 +1861,12 @@ func TestTaskGroupDiff(t *testing.T) {
 								Old:  "",
 								New:  "fail",
 							},
+							{
+								Type: DiffTypeAdded,
+								Name: "RenderTemplates",
+								Old:  "",
+								New:  "false",
+							},
 						},
 					},
 				},
@@ -1908,6 +1914,12 @@ func TestTaskGroupDiff(t *testing.T) {
 								Old:  "fail",
 								New:  "",
 							},
+							{
+								Type: DiffTypeDeleted,
+								Name: "RenderTemplates",
+								Old:  "false",
+								New:  "",
+							},
 						},
 					},
 				},
@@ -1929,6 +1941,7 @@ func TestTaskGroupDiff(t *testing.T) {
 					Interval:        2 * time.Second,
 					Delay:           2 * time.Second,
 					Mode:            "delay",
+					RenderTemplates: true,
 				},
 			},
 			Expected: &TaskGroupDiff{
@@ -1962,6 +1975,12 @@ func TestTaskGroupDiff(t *testing.T) {
 								Old:  "fail",
 								New:  "delay",
 							},
+							{
+								Type: DiffTypeEdited,
+								Name: "RenderTemplates",
+								Old:  "false",
+								New:  "true",
+							},
 						},
 					},
 				},
@@ -1976,6 +1995,7 @@ func TestTaskGroupDiff(t *testing.T) {
 					Interval:        1 * time.Second,
 					Delay:           1 * time.Second,
 					Mode:            "fail",
+					RenderTemplates: false,
 				},
 			},
 			New: &TaskGroup{
@@ -1984,6 +2004,7 @@ func TestTaskGroupDiff(t *testing.T) {
 					Interval:        2 * time.Second,
 					Delay:           1 * time.Second,
 					Mode:            "fail",
+					RenderTemplates: true,
 				},
 			},
 			Expected: &TaskGroupDiff{
@@ -2017,6 +2038,12 @@ func TestTaskGroupDiff(t *testing.T) {
 								Old:  "fail",
 								New:  "fail",
 							},
+							{
+								Type: DiffTypeEdited,
+								Name: "RenderTemplates",
+								Old:  "false",
+								New:  "true",
+							},
 						},
 					},
 				},

nomad/structs/structs.go

@@ -5869,12 +5869,14 @@ var (
 		Attempts:        2,
 		Interval:        30 * time.Minute,
 		Mode:            RestartPolicyModeFail,
+		RenderTemplates: false,
 	}
 	DefaultBatchJobRestartPolicy = RestartPolicy{
 		Delay:           15 * time.Second,
 		Attempts:        3,
 		Interval:        24 * time.Hour,
 		Mode:            RestartPolicyModeFail,
+		RenderTemplates: false,
 	}
 )
 
@@ -6207,6 +6209,9 @@ type RestartPolicy struct {
 	// Mode controls what happens when the task restarts more than attempt times
 	// in an interval.
 	Mode string
+
+	// RenderTemplates is flag to explicitly render all templates on task restart
+	RenderTemplates bool
 }
 
 func (r *RestartPolicy) Copy() *RestartPolicy {

nomad/structs/structs_test.go

@@ -4310,6 +4310,7 @@ func TestRestartPolicy_Validate(t *testing.T) {
 		Mode:            RestartPolicyModeFail,
 		Attempts:        0,
 		Interval:        5 * time.Second,
+		RenderTemplates: true,
 	}
 	if err := p.Validate(); err != nil {
 		t.Fatalf("err: %v", err)

scheduler/util.go

@@ -258,6 +258,12 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison {
 		return difference("volume request", a.Volumes, b.Volumes)
 	}
 
+	// Check if restart.render_templates is updated
+	// this requires a destructive update for template hook to receive the new config
+	if a.RestartPolicy.RenderTemplates != b.RestartPolicy.RenderTemplates {
+		return difference("group restart render_templates", a.RestartPolicy.RenderTemplates, b.RestartPolicy.RenderTemplates)
+	}
+
 	// Check each task
 	for _, at := range a.Tasks {
 		bt := b.LookupTask(at.Name)
@@ -319,6 +325,11 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison {
 		if at.LogConfig.Disabled != bt.LogConfig.Disabled {
 			return difference("task log disabled", at.LogConfig.Disabled, bt.LogConfig.Disabled)
 		}
+
+		// Check if restart.render_templates is updated
+		if at.RestartPolicy.RenderTemplates != bt.RestartPolicy.RenderTemplates {
+			return difference("task restart render_templates", at.RestartPolicy.RenderTemplates, bt.RestartPolicy.RenderTemplates)
+		}
 	}
 
 	// none of the fields that trigger a destructive update were modified,

scheduler/util_test.go

@@ -1407,3 +1407,19 @@ func TestUtil_UpdateNonTerminalAllocsToLost(t *testing.T) {
 	expected = []string{}
 	require.True(t, reflect.DeepEqual(allocsLost, expected), "actual: %v, expected: %v", allocsLost, expected)
 }
+
+func TestTaskGroupUpdated_Restart(t *testing.T) {
+	ci.Parallel(t)
+
+	j1 := mock.Job()
+	name := j1.TaskGroups[0].Name
+	j2 := j1.Copy()
+	j3 := j1.Copy()
+
+	must.False(t, tasksUpdated(j1, j2, name).modified)
+	j2.TaskGroups[0].RestartPolicy.RenderTemplates = true
+	must.True(t, tasksUpdated(j1, j2, name).modified)
+
+	j3.TaskGroups[0].Tasks[0].RestartPolicy.RenderTemplates = true
+	must.True(t, tasksUpdated(j1, j3, name).modified)
+}

website/content/docs/job-specification/restart.mdx

@@ -40,6 +40,7 @@ restart {
   attempts         = 2
   delay            = "15s"
   mode             = "fail"
+  render_templates = true
 }
 ```
 
@@ -59,6 +60,7 @@ restart {
   attempts         = 5
   delay            = "15s"
   mode             = "fail"
+  render_templates = true
 }
 ```
 
@@ -86,6 +88,11 @@ level, so that the Connect sidecar can inherit the default `restart`.
   than `attempts` times in an interval. For a detailed explanation of these
   values and their behavior, please see the [mode values section](#mode-values).
 
+- `render_templates` `(bool: false)` - Specifies whether to re-render all 
+templates when a task is restarted. If set to `true`, all templates will be re-rendered
+when the task restarts. This can be useful for re-fetching Vault secrets, even if the
+lease on the existing secrets has not yet expired.
+
 ### `restart` Parameter Defaults
 
 The values for many of the `restart` parameters vary by job type. Here are the
@@ -99,6 +106,7 @@ defaults by job type:
     delay            = "15s"
     interval         = "24h"
     mode             = "fail"
+    render_templates = false
   }
   ```
 
@@ -110,6 +118,7 @@ defaults by job type:
     attempts         = 2
     delay            = "15s"
     mode             = "fail"
+    render_templates = false
   }
   ```