luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

template: custom change_mode scripts (#13972) 

This PR adds the functionality of allowing custom scripts to be executed on template change. Resolves #2707
This commit is contained in:
Piotr Kazmierczak 2022-08-24 17:43:01 +02:00 committed by GitHub
parent 848f2dcc22
commit 7077d1f9aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 775 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/13972.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
template: add script change_mode that allows scripts to be executed on template change
```

26
api/tasks.go
View File

@ -809,11 +809,34 @@ func (wc *WaitConfig) Copy() *WaitConfig {
return nwc
}
type ChangeScript struct {
Command *string `mapstructure:"command" hcl:"command"`
Args []string `mapstructure:"args" hcl:"args,optional"`
Timeout *time.Duration `mapstructure:"timeout" hcl:"timeout,optional"`
FailOnError *bool `mapstructure:"fail_on_error" hcl:"fail_on_error"`
}
func (ch *ChangeScript) Canonicalize() {
if ch.Command == nil {
ch.Command = pointerOf("")
}
if ch.Args == nil {
ch.Args = []string{}
}
if ch.Timeout == nil {
ch.Timeout = pointerOf(5 * time.Second)
}
if ch.FailOnError == nil {
ch.FailOnError = pointerOf(false)
}
}
type Template struct {
SourcePath *string `mapstructure:"source" hcl:"source,optional"`
DestPath *string `mapstructure:"destination" hcl:"destination,optional"`
EmbeddedTmpl *string `mapstructure:"data" hcl:"data,optional"`
ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"`
ChangeScript *ChangeScript `mapstructure:"change_script" hcl:"change_script,block"`
ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"`
Splay *time.Duration `mapstructure:"splay" hcl:"splay,optional"`
Perms *string `mapstructure:"perms" hcl:"perms,optional"`
@ -849,6 +872,9 @@ func (tmpl *Template) Canonicalize() {
sig := *tmpl.ChangeSignal
tmpl.ChangeSignal = pointerOf(strings.ToUpper(sig))
}
if tmpl.ChangeScript != nil {
tmpl.ChangeScript.Canonicalize()
}
if tmpl.Splay == nil {
tmpl.Splay = pointerOf(5 * time.Second)
}

81
client/allocrunner/taskrunner/template/template.go
View File

@ -54,6 +54,10 @@ type TaskTemplateManager struct {
// runner is the consul-template runner
runner *manager.Runner
// handle is used to execute scripts
handle interfaces.ScriptExecutor
handleLock sync.Mutex
// signals is a lookup map from the string representation of a signal to its
// actual signal
signals map[string]os.Signal
@ -192,6 +196,14 @@ func (tm *TaskTemplateManager) Stop() {
}
}
// SetDriverHandle sets the executor
func (tm *TaskTemplateManager) SetDriverHandle(executor interfaces.ScriptExecutor) {
tm.handleLock.Lock()
defer tm.handleLock.Unlock()
tm.handle = executor
}
// run is the long lived loop that handles errors and templates being rendered
func (tm *TaskTemplateManager) run() {
// Runner is nil if there are no templates
@ -392,6 +404,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
var handling []string
signals := make(map[string]struct{})
scripts := []*structs.ChangeScript{}
restart := false
var splay time.Duration
@ -436,6 +449,8 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
signals[tmpl.ChangeSignal] = struct{}{}
case structs.TemplateChangeModeRestart:
restart = true
case structs.TemplateChangeModeScript:
scripts = append(scripts, tmpl.ChangeScript)
case structs.TemplateChangeModeNoop:
continue
}
@ -494,6 +509,72 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
}
}
// process script execution concurrently
var wg sync.WaitGroup
for _, script := range scripts {
wg.Add(1)
go tm.processScript(script, &wg)
}
wg.Wait()
}
// handleScriptError is a helper function that produces a TaskKilling event and
// emits a message
func (tm *TaskTemplateManager) handleScriptError(script *structs.ChangeScript, msg string) {
ev := structs.NewTaskEvent(structs.TaskHookFailed).SetDisplayMessage(msg)
tm.config.Events.EmitEvent(ev)
if script.FailOnError {
tm.config.Lifecycle.Kill(context.Background(),
structs.NewTaskEvent(structs.TaskKilling).
SetFailsTask().
SetDisplayMessage("Template script failed, task is being killed"))
}
}
// processScript is used for executing change_mode script and handling errors
func (tm *TaskTemplateManager) processScript(script *structs.ChangeScript, wg *sync.WaitGroup) {
defer wg.Done()
if tm.handle == nil {
failureMsg := fmt.Sprintf(
"Template failed to run script %v with arguments %v because task driver doesn't support the exec operation",
script.Command,
script.Args,
)
tm.handleScriptError(script, failureMsg)
return
}
_, exitCode, err := tm.handle.Exec(script.Timeout, script.Command, script.Args)
if err != nil {
failureMsg := fmt.Sprintf(
"Template failed to run script %v with arguments %v on change: %v Exit code: %v",
script.Command,
script.Args,
err,
exitCode,
)
tm.handleScriptError(script, failureMsg)
return
}
if exitCode != 0 {
failureMsg := fmt.Sprintf(
"Template ran script %v with arguments %v on change but it exited with code code: %v",
script.Command,
script.Args,
exitCode,
)
tm.handleScriptError(script, failureMsg)
return
}
tm.config.Events.EmitEvent(structs.NewTaskEvent(structs.TaskHookMessage).
SetDisplayMessage(
fmt.Sprintf(
"Template successfully ran script %v with arguments: %v. Exit code: %v",
script.Command,
script.Args,
exitCode,
)))
}
// allTemplatesNoop returns whether all the managed templates have change mode noop.

224
client/allocrunner/taskrunner/template/template_test.go
View File

@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"os/user"
"path/filepath"
@ -123,6 +122,16 @@ func (m *MockTaskHooks) EmitEvent(event *structs.TaskEvent) {
func (m *MockTaskHooks) SetState(state string, event *structs.TaskEvent) {}
// mockExecutor implements script executor interface
type mockExecutor struct {
DesiredExit int
DesiredErr error
}
func (m *mockExecutor) Exec(timeout time.Duration, cmd string, args []string) ([]byte, int, error) {
return []byte{}, m.DesiredExit, m.DesiredErr
}
// testHarness is used to test the TaskTemplateManager by spinning up
// Consul/Vault as needed
type testHarness struct {
@ -213,7 +222,6 @@ func (h *testHarness) startWithErr() error {
EnvBuilder: h.envBuilder,
MaxTemplateEventRate: h.emitRate,
})
return err
}
@ -381,7 +389,7 @@ func TestTaskTemplateManager_InvalidConfig(t *testing.T) {
func TestTaskTemplateManager_HostPath(t *testing.T) {
ci.Parallel(t)
// Make a template that will render immediately and write it to a tmp file
f, err := ioutil.TempFile("", "")
f, err := os.CreateTemp("", "")
if err != nil {
t.Fatalf("Bad: %v", err)
}
@ -417,7 +425,7 @@ func TestTaskTemplateManager_HostPath(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -494,7 +502,7 @@ func TestTaskTemplateManager_Unblock_Static(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -573,7 +581,7 @@ func TestTaskTemplateManager_Unblock_Static_NomadEnv(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -598,7 +606,7 @@ func TestTaskTemplateManager_Unblock_Static_AlreadyRendered(t *testing.T) {
// Write the contents
path := filepath.Join(harness.taskDir, file)
if err := ioutil.WriteFile(path, []byte(content), 0777); err != nil {
if err := os.WriteFile(path, []byte(content), 0777); err != nil {
t.Fatalf("Failed to write data: %v", err)
}
@ -614,7 +622,7 @@ func TestTaskTemplateManager_Unblock_Static_AlreadyRendered(t *testing.T) {
// Check the file is there
path = filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -660,7 +668,7 @@ func TestTaskTemplateManager_Unblock_Consul(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -710,7 +718,7 @@ func TestTaskTemplateManager_Unblock_Vault(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -755,7 +763,7 @@ func TestTaskTemplateManager_Unblock_Multi_Template(t *testing.T) {
// Check that the static file has been rendered
path := filepath.Join(harness.taskDir, staticFile)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -776,7 +784,7 @@ func TestTaskTemplateManager_Unblock_Multi_Template(t *testing.T) {
// Check the consul file is there
path = filepath.Join(harness.taskDir, consulFile)
raw, err = ioutil.ReadFile(path)
raw, err = os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -828,7 +836,7 @@ func TestTaskTemplateManager_FirstRender_Restored(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
require.NoError(err, "Failed to read rendered template from %q", path)
require.Equal(content, string(raw), "Unexpected template data; got %s, want %q", raw, content)
@ -922,7 +930,7 @@ func TestTaskTemplateManager_Rerender_Noop(t *testing.T) {
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -944,7 +952,7 @@ func TestTaskTemplateManager_Rerender_Noop(t *testing.T) {
// Check the file has been updated
path = filepath.Join(harness.taskDir, file)
raw, err = ioutil.ReadFile(path)
raw, err = os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -1034,7 +1042,7 @@ OUTER:
// Check the files have been updated
path := filepath.Join(harness.taskDir, file1)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -1044,7 +1052,7 @@ OUTER:
}
path = filepath.Join(harness.taskDir, file2)
raw, err = ioutil.ReadFile(path)
raw, err = os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -1108,7 +1116,7 @@ OUTER:
// Check the files have been updated
path := filepath.Join(harness.taskDir, file1)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -1143,7 +1151,7 @@ func TestTaskTemplateManager_Interpolate_Destination(t *testing.T) {
// Check the file is there
actual := fmt.Sprintf("%s.tmpl", harness.node.ID)
path := filepath.Join(harness.taskDir, actual)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
@ -1201,6 +1209,168 @@ func TestTaskTemplateManager_Signal_Error(t *testing.T) {
require.Contains(harness.mockHooks.KillEvent.DisplayMessage, "failed to send signals")
}
func TestTaskTemplateManager_ScriptExecution(t *testing.T) {
ci.Parallel(t)
// Make a template that renders based on a key in Consul and triggers script
key1 := "bam"
key2 := "bar"
content1_1 := "cat"
content1_2 := "dog"
t1 := &structs.Template{
EmbeddedTmpl: `
FOO={{key "bam"}}
`,
DestPath: "test.env",
ChangeMode: structs.TemplateChangeModeScript,
ChangeScript: &structs.ChangeScript{
Command: "/bin/foo",
Args: []string{},
Timeout: 5 * time.Second,
FailOnError: false,
},
Envvars: true,
}
t2 := &structs.Template{
EmbeddedTmpl: `
BAR={{key "bar"}}
`,
DestPath: "test2.env",
ChangeMode: structs.TemplateChangeModeScript,
ChangeScript: &structs.ChangeScript{
Command: "/bin/foo",
Args: []string{},
Timeout: 5 * time.Second,
FailOnError: false,
},
Envvars: true,
}
me := mockExecutor{DesiredExit: 0, DesiredErr: nil}
harness := newTestHarness(t, []*structs.Template{t1, t2}, true, false)
harness.start(t)
harness.manager.SetDriverHandle(&me)
defer harness.stop()
// Ensure no unblock
select {
case <-harness.mockHooks.UnblockCh:
require.Fail(t, "Task unblock should not have been called")
case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
}
// Write the key to Consul
harness.consul.SetKV(t, key1, []byte(content1_1))
harness.consul.SetKV(t, key2, []byte(content1_1))
// Wait for the unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
require.Fail(t, "Task unblock should have been called")
}
// Update the keys in Consul
harness.consul.SetKV(t, key1, []byte(content1_2))
// Wait for restart
timeout := time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second)
OUTER:
for {
select {
case <-harness.mockHooks.RestartCh:
require.Fail(t, "restart not expected")
case ev := <-harness.mockHooks.EmitEventCh:
if strings.Contains(ev.DisplayMessage, t1.ChangeScript.Command) {
break OUTER
}
case <-harness.mockHooks.SignalCh:
require.Fail(t, "signal not expected")
case <-timeout:
require.Fail(t, "should have received an event")
}
}
}
// TestTaskTemplateManager_ScriptExecutionFailTask tests whether we fail the
// task upon script execution failure if that's how it's configured.
func TestTaskTemplateManager_ScriptExecutionFailTask(t *testing.T) {
ci.Parallel(t)
require := require.New(t)
// Make a template that renders based on a key in Consul and triggers script
key1 := "bam"
key2 := "bar"
content1_1 := "cat"
content1_2 := "dog"
t1 := &structs.Template{
EmbeddedTmpl: `
FOO={{key "bam"}}
`,
DestPath: "test.env",
ChangeMode: structs.TemplateChangeModeScript,
ChangeScript: &structs.ChangeScript{
Command: "/bin/foo",
Args: []string{},
Timeout: 5 * time.Second,
FailOnError: true,
},
Envvars: true,
}
t2 := &structs.Template{
EmbeddedTmpl: `
BAR={{key "bar"}}
`,
DestPath: "test2.env",
ChangeMode: structs.TemplateChangeModeScript,
ChangeScript: &structs.ChangeScript{
Command: "/bin/foo",
Args: []string{},
Timeout: 5 * time.Second,
FailOnError: false,
},
Envvars: true,
}
me := mockExecutor{DesiredExit: 1, DesiredErr: fmt.Errorf("Script failed")}
harness := newTestHarness(t, []*structs.Template{t1, t2}, true, false)
harness.start(t)
harness.manager.SetDriverHandle(&me)
defer harness.stop()
// Ensure no unblock
select {
case <-harness.mockHooks.UnblockCh:
require.Fail("Task unblock should not have been called")
case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
}
// Write the key to Consul
harness.consul.SetKV(t, key1, []byte(content1_1))
harness.consul.SetKV(t, key2, []byte(content1_1))
// Wait for the unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
require.Fail("Task unblock should have been called")
}
// Update the keys in Consul
harness.consul.SetKV(t, key1, []byte(content1_2))
// Wait for kill channel
select {
case <-harness.mockHooks.KillCh:
break
case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
require.Fail("Should have received a signals: %+v", harness.mockHooks)
}
require.NotNil(harness.mockHooks.KillEvent)
require.Contains(harness.mockHooks.KillEvent.DisplayMessage, "task is being killed")
}
// TestTaskTemplateManager_FiltersProcessEnvVars asserts that we only render
// environment variables found in task env-vars and not read the nomad host
// process environment variables. nomad host process environment variables
@ -1241,7 +1411,7 @@ TEST_ENV_NOT_FOUND: {{env "` + testenv + `_NOTFOUND" }}`
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
raw, err := os.ReadFile(path)
require.NoError(t, err)
require.Equal(t, expected, string(raw))
@ -1297,7 +1467,7 @@ func TestTaskTemplateManager_Env_Missing(t *testing.T) {
d := t.TempDir()
// Fake writing the file so we don't have to run the whole template manager
err := ioutil.WriteFile(filepath.Join(d, "exists.env"), []byte("FOO=bar\n"), 0644)
err := os.WriteFile(filepath.Join(d, "exists.env"), []byte("FOO=bar\n"), 0644)
if err != nil {
t.Fatalf("error writing template file: %v", err)
}
@ -1330,7 +1500,7 @@ func TestTaskTemplateManager_Env_InterpolatedDest(t *testing.T) {
d := t.TempDir()
// Fake writing the file so we don't have to run the whole template manager
err := ioutil.WriteFile(filepath.Join(d, "exists.env"), []byte("FOO=bar\n"), 0644)
err := os.WriteFile(filepath.Join(d, "exists.env"), []byte("FOO=bar\n"), 0644)
if err != nil {
t.Fatalf("error writing template file: %v", err)
}
@ -1365,11 +1535,11 @@ func TestTaskTemplateManager_Env_Multi(t *testing.T) {
d := t.TempDir()
// Fake writing the files so we don't have to run the whole template manager
err := ioutil.WriteFile(filepath.Join(d, "zzz.env"), []byte("FOO=bar\nSHARED=nope\n"), 0644)
err := os.WriteFile(filepath.Join(d, "zzz.env"), []byte("FOO=bar\nSHARED=nope\n"), 0644)
if err != nil {
t.Fatalf("error writing template file 1: %v", err)
}
err = ioutil.WriteFile(filepath.Join(d, "aaa.env"), []byte("BAR=foo\nSHARED=yup\n"), 0644)
err = os.WriteFile(filepath.Join(d, "aaa.env"), []byte("BAR=foo\nSHARED=yup\n"), 0644)
if err != nil {
t.Fatalf("error writing template file 2: %v", err)
}
@ -2209,7 +2379,7 @@ func TestTaskTemplateManager_writeToFile_Disabled(t *testing.T) {
// Check the file is not there
path := filepath.Join(harness.taskDir, file)
_, err := ioutil.ReadFile(path)
_, err := os.ReadFile(path)
require.Error(t, err)
}
@ -2262,13 +2432,13 @@ func TestTaskTemplateManager_writeToFile(t *testing.T) {
// Check the templated file is there
path := filepath.Join(harness.taskDir, file)
r, err := ioutil.ReadFile(path)
r, err := os.ReadFile(path)
require.NoError(t, err)
require.True(t, bytes.HasSuffix(r, []byte("...done\n")), string(r))
// Check that writeToFile was allowed
path = filepath.Join(harness.taskDir, "writetofile.out")
r, err = ioutil.ReadFile(path)
r, err = os.ReadFile(path)
require.NoError(t, err)
require.Equal(t, "hello", string(r))
}

13
client/allocrunner/taskrunner/template_hook.go
View File

@ -115,6 +115,19 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
return nil
}
func (h *templateHook) Poststart(ctx context.Context, req *interfaces.TaskPoststartRequest, resp *interfaces.TaskPoststartResponse) error {
if req.DriverExec != nil {
h.templateManager.SetDriverHandle(req.DriverExec)
} else {
for _, template := range h.config.templates {
if template.ChangeMode == structs.TemplateChangeModeScript {
return fmt.Errorf("template has change mode set to 'script' but the driver it uses does not provide exec capability")
}
}
}
return nil
}
func (h *templateHook) newManager() (unblock chan struct{}, err error) {
unblock = make(chan struct{})
m, err := template.NewTaskTemplateManager(&template.TaskTemplateManagerConfig{

24
command/agent/job_endpoint.go
View File

@ -1216,6 +1216,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
EmbeddedTmpl: *template.EmbeddedTmpl,
ChangeMode: *template.ChangeMode,
ChangeSignal: *template.ChangeSignal,
ChangeScript: apiChangeScriptToStructsChangeScript(template.ChangeScript),
Splay: *template.Splay,
Perms: *template.Perms,
Uid: template.Uid,
@ -1224,7 +1225,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
RightDelim: *template.RightDelim,
Envvars: *template.Envvars,
VaultGrace: *template.VaultGrace,
Wait: ApiWaitConfigToStructsWaitConfig(template.Wait),
Wait: apiWaitConfigToStructsWaitConfig(template.Wait),
})
}
}
@ -1243,16 +1244,29 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
}
}
// ApiWaitConfigToStructsWaitConfig is a copy and type conversion between the API
// apiWaitConfigToStructsWaitConfig is a copy and type conversion between the API
// representation of a WaitConfig from a struct representation of a WaitConfig.
func ApiWaitConfigToStructsWaitConfig(waitConfig *api.WaitConfig) *structs.WaitConfig {
func apiWaitConfigToStructsWaitConfig(waitConfig *api.WaitConfig) *structs.WaitConfig {
if waitConfig == nil {
return nil
}
return &structs.WaitConfig{
Min: &*waitConfig.Min,
Max: &*waitConfig.Max,
Min: waitConfig.Min,
Max: waitConfig.Max,
}
}
func apiChangeScriptToStructsChangeScript(changeScript *api.ChangeScript) *structs.ChangeScript {
if changeScript == nil {
return nil
}
return &structs.ChangeScript{
Command: *changeScript.Command,
Args: changeScript.Args,
Timeout: *changeScript.Timeout,
FailOnError: *changeScript.FailOnError,
}
}

40
command/agent/job_endpoint_test.go
View File

@ -2730,13 +2730,19 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
EmbeddedTmpl: pointer.Of("embedded"),
ChangeMode: pointer.Of("change"),
ChangeSignal: pointer.Of("signal"),
Splay: pointer.Of(1 * time.Minute),
Perms: pointer.Of("666"),
Uid: pointer.Of(1000),
Gid: pointer.Of(1000),
LeftDelim: pointer.Of("abc"),
RightDelim: pointer.Of("def"),
Envvars: pointer.Of(true),
ChangeScript: &api.ChangeScript{
Command: pointer.Of("/bin/foo"),
Args: []string{"-h"},
Timeout: pointer.Of(5 * time.Second),
FailOnError: pointer.Of(false),
},
Splay: pointer.Of(1 * time.Minute),
Perms: pointer.Of("666"),
Uid: pointer.Of(1000),
Gid: pointer.Of(1000),
LeftDelim: pointer.Of("abc"),
RightDelim: pointer.Of("def"),
Envvars: pointer.Of(true),
Wait: &api.WaitConfig{
Min: pointer.Of(5 * time.Second),
Max: pointer.Of(10 * time.Second),
@ -3137,13 +3143,19 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
EmbeddedTmpl: "embedded",
ChangeMode: "change",
ChangeSignal: "SIGNAL",
Splay: 1 * time.Minute,
Perms: "666",
Uid: pointer.Of(1000),
Gid: pointer.Of(1000),
LeftDelim: "abc",
RightDelim: "def",
Envvars: true,
ChangeScript: &structs.ChangeScript{
Command: "/bin/foo",
Args: []string{"-h"},
Timeout: 5 * time.Second,
FailOnError: false,
},
Splay: 1 * time.Minute,
Perms: "666",
Uid: pointer.Of(1000),
Gid: pointer.Of(1000),
LeftDelim: "abc",
RightDelim: "def",
Envvars: true,
Wait: &structs.WaitConfig{
Min: pointer.Of(5 * time.Second),
Max: pointer.Of(10 * time.Second),

44
jobspec/parse_task.go
View File

@ -433,10 +433,19 @@ func parseArtifactOption(result map[string]string, list *ast.ObjectList) error {
func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error {
for _, o := range list.Elem().Items {
// we'll need a list of all ast objects for later
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("should be an object")
}
// Check for invalid keys
valid := []string{
"change_mode",
"change_signal",
"change_script",
"data",
"destination",
"left_delimiter",
@ -457,6 +466,7 @@ func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
delete(m, "change_script") // change_script is its own object
templ := &api.Template{
ChangeMode: stringToPtr("restart"),
@ -476,6 +486,40 @@ func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error {
return err
}
// If we have change_script, parse it
if o := listVal.Filter("change_script"); len(o.Items) > 0 {
if len(o.Items) != 1 {
return fmt.Errorf(
"change_script -> expected single stanza, got %d", len(o.Items),
)
}
var m map[string]interface{}
changeScriptBlock := o.Items[0]
// check for invalid fields
valid := []string{"command", "args", "timeout", "fail_on_error"}
if err := checkHCLKeys(changeScriptBlock.Val, valid); err != nil {
return multierror.Prefix(err, "change_script ->")
}
if err := hcl.DecodeObject(&m, changeScriptBlock.Val); err != nil {
return err
}
templ.ChangeScript = &api.ChangeScript{}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: templ.ChangeScript,
})
if err != nil {
return err
}
if err := dec.Decode(m); err != nil {
return err
}
}
*result = append(*result, templ)
}

12
jobspec/parse_test.go
View File

@ -22,6 +22,10 @@ const (
// templateChangeModeRestart marks that the task should be restarted if the
templateChangeModeRestart = "restart"
// templateChangeModeScript marks that ac script should be executed on
// template re-render
templateChangeModeScript = "script"
)
// Helper functions below are only used by this test suite
@ -380,7 +384,13 @@ func TestParse(t *testing.T) {
{
SourcePath: stringToPtr("bar"),
DestPath: stringToPtr("bar"),
ChangeMode: stringToPtr(templateChangeModeRestart),
ChangeMode: stringToPtr(templateChangeModeScript),
ChangeScript: &api.ChangeScript{
Args: []string{"-debug", "-verbose"},
Command: stringToPtr("/bin/foo"),
Timeout: timeToPtr(5 * time.Second),
FailOnError: boolToPtr(false),
},
Splay: timeToPtr(5 * time.Second),
Perms: stringToPtr("777"),
Uid: intToPtr(1001),

11
jobspec/test-fixtures/basic.hcl
View File

@ -315,8 +315,15 @@ job "binstore-storagelocker" {
}
template {
source = "bar"
destination = "bar"
source = "bar"
destination = "bar"
change_mode = "script"
change_script {
command = "/bin/foo"
args = ["-debug", "-verbose"]
timeout = "5s"
fail_on_error = false
}
perms = "777"
uid = 1001
gid = 20

3
jobspec2/hcl_conversions.go
View File

@ -10,6 +10,7 @@ import (
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/helper/pointer"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
)
@ -116,7 +117,7 @@ func decodeAffinity(body hcl.Body, ctx *hcl.EvalContext, val interface{}) hcl.Di
weight := v.GetAttr("weight")
if !weight.IsNull() {
w, _ := weight.AsBigFloat().Int64()
a.Weight = int8ToPtr(int8(w))
a.Weight = pointer.Of(int8(w))
}
// If "version" is provided, set the operand

36
jobspec2/parse_job.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/helper/pointer"
)
func normalizeJob(jc *jobConfig) {
@ -59,10 +60,10 @@ func normalizeVault(v *api.Vault) {
}
if v.Env == nil {
v.Env = boolToPtr(true)
v.Env = pointer.Of(true)
}
if v.ChangeMode == nil {
v.ChangeMode = stringToPtr("restart")
v.ChangeMode = pointer.Of("restart")
}
}
@ -102,29 +103,32 @@ func normalizeTemplates(templates []*api.Template) {
for _, t := range templates {
if t.ChangeMode == nil {
t.ChangeMode = stringToPtr("restart")
t.ChangeMode = pointer.Of("restart")
}
if t.Perms == nil {
t.Perms = stringToPtr("0644")
t.Perms = pointer.Of("0644")
}
if t.Splay == nil {
t.Splay = durationToPtr(5 * time.Second)
t.Splay = pointer.Of(5 * time.Second)
}
normalizeChangeScript(t.ChangeScript)
}
}
func int8ToPtr(v int8) *int8 {
return &v
}
func normalizeChangeScript(ch *api.ChangeScript) {
if ch == nil {
return
}
func boolToPtr(v bool) *bool {
return &v
}
if ch.Args == nil {
ch.Args = []string{}
}
func stringToPtr(v string) *string {
return &v
}
if ch.Timeout == nil {
ch.Timeout = pointer.Of(5 * time.Second)
}
func durationToPtr(v time.Duration) *time.Duration {
return &v
if ch.FailOnError == nil {
ch.FailOnError = pointer.Of(false)
}
}

31
jobspec2/parse_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/ci"
"github.com/hashicorp/nomad/helper/pointer"
"github.com/hashicorp/nomad/jobspec"
"github.com/stretchr/testify/require"
)
@ -644,13 +645,13 @@ job "job-webserver" {
{
"prod",
&api.Job{
ID: stringToPtr("job-webserver"),
Name: stringToPtr("job-webserver"),
ID: pointer.Of("job-webserver"),
Name: pointer.Of("job-webserver"),
Datacenters: []string{"prod-dc1", "prod-dc2"},
TaskGroups: []*api.TaskGroup{
{
Name: stringToPtr("group-webserver"),
Count: intToPtr(20),
Name: pointer.Of("group-webserver"),
Count: pointer.Of(20),
Tasks: []*api.Task{
{
@ -670,13 +671,13 @@ job "job-webserver" {
{
"staging",
&api.Job{
ID: stringToPtr("job-webserver"),
Name: stringToPtr("job-webserver"),
ID: pointer.Of("job-webserver"),
Name: pointer.Of("job-webserver"),
Datacenters: []string{"dc1"},
TaskGroups: []*api.TaskGroup{
{
Name: stringToPtr("group-webserver"),
Count: intToPtr(3),
Name: pointer.Of("group-webserver"),
Count: pointer.Of(3),
Tasks: []*api.Task{
{
@ -696,13 +697,13 @@ job "job-webserver" {
{
"unknown",
&api.Job{
ID: stringToPtr("job-webserver"),
Name: stringToPtr("job-webserver"),
ID: pointer.Of("job-webserver"),
Name: pointer.Of("job-webserver"),
Datacenters: []string{},
TaskGroups: []*api.TaskGroup{
{
Name: stringToPtr("group-webserver"),
Count: intToPtr(0),
Name: pointer.Of("group-webserver"),
Count: pointer.Of(0),
Tasks: []*api.Task{
{
@ -1005,11 +1006,11 @@ func TestParseServiceCheck(t *testing.T) {
require.NoError(t, err)
expectedJob := &api.Job{
ID: stringToPtr("group_service_check_script"),
Name: stringToPtr("group_service_check_script"),
ID: pointer.Of("group_service_check_script"),
Name: pointer.Of("group_service_check_script"),
TaskGroups: []*api.TaskGroup{
{
Name: stringToPtr("group"),
Name: pointer.Of("group"),
Services: []*api.Service{
{
Name: "foo-service",

40
nomad/structs/diff.go
View File

@ -1649,6 +1649,39 @@ func waitConfigDiff(old, new *WaitConfig, contextual bool) *ObjectDiff {
return diff
}
// changeScriptDiff returns the diff of two ChangeScript objects. If contextual
// diff is enabled, all fields will be returned, even if no diff occurred.
func changeScriptDiff(old, new *ChangeScript, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "ChangeScript"}
var oldPrimitiveFlat, newPrimitiveFlat map[string]string
if reflect.DeepEqual(old, new) {
return nil
} else if old == nil {
old = &ChangeScript{}
diff.Type = DiffTypeAdded
newPrimitiveFlat = flatmap.Flatten(new, nil, true)
} else if new == nil {
new = &ChangeScript{}
diff.Type = DiffTypeDeleted
oldPrimitiveFlat = flatmap.Flatten(old, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimitiveFlat = flatmap.Flatten(old, nil, true)
newPrimitiveFlat = flatmap.Flatten(new, nil, true)
}
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
// Args diffs
if setDiff := stringSetDiff(old.Args, new.Args, "Args", contextual); setDiff != nil {
diff.Objects = append(diff.Objects, setDiff)
}
return diff
}
// templateDiff returns the diff of two Consul Template objects. If contextual diff is
// enabled, all fields will be returned, even if no diff occurred.
func templateDiff(old, new *Template, contextual bool) *ObjectDiff {
@ -1697,6 +1730,13 @@ func templateDiff(old, new *Template, contextual bool) *ObjectDiff {
diff.Objects = append(diff.Objects, waitDiffs)
}
// ChangeScript diffs
if changeScriptDiffs := changeScriptDiff(
old.ChangeScript, new.ChangeScript, contextual,
); changeScriptDiffs != nil {
diff.Objects = append(diff.Objects, changeScriptDiffs)
}
return diff
}

136
nomad/structs/diff_test.go
View File

@ -7042,10 +7042,16 @@ func TestTaskDiff(t *testing.T) {
EmbeddedTmpl: "baz",
ChangeMode: "bam",
ChangeSignal: "SIGHUP",
Splay: 1,
Perms: "0644",
Uid: pointer.Of(1001),
Gid: pointer.Of(21),
ChangeScript: &ChangeScript{
Command: "/bin/foo",
Args: []string{"-debug"},
Timeout: 5,
FailOnError: false,
},
Splay: 1,
Perms: "0644",
Uid: pointer.Of(1001),
Gid: pointer.Of(21),
Wait: &WaitConfig{
Min: pointer.Of(5 * time.Second),
Max: pointer.Of(5 * time.Second),
@ -7057,11 +7063,17 @@ func TestTaskDiff(t *testing.T) {
EmbeddedTmpl: "baz2",
ChangeMode: "bam2",
ChangeSignal: "SIGHUP2",
Splay: 2,
Perms: "0666",
Uid: pointer.Of(1000),
Gid: pointer.Of(20),
Envvars: true,
ChangeScript: &ChangeScript{
Command: "/bin/foo2",
Args: []string{"-debugs"},
Timeout: 6,
FailOnError: false,
},
Splay: 2,
Perms: "0666",
Uid: pointer.Of(1000),
Gid: pointer.Of(20),
Envvars: true,
},
},
},
@ -7073,10 +7085,16 @@ func TestTaskDiff(t *testing.T) {
EmbeddedTmpl: "baz new",
ChangeMode: "bam",
ChangeSignal: "SIGHUP",
Splay: 1,
Perms: "0644",
Uid: pointer.Of(1001),
Gid: pointer.Of(21),
ChangeScript: &ChangeScript{
Command: "/bin/foo",
Args: []string{"-debug"},
Timeout: 5,
FailOnError: false,
},
Splay: 1,
Perms: "0644",
Uid: pointer.Of(1001),
Gid: pointer.Of(21),
Wait: &WaitConfig{
Min: pointer.Of(5 * time.Second),
Max: pointer.Of(10 * time.Second),
@ -7088,10 +7106,16 @@ func TestTaskDiff(t *testing.T) {
EmbeddedTmpl: "baz3",
ChangeMode: "bam3",
ChangeSignal: "SIGHUP3",
Splay: 3,
Perms: "0776",
Uid: pointer.Of(1002),
Gid: pointer.Of(22),
ChangeScript: &ChangeScript{
Command: "/bin/foo3",
Args: []string{"-debugss"},
Timeout: 7,
FailOnError: false,
},
Splay: 3,
Perms: "0776",
Uid: pointer.Of(1002),
Gid: pointer.Of(22),
Wait: &WaitConfig{
Min: pointer.Of(5 * time.Second),
Max: pointer.Of(10 * time.Second),
@ -7218,6 +7242,44 @@ func TestTaskDiff(t *testing.T) {
},
},
},
{
Type: DiffTypeAdded,
Name: "ChangeScript",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "Command",
Old: "",
New: "/bin/foo3",
},
{
Type: DiffTypeAdded,
Name: "FailOnError",
Old: "",
New: "false",
},
{
Type: DiffTypeAdded,
Name: "Timeout",
Old: "",
New: "7",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeAdded,
Name: "Args",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "Args",
Old: "",
New: "-debugss",
},
},
},
},
},
},
},
{
@ -7291,6 +7353,46 @@ func TestTaskDiff(t *testing.T) {
New: "",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeDeleted,
Name: "ChangeScript",
Fields: []*FieldDiff{
{
Type: DiffTypeDeleted,
Name: "Command",
Old: "/bin/foo2",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "FailOnError",
Old: "false",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "Timeout",
Old: "6",
New: "",
},
},
Objects: []*ObjectDiff{
{
Type: DiffTypeDeleted,
Name: "Args",
Fields: []*FieldDiff{
{
Type: DiffTypeDeleted,
Name: "Args",
Old: "-debugs",
New: "",
},
},
},
},
},
},
},
},
},

32
nomad/structs/structs.go
View File

@ -7631,14 +7631,32 @@ const (
// TemplateChangeModeRestart marks that the task should be restarted if the
// template is re-rendered
TemplateChangeModeRestart = "restart"
// TemplateChangeModeScript marks that the task should trigger a script if
// the template is re-rendered
TemplateChangeModeScript = "script"
)
var (
// TemplateChangeModeInvalidError is the error for when an invalid change
// mode is given
TemplateChangeModeInvalidError = errors.New("Invalid change mode. Must be one of the following: noop, signal, restart")
TemplateChangeModeInvalidError = errors.New("Invalid change mode. Must be one of the following: noop, signal, script, restart")
)
// ChangeScript holds the configuration for the script that is executed if
// change mode is set to script
type ChangeScript struct {
// Command is the full path to the script
Command string
// Args is a slice of arguments passed to the script
Args []string
// Timeout is the amount of seconds we wait for the script to finish
Timeout time.Duration
// FailOnError indicates whether a task should fail in case script execution
// fails or log script failure and don't interrupt the task
FailOnError bool
}
// Template represents a template configuration to be rendered for a given task
type Template struct {
// SourcePath is the path to the template to be rendered
@ -7658,6 +7676,10 @@ type Template struct {
// requires it.
ChangeSignal string
// ChangeScript is the configuration of the script. It's required if
// ChangeMode is set to script.
ChangeScript *ChangeScript
// Splay is used to avoid coordinated restarts of processes by applying a
// random wait between 0 and the given splay value before signalling the
// application of a change
@ -7756,6 +7778,10 @@ func (t *Template) Validate() error {
if t.Envvars {
_ = multierror.Append(&mErr, fmt.Errorf("cannot use signals with env var templates"))
}
case TemplateChangeModeScript:
if t.ChangeScript.Command == "" {
_ = multierror.Append(&mErr, fmt.Errorf("must specify script path value when change mode is script"))
}
default:
_ = multierror.Append(&mErr, TemplateChangeModeInvalidError)
}
@ -8061,6 +8087,10 @@ const (
// TaskHookFailed indicates that one of the hooks for a task failed.
TaskHookFailed = "Task hook failed"
// TaskHookMessage indicates that one of the hooks for a task emitted a
// message.
TaskHookMessage = "Task hook message"
// TaskRestoreFailed indicates Nomad was unable to reattach to a
// restored task.
TaskRestoreFailed = "Failed Restoring Task"

25
website/content/api-docs/json-jobs.mdx
View File

@ -1055,11 +1055,32 @@ README][ct].
- `"noop"` - take no action (continue running the task)
- `"restart"` - restart the task
- `"signal"` - send a configurable signal to the task
- `"script"` - run a script
- `ChangeSignal` - Specifies the signal to send to the task as a string like
"SIGUSR1" or "SIGINT". This option is required if the `ChangeMode` is
`signal`.
- `ChangeScript` - Configures the script triggered on template change. This
option is required if the `ChangeMode` is `script`.
The `ChangeScript` object supports the following attributes:
- `Command` - Specifies the full path to a script or executable that is to be
executed on template change. Path is relative to the driver, e.g., if running
with a container driver the path must be existing in the container. This
option is required is the `change_mode` is `script`.
- `Args` - List of arguments that are passed to the script that is to be
executed on template change.
- `Timeout` - Timeout for script execution specified using a label suffix like
"30s" or "1h". Default value is `"5s"`.
- `FailOnError` - If `true`, Nomad will kill the task if the script execution
fails. If `false`, script failure will be logged but the task will continue
uninterrupted. Default value is `false`.
- `DestPath` - Specifies the location where the resulting template should be
rendered, relative to the task directory.
@ -1080,14 +1101,14 @@ README][ct].
- `Uid` - Specifies the rendered template owner's user ID.
~> **Caveat:** Works only on Unix-based systems. Be careful when using
containerized drivers, suck as `docker` or `podman`, as groups and users
containerized drivers, such as `docker` or `podman`, as groups and users
inside the container may have different IDs than on the host system. This
feature will also **not** work with Docker Desktop.
- `Gid` - Specifies the rendered template owner's group ID.
~> **Caveat:** Works only on Unix-based systems. Be careful when using
containerized drivers, suck as `docker` or `podman`, as groups and users
containerized drivers, such as `docker` or `podman`, as groups and users
inside the container may have different IDs than on the host system. This
feature will also **not** work with Docker Desktop.

85
website/content/docs/job-specification/change_script.mdx Normal file
View File

@ -0,0 +1,85 @@
---
layout: docs
page_title: change_script Stanza - Job Specification
description: The "change_script" stanza configures a script to be run on template re-render.
---
# `change_script` Stanza
<Placement groups={['job', 'group', 'task', 'template', 'change_script']} />
The `change_script` stanza allows operators to configure scripts that
will be executed on template change. This stanza is only used when template
`change_mode` is set to `script`.
```hcl
job "docs" {
group "example" {
task "server" {
template {
source = "local/redis.conf.tpl"
destination = "local/redis.conf"
change_mode = "script"
change_script {
command = "/bin/foo"
args = ["-verbose", "-debug"]
timeout = "5s"
fail_on_error = false
}
}
}
}
}
```
## `change_script` Parameters
- `command` `(string: "")` - Specifies the full path to a script or executable
that is to be executed on template change. The command must return exit code 0
to be considered successful. Path is relative to the driver, e.g., if running
with a container driver the path must be existing in the container. This option
is required if `change_mode` is `script`.
- `args` `(array<string>: [])` - List of arguments that are passed to the script
that is to be executed on template change.
- `timeout` `(string: "5s")` - Timeout for script execution specified using a
label suffix like `"30s"` or `"1h"`.
- `fail_on_error` `(bool: false)` - If `true`, Nomad will kill the task if the
script execution fails. If `false`, script failure will be logged but the task
will continue uninterrupted.
### Template as a script example
Below is an example of how a script can be embedded in a `data` block of another
`template` stanza:
```hcl
job "docs" {
group "example" {
task "server" {
template {
data = "{{key \"my_key\"}}"
destination = "local/test"
change_mode = "script"
change_script {
path = "/local/script.sh"
}
}
template {
data = <<EOF
#!/usr/bin/env bash
echo "Running change_mode script"
sleep 10
echo "Done"
EOF
destination = "local/script.sh"
perms = "777"
}
}
}
}
```

6
website/content/docs/job-specification/template.mdx
View File

@ -53,11 +53,16 @@ refer to the [Learn Go Template Syntax][gt_learn] Learn guide.
- `"noop"` - take no action (continue running the task)
- `"restart"` - restart the task
- `"signal"` - send a configurable signal to the task
- `"script"` - run a script
- `change_signal` `(string: "")` - Specifies the signal to send to the task as a
string like `"SIGUSR1"` or `"SIGINT"`. This option is required if the
`change_mode` is `signal`.
- `change_script` <code>([ChangeScript][]: nil)</code> - Configures the script
triggered on template change. This option is required if the `change_mode` is
`script`.
- `data` `(string: "")` - Specifies the raw template to execute. One of `source`
or `data` must be specified, but not both. This is useful for smaller
templates, but we recommend using `source` for larger templates.
@ -561,6 +566,7 @@ options](/docs/configuration/client#options):
files on the client host via the `file` function. By default templates can
access files only within the [task working directory].
[changescript]: /docs/job-specification/change_script 'Nomad change_script Job Specification'
[ct]: https://github.com/hashicorp/consul-template 'Consul Template by HashiCorp'
[ct_api]: https://github.com/hashicorp/consul-template/blob/master/docs/templating-language.md 'Consul Template API by HashiCorp'
[ct_api_connect]: https://github.com/hashicorp/consul-template/blob/master/docs/templating-language.md#connect 'Consul Template API by HashiCorp - connect'

4
website/data/docs-nav-data.json
View File

@ -1358,6 +1358,10 @@
"title": "affinity",
"path": "job-specification/affinity"
},
{
"title": "change_script",
"path": "job-specification/change_script"
},
{
"title": "check",
"path": "job-specification/check"