diff --git a/client/allocrunner/taskrunner/artifact_hook.go b/client/allocrunner/taskrunner/artifact_hook.go index 689aeed50..481c098e2 100644 --- a/client/allocrunner/taskrunner/artifact_hook.go +++ b/client/allocrunner/taskrunner/artifact_hook.go @@ -52,7 +52,8 @@ func (h *artifactHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar h.logger.Debug("downloading artifact", "artifact", artifact.GetterSource) //XXX add ctx to GetArtifact to allow cancelling long downloads - if err := getter.GetArtifact(req.TaskEnv, artifact, req.TaskDir.Dir); err != nil { + if err := getter.GetArtifact(req.TaskEnv, artifact); err != nil { + wrapped := structs.NewRecoverableError( fmt.Errorf("failed to download artifact %q: %v", artifact.GetterSource, err), true, diff --git a/client/allocrunner/taskrunner/artifact_hook_test.go b/client/allocrunner/taskrunner/artifact_hook_test.go index b4e36f229..121370867 100644 --- a/client/allocrunner/taskrunner/artifact_hook_test.go +++ b/client/allocrunner/taskrunner/artifact_hook_test.go @@ -94,7 +94,7 @@ func TestTaskRunner_ArtifactHook_PartialDone(t *testing.T) { }() req := &interfaces.TaskPrestartRequest{ - TaskEnv: taskenv.NewEmptyTaskEnv(), + TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, destdir, ""), TaskDir: &allocdir.TaskDir{Dir: destdir}, Task: &structs.Task{ Artifacts: []*structs.TaskArtifact{ diff --git a/client/allocrunner/taskrunner/envoy_version_hook_test.go b/client/allocrunner/taskrunner/envoy_version_hook_test.go index 35a1b2830..26d739bd7 100644 --- a/client/allocrunner/taskrunner/envoy_version_hook_test.go +++ b/client/allocrunner/taskrunner/envoy_version_hook_test.go @@ -17,10 +17,10 @@ import ( ) var ( - taskEnvDefault = taskenv.NewTaskEnv(nil, nil, map[string]string{ + taskEnvDefault = taskenv.NewTaskEnv(nil, nil, nil, map[string]string{ "meta.connect.sidecar_image": envoy.ImageFormat, "meta.connect.gateway_image": envoy.ImageFormat, - }) + }, "", "") ) func TestEnvoyVersionHook_semver(t *testing.T) { @@ -140,7 +140,9 @@ func TestEnvoyVersionHook_interpolateImage(t *testing.T) { } hook.interpolateImage(task, taskenv.NewTaskEnv(map[string]string{ "MY_ENVOY": "my/envoy", - }, nil, nil)) + }, map[string]string{ + "MY_ENVOY": "my/envoy", + }, nil, nil, "", "")) require.Equal(t, "my/envoy", task.Config["image"]) }) diff --git a/client/allocrunner/taskrunner/getter/getter.go b/client/allocrunner/taskrunner/getter/getter.go index 3a8ce82b8..f198a7aae 100644 --- a/client/allocrunner/taskrunner/getter/getter.go +++ b/client/allocrunner/taskrunner/getter/getter.go @@ -5,12 +5,11 @@ import ( "fmt" "net/http" "net/url" - "path/filepath" "strings" "sync" gg "github.com/hashicorp/go-getter" - "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/nomad/structs" ) @@ -33,6 +32,7 @@ const ( // is usually satisfied by taskenv.TaskEnv. type EnvReplacer interface { ReplaceEnv(string) string + ClientPath(string, bool) (string, bool) } func makeGetters(headers http.Header) map[string]gg.Getter { @@ -130,21 +130,18 @@ func getHeaders(env EnvReplacer, m map[string]string) http.Header { } // GetArtifact downloads an artifact into the specified task directory. -func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir string) error { +func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact) error { ggURL, err := getGetterUrl(taskEnv, artifact) if err != nil { return newGetError(artifact.GetterSource, err, false) } + dest, escapes := taskEnv.ClientPath(artifact.RelativeDest, true) // Verify the destination is still in the task sandbox after interpolation - // Note: we *always* join here even if we get passed an absolute path so - // that $NOMAD_SECRETS_DIR and friends can be used and always fall inside - // the task working directory - dest := filepath.Join(taskDir, artifact.RelativeDest) - escapes := helper.PathEscapesSandbox(taskDir, dest) if escapes { return newGetError(artifact.RelativeDest, - errors.New("artifact destination path escapes the alloc directory"), false) + errors.New("artifact destination path escapes the alloc directory"), + false) } // Convert from string getter mode to go-getter const diff --git a/client/allocrunner/taskrunner/getter/getter_test.go b/client/allocrunner/taskrunner/getter/getter_test.go index 3931bc52c..274b2b71d 100644 --- a/client/allocrunner/taskrunner/getter/getter_test.go +++ b/client/allocrunner/taskrunner/getter/getter_test.go @@ -15,39 +15,69 @@ import ( "testing" "github.com/hashicorp/nomad/client/taskenv" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" ) // noopReplacer is a noop version of taskenv.TaskEnv.ReplaceEnv. -type noopReplacer struct{} +type noopReplacer struct { + taskDir string +} + +func clientPath(taskDir, path string, join bool) (string, bool) { + if !filepath.IsAbs(path) || (helper.PathEscapesSandbox(taskDir, path) && join) { + path = filepath.Join(taskDir, path) + } + path = filepath.Clean(path) + if taskDir != "" && !helper.PathEscapesSandbox(taskDir, path) { + return path, false + } + return path, true +} func (noopReplacer) ReplaceEnv(s string) string { return s } -var noopTaskEnv = noopReplacer{} +func (r noopReplacer) ClientPath(p string, join bool) (string, bool) { + path, escapes := clientPath(r.taskDir, r.ReplaceEnv(p), join) + return path, escapes +} + +func noopTaskEnv(taskDir string) EnvReplacer { + return noopReplacer{ + taskDir: taskDir, + } +} // upperReplacer is a version of taskenv.TaskEnv.ReplaceEnv that upper-cases // the given input. -type upperReplacer struct{} +type upperReplacer struct { + taskDir string +} func (upperReplacer) ReplaceEnv(s string) string { return strings.ToUpper(s) } +func (u upperReplacer) ClientPath(p string, join bool) (string, bool) { + path, escapes := clientPath(u.taskDir, u.ReplaceEnv(p), join) + return path, escapes +} + func removeAllT(t *testing.T, path string) { require.NoError(t, os.RemoveAll(path)) } func TestGetArtifact_getHeaders(t *testing.T) { t.Run("nil", func(t *testing.T) { - require.Nil(t, getHeaders(noopTaskEnv, nil)) + require.Nil(t, getHeaders(noopTaskEnv(""), nil)) }) t.Run("empty", func(t *testing.T) { - require.Nil(t, getHeaders(noopTaskEnv, make(map[string]string))) + require.Nil(t, getHeaders(noopTaskEnv(""), make(map[string]string))) }) t.Run("set", func(t *testing.T) { @@ -94,12 +124,14 @@ func TestGetArtifact_Headers(t *testing.T) { } // Download the artifact. - taskEnv := new(upperReplacer) - err = GetArtifact(taskEnv, artifact, taskDir) + taskEnv := upperReplacer{ + taskDir: taskDir, + } + err = GetArtifact(taskEnv, artifact) require.NoError(t, err) // Verify artifact exists. - b, err := ioutil.ReadFile(filepath.Join(taskDir, file)) + b, err := ioutil.ReadFile(filepath.Join(taskDir, taskEnv.ReplaceEnv(file))) require.NoError(t, err) // Verify we wrote the interpolated header value into the file that is our @@ -129,7 +161,7 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) { } // Download the artifact - if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil { + if err := GetArtifact(noopTaskEnv(taskDir), artifact); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -163,7 +195,7 @@ func TestGetArtifact_File_RelativeDest(t *testing.T) { } // Download the artifact - if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil { + if err := GetArtifact(noopTaskEnv(taskDir), artifact); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -197,7 +229,7 @@ func TestGetArtifact_File_EscapeDest(t *testing.T) { } // attempt to download the artifact - err = GetArtifact(noopTaskEnv, artifact, taskDir) + err = GetArtifact(noopTaskEnv(taskDir), artifact) if err == nil || !strings.Contains(err.Error(), "escapes") { t.Fatalf("expected GetArtifact to disallow sandbox escape: %v", err) } @@ -247,7 +279,7 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) { } // Download the artifact and expect an error - if err := GetArtifact(noopTaskEnv, artifact, taskDir); err == nil { + if err := GetArtifact(noopTaskEnv(taskDir), artifact); err == nil { t.Fatalf("GetArtifact should have failed") } } @@ -312,7 +344,7 @@ func TestGetArtifact_Archive(t *testing.T) { }, } - if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil { + if err := GetArtifact(noopTaskEnv(taskDir), artifact); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -345,7 +377,7 @@ func TestGetArtifact_Setuid(t *testing.T) { }, } - require.NoError(t, GetArtifact(noopTaskEnv, artifact, taskDir)) + require.NoError(t, GetArtifact(noopTaskEnv(taskDir), artifact)) var expected map[string]int @@ -470,7 +502,7 @@ func TestGetGetterUrl_Queries(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - act, err := getGetterUrl(noopTaskEnv, c.artifact) + act, err := getGetterUrl(noopTaskEnv(""), c.artifact) if err != nil { t.Fatalf("want %q; got err %v", c.output, err) } else if act != c.output { diff --git a/client/allocrunner/taskrunner/task_dir_hook.go b/client/allocrunner/taskrunner/task_dir_hook.go index cc8769db9..ac2d4338d 100644 --- a/client/allocrunner/taskrunner/task_dir_hook.go +++ b/client/allocrunner/taskrunner/task_dir_hook.go @@ -5,6 +5,7 @@ import ( "strings" log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/allocrunner/interfaces" cconfig "github.com/hashicorp/nomad/client/config" @@ -76,6 +77,12 @@ func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart // setEnvvars sets path and host env vars depending on the FS isolation used. func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *cconfig.Config) { + + envBuilder.SetClientTaskRoot(taskDir.Dir) + envBuilder.SetClientSharedAllocDir(taskDir.SharedAllocDir) + envBuilder.SetClientTaskLocalDir(taskDir.LocalDir) + envBuilder.SetClientTaskSecretsDir(taskDir.SecretsDir) + // Set driver-specific environment variables switch fsi { case drivers.FSIsolationNone: @@ -93,11 +100,10 @@ func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *a // Set the host environment variables for non-image based drivers if fsi != drivers.FSIsolationImage { // COMPAT(1.0) using inclusive language, blacklist is kept for backward compatibility. - denylist := conf.ReadAlternativeDefault( + filter := strings.Split(conf.ReadAlternativeDefault( []string{"env.denylist", "env.blacklist"}, cconfig.DefaultEnvDenylist, - ) - filter := strings.Split(denylist, ",") + ), ",") envBuilder.SetHostEnvvars(filter) } } diff --git a/client/allocrunner/taskrunner/template/template.go b/client/allocrunner/taskrunner/template/template.go index 3f6c8f1b1..e1e49de99 100644 --- a/client/allocrunner/taskrunner/template/template.go +++ b/client/allocrunner/taskrunner/template/template.go @@ -6,7 +6,6 @@ import ( "fmt" "math/rand" "os" - "path/filepath" "sort" "strconv" "strings" @@ -18,7 +17,6 @@ import ( "github.com/hashicorp/consul-template/signals" envparse "github.com/hashicorp/go-envparse" multierror "github.com/hashicorp/go-multierror" - "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/taskenv" @@ -211,7 +209,7 @@ func (tm *TaskTemplateManager) run() { } // Read environment variables from env templates before we unblock - envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build()) + envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.EnvBuilder.Build()) if err != nil { tm.config.Lifecycle.Kill(context.Background(), structs.NewTaskEvent(structs.TaskKilling). @@ -416,7 +414,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time } // Read environment variables from templates - envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build()) + envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.EnvBuilder.Build()) if err != nil { tm.config.Lifecycle.Kill(context.Background(), structs.NewTaskEvent(structs.TaskKilling). @@ -571,41 +569,20 @@ func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.Templa sandboxEnabled := !config.ClientConfig.TemplateConfig.DisableSandbox taskEnv := config.EnvBuilder.Build() - // Make NOMAD_{ALLOC,TASK,SECRETS}_DIR relative paths to avoid treating - // them as sandbox escapes when using containers. - if taskEnv.EnvMap[taskenv.AllocDir] == allocdir.SharedAllocContainerPath { - taskEnv.EnvMap[taskenv.AllocDir] = allocdir.SharedAllocName - } - if taskEnv.EnvMap[taskenv.TaskLocalDir] == allocdir.TaskLocalContainerPath { - taskEnv.EnvMap[taskenv.TaskLocalDir] = allocdir.TaskLocal - } - if taskEnv.EnvMap[taskenv.SecretsDir] == allocdir.TaskSecretsContainerPath { - taskEnv.EnvMap[taskenv.SecretsDir] = allocdir.TaskSecrets - } - ctmpls := make(map[*ctconf.TemplateConfig]*structs.Template, len(config.Templates)) for _, tmpl := range config.Templates { var src, dest string if tmpl.SourcePath != "" { - src = taskEnv.ReplaceEnv(tmpl.SourcePath) - if !filepath.IsAbs(src) { - src = filepath.Join(config.TaskDir, src) - } else { - src = filepath.Clean(src) - } - escapes := helper.PathEscapesSandbox(config.TaskDir, src) + var escapes bool + src, escapes = taskEnv.ClientPath(tmpl.SourcePath, false) if escapes && sandboxEnabled { return nil, sourceEscapesErr } } if tmpl.DestPath != "" { - dest = taskEnv.ReplaceEnv(tmpl.DestPath) - // Note: we *always* join here even if we get passed an absolute - // path so that $NOMAD_SECRETS_DIR and friends can be used and - // always fall inside the task working directory - dest = filepath.Join(config.TaskDir, dest) - escapes := helper.PathEscapesSandbox(config.TaskDir, dest) + var escapes bool + dest, escapes = taskEnv.ClientPath(tmpl.DestPath, true) if escapes && sandboxEnabled { return nil, destEscapesErr } @@ -740,14 +717,15 @@ func newRunnerConfig(config *TaskTemplateManagerConfig, } // loadTemplateEnv loads task environment variables from all templates. -func loadTemplateEnv(tmpls []*structs.Template, taskDir string, taskEnv *taskenv.TaskEnv) (map[string]string, error) { +func loadTemplateEnv(tmpls []*structs.Template, taskEnv *taskenv.TaskEnv) (map[string]string, error) { all := make(map[string]string, 50) for _, t := range tmpls { if !t.Envvars { continue } - dest := filepath.Join(taskDir, taskEnv.ReplaceEnv(t.DestPath)) + // we checked escape before we rendered the file + dest, _ := taskEnv.ClientPath(t.DestPath, true) f, err := os.Open(dest) if err != nil { return nil, fmt.Errorf("error opening env template: %v", err) diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index ff6fa682d..e71178cf8 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -161,6 +161,7 @@ func newTestHarness(t *testing.T, templates []*structs.Template, consul, vault b t.Fatalf("Failed to make tmpdir: %v", err) } harness.taskDir = d + harness.envBuilder.SetClientTaskRoot(harness.taskDir) if consul { harness.consul, err = ctestutil.NewTestServerConfigT(t, func(c *ctestutil.TestServerConfig) { @@ -1300,7 +1301,8 @@ func TestTaskTemplateManager_Env_Missing(t *testing.T) { }, } - if vars, err := loadTemplateEnv(templates, d, taskenv.NewEmptyTaskEnv()); err == nil { + taskEnv := taskenv.NewEmptyBuilder().SetClientTaskRoot(d).Build() + if vars, err := loadTemplateEnv(templates, taskEnv); err == nil { t.Fatalf("expected an error but instead got env vars: %#v", vars) } } @@ -1334,9 +1336,12 @@ func TestTaskTemplateManager_Env_InterpolatedDest(t *testing.T) { // Build the env taskEnv := taskenv.NewTaskEnv( map[string]string{"NOMAD_META_path": "exists"}, - map[string]string{}, map[string]string{}) + map[string]string{"NOMAD_META_path": "exists"}, + map[string]string{}, + map[string]string{}, + d, "") - vars, err := loadTemplateEnv(templates, d, taskEnv) + vars, err := loadTemplateEnv(templates, taskEnv) require.NoError(err) require.Contains(vars, "FOO") require.Equal(vars["FOO"], "bar") @@ -1375,7 +1380,8 @@ func TestTaskTemplateManager_Env_Multi(t *testing.T) { }, } - vars, err := loadTemplateEnv(templates, d, taskenv.NewEmptyTaskEnv()) + taskEnv := taskenv.NewEmptyBuilder().SetClientTaskRoot(d).Build() + vars, err := loadTemplateEnv(templates, taskEnv) if err != nil { t.Fatalf("expected no error: %v", err) } @@ -1585,6 +1591,10 @@ func TestTaskTemplateManager_Escapes(t *testing.T) { b.SetAllocDir(allocdir.SharedAllocContainerPath) b.SetTaskLocalDir(allocdir.TaskLocalContainerPath) b.SetSecretsDir(allocdir.TaskSecretsContainerPath) + b.SetClientTaskRoot(taskDir.Dir) + b.SetClientSharedAllocDir(taskDir.SharedAllocDir) + b.SetClientTaskLocalDir(taskDir.LocalDir) + b.SetClientTaskSecretsDir(taskDir.SecretsDir) return b } @@ -1595,6 +1605,10 @@ func TestTaskTemplateManager_Escapes(t *testing.T) { b.SetAllocDir(taskDir.SharedAllocDir) b.SetTaskLocalDir(taskDir.LocalDir) b.SetSecretsDir(taskDir.SecretsDir) + b.SetClientTaskRoot(taskDir.Dir) + b.SetClientSharedAllocDir(taskDir.SharedAllocDir) + b.SetClientTaskLocalDir(taskDir.LocalDir) + b.SetClientTaskSecretsDir(taskDir.SecretsDir) return b } diff --git a/client/taskenv/env.go b/client/taskenv/env.go index cdea2f5e4..01a283e5d 100644 --- a/client/taskenv/env.go +++ b/client/taskenv/env.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "os" + "path/filepath" "strconv" "strings" "sync" @@ -135,15 +136,31 @@ type TaskEnv struct { // envList is a memoized list created by List() envList []string + + // EnvMap is the map of environment variables with client-specific + // task directories + // See https://github.com/hashicorp/nomad/pull/9671 + EnvMapClient map[string]string + + // clientTaskDir is the absolute path to the task root directory on the host + // / + clientTaskDir string + + // clientSharedAllocDir is the path to shared alloc directory on the host + // /alloc/ + clientSharedAllocDir string } // NewTaskEnv creates a new task environment with the given environment, device // environment and node attribute maps. -func NewTaskEnv(env, deviceEnv, node map[string]string) *TaskEnv { +func NewTaskEnv(env, envClient, deviceEnv, node map[string]string, clientTaskDir, clientAllocDir string) *TaskEnv { return &TaskEnv{ - NodeAttrs: node, - deviceEnv: deviceEnv, - EnvMap: env, + NodeAttrs: node, + deviceEnv: deviceEnv, + EnvMap: env, + EnvMapClient: envClient, + clientTaskDir: clientTaskDir, + clientSharedAllocDir: clientAllocDir, } } @@ -290,6 +307,53 @@ func (t *TaskEnv) ReplaceEnv(arg string) string { return hargs.ReplaceEnv(arg, t.EnvMap, t.NodeAttrs) } +// replaceEnvClient takes an arg and replaces all occurrences of client-specific +// environment variables and Nomad variables. If the variable is found in the +// passed map it is replaced, otherwise the original string is returned. +// The difference from ReplaceEnv client is potentially different values for +// the following variables: +// * NOMAD_ALLOC_DIR +// * NOMAD_TASK_DIR +// * NOMAD_SECRETS_DIR +// and anything that was interpolated using them. +// +// See https://github.com/hashicorp/nomad/pull/9671 +func (t *TaskEnv) replaceEnvClient(arg string) string { + return hargs.ReplaceEnv(arg, t.EnvMapClient, t.NodeAttrs) +} + +// checkEscape returns true if the absolute path testPath escapes both the +// task directory and shared allocation directory specified in the +// directory path fields of this TaskEnv +func (t *TaskEnv) checkEscape(testPath string) bool { + for _, p := range []string{t.clientTaskDir, t.clientSharedAllocDir} { + if p != "" && !helper.PathEscapesSandbox(p, testPath) { + return false + } + } + return true +} + +// ClientPath interpolates the argument as a path, using the +// environment variables with client-relative directories. The +// result is an absolute path on the client filesystem. +// +// If the interpolated result is a relative path, it is made absolute +// wrt to the task working directory. +// If joinEscape, an interpolated path that escapes will be joined with the +// task dir. +// The result is checked to see whether it (still) escapes both the task working +// directory and the shared allocation directory. +func (t *TaskEnv) ClientPath(rawPath string, joinEscape bool) (string, bool) { + path := t.replaceEnvClient(rawPath) + if !filepath.IsAbs(path) || (t.checkEscape(path) && joinEscape) { + path = filepath.Join(t.clientTaskDir, path) + } + path = filepath.Clean(path) + escapes := t.checkEscape(path) + return path, escapes +} + // Builder is used to build task environment's and is safe for concurrent use. type Builder struct { // envvars are custom set environment variables @@ -316,6 +380,18 @@ type Builder struct { // secretsDir from task's perspective; eg /secrets secretsDir string + // clientSharedAllocDir is the shared alloc dir from the client's perspective; eg, //alloc + clientSharedAllocDir string + + // clientTaskRoot is the task working directory from the client's perspective; eg // + clientTaskRoot string + + // clientTaskLocalDir is the local dir from the client's perspective; eg /local + clientTaskLocalDir string + + // clientTaskSecretsDir is the secrets dir from the client's perspective; eg /secrets + clientTaskSecretsDir string + cpuLimit int64 memLimit int64 taskName string @@ -381,24 +457,23 @@ func NewEmptyBuilder() *Builder { } } -// Build must be called after all the tasks environment values have been set. -func (b *Builder) Build() *TaskEnv { - nodeAttrs := make(map[string]string) +// buildEnv returns the environment variables and device environment +// variables with respect to the task directories passed in the arguments. +func (b *Builder) buildEnv(allocDir, localDir, secretsDir string, + nodeAttrs map[string]string) (map[string]string, map[string]string) { + envMap := make(map[string]string) var deviceEnvs map[string]string - b.mu.RLock() - defer b.mu.RUnlock() - // Add the directories - if b.allocDir != "" { - envMap[AllocDir] = b.allocDir + if allocDir != "" { + envMap[AllocDir] = allocDir } - if b.localDir != "" { - envMap[TaskLocalDir] = b.localDir + if localDir != "" { + envMap[TaskLocalDir] = localDir } - if b.secretsDir != "" { - envMap[SecretsDir] = b.secretsDir + if secretsDir != "" { + envMap[SecretsDir] = secretsDir } // Add the resource limits @@ -442,9 +517,6 @@ func (b *Builder) Build() *TaskEnv { } if b.region != "" { envMap[Region] = b.region - - // Copy region over to node attrs - nodeAttrs[nodeRegionKey] = b.region } // Build the network related env vars @@ -473,11 +545,6 @@ func (b *Builder) Build() *TaskEnv { envMap[k] = v } - // Copy node attributes - for k, v := range b.nodeAttrs { - nodeAttrs[k] = v - } - // Interpolate and add environment variables for k, v := range b.hostEnv { envMap[k] = hargs.ReplaceEnv(v, nodeAttrs, envMap) @@ -522,7 +589,29 @@ func (b *Builder) Build() *TaskEnv { cleanedEnv[cleanedK] = v } - return NewTaskEnv(cleanedEnv, deviceEnvs, nodeAttrs) + return cleanedEnv, deviceEnvs +} + +// Build must be called after all the tasks environment values have been set. +func (b *Builder) Build() *TaskEnv { + nodeAttrs := make(map[string]string) + + b.mu.RLock() + defer b.mu.RUnlock() + + if b.region != "" { + // Copy region over to node attrs + nodeAttrs[nodeRegionKey] = b.region + } + // Copy node attributes + for k, v := range b.nodeAttrs { + nodeAttrs[k] = v + } + + envMap, deviceEnvs := b.buildEnv(b.allocDir, b.localDir, b.secretsDir, nodeAttrs) + envMapClient, _ := b.buildEnv(b.clientSharedAllocDir, b.clientTaskLocalDir, b.clientTaskSecretsDir, nodeAttrs) + + return NewTaskEnv(envMap, envMapClient, deviceEnvs, nodeAttrs, b.clientTaskRoot, b.clientSharedAllocDir) } // Update task updates the environment based on a new alloc and task. @@ -726,6 +815,34 @@ func (b *Builder) SetTaskLocalDir(dir string) *Builder { return b } +func (b *Builder) SetClientSharedAllocDir(dir string) *Builder { + b.mu.Lock() + b.clientSharedAllocDir = dir + b.mu.Unlock() + return b +} + +func (b *Builder) SetClientTaskRoot(dir string) *Builder { + b.mu.Lock() + b.clientTaskRoot = dir + b.mu.Unlock() + return b +} + +func (b *Builder) SetClientTaskLocalDir(dir string) *Builder { + b.mu.Lock() + b.clientTaskLocalDir = dir + b.mu.Unlock() + return b +} + +func (b *Builder) SetClientTaskSecretsDir(dir string) *Builder { + b.mu.Lock() + b.clientTaskSecretsDir = dir + b.mu.Unlock() + return b +} + func (b *Builder) SetSecretsDir(dir string) *Builder { b.mu.Lock() b.secretsDir = dir diff --git a/client/taskenv/services_test.go b/client/taskenv/services_test.go index 1a94c0e7e..b376197b7 100644 --- a/client/taskenv/services_test.go +++ b/client/taskenv/services_test.go @@ -103,9 +103,8 @@ func TestInterpolateServices(t *testing.T) { var testEnv = NewTaskEnv( map[string]string{"foo": "bar", "baz": "blah"}, - nil, - nil, -) + map[string]string{"foo": "bar", "baz": "blah"}, + nil, nil, "", "") func TestInterpolate_interpolateMapStringSliceString(t *testing.T) { t.Parallel() @@ -164,7 +163,7 @@ func TestInterpolate_interpolateMapStringInterface(t *testing.T) { func TestInterpolate_interpolateConnect(t *testing.T) { t.Parallel() - env := NewTaskEnv(map[string]string{ + e := map[string]string{ "tag1": "_tag1", "port1": "12345", "address1": "1.2.3.4", @@ -200,7 +199,8 @@ func TestInterpolate_interpolateConnect(t *testing.T) { "protocol2": "_protocol2", "service1": "_service1", "host1": "_host1", - }, nil, nil) + } + env := NewTaskEnv(e, e, nil, nil, "", "") connect := &structs.ConsulConnect{ Native: false, diff --git a/e2e/consultemplate/consultemplate.go b/e2e/consultemplate/consultemplate.go index 7bc158282..174fe9d6d 100644 --- a/e2e/consultemplate/consultemplate.go +++ b/e2e/consultemplate/consultemplate.go @@ -244,6 +244,11 @@ func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_Ok(f *framework.F) { func(out string) bool { return len(out) > 0 }, nil), "expected file to have contents") + + f.NoError(waitForTemplateRender(allocID, "alloc/shared.txt", + func(out string) bool { + return len(out) > 0 + }, nil), "expected shared-alloc-dir file to have contents") } // TestTemplatePathInterpolation_Bad asserts that template.source paths are not @@ -287,6 +292,73 @@ func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_Bad(f *framework.F) f.True(found, "alloc failed but NOT due to expected source path escape error") } +// TestTemplatePathInterpolation_SharedAlloc asserts that NOMAD_ALLOC_DIR +// is supported as a destination for artifact and template blocks, and +// that it is properly interpolated for task drivers with varying +// filesystem isolation +func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_SharedAllocDir(f *framework.F) { + jobID := "template-shared-alloc-" + uuid.Generate()[:8] + tc.jobIDs = append(tc.jobIDs, jobID) + + allocStubs := e2eutil.RegisterAndWaitForAllocs( + f.T(), tc.Nomad(), "consultemplate/input/template_shared_alloc.nomad", jobID, "") + f.Len(allocStubs, 1) + allocID := allocStubs[0].ID + + e2eutil.WaitForAllocRunning(f.T(), tc.Nomad(), allocID) + + for _, task := range []string{"docker", "exec", "raw_exec"} { + + // tests that we can render templates into the shared alloc directory + f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/raw_exec.env", + func(out string) bool { + return len(out) > 0 && strings.TrimSpace(out) != "/alloc" + }, nil), "expected raw_exec.env to not be '/alloc'") + + f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/exec.env", + func(out string) bool { + return strings.TrimSpace(out) == "/alloc" + }, nil), "expected shared exec.env to contain '/alloc'") + + f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/docker.env", + func(out string) bool { + return strings.TrimSpace(out) == "/alloc" + }, nil), "expected shared docker.env to contain '/alloc'") + + // test that we can fetch artifacts into the shared alloc directory + for _, a := range []string{"google1.html", "google2.html", "google3.html"} { + f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/"+a, + func(out string) bool { + return len(out) > 0 + }, nil), "expected artifact in alloc dir") + } + + // test that we can load environment variables rendered with templates using interpolated paths + out, err := e2e.Command("nomad", "alloc", "exec", "-task", task, allocID, "sh", "-c", "env") + f.NoError(err) + f.Contains(out, "HELLO_FROM=raw_exec") + } +} + +func waitForTaskFile(allocID, task, path string, test func(out string) bool, wc *e2e.WaitConfig) error { + var err error + var out string + interval, retries := wc.OrDefault() + + testutil.WaitForResultRetries(retries, func() (bool, error) { + time.Sleep(interval) + out, err = e2e.Command("nomad", "alloc", "exec", "-task", task, allocID, "sh", "-c", "cat "+path) + if err != nil { + return false, fmt.Errorf("could not cat file %q from task %q in allocation %q: %v", + path, task, allocID, err) + } + return test(out), nil + }, func(e error) { + err = fmt.Errorf("test for file content failed: got %#v\nerror: %v", out, e) + }) + return err +} + // waitForTemplateRender is a helper that grabs a file via alloc fs // and tests it for func waitForTemplateRender(allocID, path string, test func(string) bool, wc *e2e.WaitConfig) error { diff --git a/e2e/consultemplate/input/template_paths.nomad b/e2e/consultemplate/input/template_paths.nomad index d52e26282..d09232380 100644 --- a/e2e/consultemplate/input/template_paths.nomad +++ b/e2e/consultemplate/input/template_paths.nomad @@ -28,6 +28,13 @@ job "template-paths" { destination = "${NOMAD_SECRETS_DIR}/foo/dst" } + template { + destination = "${NOMAD_ALLOC_DIR}/shared.txt" + data = <