luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

update template and artifact interpolation to use client-relative paths 

resolves #9839
resolves #6929
resolves #6910

e2e: template env interpolation path testing
This commit is contained in:
Chris Baker 2020-12-14 17:56:34 +00:00
parent 724f82e32d
commit 9b125b8837
14 changed files with 443 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/allocrunner/taskrunner/artifact_hook.go
View File

@ -52,7 +52,8 @@ func (h *artifactHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
h.logger.Debug("downloading artifact", "artifact", artifact.GetterSource)
//XXX add ctx to GetArtifact to allow cancelling long downloads
if err := getter.GetArtifact(req.TaskEnv, artifact, req.TaskDir.Dir); err != nil {
if err := getter.GetArtifact(req.TaskEnv, artifact); err != nil {
wrapped := structs.NewRecoverableError(
fmt.Errorf("failed to download artifact %q: %v", artifact.GetterSource, err),
true,

2
client/allocrunner/taskrunner/artifact_hook_test.go
View File

@ -94,7 +94,7 @@ func TestTaskRunner_ArtifactHook_PartialDone(t *testing.T) {
}()
req := &interfaces.TaskPrestartRequest{
TaskEnv: taskenv.NewEmptyTaskEnv(),
TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, destdir, ""),
TaskDir: &allocdir.TaskDir{Dir: destdir},
Task: &structs.Task{
Artifacts: []*structs.TaskArtifact{

8
client/allocrunner/taskrunner/envoy_version_hook_test.go
View File

@ -17,10 +17,10 @@ import (
)
var (
taskEnvDefault = taskenv.NewTaskEnv(nil, nil, map[string]string{
taskEnvDefault = taskenv.NewTaskEnv(nil, nil, nil, map[string]string{
"meta.connect.sidecar_image": envoy.ImageFormat,
"meta.connect.gateway_image": envoy.ImageFormat,
})
}, "", "")
)
func TestEnvoyVersionHook_semver(t *testing.T) {
@ -140,7 +140,9 @@ func TestEnvoyVersionHook_interpolateImage(t *testing.T) {
}
hook.interpolateImage(task, taskenv.NewTaskEnv(map[string]string{
"MY_ENVOY": "my/envoy",
}, nil, nil))
}, map[string]string{
"MY_ENVOY": "my/envoy",
}, nil, nil, "", ""))
require.Equal(t, "my/envoy", task.Config["image"])
})

15
client/allocrunner/taskrunner/getter/getter.go
View File

@ -5,12 +5,11 @@ import (
"fmt"
"net/http"
"net/url"
"path/filepath"
"strings"
"sync"
gg "github.com/hashicorp/go-getter"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
)
@ -33,6 +32,7 @@ const (
// is usually satisfied by taskenv.TaskEnv.
type EnvReplacer interface {
ReplaceEnv(string) string
ClientPath(string, bool) (string, bool)
}
func makeGetters(headers http.Header) map[string]gg.Getter {
@ -130,21 +130,18 @@ func getHeaders(env EnvReplacer, m map[string]string) http.Header {
}
// GetArtifact downloads an artifact into the specified task directory.
func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir string) error {
func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact) error {
ggURL, err := getGetterUrl(taskEnv, artifact)
if err != nil {
return newGetError(artifact.GetterSource, err, false)
}
dest, escapes := taskEnv.ClientPath(artifact.RelativeDest, true)
// Verify the destination is still in the task sandbox after interpolation
// Note: we *always* join here even if we get passed an absolute path so
// that $NOMAD_SECRETS_DIR and friends can be used and always fall inside
// the task working directory
dest := filepath.Join(taskDir, artifact.RelativeDest)
escapes := helper.PathEscapesSandbox(taskDir, dest)
if escapes {
return newGetError(artifact.RelativeDest,
errors.New("artifact destination path escapes the alloc directory"), false)
errors.New("artifact destination path escapes the alloc directory"),
false)
}
// Convert from string getter mode to go-getter const

62
client/allocrunner/taskrunner/getter/getter_test.go
View File

@ -15,39 +15,69 @@ import (
"testing"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/stretchr/testify/require"
)
// noopReplacer is a noop version of taskenv.TaskEnv.ReplaceEnv.
type noopReplacer struct{}
type noopReplacer struct {
taskDir string
}
func clientPath(taskDir, path string, join bool) (string, bool) {
if !filepath.IsAbs(path) || (helper.PathEscapesSandbox(taskDir, path) && join) {
path = filepath.Join(taskDir, path)
}
path = filepath.Clean(path)
if taskDir != "" && !helper.PathEscapesSandbox(taskDir, path) {
return path, false
}
return path, true
}
func (noopReplacer) ReplaceEnv(s string) string {
return s
}
var noopTaskEnv = noopReplacer{}
func (r noopReplacer) ClientPath(p string, join bool) (string, bool) {
path, escapes := clientPath(r.taskDir, r.ReplaceEnv(p), join)
return path, escapes
}
func noopTaskEnv(taskDir string) EnvReplacer {
return noopReplacer{
taskDir: taskDir,
}
}
// upperReplacer is a version of taskenv.TaskEnv.ReplaceEnv that upper-cases
// the given input.
type upperReplacer struct{}
type upperReplacer struct {
taskDir string
}
func (upperReplacer) ReplaceEnv(s string) string {
return strings.ToUpper(s)
}
func (u upperReplacer) ClientPath(p string, join bool) (string, bool) {
path, escapes := clientPath(u.taskDir, u.ReplaceEnv(p), join)
return path, escapes
}
func removeAllT(t *testing.T, path string) {
require.NoError(t, os.RemoveAll(path))
}
func TestGetArtifact_getHeaders(t *testing.T) {
t.Run("nil", func(t *testing.T) {
require.Nil(t, getHeaders(noopTaskEnv, nil))
require.Nil(t, getHeaders(noopTaskEnv(""), nil))
})
t.Run("empty", func(t *testing.T) {
require.Nil(t, getHeaders(noopTaskEnv, make(map[string]string)))
require.Nil(t, getHeaders(noopTaskEnv(""), make(map[string]string)))
})
t.Run("set", func(t *testing.T) {
@ -94,12 +124,14 @@ func TestGetArtifact_Headers(t *testing.T) {
}
// Download the artifact.
taskEnv := new(upperReplacer)
err = GetArtifact(taskEnv, artifact, taskDir)
taskEnv := upperReplacer{
taskDir: taskDir,
}
err = GetArtifact(taskEnv, artifact)
require.NoError(t, err)
// Verify artifact exists.
b, err := ioutil.ReadFile(filepath.Join(taskDir, file))
b, err := ioutil.ReadFile(filepath.Join(taskDir, taskEnv.ReplaceEnv(file)))
require.NoError(t, err)
// Verify we wrote the interpolated header value into the file that is our
@ -129,7 +161,7 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) {
}
// Download the artifact
if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil {
if err := GetArtifact(noopTaskEnv(taskDir), artifact); err != nil {
t.Fatalf("GetArtifact failed: %v", err)
}
@ -163,7 +195,7 @@ func TestGetArtifact_File_RelativeDest(t *testing.T) {
}
// Download the artifact
if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil {
if err := GetArtifact(noopTaskEnv(taskDir), artifact); err != nil {
t.Fatalf("GetArtifact failed: %v", err)
}
@ -197,7 +229,7 @@ func TestGetArtifact_File_EscapeDest(t *testing.T) {
}
// attempt to download the artifact
err = GetArtifact(noopTaskEnv, artifact, taskDir)
err = GetArtifact(noopTaskEnv(taskDir), artifact)
if err == nil || !strings.Contains(err.Error(), "escapes") {
t.Fatalf("expected GetArtifact to disallow sandbox escape: %v", err)
}
@ -247,7 +279,7 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) {
}
// Download the artifact and expect an error
if err := GetArtifact(noopTaskEnv, artifact, taskDir); err == nil {
if err := GetArtifact(noopTaskEnv(taskDir), artifact); err == nil {
t.Fatalf("GetArtifact should have failed")
}
}
@ -312,7 +344,7 @@ func TestGetArtifact_Archive(t *testing.T) {
},
}
if err := GetArtifact(noopTaskEnv, artifact, taskDir); err != nil {
if err := GetArtifact(noopTaskEnv(taskDir), artifact); err != nil {
t.Fatalf("GetArtifact failed: %v", err)
}
@ -345,7 +377,7 @@ func TestGetArtifact_Setuid(t *testing.T) {
},
}
require.NoError(t, GetArtifact(noopTaskEnv, artifact, taskDir))
require.NoError(t, GetArtifact(noopTaskEnv(taskDir), artifact))
var expected map[string]int
@ -470,7 +502,7 @@ func TestGetGetterUrl_Queries(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
act, err := getGetterUrl(noopTaskEnv, c.artifact)
act, err := getGetterUrl(noopTaskEnv(""), c.artifact)
if err != nil {
t.Fatalf("want %q; got err %v", c.output, err)
} else if act != c.output {

12
client/allocrunner/taskrunner/task_dir_hook.go
View File

@ -5,6 +5,7 @@ import (
"strings"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
cconfig "github.com/hashicorp/nomad/client/config"
@ -76,6 +77,12 @@ func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart
// setEnvvars sets path and host env vars depending on the FS isolation used.
func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *cconfig.Config) {
envBuilder.SetClientTaskRoot(taskDir.Dir)
envBuilder.SetClientSharedAllocDir(taskDir.SharedAllocDir)
envBuilder.SetClientTaskLocalDir(taskDir.LocalDir)
envBuilder.SetClientTaskSecretsDir(taskDir.SecretsDir)
// Set driver-specific environment variables
switch fsi {
case drivers.FSIsolationNone:
@ -93,11 +100,10 @@ func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *a
// Set the host environment variables for non-image based drivers
if fsi != drivers.FSIsolationImage {
// COMPAT(1.0) using inclusive language, blacklist is kept for backward compatibility.
denylist := conf.ReadAlternativeDefault(
filter := strings.Split(conf.ReadAlternativeDefault(
[]string{"env.denylist", "env.blacklist"},
cconfig.DefaultEnvDenylist,
)
filter := strings.Split(denylist, ",")
), ",")
envBuilder.SetHostEnvvars(filter)
}
}

40
client/allocrunner/taskrunner/template/template.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"math/rand"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
@ -18,7 +17,6 @@ import (
"github.com/hashicorp/consul-template/signals"
envparse "github.com/hashicorp/go-envparse"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/taskenv"
@ -211,7 +209,7 @@ func (tm *TaskTemplateManager) run() {
}
// Read environment variables from env templates before we unblock
envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build())
envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.EnvBuilder.Build())
if err != nil {
tm.config.Lifecycle.Kill(context.Background(),
structs.NewTaskEvent(structs.TaskKilling).
@ -416,7 +414,7 @@ func (tm *TaskTemplateManager) onTemplateRendered(handledRenders map[string]time
}
// Read environment variables from templates
envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir, tm.config.EnvBuilder.Build())
envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.EnvBuilder.Build())
if err != nil {
tm.config.Lifecycle.Kill(context.Background(),
structs.NewTaskEvent(structs.TaskKilling).
@ -571,41 +569,20 @@ func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[*ctconf.Templa
sandboxEnabled := !config.ClientConfig.TemplateConfig.DisableSandbox
taskEnv := config.EnvBuilder.Build()
// Make NOMAD_{ALLOC,TASK,SECRETS}_DIR relative paths to avoid treating
// them as sandbox escapes when using containers.
if taskEnv.EnvMap[taskenv.AllocDir] == allocdir.SharedAllocContainerPath {
taskEnv.EnvMap[taskenv.AllocDir] = allocdir.SharedAllocName
}
if taskEnv.EnvMap[taskenv.TaskLocalDir] == allocdir.TaskLocalContainerPath {
taskEnv.EnvMap[taskenv.TaskLocalDir] = allocdir.TaskLocal
}
if taskEnv.EnvMap[taskenv.SecretsDir] == allocdir.TaskSecretsContainerPath {
taskEnv.EnvMap[taskenv.SecretsDir] = allocdir.TaskSecrets
}
ctmpls := make(map[*ctconf.TemplateConfig]*structs.Template, len(config.Templates))
for _, tmpl := range config.Templates {
var src, dest string
if tmpl.SourcePath != "" {
src = taskEnv.ReplaceEnv(tmpl.SourcePath)
if !filepath.IsAbs(src) {
src = filepath.Join(config.TaskDir, src)
} else {
src = filepath.Clean(src)
}
escapes := helper.PathEscapesSandbox(config.TaskDir, src)
var escapes bool
src, escapes = taskEnv.ClientPath(tmpl.SourcePath, false)
if escapes && sandboxEnabled {
return nil, sourceEscapesErr
}
}
if tmpl.DestPath != "" {
dest = taskEnv.ReplaceEnv(tmpl.DestPath)
// Note: we *always* join here even if we get passed an absolute
// path so that $NOMAD_SECRETS_DIR and friends can be used and
// always fall inside the task working directory
dest = filepath.Join(config.TaskDir, dest)
escapes := helper.PathEscapesSandbox(config.TaskDir, dest)
var escapes bool
dest, escapes = taskEnv.ClientPath(tmpl.DestPath, true)
if escapes && sandboxEnabled {
return nil, destEscapesErr
}
@ -740,14 +717,15 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
}
// loadTemplateEnv loads task environment variables from all templates.
func loadTemplateEnv(tmpls []*structs.Template, taskDir string, taskEnv *taskenv.TaskEnv) (map[string]string, error) {
func loadTemplateEnv(tmpls []*structs.Template, taskEnv *taskenv.TaskEnv) (map[string]string, error) {
all := make(map[string]string, 50)
for _, t := range tmpls {
if !t.Envvars {
continue
}
dest := filepath.Join(taskDir, taskEnv.ReplaceEnv(t.DestPath))
// we checked escape before we rendered the file
dest, _ := taskEnv.ClientPath(t.DestPath, true)
f, err := os.Open(dest)
if err != nil {
return nil, fmt.Errorf("error opening env template: %v", err)

22
client/allocrunner/taskrunner/template/template_test.go
View File

@ -161,6 +161,7 @@ func newTestHarness(t *testing.T, templates []*structs.Template, consul, vault b
t.Fatalf("Failed to make tmpdir: %v", err)
}
harness.taskDir = d
harness.envBuilder.SetClientTaskRoot(harness.taskDir)
if consul {
harness.consul, err = ctestutil.NewTestServerConfigT(t, func(c *ctestutil.TestServerConfig) {
@ -1300,7 +1301,8 @@ func TestTaskTemplateManager_Env_Missing(t *testing.T) {
},
}
if vars, err := loadTemplateEnv(templates, d, taskenv.NewEmptyTaskEnv()); err == nil {
taskEnv := taskenv.NewEmptyBuilder().SetClientTaskRoot(d).Build()
if vars, err := loadTemplateEnv(templates, taskEnv); err == nil {
t.Fatalf("expected an error but instead got env vars: %#v", vars)
}
}
@ -1334,9 +1336,12 @@ func TestTaskTemplateManager_Env_InterpolatedDest(t *testing.T) {
// Build the env
taskEnv := taskenv.NewTaskEnv(
map[string]string{"NOMAD_META_path": "exists"},
map[string]string{}, map[string]string{})
map[string]string{"NOMAD_META_path": "exists"},
map[string]string{},
map[string]string{},
d, "")
vars, err := loadTemplateEnv(templates, d, taskEnv)
vars, err := loadTemplateEnv(templates, taskEnv)
require.NoError(err)
require.Contains(vars, "FOO")
require.Equal(vars["FOO"], "bar")
@ -1375,7 +1380,8 @@ func TestTaskTemplateManager_Env_Multi(t *testing.T) {
},
}
vars, err := loadTemplateEnv(templates, d, taskenv.NewEmptyTaskEnv())
taskEnv := taskenv.NewEmptyBuilder().SetClientTaskRoot(d).Build()
vars, err := loadTemplateEnv(templates, taskEnv)
if err != nil {
t.Fatalf("expected no error: %v", err)
}
@ -1585,6 +1591,10 @@ func TestTaskTemplateManager_Escapes(t *testing.T) {
b.SetAllocDir(allocdir.SharedAllocContainerPath)
b.SetTaskLocalDir(allocdir.TaskLocalContainerPath)
b.SetSecretsDir(allocdir.TaskSecretsContainerPath)
b.SetClientTaskRoot(taskDir.Dir)
b.SetClientSharedAllocDir(taskDir.SharedAllocDir)
b.SetClientTaskLocalDir(taskDir.LocalDir)
b.SetClientTaskSecretsDir(taskDir.SecretsDir)
return b
}
@ -1595,6 +1605,10 @@ func TestTaskTemplateManager_Escapes(t *testing.T) {
b.SetAllocDir(taskDir.SharedAllocDir)
b.SetTaskLocalDir(taskDir.LocalDir)
b.SetSecretsDir(taskDir.SecretsDir)
b.SetClientTaskRoot(taskDir.Dir)
b.SetClientSharedAllocDir(taskDir.SharedAllocDir)
b.SetClientTaskLocalDir(taskDir.LocalDir)
b.SetClientTaskSecretsDir(taskDir.SecretsDir)
return b
}

167
client/taskenv/env.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
@ -135,15 +136,31 @@ type TaskEnv struct {
// envList is a memoized list created by List()
envList []string
// EnvMap is the map of environment variables with client-specific
// task directories
// See https://github.com/hashicorp/nomad/pull/9671
EnvMapClient map[string]string
// clientTaskDir is the absolute path to the task root directory on the host
// <alloc_dir>/<task>
clientTaskDir string
// clientSharedAllocDir is the path to shared alloc directory on the host
// <alloc_dir>/alloc/
clientSharedAllocDir string
}
// NewTaskEnv creates a new task environment with the given environment, device
// environment and node attribute maps.
func NewTaskEnv(env, deviceEnv, node map[string]string) *TaskEnv {
func NewTaskEnv(env, envClient, deviceEnv, node map[string]string, clientTaskDir, clientAllocDir string) *TaskEnv {
return &TaskEnv{
NodeAttrs: node,
deviceEnv: deviceEnv,
EnvMap: env,
NodeAttrs: node,
deviceEnv: deviceEnv,
EnvMap: env,
EnvMapClient: envClient,
clientTaskDir: clientTaskDir,
clientSharedAllocDir: clientAllocDir,
}
}
@ -290,6 +307,53 @@ func (t *TaskEnv) ReplaceEnv(arg string) string {
return hargs.ReplaceEnv(arg, t.EnvMap, t.NodeAttrs)
}
// replaceEnvClient takes an arg and replaces all occurrences of client-specific
// environment variables and Nomad variables. If the variable is found in the
// passed map it is replaced, otherwise the original string is returned.
// The difference from ReplaceEnv client is potentially different values for
// the following variables:
// * NOMAD_ALLOC_DIR
// * NOMAD_TASK_DIR
// * NOMAD_SECRETS_DIR
// and anything that was interpolated using them.
//
// See https://github.com/hashicorp/nomad/pull/9671
func (t *TaskEnv) replaceEnvClient(arg string) string {
return hargs.ReplaceEnv(arg, t.EnvMapClient, t.NodeAttrs)
}
// checkEscape returns true if the absolute path testPath escapes both the
// task directory and shared allocation directory specified in the
// directory path fields of this TaskEnv
func (t *TaskEnv) checkEscape(testPath string) bool {
for _, p := range []string{t.clientTaskDir, t.clientSharedAllocDir} {
if p != "" && !helper.PathEscapesSandbox(p, testPath) {
return false
}
}
return true
}
// ClientPath interpolates the argument as a path, using the
// environment variables with client-relative directories. The
// result is an absolute path on the client filesystem.
//
// If the interpolated result is a relative path, it is made absolute
// wrt to the task working directory.
// If joinEscape, an interpolated path that escapes will be joined with the
// task dir.
// The result is checked to see whether it (still) escapes both the task working
// directory and the shared allocation directory.
func (t *TaskEnv) ClientPath(rawPath string, joinEscape bool) (string, bool) {
path := t.replaceEnvClient(rawPath)
if !filepath.IsAbs(path) || (t.checkEscape(path) && joinEscape) {
path = filepath.Join(t.clientTaskDir, path)
}
path = filepath.Clean(path)
escapes := t.checkEscape(path)
return path, escapes
}
// Builder is used to build task environment's and is safe for concurrent use.
type Builder struct {
// envvars are custom set environment variables
@ -316,6 +380,18 @@ type Builder struct {
// secretsDir from task's perspective; eg /secrets
secretsDir string
// clientSharedAllocDir is the shared alloc dir from the client's perspective; eg, <alloc_dir>/<alloc_id>/alloc
clientSharedAllocDir string
// clientTaskRoot is the task working directory from the client's perspective; eg <alloc_dir>/<alloc_id>/<task>
clientTaskRoot string
// clientTaskLocalDir is the local dir from the client's perspective; eg <client_task_root>/local
clientTaskLocalDir string
// clientTaskSecretsDir is the secrets dir from the client's perspective; eg <client_task_root>/secrets
clientTaskSecretsDir string
cpuLimit int64
memLimit int64
taskName string
@ -381,24 +457,23 @@ func NewEmptyBuilder() *Builder {
}
}
// Build must be called after all the tasks environment values have been set.
func (b *Builder) Build() *TaskEnv {
nodeAttrs := make(map[string]string)
// buildEnv returns the environment variables and device environment
// variables with respect to the task directories passed in the arguments.
func (b *Builder) buildEnv(allocDir, localDir, secretsDir string,
nodeAttrs map[string]string) (map[string]string, map[string]string) {
envMap := make(map[string]string)
var deviceEnvs map[string]string
b.mu.RLock()
defer b.mu.RUnlock()
// Add the directories
if b.allocDir != "" {
envMap[AllocDir] = b.allocDir
if allocDir != "" {
envMap[AllocDir] = allocDir
}
if b.localDir != "" {
envMap[TaskLocalDir] = b.localDir
if localDir != "" {
envMap[TaskLocalDir] = localDir
}
if b.secretsDir != "" {
envMap[SecretsDir] = b.secretsDir
if secretsDir != "" {
envMap[SecretsDir] = secretsDir
}
// Add the resource limits
@ -442,9 +517,6 @@ func (b *Builder) Build() *TaskEnv {
}
if b.region != "" {
envMap[Region] = b.region
// Copy region over to node attrs
nodeAttrs[nodeRegionKey] = b.region
}
// Build the network related env vars
@ -473,11 +545,6 @@ func (b *Builder) Build() *TaskEnv {
envMap[k] = v
}
// Copy node attributes
for k, v := range b.nodeAttrs {
nodeAttrs[k] = v
}
// Interpolate and add environment variables
for k, v := range b.hostEnv {
envMap[k] = hargs.ReplaceEnv(v, nodeAttrs, envMap)
@ -522,7 +589,29 @@ func (b *Builder) Build() *TaskEnv {
cleanedEnv[cleanedK] = v
}
return NewTaskEnv(cleanedEnv, deviceEnvs, nodeAttrs)
return cleanedEnv, deviceEnvs
}
// Build must be called after all the tasks environment values have been set.
func (b *Builder) Build() *TaskEnv {
nodeAttrs := make(map[string]string)
b.mu.RLock()
defer b.mu.RUnlock()
if b.region != "" {
// Copy region over to node attrs
nodeAttrs[nodeRegionKey] = b.region
}
// Copy node attributes
for k, v := range b.nodeAttrs {
nodeAttrs[k] = v
}
envMap, deviceEnvs := b.buildEnv(b.allocDir, b.localDir, b.secretsDir, nodeAttrs)
envMapClient, _ := b.buildEnv(b.clientSharedAllocDir, b.clientTaskLocalDir, b.clientTaskSecretsDir, nodeAttrs)
return NewTaskEnv(envMap, envMapClient, deviceEnvs, nodeAttrs, b.clientTaskRoot, b.clientSharedAllocDir)
}
// Update task updates the environment based on a new alloc and task.
@ -726,6 +815,34 @@ func (b *Builder) SetTaskLocalDir(dir string) *Builder {
return b
}
func (b *Builder) SetClientSharedAllocDir(dir string) *Builder {
b.mu.Lock()
b.clientSharedAllocDir = dir
b.mu.Unlock()
return b
}
func (b *Builder) SetClientTaskRoot(dir string) *Builder {
b.mu.Lock()
b.clientTaskRoot = dir
b.mu.Unlock()
return b
}
func (b *Builder) SetClientTaskLocalDir(dir string) *Builder {
b.mu.Lock()
b.clientTaskLocalDir = dir
b.mu.Unlock()
return b
}
func (b *Builder) SetClientTaskSecretsDir(dir string) *Builder {
b.mu.Lock()
b.clientTaskSecretsDir = dir
b.mu.Unlock()
return b
}
func (b *Builder) SetSecretsDir(dir string) *Builder {
b.mu.Lock()
b.secretsDir = dir

10
client/taskenv/services_test.go
View File

@ -103,9 +103,8 @@ func TestInterpolateServices(t *testing.T) {
var testEnv = NewTaskEnv(
map[string]string{"foo": "bar", "baz": "blah"},
nil,
nil,
)
map[string]string{"foo": "bar", "baz": "blah"},
nil, nil, "", "")
func TestInterpolate_interpolateMapStringSliceString(t *testing.T) {
t.Parallel()
@ -164,7 +163,7 @@ func TestInterpolate_interpolateMapStringInterface(t *testing.T) {
func TestInterpolate_interpolateConnect(t *testing.T) {
t.Parallel()
env := NewTaskEnv(map[string]string{
e := map[string]string{
"tag1": "_tag1",
"port1": "12345",
"address1": "1.2.3.4",
@ -200,7 +199,8 @@ func TestInterpolate_interpolateConnect(t *testing.T) {
"protocol2": "_protocol2",
"service1": "_service1",
"host1": "_host1",
}, nil, nil)
}
env := NewTaskEnv(e, e, nil, nil, "", "")
connect := &structs.ConsulConnect{
Native: false,

72
e2e/consultemplate/consultemplate.go
View File

@ -244,6 +244,11 @@ func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_Ok(f *framework.F) {
func(out string) bool {
return len(out) > 0
}, nil), "expected file to have contents")
f.NoError(waitForTemplateRender(allocID, "alloc/shared.txt",
func(out string) bool {
return len(out) > 0
}, nil), "expected shared-alloc-dir file to have contents")
}
// TestTemplatePathInterpolation_Bad asserts that template.source paths are not
@ -287,6 +292,73 @@ func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_Bad(f *framework.F)
f.True(found, "alloc failed but NOT due to expected source path escape error")
}
// TestTemplatePathInterpolation_SharedAlloc asserts that NOMAD_ALLOC_DIR
// is supported as a destination for artifact and template blocks, and
// that it is properly interpolated for task drivers with varying
// filesystem isolation
func (tc *ConsulTemplateTest) TestTemplatePathInterpolation_SharedAllocDir(f *framework.F) {
jobID := "template-shared-alloc-" + uuid.Generate()[:8]
tc.jobIDs = append(tc.jobIDs, jobID)
allocStubs := e2eutil.RegisterAndWaitForAllocs(
f.T(), tc.Nomad(), "consultemplate/input/template_shared_alloc.nomad", jobID, "")
f.Len(allocStubs, 1)
allocID := allocStubs[0].ID
e2eutil.WaitForAllocRunning(f.T(), tc.Nomad(), allocID)
for _, task := range []string{"docker", "exec", "raw_exec"} {
// tests that we can render templates into the shared alloc directory
f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/raw_exec.env",
func(out string) bool {
return len(out) > 0 && strings.TrimSpace(out) != "/alloc"
}, nil), "expected raw_exec.env to not be '/alloc'")
f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/exec.env",
func(out string) bool {
return strings.TrimSpace(out) == "/alloc"
}, nil), "expected shared exec.env to contain '/alloc'")
f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/docker.env",
func(out string) bool {
return strings.TrimSpace(out) == "/alloc"
}, nil), "expected shared docker.env to contain '/alloc'")
// test that we can fetch artifacts into the shared alloc directory
for _, a := range []string{"google1.html", "google2.html", "google3.html"} {
f.NoError(waitForTaskFile(allocID, task, "${NOMAD_ALLOC_DIR}/"+a,
func(out string) bool {
return len(out) > 0
}, nil), "expected artifact in alloc dir")
}
// test that we can load environment variables rendered with templates using interpolated paths
out, err := e2e.Command("nomad", "alloc", "exec", "-task", task, allocID, "sh", "-c", "env")
f.NoError(err)
f.Contains(out, "HELLO_FROM=raw_exec")
}
}
func waitForTaskFile(allocID, task, path string, test func(out string) bool, wc *e2e.WaitConfig) error {
var err error
var out string
interval, retries := wc.OrDefault()
testutil.WaitForResultRetries(retries, func() (bool, error) {
time.Sleep(interval)
out, err = e2e.Command("nomad", "alloc", "exec", "-task", task, allocID, "sh", "-c", "cat "+path)
if err != nil {
return false, fmt.Errorf("could not cat file %q from task %q in allocation %q: %v",
path, task, allocID, err)
}
return test(out), nil
}, func(e error) {
err = fmt.Errorf("test for file content failed: got %#v\nerror: %v", out, e)
})
return err
}
// waitForTemplateRender is a helper that grabs a file via alloc fs
// and tests it for
func waitForTemplateRender(allocID, path string, test func(string) bool, wc *e2e.WaitConfig) error {

7
e2e/consultemplate/input/template_paths.nomad
View File

@ -28,6 +28,13 @@ job "template-paths" {
destination = "${NOMAD_SECRETS_DIR}/foo/dst"
}
template {
destination = "${NOMAD_ALLOC_DIR}/shared.txt"
data = <<EOH
Data shared between all task in alloc dir.
EOH
}
resources {
cpu = 128
memory = 64

114
e2e/consultemplate/input/template_shared_alloc.nomad Normal file
View File

@ -0,0 +1,114 @@
job "template-shared-alloc" {
datacenters = ["dc1", "dc2"]
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "template-paths" {
task "raw_exec" {
driver = "raw_exec"
config {
command = "/bin/sh"
args = ["-c", "sleep 300"]
}
lifecycle {
hook = "prestart"
sidecar = true
}
artifact {
source = "https://google.com"
destination = "../alloc/google1.html"
}
template {
destination = "${NOMAD_ALLOC_DIR}/${NOMAD_TASK_NAME}.env"
data = <<EOH
{{env "NOMAD_ALLOC_DIR"}}
EOH
}
template {
destination = "${NOMAD_ALLOC_DIR}/hello-from-raw.env"
data = <<EOH
HELLO_FROM={{env "NOMAD_TASK_NAME"}}
EOH
env = true
}
resources {
cpu = 128
memory = 64
}
}
task "docker" {
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["-c", "sleep 300"]
}
artifact {
source = "https://google.com"
destination = "../alloc/google2.html"
}
template {
destination = "${NOMAD_ALLOC_DIR}/${NOMAD_TASK_NAME}.env"
data = <<EOH
{{env "NOMAD_ALLOC_DIR"}}
EOH
}
template {
source = "${NOMAD_ALLOC_DIR}/hello-from-raw.env"
destination = "${NOMAD_LOCAL_DIR}/hello-from-raw.env"
env = true
}
resources {
cpu = 128
memory = 64
}
}
task "exec" {
driver = "exec"
config {
command = "/bin/sh"
args = ["-c", "sleep 300"]
}
artifact {
source = "https://google.com"
destination = "${NOMAD_ALLOC_DIR}/google3.html"
}
template {
destination = "${NOMAD_ALLOC_DIR}/${NOMAD_TASK_NAME}.env"
data = <<EOH
{{env "NOMAD_ALLOC_DIR"}}
EOH
}
template {
source = "${NOMAD_ALLOC_DIR}/hello-from-raw.env"
destination = "${NOMAD_LOCAL_DIR}/hello-from-raw.env"
env = true
}
resources {
cpu = 128
memory = 64
}
}
}
}

6
plugins/drivers/testutils/testing.go
View File

@ -250,6 +250,12 @@ func (d *MockDriver) ExecTaskStreaming(ctx context.Context, taskID string, execO
// SetEnvvars sets path and host env vars depending on the FS isolation used.
func SetEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *config.Config) {
envBuilder.SetClientTaskRoot(taskDir.Dir)
envBuilder.SetClientSharedAllocDir(taskDir.SharedAllocDir)
envBuilder.SetClientTaskLocalDir(taskDir.LocalDir)
envBuilder.SetClientTaskSecretsDir(taskDir.SecretsDir)
// Set driver-specific environment variables
switch fsi {
case drivers.FSIsolationNone: