From 7936c1e33f4f7a2ea5b4c8279c5eac41c79d287a Mon Sep 17 00:00:00 2001 From: grembo Date: Fri, 23 Jun 2023 21:15:04 +0200 Subject: [PATCH] Add `disable_file` parameter to job's `vault` stanza (#13343) This complements the `env` parameter, so that the operator can author tasks that don't share their Vault token with the workload when using `image` filesystem isolation. As a result, more powerful tokens can be used in a job definition, allowing it to use template stanzas to issue all kinds of secrets (database secrets, Vault tokens with very specific policies, etc.), without sharing that issuing power with the task itself. This is accomplished by creating a directory called `private` within the task's working directory, which shares many properties of the `secrets` directory (tmpfs where possible, not accessible by `nomad alloc fs` or Nomad's web UI), but isn't mounted into/bound to the container. If the `disable_file` parameter is set to `false` (its default), the Vault token is also written to the NOMAD_SECRETS_DIR, so the default behavior is backwards compatible. Even if the operator never changes the default, they will still benefit from the improved behavior of Nomad never reading the token back in from that - potentially altered - location. --- .changelog/13343.txt | 3 + api/tasks.go | 4 + api/tasks_test.go | 1 + client/allocdir/alloc_dir.go | 15 ++++ client/allocdir/task_dir.go | 14 +++ .../taskrunner/task_runner_linux_test.go | 85 +++++++++++++++++++ .../taskrunner/task_runner_test.go | 60 ++++++++++++- client/allocrunner/taskrunner/vault_hook.go | 55 +++++++++--- command/agent/job_endpoint.go | 1 + command/agent/job_endpoint_test.go | 2 + .../shared/executor/executor_linux_test.go | 1 + jobspec/parse.go | 1 + jobspec/parse_group.go | 5 +- jobspec/parse_job.go | 5 +- jobspec/parse_task.go | 5 +- jobspec/parse_test.go | 31 ++++--- jobspec/test-fixtures/basic.hcl | 1 + jobspec/test-fixtures/vault_inheritance.hcl | 5 +- jobspec2/parse_job.go | 3 + nomad/structs/diff_test.go | 30 +++++++ nomad/structs/structs.go | 13 ++- website/content/docs/concepts/filesystem.mdx | 21 +++++ .../content/docs/job-specification/vault.mdx | 69 ++++++++++++++- 23 files changed, 385 insertions(+), 45 deletions(-) create mode 100644 .changelog/13343.txt create mode 100644 client/allocrunner/taskrunner/task_runner_linux_test.go diff --git a/.changelog/13343.txt b/.changelog/13343.txt new file mode 100644 index 000000000..0de61d8b8 --- /dev/null +++ b/.changelog/13343.txt @@ -0,0 +1,3 @@ +```release-note:improvement +vault: Add new configuration `disable_file` to prevent access to the Vault token by tasks that use `image` filesystem isolation +``` diff --git a/api/tasks.go b/api/tasks.go index 653442905..188fa8649 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -919,6 +919,7 @@ type Vault struct { Policies []string `hcl:"policies,optional"` Namespace *string `mapstructure:"namespace" hcl:"namespace,optional"` Env *bool `hcl:"env,optional"` + DisableFile *bool `mapstructure:"disable_file" hcl:"disable_file,optional"` ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"` ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"` } @@ -927,6 +928,9 @@ func (v *Vault) Canonicalize() { if v.Env == nil { v.Env = pointerOf(true) } + if v.DisableFile == nil { + v.DisableFile = pointerOf(false) + } if v.Namespace == nil { v.Namespace = pointerOf("") } diff --git a/api/tasks_test.go b/api/tasks_test.go index 10b917b43..231993906 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -460,6 +460,7 @@ func TestTask_Canonicalize_Vault(t *testing.T) { input: &Vault{}, expected: &Vault{ Env: pointerOf(true), + DisableFile: pointerOf(false), Namespace: pointerOf(""), ChangeMode: pointerOf("restart"), ChangeSignal: pointerOf("SIGHUP"), diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 637524761..8ef9c6c65 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -60,6 +60,10 @@ var ( // directory TaskSecrets = "secrets" + // TaskPrivate is the name of the private directory inside each task + // directory + TaskPrivate = "private" + // TaskDirs is the set of directories created in each tasks directory. TaskDirs = map[string]os.FileMode{TmpDirName: os.ModeSticky | 0777} @@ -306,6 +310,13 @@ func (d *AllocDir) UnmountAll() error { } } + if pathExists(dir.PrivateDir) { + if err := removeSecretDir(dir.PrivateDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the private dir %q: %v", dir.PrivateDir, err)) + } + } + // Unmount dev/ and proc/ have been mounted. if err := dir.unmountSpecialDirs(); err != nil { mErr.Errors = append(mErr.Errors, err) @@ -447,6 +458,10 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) { d.mu.RUnlock() return nil, fmt.Errorf("Reading secret file prohibited: %s", path) } + if filepath.HasPrefix(p, dir.PrivateDir) { + d.mu.RUnlock() + return nil, fmt.Errorf("Reading private file prohibited: %s", path) + } } d.mu.RUnlock() diff --git a/client/allocdir/task_dir.go b/client/allocdir/task_dir.go index 120212bf2..eb6afc622 100644 --- a/client/allocdir/task_dir.go +++ b/client/allocdir/task_dir.go @@ -41,6 +41,10 @@ type TaskDir struct { // /secrets/ SecretsDir string + // PrivateDir is the path to private/ directory on the host + // /private/ + PrivateDir string + // skip embedding these paths in chroots. Used for avoiding embedding // client.alloc_dir recursively. skip map[string]struct{} @@ -68,6 +72,7 @@ func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string) SharedTaskDir: filepath.Join(taskDir, SharedAllocName), LocalDir: filepath.Join(taskDir, TaskLocal), SecretsDir: filepath.Join(taskDir, TaskSecrets), + PrivateDir: filepath.Join(taskDir, TaskPrivate), skip: skip, logger: logger, } @@ -130,6 +135,15 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { return err } + // Create the private directory + if err := createSecretDir(t.PrivateDir); err != nil { + return err + } + + if err := dropDirPermissions(t.PrivateDir, os.ModePerm); err != nil { + return err + } + // Build chroot if chroot filesystem isolation is going to be used if createChroot { if err := t.buildChroot(chroot); err != nil { diff --git a/client/allocrunner/taskrunner/task_runner_linux_test.go b/client/allocrunner/taskrunner/task_runner_linux_test.go new file mode 100644 index 000000000..05de054a6 --- /dev/null +++ b/client/allocrunner/taskrunner/task_runner_linux_test.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package taskrunner + +import ( + "context" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/client/vaultclient" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/shoenig/test/must" +) + +func TestTaskRunner_DisableFileForVaultToken_UpgradePath(t *testing.T) { + ci.Parallel(t) + ci.SkipTestWithoutRootAccess(t) + + // Create test allocation with a Vault block. + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Config = map[string]any{ + "run_for": "0s", + } + task.Vault = &structs.Vault{ + Policies: []string{"default"}, + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + // Remove private dir and write the Vault token to the secrets dir to + // simulate an old task. + err := conf.TaskDir.Build(false, nil) + must.NoError(t, err) + + err = syscall.Unmount(conf.TaskDir.PrivateDir, 0) + must.NoError(t, err) + err = os.Remove(conf.TaskDir.PrivateDir) + must.NoError(t, err) + + token := "1234" + tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + err = os.WriteFile(tokenPath, []byte(token), 0666) + must.NoError(t, err) + + // Setup a test Vault client. + handler := func(*structs.Allocation, []string) (map[string]string, error) { + return map[string]string{task.Name: token}, nil + } + vaultClient := conf.Vault.(*vaultclient.MockVaultClient) + vaultClient.DeriveTokenFn = handler + + // Start task runner and wait for task to finish. + tr, err := NewTaskRunner(conf) + must.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + time.Sleep(500 * time.Millisecond) + + testWaitForTaskToDie(t, tr) + + // Verify task exited successfully. + finalState := tr.TaskState() + must.Eq(t, structs.TaskStateDead, finalState.State) + must.False(t, finalState.Failed) + + // Verfiry token is in secrets dir. + tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + data, err := os.ReadFile(tokenPath) + must.NoError(t, err) + must.Eq(t, token, string(data)) + + // Varify token is not in private dir since the allocation doesn't have + // this path. + tokenPath = filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) + _, err = os.Stat(tokenPath) + must.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index b25a261f3..24757b6ce 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1644,11 +1644,16 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { require.False(t, finalState.Failed) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := os.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) + tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + data, err = os.ReadFile(tokenPath) + require.NoError(t, err) + require.Equal(t, token, string(data)) + // Kill task runner to trigger stop hooks tr.Kill(context.Background(), structs.NewTaskEvent("kill")) select { @@ -1672,6 +1677,57 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { }) } +func TestTaskRunner_DisableFileForVaultToken(t *testing.T) { + ci.Parallel(t) + + // Create test allocation with a Vault block disabling the token file in + // the secrets dir. + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Config = map[string]any{ + "run_for": "0s", + } + task.Vault = &structs.Vault{ + Policies: []string{"default"}, + DisableFile: true, + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + // Setup a test Vault client + token := "1234" + handler := func(*structs.Allocation, []string) (map[string]string, error) { + return map[string]string{task.Name: token}, nil + } + vaultClient := conf.Vault.(*vaultclient.MockVaultClient) + vaultClient.DeriveTokenFn = handler + + // Start task runner and wait for it to complete. + tr, err := NewTaskRunner(conf) + must.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + + testWaitForTaskToDie(t, tr) + + // Verify task exited successfully. + finalState := tr.TaskState() + must.Eq(t, structs.TaskStateDead, finalState.State) + must.False(t, finalState.Failed) + + // Verify token is in the private dir. + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) + data, err := os.ReadFile(tokenPath) + must.NoError(t, err) + must.Eq(t, token, string(data)) + + // Verify token is not in secrets dir. + tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + _, err = os.Stat(tokenPath) + must.ErrorIs(t, err, os.ErrNotExist) +} + // TestTaskRunner_DeriveToken_Retry asserts that if a recoverable error is // returned when deriving a vault token a task will continue to block while // it's retried. @@ -1721,7 +1777,7 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) { require.Equal(t, 1, count) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := os.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index 2a2ce87af..b979e055c 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "path" "path/filepath" "sync" "time" @@ -80,8 +81,13 @@ type vaultHook struct { ctx context.Context cancel context.CancelFunc - // tokenPath is the path in which to read and write the token - tokenPath string + // privateDirTokenPath is the path inside the task's private directory where + // the Vault token is read and written. + privateDirTokenPath string + + // secretsDirTokenPath is the path inside the task's secret directory where the + // Vault token is written unless disabled by the task. + secretsDirTokenPath string // alloc is the allocation alloc *structs.Allocation @@ -131,17 +137,24 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe // Try to recover a token if it was previously written in the secrets // directory recoveredToken := "" - h.tokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) - data, err := os.ReadFile(h.tokenPath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to recover vault token: %v", err) - } + h.privateDirTokenPath = filepath.Join(req.TaskDir.PrivateDir, vaultTokenFile) + h.secretsDirTokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) - // Token file doesn't exist - } else { - // Store the recovered token - recoveredToken = string(data) + // Handle upgrade path by searching for the previous token in all possible + // paths where the token may be. + for _, path := range []string{h.privateDirTokenPath, h.secretsDirTokenPath} { + data, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to recover vault token from %s: %v", path, err) + } + + // Token file doesn't exist in this path. + } else { + // Store the recovered token + recoveredToken = string(data) + break + } } // Launch the token manager @@ -345,9 +358,25 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) { // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { - if err := os.WriteFile(h.tokenPath, []byte(token), 0666); err != nil { + // Handle upgrade path by first checking if the tasks private directory + // exists. If it doesn't, this allocation probably existed before the + // private directory was introduced, so keep using the secret directory to + // prevent unnecessary errors during task recovery. + if _, err := os.Stat(path.Dir(h.privateDirTokenPath)); os.IsNotExist(err) { + if err := os.WriteFile(h.secretsDirTokenPath, []byte(token), 0666); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + return nil + } + + if err := os.WriteFile(h.privateDirTokenPath, []byte(token), 0600); err != nil { return fmt.Errorf("failed to write vault token: %v", err) } + if !h.vaultBlock.DisableFile { + if err := os.WriteFile(h.secretsDirTokenPath, []byte(token), 0666); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + } return nil } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index eeb114a40..b12599141 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1265,6 +1265,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, Policies: apiTask.Vault.Policies, Namespace: *apiTask.Vault.Namespace, Env: *apiTask.Vault.Env, + DisableFile: *apiTask.Vault.DisableFile, ChangeMode: *apiTask.Vault.ChangeMode, ChangeSignal: *apiTask.Vault.ChangeSignal, } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 056f4a686..9b6d9b0a8 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2785,6 +2785,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Namespace: pointer.Of("ns1"), Policies: []string{"a", "b", "c"}, Env: pointer.Of(true), + DisableFile: pointer.Of(false), ChangeMode: pointer.Of("c"), ChangeSignal: pointer.Of("sighup"), }, @@ -3204,6 +3205,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Namespace: "ns1", Policies: []string{"a", "b", "c"}, Env: true, + DisableFile: false, ChangeMode: "c", ChangeSignal: "sighup", }, diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index 7172e1136..edd4e4b4c 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -244,6 +244,7 @@ etc/ lib/ lib64/ local/ +private/ proc/ secrets/ sys/ diff --git a/jobspec/parse.go b/jobspec/parse.go index b840a247f..7286b6e33 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -511,6 +511,7 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { "namespace", "policies", "env", + "disable_file", "change_mode", "change_signal", } diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 7a4fdf746..213e286a5 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -213,8 +213,9 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { // If we have a vault block, then parse that if o := listVal.Filter("vault"); len(o.Items) > 0 { tgVault := &api.Vault{ - Env: boolToPtr(true), - ChangeMode: stringToPtr("restart"), + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr("restart"), } if err := parseVault(tgVault, o); err != nil { diff --git a/jobspec/parse_job.go b/jobspec/parse_job.go index b51b25666..46c3dd757 100644 --- a/jobspec/parse_job.go +++ b/jobspec/parse_job.go @@ -194,8 +194,9 @@ func parseJob(result *api.Job, list *ast.ObjectList) error { // If we have a vault block, then parse that if o := listVal.Filter("vault"); len(o.Items) > 0 { jobVault := &api.Vault{ - Env: boolToPtr(true), - ChangeMode: stringToPtr("restart"), + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr("restart"), } if err := parseVault(jobVault, o); err != nil { diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index a7a091c32..5d596738d 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -314,8 +314,9 @@ func parseTask(item *ast.ObjectItem, keys []string) (*api.Task, error) { // If we have a vault block, then parse that if o := listVal.Filter("vault"); len(o.Items) > 0 { v := &api.Vault{ - Env: boolToPtr(true), - ChangeMode: stringToPtr("restart"), + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr("restart"), } if err := parseVault(v, o); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 6ccc72254..3fe453c21 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -369,10 +369,11 @@ func TestParse(t *testing.T) { }, }, Vault: &api.Vault{ - Namespace: stringToPtr("ns1"), - Policies: []string{"foo", "bar"}, - Env: boolToPtr(true), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Namespace: stringToPtr("ns1"), + Policies: []string{"foo", "bar"}, + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, Templates: []*api.Template{ { @@ -435,6 +436,7 @@ func TestParse(t *testing.T) { Vault: &api.Vault{ Policies: []string{"foo", "bar"}, Env: boolToPtr(false), + DisableFile: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeSignal), ChangeSignal: stringToPtr("SIGUSR1"), }, @@ -801,17 +803,19 @@ func TestParse(t *testing.T) { { Name: "redis", Vault: &api.Vault{ - Policies: []string{"group"}, - Env: boolToPtr(true), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Policies: []string{"group"}, + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, }, { Name: "redis2", Vault: &api.Vault{ - Policies: []string{"task"}, - Env: boolToPtr(false), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Policies: []string{"task"}, + Env: boolToPtr(false), + DisableFile: boolToPtr(true), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, }, }, @@ -822,9 +826,10 @@ func TestParse(t *testing.T) { { Name: "redis", Vault: &api.Vault{ - Policies: []string{"job"}, - Env: boolToPtr(true), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Policies: []string{"job"}, + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 17bd0a0ff..66695c1c3 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -362,6 +362,7 @@ job "binstore-storagelocker" { vault { policies = ["foo", "bar"] env = false + disable_file = false change_mode = "signal" change_signal = "SIGUSR1" } diff --git a/jobspec/test-fixtures/vault_inheritance.hcl b/jobspec/test-fixtures/vault_inheritance.hcl index c844f8d1b..4ee592187 100644 --- a/jobspec/test-fixtures/vault_inheritance.hcl +++ b/jobspec/test-fixtures/vault_inheritance.hcl @@ -15,8 +15,9 @@ job "example" { task "redis2" { vault { - policies = ["task"] - env = false + policies = ["task"] + env = false + disable_file = true } } } diff --git a/jobspec2/parse_job.go b/jobspec2/parse_job.go index e1cd81cb4..001d23b0c 100644 --- a/jobspec2/parse_job.go +++ b/jobspec2/parse_job.go @@ -65,6 +65,9 @@ func normalizeVault(v *api.Vault) { if v.Env == nil { v.Env = pointer.Of(true) } + if v.DisableFile == nil { + v.DisableFile = pointer.Of(false) + } if v.ChangeMode == nil { v.ChangeMode = pointer.Of("restart") } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index a93fa98a5..7d03c1fd4 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -6887,6 +6887,7 @@ func TestTaskDiff(t *testing.T) { Vault: &Vault{ Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -6910,6 +6911,12 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "SIGUSR1", }, + { + Type: DiffTypeAdded, + Name: "DisableFile", + Old: "", + New: "true", + }, { Type: DiffTypeAdded, Name: "Env", @@ -6947,6 +6954,7 @@ func TestTaskDiff(t *testing.T) { Vault: &Vault{ Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -6971,6 +6979,12 @@ func TestTaskDiff(t *testing.T) { Old: "SIGUSR1", New: "", }, + { + Type: DiffTypeDeleted, + Name: "DisableFile", + Old: "true", + New: "", + }, { Type: DiffTypeDeleted, Name: "Env", @@ -7009,6 +7023,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns1", Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -7018,6 +7033,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns2", Policies: []string{"bar", "baz"}, Env: false, + DisableFile: false, ChangeMode: "restart", ChangeSignal: "foo", }, @@ -7041,6 +7057,12 @@ func TestTaskDiff(t *testing.T) { Old: "SIGUSR1", New: "foo", }, + { + Type: DiffTypeEdited, + Name: "DisableFile", + Old: "true", + New: "false", + }, { Type: DiffTypeEdited, Name: "Env", @@ -7086,6 +7108,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns1", Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -7095,6 +7118,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns1", Policies: []string{"bar", "baz"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -7118,6 +7142,12 @@ func TestTaskDiff(t *testing.T) { Old: "SIGUSR1", New: "SIGUSR1", }, + { + Type: DiffTypeNone, + Name: "DisableFile", + Old: "true", + New: "true", + }, { Type: DiffTypeNone, Name: "Env", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 83ef6ce88..af9ea2a3b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -9760,6 +9760,10 @@ type Vault struct { // variable Env bool + // DisableFile marks whether the Vault Token should be exposed in the file + // vault_token in the task's secrets directory. + DisableFile bool + // ChangeMode is used to configure the task's behavior when the Vault // token changes because the original token could not be renewed in time. ChangeMode string @@ -9769,13 +9773,6 @@ type Vault struct { ChangeSignal string } -func DefaultVaultBlock() *Vault { - return &Vault{ - Env: true, - ChangeMode: VaultChangeModeRestart, - } -} - func (v *Vault) Equal(o *Vault) bool { if v == nil || o == nil { return v == o @@ -9787,6 +9784,8 @@ func (v *Vault) Equal(o *Vault) bool { return false case v.Env != o.Env: return false + case v.DisableFile != o.DisableFile: + return false case v.ChangeMode != o.ChangeMode: return false case v.ChangeSignal != o.ChangeSignal: diff --git a/website/content/docs/concepts/filesystem.mdx b/website/content/docs/concepts/filesystem.mdx index c0514750e..74dff6d50 100644 --- a/website/content/docs/concepts/filesystem.mdx +++ b/website/content/docs/concepts/filesystem.mdx @@ -29,10 +29,12 @@ allocation directory like the one below. │ └── tmp ├── task1 │ ├── local +│ ├── private │ ├── secrets │ └── tmp └── task2 ├── local + ├── private ├── secrets └── tmp ``` @@ -68,6 +70,17 @@ allocation directory like the one below. `NOMAD_TASK_DIR`. Note this is not the same as the "task working directory". This directory is private to the task. + - **«taskname»/private/**: This directory is used by Nomad to store private files + related to the allocation, such as Vault tokens, that are not shared with tasks + when using [`image` isolation](#image-isolation). The contents of files in this + directory cannot be read by the `nomad alloc fs` command or the via Nomad's + API. + + While not shared with tasks that use image isolation, this + path is still accessible by tasks using + chroot or none isolation + + - **«taskname»/secrets/**: This directory is the location provided to the task as `NOMAD_SECRETS_DIR`. The contents of files in this directory cannot be read by the `nomad alloc fs` command. It can be used to store secret data that @@ -97,6 +110,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/ $ nomad alloc fs c0b2245f task1/ Mode Size Modified Time Name drwxrwxrwx 4.0 KiB 2020-10-27T18:00:33Z local/ +drwxrwxrwx 60 B 2020-10-27T18:00:32Z private/ drwxrwxrwx 60 B 2020-10-27T18:00:32Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/ ``` @@ -150,6 +164,7 @@ minimal filesystem tree: │ └── tmp └── task1 ├── local + ├── private ├── secrets └── tmp ``` @@ -165,6 +180,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z task1/ $ nomad alloc fs b0686b27 task1 Mode Size Modified Time Name drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z local/ +drwxrwxrwx 60 B 2020-10-27T18:51:54Z private/ drwxrwxrwx 60 B 2020-10-27T18:51:54Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z tmp/ @@ -287,6 +303,7 @@ contents], in addition to the `NOMAD_ALLOC_DIR`, `NOMAD_TASK_DIR`, and ├── lib32 ├── lib64 ├── local + ├── private ├── proc ├── run ├── sbin @@ -315,6 +332,7 @@ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib32/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib64/ drwxrwxrwx 4.0 KiB 2020-10-27T19:05:22Z local/ +drwxrwxrwx 60 B 2020-10-27T19:05:22Z private/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:24Z proc/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z run/ drwxr-xr-x 12 KiB 2020-10-27T19:05:22Z sbin/ @@ -334,6 +352,7 @@ $ nomad alloc exec eebd13a7 /bin/sh $ mount ... /dev/mapper/root on /alloc type ext4 (rw,relatime,errors=remount-ro,data=ordered) +tmpfs on /private type tmpfs (rw,noexec,relatime,size=1024k) tmpfs on /secrets type tmpfs (rw,noexec,relatime,size=1024k) ... ``` @@ -377,6 +396,7 @@ minimal filesystem tree: └── task3 ├── executor.out ├── local + ├── private ├── secrets └── tmp ``` @@ -388,6 +408,7 @@ $ nomad alloc fs 87ec7d12 task3 Mode Size Modified Time Name -rw-r--r-- 140 B 2020-10-27T19:15:33Z executor.out drwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z local/ +drwxrwxrwx 60 B 2020-10-27T19:15:33Z private/ drwxrwxrwx 60 B 2020-10-27T19:15:33Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z tmp/ ``` diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index 20fe3ab27..cfe5a1893 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -45,6 +45,7 @@ to the secret directory at `secrets/vault_token` and by injecting a `VAULT_TOKEN environment variable. If the Nomad cluster is [configured](/nomad/docs/configuration/vault#namespace) to use [Vault Namespaces](/vault/docs/enterprise/namespaces), a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` is set. +This behavior can be altered using the `env` and `file` parameters. If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or network error), the client will attempt to retrieve a new Vault token. If successful, the @@ -70,6 +71,19 @@ with Vault as well. - `env` `(bool: true)` - Specifies if the `VAULT_TOKEN` and `VAULT_NAMESPACE` environment variables should be set when starting the task. +- `disable_file` `(bool: false)` - Specifies if the Vault token should be + written to `secrets/vault_token`. + + While the secrets path is not shared with tasks that + use + image + filesystem isolation, it is still accessible by tasks using + chroot + or none + isolation. + + + - `namespace` `(string: "")` - Specifies the Vault Namespace to use for the task. The Nomad client will retrieve a Vault token that is scoped to this particular namespace. @@ -109,6 +123,58 @@ vault { } ``` +### Private Token and Change Modes + +This example retrieves a Vault token that is not shared with the task when using +a driver that provides `image` isolation like [Docker][docker]. + +This allows Nomad to use a powerful Vault token that interacts with the task's +[`template`][template] stanzas to issue all kinds of secrets (e.g., database +secrets, other vault tokens, etc.), without sharing that issuing power with +the task itself: + +```hcl +vault { + policies = ["tls-policy", "nomad-job-policy"] + change_mode = "noop" + env = false + file = false +} + +template { + data = <<-EOH +{{with secret "auth/token/create/nomad-job" "policies=examplepolicy"}}{{.Auth.ClientToken}}{{ end }} +EOH + + destination = "${NOMAD_SECRETS_DIR}/examplepolicy.token" + change_mode = "noop" + perms = "600" +} + +template { + data = <<-EOH +{{ with secret "pki_int/issue/nomad-task" + "common_name=example.service.consul" "ttl=72h" + "alt_names=localhost" "ip_sans=127.0.0.1"}} +{{ .Data.certificate }} +{{ .Data.private_key }} +{{ end }} +EOH + + destination = "${NOMAD_SECRETS_DIR}/client.crt" + change_mode = "restart" + perms = "600" +} +``` + +The example above uses `change_mode = "noop"` in the `template` stanza for +`examplepolicy.token`, which means that the task's workload is responsible for +detecting and handling changes to that file. In contrast, the `template` stanza +for `client.crt` is configured so that Nomad will restart the task whenever +the certificate is reissued, as indicated by `change_mode = "restart"` +(which is the default value for `change_mode`). + + ### Vault Namespace This example shows specifying a particular Vault namespace for a given task. @@ -125,8 +191,7 @@ vault { } ``` +[docker]: /nomad/docs/drivers/docker "Docker Driver" [restart]: /nomad/docs/job-specification/restart "Nomad restart Job Specification" - [template]: /nomad/docs/job-specification/template "Nomad template Job Specification" - [vault]: https://www.vaultproject.io/ "Vault by HashiCorp"