luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Add `disable_file` parameter to job's `vault` stanza (#13343) 

This complements the `env` parameter, so that the operator can author
tasks that don't share their Vault token with the workload when using 
`image` filesystem isolation. As a result, more powerful tokens can be used 
in a job definition, allowing it to use template stanzas to issue all kinds of 
secrets (database secrets, Vault tokens with very specific policies, etc.), 
without sharing that issuing power with the task itself.

This is accomplished by creating a directory called `private` within
the task's working directory, which shares many properties of
the `secrets` directory (tmpfs where possible, not accessible by
`nomad alloc fs` or Nomad's web UI), but isn't mounted into/bound to the
container.

If the `disable_file` parameter is set to `false` (its default), the Vault token
is also written to the NOMAD_SECRETS_DIR, so the default behavior is
backwards compatible. Even if the operator never changes the default,
they will still benefit from the improved behavior of Nomad never reading
the token back in from that - potentially altered - location.
This commit is contained in:
grembo 2023-06-23 21:15:04 +02:00 committed by GitHub
parent faa3377a56
commit 7936c1e33f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 385 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/13343.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
vault: Add new configuration `disable_file` to prevent access to the Vault token by tasks that use `image` filesystem isolation
```

4
api/tasks.go
View File

@ -919,6 +919,7 @@ type Vault struct {
Policies []string `hcl:"policies,optional"`
Namespace *string `mapstructure:"namespace" hcl:"namespace,optional"`
Env *bool `hcl:"env,optional"`
DisableFile *bool `mapstructure:"disable_file" hcl:"disable_file,optional"`
ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"`
ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"`
}
@ -927,6 +928,9 @@ func (v *Vault) Canonicalize() {
if v.Env == nil {
v.Env = pointerOf(true)
}
if v.DisableFile == nil {
v.DisableFile = pointerOf(false)
}
if v.Namespace == nil {
v.Namespace = pointerOf("")
}

1
api/tasks_test.go
View File

@ -460,6 +460,7 @@ func TestTask_Canonicalize_Vault(t *testing.T) {
input: &Vault{},
expected: &Vault{
Env: pointerOf(true),
DisableFile: pointerOf(false),
Namespace: pointerOf(""),
ChangeMode: pointerOf("restart"),
ChangeSignal: pointerOf("SIGHUP"),

15
client/allocdir/alloc_dir.go
View File

@ -60,6 +60,10 @@ var (
// directory
TaskSecrets = "secrets"
// TaskPrivate is the name of the private directory inside each task
// directory
TaskPrivate = "private"
// TaskDirs is the set of directories created in each tasks directory.
TaskDirs = map[string]os.FileMode{TmpDirName: os.ModeSticky | 0777}
@ -306,6 +310,13 @@ func (d *AllocDir) UnmountAll() error {
}
}
if pathExists(dir.PrivateDir) {
if err := removeSecretDir(dir.PrivateDir); err != nil {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("failed to remove the private dir %q: %v", dir.PrivateDir, err))
}
}
// Unmount dev/ and proc/ have been mounted.
if err := dir.unmountSpecialDirs(); err != nil {
mErr.Errors = append(mErr.Errors, err)
@ -447,6 +458,10 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) {
d.mu.RUnlock()
return nil, fmt.Errorf("Reading secret file prohibited: %s", path)
}
if filepath.HasPrefix(p, dir.PrivateDir) {
d.mu.RUnlock()
return nil, fmt.Errorf("Reading private file prohibited: %s", path)
}
}
d.mu.RUnlock()

14
client/allocdir/task_dir.go
View File

@ -41,6 +41,10 @@ type TaskDir struct {
// <task_dir>/secrets/
SecretsDir string
// PrivateDir is the path to private/ directory on the host
// <task_dir>/private/
PrivateDir string
// skip embedding these paths in chroots. Used for avoiding embedding
// client.alloc_dir recursively.
skip map[string]struct{}
@ -68,6 +72,7 @@ func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string)
SharedTaskDir: filepath.Join(taskDir, SharedAllocName),
LocalDir: filepath.Join(taskDir, TaskLocal),
SecretsDir: filepath.Join(taskDir, TaskSecrets),
PrivateDir: filepath.Join(taskDir, TaskPrivate),
skip: skip,
logger: logger,
}
@ -130,6 +135,15 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error {
return err
}
// Create the private directory
if err := createSecretDir(t.PrivateDir); err != nil {
return err
}
if err := dropDirPermissions(t.PrivateDir, os.ModePerm); err != nil {
return err
}
// Build chroot if chroot filesystem isolation is going to be used
if createChroot {
if err := t.buildChroot(chroot); err != nil {

85
client/allocrunner/taskrunner/task_runner_linux_test.go Normal file
View File

@ -0,0 +1,85 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package taskrunner
import (
"context"
"os"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/hashicorp/nomad/ci"
"github.com/hashicorp/nomad/client/vaultclient"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/shoenig/test/must"
)
func TestTaskRunner_DisableFileForVaultToken_UpgradePath(t *testing.T) {
ci.Parallel(t)
ci.SkipTestWithoutRootAccess(t)
// Create test allocation with a Vault block.
alloc := mock.BatchAlloc()
task := alloc.Job.TaskGroups[0].Tasks[0]
task.Config = map[string]any{
"run_for": "0s",
}
task.Vault = &structs.Vault{
Policies: []string{"default"},
}
conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name)
defer cleanup()
// Remove private dir and write the Vault token to the secrets dir to
// simulate an old task.
err := conf.TaskDir.Build(false, nil)
must.NoError(t, err)
err = syscall.Unmount(conf.TaskDir.PrivateDir, 0)
must.NoError(t, err)
err = os.Remove(conf.TaskDir.PrivateDir)
must.NoError(t, err)
token := "1234"
tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile)
err = os.WriteFile(tokenPath, []byte(token), 0666)
must.NoError(t, err)
// Setup a test Vault client.
handler := func(*structs.Allocation, []string) (map[string]string, error) {
return map[string]string{task.Name: token}, nil
}
vaultClient := conf.Vault.(*vaultclient.MockVaultClient)
vaultClient.DeriveTokenFn = handler
// Start task runner and wait for task to finish.
tr, err := NewTaskRunner(conf)
must.NoError(t, err)
defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
go tr.Run()
time.Sleep(500 * time.Millisecond)
testWaitForTaskToDie(t, tr)
// Verify task exited successfully.
finalState := tr.TaskState()
must.Eq(t, structs.TaskStateDead, finalState.State)
must.False(t, finalState.Failed)
// Verfiry token is in secrets dir.
tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile)
data, err := os.ReadFile(tokenPath)
must.NoError(t, err)
must.Eq(t, token, string(data))
// Varify token is not in private dir since the allocation doesn't have
// this path.
tokenPath = filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile)
_, err = os.Stat(tokenPath)
must.ErrorIs(t, err, os.ErrNotExist)
}

60
client/allocrunner/taskrunner/task_runner_test.go
View File

@ -1644,11 +1644,16 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) {
require.False(t, finalState.Failed)
// Check that the token is on disk
tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile)
tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile)
data, err := os.ReadFile(tokenPath)
require.NoError(t, err)
require.Equal(t, token, string(data))
tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile)
data, err = os.ReadFile(tokenPath)
require.NoError(t, err)
require.Equal(t, token, string(data))
// Kill task runner to trigger stop hooks
tr.Kill(context.Background(), structs.NewTaskEvent("kill"))
select {
@ -1672,6 +1677,57 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) {
})
}
func TestTaskRunner_DisableFileForVaultToken(t *testing.T) {
ci.Parallel(t)
// Create test allocation with a Vault block disabling the token file in
// the secrets dir.
alloc := mock.BatchAlloc()
task := alloc.Job.TaskGroups[0].Tasks[0]
task.Config = map[string]any{
"run_for": "0s",
}
task.Vault = &structs.Vault{
Policies: []string{"default"},
DisableFile: true,
}
conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name)
defer cleanup()
// Setup a test Vault client
token := "1234"
handler := func(*structs.Allocation, []string) (map[string]string, error) {
return map[string]string{task.Name: token}, nil
}
vaultClient := conf.Vault.(*vaultclient.MockVaultClient)
vaultClient.DeriveTokenFn = handler
// Start task runner and wait for it to complete.
tr, err := NewTaskRunner(conf)
must.NoError(t, err)
defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
go tr.Run()
testWaitForTaskToDie(t, tr)
// Verify task exited successfully.
finalState := tr.TaskState()
must.Eq(t, structs.TaskStateDead, finalState.State)
must.False(t, finalState.Failed)
// Verify token is in the private dir.
tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile)
data, err := os.ReadFile(tokenPath)
must.NoError(t, err)
must.Eq(t, token, string(data))
// Verify token is not in secrets dir.
tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile)
_, err = os.Stat(tokenPath)
must.ErrorIs(t, err, os.ErrNotExist)
}
// TestTaskRunner_DeriveToken_Retry asserts that if a recoverable error is
// returned when deriving a vault token a task will continue to block while
// it's retried.
@ -1721,7 +1777,7 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) {
require.Equal(t, 1, count)
// Check that the token is on disk
tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile)
tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile)
data, err := os.ReadFile(tokenPath)
require.NoError(t, err)
require.Equal(t, token, string(data))

55
client/allocrunner/taskrunner/vault_hook.go
View File

@ -7,6 +7,7 @@ import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"sync"
"time"
@ -80,8 +81,13 @@ type vaultHook struct {
ctx context.Context
cancel context.CancelFunc
// tokenPath is the path in which to read and write the token
tokenPath string
// privateDirTokenPath is the path inside the task's private directory where
// the Vault token is read and written.
privateDirTokenPath string
// secretsDirTokenPath is the path inside the task's secret directory where the
// Vault token is written unless disabled by the task.
secretsDirTokenPath string
// alloc is the allocation
alloc *structs.Allocation
@ -131,17 +137,24 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe
// Try to recover a token if it was previously written in the secrets
// directory
recoveredToken := ""
h.tokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile)
data, err := os.ReadFile(h.tokenPath)
if err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("failed to recover vault token: %v", err)
}
h.privateDirTokenPath = filepath.Join(req.TaskDir.PrivateDir, vaultTokenFile)
h.secretsDirTokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile)
// Token file doesn't exist
} else {
// Store the recovered token
recoveredToken = string(data)
// Handle upgrade path by searching for the previous token in all possible
// paths where the token may be.
for _, path := range []string{h.privateDirTokenPath, h.secretsDirTokenPath} {
data, err := os.ReadFile(path)
if err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("failed to recover vault token from %s: %v", path, err)
}
// Token file doesn't exist in this path.
} else {
// Store the recovered token
recoveredToken = string(data)
break
}
}
// Launch the token manager
@ -345,9 +358,25 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) {
// writeToken writes the given token to disk
func (h *vaultHook) writeToken(token string) error {
if err := os.WriteFile(h.tokenPath, []byte(token), 0666); err != nil {
// Handle upgrade path by first checking if the tasks private directory
// exists. If it doesn't, this allocation probably existed before the
// private directory was introduced, so keep using the secret directory to
// prevent unnecessary errors during task recovery.
if _, err := os.Stat(path.Dir(h.privateDirTokenPath)); os.IsNotExist(err) {
if err := os.WriteFile(h.secretsDirTokenPath, []byte(token), 0666); err != nil {
return fmt.Errorf("failed to write vault token to secrets dir: %v", err)
}
return nil
}
if err := os.WriteFile(h.privateDirTokenPath, []byte(token), 0600); err != nil {
return fmt.Errorf("failed to write vault token: %v", err)
}
if !h.vaultBlock.DisableFile {
if err := os.WriteFile(h.secretsDirTokenPath, []byte(token), 0666); err != nil {
return fmt.Errorf("failed to write vault token to secrets dir: %v", err)
}
}
return nil
}

1
command/agent/job_endpoint.go
View File

@ -1265,6 +1265,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
Policies: apiTask.Vault.Policies,
Namespace: *apiTask.Vault.Namespace,
Env: *apiTask.Vault.Env,
DisableFile: *apiTask.Vault.DisableFile,
ChangeMode: *apiTask.Vault.ChangeMode,
ChangeSignal: *apiTask.Vault.ChangeSignal,
}

2
command/agent/job_endpoint_test.go
View File

@ -2785,6 +2785,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Namespace: pointer.Of("ns1"),
Policies: []string{"a", "b", "c"},
Env: pointer.Of(true),
DisableFile: pointer.Of(false),
ChangeMode: pointer.Of("c"),
ChangeSignal: pointer.Of("sighup"),
},
@ -3204,6 +3205,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Namespace: "ns1",
Policies: []string{"a", "b", "c"},
Env: true,
DisableFile: false,
ChangeMode: "c",
ChangeSignal: "sighup",
},

1
drivers/shared/executor/executor_linux_test.go
View File

@ -244,6 +244,7 @@ etc/
lib/
lib64/
local/
private/
proc/
secrets/
sys/

1
jobspec/parse.go
View File

@ -511,6 +511,7 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error {
"namespace",
"policies",
"env",
"disable_file",
"change_mode",
"change_signal",
}

5
jobspec/parse_group.go
View File

@ -213,8 +213,9 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error {
// If we have a vault block, then parse that
if o := listVal.Filter("vault"); len(o.Items) > 0 {
tgVault := &api.Vault{
Env: boolToPtr(true),
ChangeMode: stringToPtr("restart"),
Env: boolToPtr(true),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr("restart"),
}
if err := parseVault(tgVault, o); err != nil {

5
jobspec/parse_job.go
View File

@ -194,8 +194,9 @@ func parseJob(result *api.Job, list *ast.ObjectList) error {
// If we have a vault block, then parse that
if o := listVal.Filter("vault"); len(o.Items) > 0 {
jobVault := &api.Vault{
Env: boolToPtr(true),
ChangeMode: stringToPtr("restart"),
Env: boolToPtr(true),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr("restart"),
}
if err := parseVault(jobVault, o); err != nil {

5
jobspec/parse_task.go
View File

@ -314,8 +314,9 @@ func parseTask(item *ast.ObjectItem, keys []string) (*api.Task, error) {
// If we have a vault block, then parse that
if o := listVal.Filter("vault"); len(o.Items) > 0 {
v := &api.Vault{
Env: boolToPtr(true),
ChangeMode: stringToPtr("restart"),
Env: boolToPtr(true),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr("restart"),
}
if err := parseVault(v, o); err != nil {

31
jobspec/parse_test.go
View File

@ -369,10 +369,11 @@ func TestParse(t *testing.T) {
},
},
Vault: &api.Vault{
Namespace: stringToPtr("ns1"),
Policies: []string{"foo", "bar"},
Env: boolToPtr(true),
ChangeMode: stringToPtr(vaultChangeModeRestart),
Namespace: stringToPtr("ns1"),
Policies: []string{"foo", "bar"},
Env: boolToPtr(true),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr(vaultChangeModeRestart),
},
Templates: []*api.Template{
{
@ -435,6 +436,7 @@ func TestParse(t *testing.T) {
Vault: &api.Vault{
Policies: []string{"foo", "bar"},
Env: boolToPtr(false),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr(vaultChangeModeSignal),
ChangeSignal: stringToPtr("SIGUSR1"),
},
@ -801,17 +803,19 @@ func TestParse(t *testing.T) {
{
Name: "redis",
Vault: &api.Vault{
Policies: []string{"group"},
Env: boolToPtr(true),
ChangeMode: stringToPtr(vaultChangeModeRestart),
Policies: []string{"group"},
Env: boolToPtr(true),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr(vaultChangeModeRestart),
},
},
{
Name: "redis2",
Vault: &api.Vault{
Policies: []string{"task"},
Env: boolToPtr(false),
ChangeMode: stringToPtr(vaultChangeModeRestart),
Policies: []string{"task"},
Env: boolToPtr(false),
DisableFile: boolToPtr(true),
ChangeMode: stringToPtr(vaultChangeModeRestart),
},
},
},
@ -822,9 +826,10 @@ func TestParse(t *testing.T) {
{
Name: "redis",
Vault: &api.Vault{
Policies: []string{"job"},
Env: boolToPtr(true),
ChangeMode: stringToPtr(vaultChangeModeRestart),
Policies: []string{"job"},
Env: boolToPtr(true),
DisableFile: boolToPtr(false),
ChangeMode: stringToPtr(vaultChangeModeRestart),
},
},
},

1
jobspec/test-fixtures/basic.hcl
View File

@ -362,6 +362,7 @@ job "binstore-storagelocker" {
vault {
policies = ["foo", "bar"]
env = false
disable_file = false
change_mode = "signal"
change_signal = "SIGUSR1"
}

5
jobspec/test-fixtures/vault_inheritance.hcl
View File

@ -15,8 +15,9 @@ job "example" {
task "redis2" {
vault {
policies = ["task"]
env = false
policies = ["task"]
env = false
disable_file = true
}
}
}

3
jobspec2/parse_job.go
View File

@ -65,6 +65,9 @@ func normalizeVault(v *api.Vault) {
if v.Env == nil {
v.Env = pointer.Of(true)
}
if v.DisableFile == nil {
v.DisableFile = pointer.Of(false)
}
if v.ChangeMode == nil {
v.ChangeMode = pointer.Of("restart")
}

30
nomad/structs/diff_test.go
View File

@ -6887,6 +6887,7 @@ func TestTaskDiff(t *testing.T) {
Vault: &Vault{
Policies: []string{"foo", "bar"},
Env: true,
DisableFile: true,
ChangeMode: "signal",
ChangeSignal: "SIGUSR1",
},
@ -6910,6 +6911,12 @@ func TestTaskDiff(t *testing.T) {
Old: "",
New: "SIGUSR1",
},
{
Type: DiffTypeAdded,
Name: "DisableFile",
Old: "",
New: "true",
},
{
Type: DiffTypeAdded,
Name: "Env",
@ -6947,6 +6954,7 @@ func TestTaskDiff(t *testing.T) {
Vault: &Vault{
Policies: []string{"foo", "bar"},
Env: true,
DisableFile: true,
ChangeMode: "signal",
ChangeSignal: "SIGUSR1",
},
@ -6971,6 +6979,12 @@ func TestTaskDiff(t *testing.T) {
Old: "SIGUSR1",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "DisableFile",
Old: "true",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "Env",
@ -7009,6 +7023,7 @@ func TestTaskDiff(t *testing.T) {
Namespace: "ns1",
Policies: []string{"foo", "bar"},
Env: true,
DisableFile: true,
ChangeMode: "signal",
ChangeSignal: "SIGUSR1",
},
@ -7018,6 +7033,7 @@ func TestTaskDiff(t *testing.T) {
Namespace: "ns2",
Policies: []string{"bar", "baz"},
Env: false,
DisableFile: false,
ChangeMode: "restart",
ChangeSignal: "foo",
},
@ -7041,6 +7057,12 @@ func TestTaskDiff(t *testing.T) {
Old: "SIGUSR1",
New: "foo",
},
{
Type: DiffTypeEdited,
Name: "DisableFile",
Old: "true",
New: "false",
},
{
Type: DiffTypeEdited,
Name: "Env",
@ -7086,6 +7108,7 @@ func TestTaskDiff(t *testing.T) {
Namespace: "ns1",
Policies: []string{"foo", "bar"},
Env: true,
DisableFile: true,
ChangeMode: "signal",
ChangeSignal: "SIGUSR1",
},
@ -7095,6 +7118,7 @@ func TestTaskDiff(t *testing.T) {
Namespace: "ns1",
Policies: []string{"bar", "baz"},
Env: true,
DisableFile: true,
ChangeMode: "signal",
ChangeSignal: "SIGUSR1",
},
@ -7118,6 +7142,12 @@ func TestTaskDiff(t *testing.T) {
Old: "SIGUSR1",
New: "SIGUSR1",
},
{
Type: DiffTypeNone,
Name: "DisableFile",
Old: "true",
New: "true",
},
{
Type: DiffTypeNone,
Name: "Env",

13
nomad/structs/structs.go
View File

@ -9760,6 +9760,10 @@ type Vault struct {
// variable
Env bool
// DisableFile marks whether the Vault Token should be exposed in the file
// vault_token in the task's secrets directory.
DisableFile bool
// ChangeMode is used to configure the task's behavior when the Vault
// token changes because the original token could not be renewed in time.
ChangeMode string
@ -9769,13 +9773,6 @@ type Vault struct {
ChangeSignal string
}
func DefaultVaultBlock() *Vault {
return &Vault{
Env: true,
ChangeMode: VaultChangeModeRestart,
}
}
func (v *Vault) Equal(o *Vault) bool {
if v == nil || o == nil {
return v == o
@ -9787,6 +9784,8 @@ func (v *Vault) Equal(o *Vault) bool {
return false
case v.Env != o.Env:
return false
case v.DisableFile != o.DisableFile:
return false
case v.ChangeMode != o.ChangeMode:
return false
case v.ChangeSignal != o.ChangeSignal:

21
website/content/docs/concepts/filesystem.mdx
View File

@ -29,10 +29,12 @@ allocation directory like the one below.
│ └── tmp
├── task1
│ ├── local
│ ├── private
│ ├── secrets
│ └── tmp
└── task2
├── local
├── private
├── secrets
└── tmp
```
@ -68,6 +70,17 @@ allocation directory like the one below.
`NOMAD_TASK_DIR`. Note this is not the same as the "task working
directory". This directory is private to the task.
- **«taskname»/private/**: This directory is used by Nomad to store private files
related to the allocation, such as Vault tokens, that are not shared with tasks
when using [`image` isolation](#image-isolation). The contents of files in this
directory cannot be read by the `nomad alloc fs` command or the via Nomad's
API.
<Warning>
While not shared with tasks that use <code>image</code> isolation, this
path is still accessible by tasks using <a href="#chroot-isolation">
<code>chroot</code></a> or <a href="#none-isolation"><code>none</code></a> isolation
</Warning>
- **«taskname»/secrets/**: This directory is the location provided to the task as
`NOMAD_SECRETS_DIR`. The contents of files in this directory cannot be read
by the `nomad alloc fs` command. It can be used to store secret data that
@ -97,6 +110,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/
$ nomad alloc fs c0b2245f task1/
Mode Size Modified Time Name
drwxrwxrwx 4.0 KiB 2020-10-27T18:00:33Z local/
drwxrwxrwx 60 B 2020-10-27T18:00:32Z private/
drwxrwxrwx 60 B 2020-10-27T18:00:32Z secrets/
dtrwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/
```
@ -150,6 +164,7 @@ minimal filesystem tree:
│ └── tmp
└── task1
├── local
├── private
├── secrets
└── tmp
```
@ -165,6 +180,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z task1/
$ nomad alloc fs b0686b27 task1
Mode Size Modified Time Name
drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z local/
drwxrwxrwx 60 B 2020-10-27T18:51:54Z private/
drwxrwxrwx 60 B 2020-10-27T18:51:54Z secrets/
dtrwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z tmp/
@ -287,6 +303,7 @@ contents], in addition to the `NOMAD_ALLOC_DIR`, `NOMAD_TASK_DIR`, and
├── lib32
├── lib64
├── local
├── private
├── proc
├── run
├── sbin
@ -315,6 +332,7 @@ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib/
drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib32/
drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib64/
drwxrwxrwx 4.0 KiB 2020-10-27T19:05:22Z local/
drwxrwxrwx 60 B 2020-10-27T19:05:22Z private/
drwxr-xr-x 4.0 KiB 2020-10-27T19:05:24Z proc/
drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z run/
drwxr-xr-x 12 KiB 2020-10-27T19:05:22Z sbin/
@ -334,6 +352,7 @@ $ nomad alloc exec eebd13a7 /bin/sh
$ mount
...
/dev/mapper/root on /alloc type ext4 (rw,relatime,errors=remount-ro,data=ordered)
tmpfs on /private type tmpfs (rw,noexec,relatime,size=1024k)
tmpfs on /secrets type tmpfs (rw,noexec,relatime,size=1024k)
...
```
@ -377,6 +396,7 @@ minimal filesystem tree:
└── task3
├── executor.out
├── local
├── private
├── secrets
└── tmp
```
@ -388,6 +408,7 @@ $ nomad alloc fs 87ec7d12 task3
Mode Size Modified Time Name
-rw-r--r-- 140 B 2020-10-27T19:15:33Z executor.out
drwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z local/
drwxrwxrwx 60 B 2020-10-27T19:15:33Z private/
drwxrwxrwx 60 B 2020-10-27T19:15:33Z secrets/
dtrwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z tmp/
```

69
website/content/docs/job-specification/vault.mdx
View File

@ -45,6 +45,7 @@ to the secret directory at `secrets/vault_token` and by injecting a `VAULT_TOKEN
environment variable. If the Nomad cluster is [configured](/nomad/docs/configuration/vault#namespace)
to use [Vault Namespaces](/vault/docs/enterprise/namespaces),
a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` is set.
This behavior can be altered using the `env` and `file` parameters.
If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or
network error), the client will attempt to retrieve a new Vault token. If successful, the
@ -70,6 +71,19 @@ with Vault as well.
- `env` `(bool: true)` - Specifies if the `VAULT_TOKEN` and `VAULT_NAMESPACE`
environment variables should be set when starting the task.
- `disable_file` `(bool: false)` - Specifies if the Vault token should be
written to `secrets/vault_token`.
<Warning>
While the <code>secrets</code> path is not shared with tasks that
use <a href="/nomad/docs/concepts/filesystem#image-isolation">
<code>image</code>
</a> filesystem isolation, it is still accessible by tasks using <a href="/nomad/docs/concepts/filesystem#chroot-isolation">
<code>chroot</code>
</a> or <a href="/nomad/docs/concepts/filesystem#none-isolation"><code>none</code>
</a> isolation.
</Warning>
- `namespace` `(string: "")` <EnterpriseAlert inline/> - Specifies the Vault Namespace
to use for the task. The Nomad client will retrieve a Vault token that is scoped to
this particular namespace.
@ -109,6 +123,58 @@ vault {
}
```
### Private Token and Change Modes
This example retrieves a Vault token that is not shared with the task when using
a driver that provides `image` isolation like [Docker][docker].
This allows Nomad to use a powerful Vault token that interacts with the task's
[`template`][template] stanzas to issue all kinds of secrets (e.g., database
secrets, other vault tokens, etc.), without sharing that issuing power with
the task itself:
```hcl
vault {
policies = ["tls-policy", "nomad-job-policy"]
change_mode = "noop"
env = false
file = false
}
template {
data = <<-EOH
{{with secret "auth/token/create/nomad-job" "policies=examplepolicy"}}{{.Auth.ClientToken}}{{ end }}
EOH
destination = "${NOMAD_SECRETS_DIR}/examplepolicy.token"
change_mode = "noop"
perms = "600"
}
template {
data = <<-EOH
{{ with secret "pki_int/issue/nomad-task"
"common_name=example.service.consul" "ttl=72h"
"alt_names=localhost" "ip_sans=127.0.0.1"}}
{{ .Data.certificate }}
{{ .Data.private_key }}
{{ end }}
EOH
destination = "${NOMAD_SECRETS_DIR}/client.crt"
change_mode = "restart"
perms = "600"
}
```
The example above uses `change_mode = "noop"` in the `template` stanza for
`examplepolicy.token`, which means that the task's workload is responsible for
detecting and handling changes to that file. In contrast, the `template` stanza
for `client.crt` is configured so that Nomad will restart the task whenever
the certificate is reissued, as indicated by `change_mode = "restart"`
(which is the default value for `change_mode`).
### Vault Namespace
This example shows specifying a particular Vault namespace for a given task.
@ -125,8 +191,7 @@ vault {
}
```
[docker]: /nomad/docs/drivers/docker "Docker Driver"
[restart]: /nomad/docs/job-specification/restart "Nomad restart Job Specification"
[template]: /nomad/docs/job-specification/template "Nomad template Job Specification"
[vault]: https://www.vaultproject.io/ "Vault by HashiCorp"