From eef786dd9da5e95e67dc768fa741591f83b4e69f Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Fri, 2 Sep 2016 12:44:05 -0700 Subject: [PATCH] Secret dir materialized in alloc/task directory --- client/allocdir/alloc_dir.go | 22 +++++++++++ client/allocdir/alloc_dir_darwin.go | 11 ++++++ client/allocdir/alloc_dir_freebsd.go | 11 ++++++ client/allocdir/alloc_dir_linux.go | 37 +++++++++++++++++++ client/allocdir/alloc_dir_test.go | 4 ++ client/allocdir/alloc_dir_unix.go | 10 ++++- client/allocdir/alloc_dir_windows.go | 20 +++++++++- client/client_test.go | 18 +++++++++ client/driver/docker.go | 1 + client/driver/docker_test.go | 3 +- client/driver/driver.go | 1 + client/driver/env/env.go | 18 +++++++++ client/driver/exec_test.go | 3 +- client/driver/executor/executor_linux_test.go | 18 ++++++++- client/driver/java_test.go | 3 +- client/driver/qemu_test.go | 3 +- client/driver/raw_exec_test.go | 3 +- client/driver/rkt_test.go | 1 + client/task_runner_test.go | 2 + 19 files changed, 179 insertions(+), 10 deletions(-) diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 6d0a41ee0..312bb5d79 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -46,6 +46,10 @@ var ( // regardless of driver. TaskLocal = "local" + // TaskSecrets is the the name of the secret directory inside each task + // directory + TaskSecrets = "secrets" + // TaskDirs is the set of directories created in each tasks directory. TaskDirs = []string{"tmp"} ) @@ -154,6 +158,14 @@ func (d *AllocDir) UnmountAll() error { } } + taskSecret := filepath.Join(dir, TaskSecrets) + if d.pathExists(taskSecret) { + if err := d.removeSecretDir(taskSecret); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the secret dir %q: %v", taskSecret, err)) + } + } + // Unmount dev/ and proc/ have been mounted. d.unmountSpecialDirs(dir) } @@ -223,6 +235,16 @@ func (d *AllocDir) Build(tasks []*structs.Task) error { return err } } + + // Create the secret directory + secret := filepath.Join(taskDir, TaskSecrets) + if err := d.createSecretDir(secret); err != nil { + return err + } + + if err := d.dropDirPermissions(secret); err != nil { + return err + } } return nil diff --git a/client/allocdir/alloc_dir_darwin.go b/client/allocdir/alloc_dir_darwin.go index 2cfdd38c3..9b94ec6e5 100644 --- a/client/allocdir/alloc_dir_darwin.go +++ b/client/allocdir/alloc_dir_darwin.go @@ -1,6 +1,7 @@ package allocdir import ( + "os" "syscall" ) @@ -14,6 +15,16 @@ func (d *AllocDir) unmountSharedDir(dir string) error { return syscall.Unlink(dir) } +// createSecretDir creates the secrets dir folder at the given path +func (d *AllocDir) createSecretDir(dir string) error { + return os.MkdirAll(dir, 0777) +} + +// removeSecretDir removes the secrets dir folder +func (d *AllocDir) removeSecretDir(dir string) error { + return os.RemoveAll(dir) +} + // MountSpecialDirs mounts the dev and proc file system on the chroot of the // task. It's a no-op on darwin. func (d *AllocDir) MountSpecialDirs(taskDir string) error { diff --git a/client/allocdir/alloc_dir_freebsd.go b/client/allocdir/alloc_dir_freebsd.go index a4d3801db..0f95e056b 100644 --- a/client/allocdir/alloc_dir_freebsd.go +++ b/client/allocdir/alloc_dir_freebsd.go @@ -1,6 +1,7 @@ package allocdir import ( + "os" "syscall" ) @@ -14,6 +15,16 @@ func (d *AllocDir) unmountSharedDir(dir string) error { return syscall.Unlink(dir) } +// createSecretDir creates the secrets dir folder at the given path +func (d *AllocDir) createSecretDir(dir string) error { + return os.MkdirAll(dir, 0777) +} + +// removeSecretDir removes the secrets dir folder +func (d *AllocDir) removeSecretDir(dir string) error { + return os.RemoveAll(dir) +} + // MountSpecialDirs mounts the dev and proc file system on the chroot of the // task. It's a no-op on FreeBSD right now. func (d *AllocDir) MountSpecialDirs(taskDir string) error { diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go index 9b2c67035..48ab1c3ad 100644 --- a/client/allocdir/alloc_dir_linux.go +++ b/client/allocdir/alloc_dir_linux.go @@ -6,9 +6,16 @@ import ( "path/filepath" "syscall" + "golang.org/x/sys/unix" + "github.com/hashicorp/go-multierror" ) +const ( + // secretDirTmpfsSize is the size of the tmpfs per task in MBs + secretDirTmpfsSize = 1 +) + // Bind mounts the shared directory into the task directory. Must be root to // run. func (d *AllocDir) mountSharedDir(taskDir string) error { @@ -23,6 +30,36 @@ func (d *AllocDir) unmountSharedDir(dir string) error { return syscall.Unmount(dir, 0) } +// createSecretDir creates the secrets dir folder at the given path using a +// tmpfs +func (d *AllocDir) createSecretDir(dir string) error { + // Only mount the tmpfs if we are root + if unix.Geteuid() == 0 { + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + + var flags uintptr + flags = syscall.MS_NOEXEC + options := fmt.Sprintf("size=%dm", secretDirTmpfsSize) + err := syscall.Mount("tmpfs", dir, "tmpfs", flags, options) + return os.NewSyscallError("mount", err) + } + + return os.MkdirAll(dir, 0777) +} + +// createSecretDir removes the secrets dir folder +func (d *AllocDir) removeSecretDir(dir string) error { + if unix.Geteuid() == 0 { + if err := syscall.Unmount(dir, 0); err != nil { + return err + } + } + + return os.RemoveAll(dir) +} + // MountSpecialDirs mounts the dev and proc file system from the host to the // chroot func (d *AllocDir) MountSpecialDirs(taskDir string) error { diff --git a/client/allocdir/alloc_dir_test.go b/client/allocdir/alloc_dir_test.go index aa0f5e49b..9c2d42c6e 100644 --- a/client/allocdir/alloc_dir_test.go +++ b/client/allocdir/alloc_dir_test.go @@ -73,6 +73,10 @@ func TestAllocDir_BuildAlloc(t *testing.T) { if _, err := os.Stat(tDir); os.IsNotExist(err) { t.Fatalf("Build(%v) didn't create TaskDir %v", tasks, tDir) } + + if _, err := os.Stat(filepath.Join(tDir, TaskSecrets)); os.IsNotExist(err) { + t.Fatalf("Build(%v) didn't create secret dir %v", tasks) + } } } diff --git a/client/allocdir/alloc_dir_unix.go b/client/allocdir/alloc_dir_unix.go index 339e59d5d..0807505c4 100644 --- a/client/allocdir/alloc_dir_unix.go +++ b/client/allocdir/alloc_dir_unix.go @@ -14,11 +14,17 @@ import ( ) var ( - //Path inside container for mounted directory shared across tasks in a task group. + // SharedAllocContainerPath is the path inside container for mounted + // directory shared across tasks in a task group. SharedAllocContainerPath = filepath.Join("/", SharedAllocName) - //Path inside container for mounted directory for local storage. + // TaskLocalContainer is the path inside a container for mounted directory + // for local storage. TaskLocalContainerPath = filepath.Join("/", TaskLocal) + + // TaskSecretsContainerPath is the path inside a container for mounted + // secrets directory + TaskSecretsContainerPath = filepath.Join("/", TaskSecrets) ) func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error { diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go index 112fe9b63..cacc9da82 100644 --- a/client/allocdir/alloc_dir_windows.go +++ b/client/allocdir/alloc_dir_windows.go @@ -7,11 +7,17 @@ import ( ) var ( - //Path inside container for mounted directory that is shared across tasks in a task group. + // SharedAllocContainerPath is the path inside container for mounted + // directory shared across tasks in a task group. SharedAllocContainerPath = filepath.Join("c:\\", SharedAllocName) - //Path inside container for mounted directory for local storage. + // TaskLocalContainer is the path inside a container for mounted directory + // for local storage. TaskLocalContainerPath = filepath.Join("c:\\", TaskLocal) + + // TaskSecretsContainerPath is the path inside a container for mounted + // secrets directory + TaskSecretsContainerPath = filepath.Join("c:\\", TaskSecrets) ) func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error { @@ -23,6 +29,16 @@ func (d *AllocDir) mountSharedDir(dir string) error { return errors.New("Mount on Windows not supported.") } +// createSecretDir creates the secrets dir folder at the given path +func (d *AllocDir) createSecretDir(dir string) error { + return os.MkdirAll(dir, 0777) +} + +// removeSecretDir removes the secrets dir folder +func (d *AllocDir) removeSecretDir(dir string) error { + return os.RemoveAll(dir) +} + // The windows version does nothing currently. func (d *AllocDir) dropDirPermissions(path string) error { return nil diff --git a/client/client_test.go b/client/client_test.go index ba98dc054..cab6e7fa1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -575,6 +575,14 @@ func TestClient_SaveRestoreState(t *testing.T) { }, func(err error) { t.Fatalf("err: %v", err) }) + + // Destroy all the allocations + c2.allocLock.Lock() + for _, ar := range c2.allocs { + ar.Destroy() + <-ar.WaitCh() + } + c2.allocLock.Unlock() } func TestClient_Init(t *testing.T) { @@ -608,6 +616,7 @@ func TestClient_BlockedAllocations(t *testing.T) { c1 := testClient(t, func(c *config.Config) { c.RPCHandler = s1 }) + defer c1.Shutdown() // Wait for the node to be ready state := s1.State() @@ -691,4 +700,13 @@ func TestClient_BlockedAllocations(t *testing.T) { }, func(err error) { t.Fatalf("err: %v", err) }) + + // Destroy all the allocations + c1.allocLock.Lock() + for _, ar := range c1.allocs { + ar.Destroy() + <-ar.WaitCh() + } + c1.allocLock.Unlock() + } diff --git a/client/driver/docker.go b/client/driver/docker.go index 6c1390aa1..c2fdc92c2 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -381,6 +381,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, // Set environment variables. d.taskEnv.SetAllocDir(allocdir.SharedAllocContainerPath) d.taskEnv.SetTaskLocalDir(allocdir.TaskLocalContainerPath) + d.taskEnv.SetTaskLocalDir(allocdir.TaskSecretsContainerPath) config := &docker.Config{ Image: driverConfig.ImageName, diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 97ed914eb..5029d9e78 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -117,7 +117,8 @@ func dockerSetup(t *testing.T, task *structs.Task) (*docker.Client, DriverHandle // This test should always pass, even if docker daemon is not available func TestDockerDriver_Fingerprint(t *testing.T) { - driverCtx, _ := testDriverContexts(&structs.Task{Name: "foo", Resources: basicResources}) + driverCtx, execCtx := testDriverContexts(&structs.Task{Name: "foo", Resources: basicResources}) + defer execCtx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) node := &structs.Node{ Attributes: make(map[string]string), diff --git a/client/driver/driver.go b/client/driver/driver.go index 2ad38835a..60adf9c7d 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -153,6 +153,7 @@ func GetTaskEnv(allocDir *allocdir.AllocDir, node *structs.Node, } env.SetTaskLocalDir(filepath.Join(taskdir, allocdir.TaskLocal)) + env.SetSecretDir(filepath.Join(taskdir, allocdir.TaskSecrets)) } if task.Resources != nil { diff --git a/client/driver/env/env.go b/client/driver/env/env.go index 1ac9b7510..e2ff660b8 100644 --- a/client/driver/env/env.go +++ b/client/driver/env/env.go @@ -21,6 +21,10 @@ const ( // removed. TaskLocalDir = "NOMAD_TASK_DIR" + // SecretDir is the environment variable with the path to the tasks secret + // directory where it can store sensitive data. + SecretDir = "NOMAD_SECRET_DIR" + // MemLimit is the environment variable with the tasks memory limit in MBs. MemLimit = "NOMAD_MEMORY_LIMIT" @@ -79,6 +83,7 @@ type TaskEnvironment struct { JobMeta map[string]string AllocDir string TaskDir string + SecretDir string CpuLimit int MemLimit int TaskName string @@ -153,6 +158,9 @@ func (t *TaskEnvironment) Build() *TaskEnvironment { if t.TaskDir != "" { t.TaskEnv[TaskLocalDir] = t.TaskDir } + if t.SecretDir != "" { + t.TaskEnv[SecretDir] = t.SecretDir + } // Build the resource limits if t.MemLimit != 0 { @@ -249,6 +257,16 @@ func (t *TaskEnvironment) ClearTaskLocalDir() *TaskEnvironment { return t } +func (t *TaskEnvironment) SetSecretDir(dir string) *TaskEnvironment { + t.SecretDir = dir + return t +} + +func (t *TaskEnvironment) ClearSecretDir() *TaskEnvironment { + t.SecretDir = "" + return t +} + func (t *TaskEnvironment) SetMemLimit(limit int) *TaskEnvironment { t.MemLimit = limit return t diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 55e4f7b9d..f2e72637f 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -26,7 +26,8 @@ func TestExecDriver_Fingerprint(t *testing.T) { Name: "foo", Resources: structs.DefaultResources(), } - driverCtx, _ := testDriverContexts(task) + driverCtx, execCtx := testDriverContexts(task) + defer execCtx.AllocDir.Destroy() d := NewExecDriver(driverCtx) node := &structs.Node{ Attributes: map[string]string{ diff --git a/client/driver/executor/executor_linux_test.go b/client/driver/executor/executor_linux_test.go index c3006071a..321e0e012 100644 --- a/client/driver/executor/executor_linux_test.go +++ b/client/driver/executor/executor_linux_test.go @@ -81,7 +81,23 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) { t.Fatalf("file %v hasn't been removed", memLimits) } - expected := "/:\nalloc/\nbin/\ndev/\netc/\nlib/\nlib64/\nlocal/\nproc/\ntmp/\nusr/\n\n/etc/:\nld.so.cache\nld.so.conf\nld.so.conf.d/" + expected := `/: +alloc/ +bin/ +dev/ +etc/ +lib/ +lib64/ +local/ +proc/ +secrets/ +tmp/ +usr/ + +/etc/: +ld.so.cache +ld.so.conf +ld.so.conf.d/` file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0") output, err := ioutil.ReadFile(file) if err != nil { diff --git a/client/driver/java_test.go b/client/driver/java_test.go index 553bb727f..40e7b46c6 100644 --- a/client/driver/java_test.go +++ b/client/driver/java_test.go @@ -35,7 +35,8 @@ func TestJavaDriver_Fingerprint(t *testing.T) { Name: "foo", Resources: structs.DefaultResources(), } - driverCtx, _ := testDriverContexts(task) + driverCtx, execCtx := testDriverContexts(task) + defer execCtx.AllocDir.Destroy() d := NewJavaDriver(driverCtx) node := &structs.Node{ Attributes: map[string]string{ diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go index cd139c72d..4f3122080 100644 --- a/client/driver/qemu_test.go +++ b/client/driver/qemu_test.go @@ -19,7 +19,8 @@ func TestQemuDriver_Fingerprint(t *testing.T) { Name: "foo", Resources: structs.DefaultResources(), } - driverCtx, _ := testDriverContexts(task) + driverCtx, execCtx := testDriverContexts(task) + defer execCtx.AllocDir.Destroy() d := NewQemuDriver(driverCtx) node := &structs.Node{ Attributes: make(map[string]string), diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go index 1a5666f79..17023393f 100644 --- a/client/driver/raw_exec_test.go +++ b/client/driver/raw_exec_test.go @@ -21,7 +21,8 @@ func TestRawExecDriver_Fingerprint(t *testing.T) { Name: "foo", Resources: structs.DefaultResources(), } - driverCtx, _ := testDriverContexts(task) + driverCtx, execCtx := testDriverContexts(task) + defer execCtx.AllocDir.Destroy() d := NewRawExecDriver(driverCtx) node := &structs.Node{ Attributes: make(map[string]string), diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index f8ed84243..fc68e4780 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -376,6 +376,7 @@ func TestRktTaskValidate(t *testing.T) { "dns_servers": []string{"8.8.8.8", "8.8.4.4"}, "dns_search_domains": []string{"example.com", "example.org", "example.net"}, }, + Resources: basicResources, } driverCtx, execCtx := testDriverContexts(task) defer execCtx.AllocDir.Destroy() diff --git a/client/task_runner_test.go b/client/task_runner_test.go index 0fc7316c2..88c2b6be4 100644 --- a/client/task_runner_test.go +++ b/client/task_runner_test.go @@ -385,6 +385,8 @@ func TestTaskRunner_Download_Retries(t *testing.T) { func TestTaskRunner_Validate_UserEnforcement(t *testing.T) { _, tr := testTaskRunner(false) + defer tr.Destroy(structs.NewTaskEvent(structs.TaskKilled)) + defer tr.ctx.AllocDir.Destroy() // Try to run as root with exec. tr.task.Driver = "exec"