Secret dir, hello world

client/alloc_runner.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver"
+	"github.com/hashicorp/nomad/client/secretdir"
 	"github.com/hashicorp/nomad/nomad/structs"
 
 	cstructs "github.com/hashicorp/nomad/client/structs"
@@ -51,17 +52,20 @@ type AllocRunner struct {
 
 	dirtyCh chan struct{}
 
-	ctx        *driver.ExecContext
-	ctxLock    sync.Mutex
-	tasks      map[string]*TaskRunner
-	taskStates map[string]*structs.TaskState
-	restored   map[string]struct{}
-	taskLock   sync.RWMutex
+	ctx     *driver.ExecContext
+	ctxLock sync.Mutex
 
+	tasks    map[string]*TaskRunner
+	restored map[string]struct{}
+	taskLock sync.RWMutex
+
+	taskStates     map[string]*structs.TaskState
 	taskStatusLock sync.RWMutex
 
 	updateCh chan *structs.Allocation
 
+	secretDir *secretdir.SecretDir
+
 	destroy     bool
 	destroyCh   chan struct{}
 	destroyLock sync.Mutex
@@ -80,12 +84,13 @@ type allocRunnerState struct {
 
 // NewAllocRunner is used to create a new allocation context
 func NewAllocRunner(logger *log.Logger, config *config.Config, updater AllocStateUpdater,
-	alloc *structs.Allocation) *AllocRunner {
+	alloc *structs.Allocation, secretDir *secretdir.SecretDir) *AllocRunner {
 	ar := &AllocRunner{
 		config:     config,
 		updater:    updater,
 		logger:     logger,
 		alloc:      alloc,
+		secretDir:  secretDir,
 		dirtyCh:    make(chan struct{}, 1),
 		tasks:      make(map[string]*TaskRunner),
 		taskStates: copyTaskStates(alloc.TaskStates),
@@ -386,9 +391,11 @@ func (r *AllocRunner) Run() {
 	// Create the execution context
 	r.ctxLock.Lock()
 	if r.ctx == nil {
-		allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID), r.Alloc().Resources.DiskMB)
+		path := filepath.Join(r.config.AllocDir, r.alloc.ID)
+		size := r.Alloc().Resources.DiskMB
+		allocDir := allocdir.NewAllocDir(r.alloc.ID, path, size, r.secretDir)
 		if err := allocDir.Build(tg.Tasks); err != nil {
-			r.logger.Printf("[WARN] client: failed to build task directories: %v", err)
+			r.logger.Printf("[ERR] client: failed to build task directories: %v", err)
 			r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup))
 			r.ctxLock.Unlock()
 			return

client/allocdir/alloc_dir.go

@@ -14,6 +14,7 @@ import (
 	"gopkg.in/tomb.v1"
 
 	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/nomad/client/secretdir"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hpcloud/tail/watch"
 )
@@ -46,11 +47,21 @@ var (
 	// regardless of driver.
 	TaskLocal = "local"
 
+	// TaskSecrets is the the name of the secret directory inside each task
+	// directory
+	TaskSecrets = "secrets"
+
 	// TaskDirs is the set of directories created in each tasks directory.
 	TaskDirs = []string{"tmp"}
 )
 
 type AllocDir struct {
+	// AllocID is the allocation ID for this directory
+	AllocID string
+
+	// SecretDir is used to build the secret directory for the allocation
+	SecretDir *secretdir.SecretDir
+
 	// AllocDir is the directory used for storing any state
 	// of this allocation. It will be purged on alloc destroy.
 	AllocDir string
@@ -110,8 +121,10 @@ type AllocDirFS interface {
 
 // NewAllocDir initializes the AllocDir struct with allocDir as base path for
 // the allocation directory and maxSize as the maximum allowed size in megabytes.
-func NewAllocDir(allocDir string, maxSize int) *AllocDir {
+func NewAllocDir(allocID, allocDir string, maxSize int, sdir *secretdir.SecretDir) *AllocDir {
 	d := &AllocDir{
+		AllocID:                   allocID,
+		SecretDir:                 sdir,
 		AllocDir:                  allocDir,
 		MaxCheckDiskInterval:      maxCheckDiskInterval,
 		MinCheckDiskInterval:      minCheckDiskInterval,
@@ -145,7 +158,7 @@ func (d *AllocDir) UnmountAll() error {
 		// Check if the directory has the shared alloc mounted.
 		taskAlloc := filepath.Join(dir, SharedAllocName)
 		if d.pathExists(taskAlloc) {
-			if err := d.unmountSharedDir(taskAlloc); err != nil {
+			if err := d.unmount(taskAlloc); err != nil {
 				mErr.Errors = append(mErr.Errors,
 					fmt.Errorf("failed to unmount shared alloc dir %q: %v", taskAlloc, err))
 			} else if err := os.RemoveAll(taskAlloc); err != nil {
@@ -154,6 +167,18 @@ func (d *AllocDir) UnmountAll() error {
 			}
 		}
 
+		// Remove the secrets dir
+		taskSecrets := filepath.Join(dir, TaskSecrets)
+		if d.pathExists(taskSecrets) {
+			if err := d.unmount(taskSecrets); err != nil {
+				mErr.Errors = append(mErr.Errors,
+					fmt.Errorf("failed to unmount secrets dir %q: %v", taskSecrets, err))
+			} else if err := os.RemoveAll(taskSecrets); err != nil {
+				mErr.Errors = append(mErr.Errors,
+					fmt.Errorf("failed to delete secrets dir %q: %v", taskSecrets, err))
+			}
+		}
+
 		// Unmount dev/ and proc/ have been mounted.
 		d.unmountSpecialDirs(dir)
 	}
@@ -223,6 +248,18 @@ func (d *AllocDir) Build(tasks []*structs.Task) error {
 				return err
 			}
 		}
+
+		// Get the secret directory
+		sdir, err := d.SecretDir.CreateFor(d.AllocID, t.Name)
+		if err != nil {
+			return fmt.Errorf("Creating secret directory for task %q failed: %v", t.Name, err)
+		}
+
+		// Mount the secret directory
+		taskSecret := filepath.Join(taskDir, TaskSecrets)
+		if err := d.mount(sdir, taskSecret); err != nil {
+			return fmt.Errorf("failed to mount secret directory: %v", err)
+		}
 	}
 
 	return nil
@@ -332,7 +369,7 @@ func (d *AllocDir) MountSharedDir(task string) error {
 	}
 
 	taskLoc := filepath.Join(taskDir, SharedAllocName)
-	if err := d.mountSharedDir(taskLoc); err != nil {
+	if err := d.mount(d.SharedDir, taskLoc); err != nil {
 		return fmt.Errorf("Failed to mount shared directory for task %v: %v", task, err)
 	}
 

client/allocdir/alloc_dir_darwin.go

@@ -4,13 +4,13 @@ import (
 	"syscall"
 )
 
-// Hardlinks the shared directory. As a side-effect the shared directory and
-// task directory must be on the same filesystem.
-func (d *AllocDir) mountSharedDir(dir string) error {
-	return syscall.Link(d.SharedDir, dir)
+// Hardlinks the shared directory. As a side-effect the src and dest directory
+// must be on the same filesystem.
+func (d *AllocDir) mount(src, dest string) error {
+	return syscall.Link(src, dest)
 }
 
-func (d *AllocDir) unmountSharedDir(dir string) error {
+func (d *AllocDir) unmount(dir string) error {
 	return syscall.Unlink(dir)
 }
 

client/allocdir/alloc_dir_freebsd.go

@@ -4,13 +4,13 @@ import (
 	"syscall"
 )
 
-// Hardlinks the shared directory. As a side-effect the shared directory and
-// task directory must be on the same filesystem.
-func (d *AllocDir) mountSharedDir(dir string) error {
-	return syscall.Link(d.SharedDir, dir)
+// Hardlinks the shared directory. As a side-effect the src and dest directory
+// must be on the same filesystem.
+func (d *AllocDir) mount(src, dest string) error {
+	return syscall.Link(src, dest)
 }
 
-func (d *AllocDir) unmountSharedDir(dir string) error {
+func (d *AllocDir) unmount(dir string) error {
 	return syscall.Unlink(dir)
 }
 

client/allocdir/alloc_dir_linux.go

@@ -9,17 +9,15 @@ import (
 	"github.com/hashicorp/go-multierror"
 )
 
-// Bind mounts the shared directory into the task directory. Must be root to
-// run.
-func (d *AllocDir) mountSharedDir(taskDir string) error {
-	if err := os.MkdirAll(taskDir, 0777); err != nil {
+func (d *AllocDir) mount(src, dest string) error {
+	if err := os.MkdirAll(dest, 0777); err != nil {
 		return err
 	}
 
-	return syscall.Mount(d.SharedDir, taskDir, "", syscall.MS_BIND, "")
+	return syscall.Mount(src, dest, "", syscall.MS_BIND, "")
 }
 
-func (d *AllocDir) unmountSharedDir(dir string) error {
+func (d *AllocDir) unmount(dir string) error {
 	return syscall.Unmount(dir, 0)
 }
 

client/allocdir/alloc_dir_windows.go

@@ -19,17 +19,17 @@ func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
 }
 
 // The windows version does nothing currently.
-func (d *AllocDir) mountSharedDir(dir string) error {
+func (d *AllocDir) mount(src, dest string) error {
 	return errors.New("Mount on Windows not supported.")
 }
 
 // The windows version does nothing currently.
-func (d *AllocDir) dropDirPermissions(path string) error {
+func (d *AllocDir) unmount(dir string) error {
 	return nil
 }
 
 // The windows version does nothing currently.
-func (d *AllocDir) unmountSharedDir(dir string) error {
+func (d *AllocDir) dropDirPermissions(path string) error {
 	return nil
 }
 

client/client.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hashicorp/nomad/client/driver"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/rpcproxy"
+	"github.com/hashicorp/nomad/client/secretdir"
 	"github.com/hashicorp/nomad/client/stats"
 	"github.com/hashicorp/nomad/command/agent/consul"
 	"github.com/hashicorp/nomad/nomad"
@@ -77,6 +78,9 @@ const (
 	// allocSyncRetryIntv is the interval on which we retry updating
 	// the status of the allocation
 	allocSyncRetryIntv = 5 * time.Second
+
+	// secretDir is the secrets directory nested under the state directory
+	secretDir = "secrets"
 )
 
 // ClientStatsReporter exposes all the APIs related to resource usage of a Nomad
@@ -107,6 +111,8 @@ type Client struct {
 
 	connPool *nomad.ConnPool
 
+	secretDir *secretdir.SecretDir
+
 	// lastHeartbeatFromQuorum is an atomic int32 acting as a bool.  When
 	// true, the last heartbeat message had a leader.  When false (0),
 	// the last heartbeat did not include the RPC address of the leader,
@@ -275,6 +281,14 @@ func (c *Client) init() error {
 	}
 
 	c.logger.Printf("[INFO] client: using alloc directory %v", c.config.AllocDir)
+
+	// Initialize the secret directory
+	sdir, err := secretdir.NewSecretDir(filepath.Join(c.config.StateDir, secretDir))
+	if err != nil {
+		return err
+	}
+	c.secretDir = sdir
+
 	return nil
 }
 
@@ -327,6 +341,10 @@ func (c *Client) Shutdown() error {
 			<-ar.WaitCh()
 		}
 		c.allocLock.Unlock()
+
+		if err := c.secretDir.Destroy(); err != nil {
+			return err
+		}
 	}
 
 	c.shutdown = true
@@ -449,7 +467,7 @@ func (c *Client) restoreState() error {
 		id := entry.Name()
 		alloc := &structs.Allocation{ID: id}
 		c.configLock.RLock()
-		ar := NewAllocRunner(c.logger, c.configCopy, c.updateAllocStatus, alloc)
+		ar := NewAllocRunner(c.logger, c.configCopy, c.updateAllocStatus, alloc, c.secretDir)
 		c.configLock.RUnlock()
 		c.allocLock.Lock()
 		c.allocs[id] = ar
@@ -1264,7 +1282,7 @@ func (c *Client) updateAlloc(exist, update *structs.Allocation) error {
 // addAlloc is invoked when we should add an allocation
 func (c *Client) addAlloc(alloc *structs.Allocation) error {
 	c.configLock.RLock()
-	ar := NewAllocRunner(c.logger, c.configCopy, c.updateAllocStatus, alloc)
+	ar := NewAllocRunner(c.logger, c.configCopy, c.updateAllocStatus, alloc, c.secretDir)
 	c.configLock.RUnlock()
 	go ar.Run()
 

client/secretdir/secret_dir.go

@@ -0,0 +1,67 @@
+package secretdir
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+const ()
+
+type SecretDir struct {
+	// Dir is the path to the secret directory
+	Dir string
+}
+
+func NewSecretDir(dir string) (*SecretDir, error) {
+	s := &SecretDir{
+		Dir: dir,
+	}
+
+	if err := s.init(); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+// init checks the secret directory exists and if it doesn't creates the secret
+// directory
+func (s *SecretDir) init() error {
+	if _, err := os.Stat(s.Dir); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to stat secret dir: %v", err)
+	}
+
+	// TODO this shouldn't be hardcoded
+	if err := s.create(32); err != nil {
+		return fmt.Errorf("failed to create secret dir: %v", err)
+	}
+
+	return nil
+}
+
+// Destroy is used to destroy the secret dir
+func (s *SecretDir) Destroy() error {
+	return s.destroy()
+}
+
+func (s *SecretDir) getPathFor(allocID, task string) string {
+	return filepath.Join(s.Dir, fmt.Sprintf("%s-%s", allocID, task))
+}
+
+// CreateFor creates a secret directory for the given allocation and task. If
+// the directory couldn't be created an error is returned, otherwise the path
+// is.
+func (s *SecretDir) CreateFor(allocID, task string) (string, error) {
+	path := s.getPathFor(allocID, task)
+	if err := os.Mkdir(path, 0777); err != nil {
+		return "", err
+	}
+	return path, nil
+}
+
+// Remove deletes the secret directory for the given allocation and task
+func (s *SecretDir) Remove(allocID, task string) error {
+	path := s.getPathFor(allocID, task)
+	return os.RemoveAll(path)
+}

client/secretdir/secret_dir_test.go

@@ -0,0 +1,70 @@
+package secretdir
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestSecretDir_CreateDestroy(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("Failed to make tmpdir: %v", err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	path := filepath.Join(tmpdir, "secret")
+	sdir, err := NewSecretDir(path)
+	if err != nil {
+		t.Fatalf("Failed to create SecretDir: %v", err)
+	}
+
+	// Check the folder exists
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("Stating path failed: %v", err)
+	}
+
+	if err := sdir.Destroy(); err != nil {
+		t.Fatalf("Destroying failed: %v", err)
+	}
+
+	// Check the folder doesn't exists
+	if _, err := os.Stat(path); err == nil || !os.IsNotExist(err) {
+		t.Fatalf("path err: %v", err)
+	}
+}
+
+func TestSecretDir_CreateFor_Remove(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("Failed to make tmpdir: %v", err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	path := filepath.Join(tmpdir, "secret")
+	sdir, err := NewSecretDir(path)
+	if err != nil {
+		t.Fatalf("Failed to create SecretDir: %v", err)
+	}
+
+	alloc, task := "123", "foo"
+	taskDir, err := sdir.CreateFor(alloc, task)
+	if err != nil {
+		t.Fatalf("CreateFor failed: %v", err)
+	}
+
+	// Check the folder exists
+	if _, err := os.Stat(taskDir); err != nil {
+		t.Fatalf("Stating path failed: %v", err)
+	}
+
+	if err := sdir.Remove(alloc, task); err != nil {
+		t.Fatalf("Destroying failed: %v", err)
+	}
+
+	// Check the folder doesn't exists
+	if _, err := os.Stat(taskDir); err == nil || !os.IsNotExist(err) {
+		t.Fatalf("path err: %v", err)
+	}
+}

client/secretdir/secret_dir_universal.go

@@ -0,0 +1,15 @@
+// +build !dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
+
+package secretdir
+
+import "os"
+
+// create creates a normal folder at the secret dir path
+func (s *SecretDir) create(sizeMB int) error {
+	return os.MkdirAll(s.Dir, 0700)
+}
+
+// destroy removes the secret dir
+func (s *SecretDir) destroy() error {
+	return os.RemoveAll(s.Dir)
+}

client/secretdir/secret_dir_unix.go

@@ -0,0 +1,31 @@
+// +build dragonfly freebsd linux netbsd openbsd solaris
+
+package secretdir
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+// create creates a tmpfs folder at the secret dir path
+func (s *SecretDir) create(sizeMB int64) error {
+	if err := os.MkdirAll(s.Dir, 0700); err != nil {
+		return err
+	}
+
+	var flags uintptr
+	flags = syscall.MS_NOEXEC
+	options := fmt.Sprintf("size=%dm", sizeMB)
+	err := syscall.Mount("tmpfs", s.Dir, "tmpfs", flags, options)
+	return os.NewSyscallError("mount", err)
+}
+
+// destroy unmounts the tmpfs folder and deletes it
+func (s *SecretDir) destroy() error {
+	if err := syscall.Unmount(s.Dir, 0); err != nil {
+		return err
+	}
+
+	return os.RemoveAll(s.Dir)
+}

client/secretdir/secret_dir_unix_test.go

@@ -0,0 +1 @@
+package secretdir