AllocDirBuilder that creates the alloc directory structure

client/alloc_runner.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -146,7 +147,7 @@ func (r *AllocRunner) DestroyState() error {
 
 // DestroyContext is used to destroy the context
 func (r *AllocRunner) DestroyContext() error {
-	return os.RemoveAll(r.ctx.AllocDir)
+	return r.ctx.AllocDir.Destroy()
 }
 
 // Alloc returns the associated allocation
@@ -278,7 +279,8 @@ func (r *AllocRunner) Run() {
 	// Create the execution context
 	if r.ctx == nil {
 		r.ctx = driver.NewExecContext()
-		r.ctx.AllocDir = filepath.Join(r.config.AllocDir, r.alloc.ID)
+		r.ctx.AllocDir = allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID))
+		r.ctx.AllocDir.Build(tg.Tasks)
 	}
 
 	// Start the task runners

client/allocdir/alloc_dir.go

@@ -0,0 +1,65 @@
+package allocdir
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+var (
+	// The name of the directory that is shared across tasks in a task group.
+	SharedAllocName = "alloc"
+
+	// The set of directories that exist inside eache shared alloc directory.
+	SharedAllocDirs = []string{"logs", "tmp", "data"}
+
+	// The name of the directory that exists inside each task directory
+	// regardless of driver.
+	TaskLocal = "local"
+)
+
+// Builds the necessary directory structure for running an alloc.
+type AllocDirBuilder interface {
+	// Given a list of a task build the correct alloc structure.
+	Build([]*structs.Task) error
+
+	// Tears down previously build directory structure.
+	Destroy() error
+
+	// Returns the directory of a task if it was created, otherwise an error is
+	// returned.
+	TaskDir(task string) (string, error)
+}
+
+type AllocDir struct {
+	// AllocDir is the directory used for storing any state
+	// of this allocation. It will be purged on alloc destroy.
+	AllocDir string
+
+	// The shared directory is available to all tasks within the same task
+	// group.
+	SharedDir string
+
+	// TaskDirs is a mapping of task names to their non-shared directory.
+	TaskDirs map[string]string
+}
+
+func NewAllocDir(allocDir string) *AllocDir {
+	d := &AllocDir{AllocDir: allocDir, TaskDirs: make(map[string]string)}
+	d.SharedDir = filepath.Join(d.AllocDir, SharedAllocName)
+	return d
+}
+
+func (d *AllocDir) Destroy() error {
+	return os.RemoveAll(d.AllocDir)
+}
+
+func (d *AllocDir) TaskDir(task string) (string, error) {
+	if dir, ok := d.TaskDirs[task]; ok {
+		return dir, nil
+	}
+
+	return "", fmt.Errorf("Task directory doesn't exist for task %v", task)
+}

client/allocdir/alloc_dir_linux.go

@@ -0,0 +1,98 @@
+// +build !windows
+
+package allocdir
+
+import (
+	"fmt"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+)
+
+func (d *AllocDir) Build(tasks []*structs.Task) error {
+	// Make the alloc directory, owned by the nomad process.
+	if err := os.Mkdir(d.AllocDir, 0700); err != nil {
+		return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err)
+	}
+
+	nobody, err := user.Lookup("nobody")
+	if err != nil {
+		return fmt.Errorf("Could not set owner/group on shared alloc directory: %v", err)
+	}
+
+	uid, err := getUid(nobody)
+	if err != nil {
+		return err
+	}
+
+	gid, err := getGid(nobody)
+	if err != nil {
+		return err
+	}
+
+	// Make the shared directory and make it availabe to all user/groups.
+	if err := mkOwnedDir(d.SharedDir, uid, gid, 0777); err != nil {
+		return err
+	}
+
+	for _, dir := range SharedAllocDirs {
+		p := filepath.Join(d.SharedDir, dir)
+		if err := mkOwnedDir(p, uid, gid, 0777); err != nil {
+			return err
+		}
+	}
+
+	// Make the task directories.
+	for _, t := range tasks {
+		p := filepath.Join(d.AllocDir, t.Name)
+		if err := mkOwnedDir(p, uid, gid, 0777); err != nil {
+			return err
+		}
+
+		// Create a local directory that each task can use.
+		local := filepath.Join(p, TaskLocal)
+		if err := mkOwnedDir(local, uid, gid, 0777); err != nil {
+			return err
+		}
+		d.TaskDirs[t.Name] = local
+
+		// TODO: Mount the shared alloc dir into each task dir.
+	}
+
+	return nil
+}
+
+// mkOwnedDir creates the directory specified by the path with the passed
+// permissions. It also sets the passed uid and gid. It returns an error if any
+// of these operations fail.
+func mkOwnedDir(path string, uid, gid int, perm os.FileMode) error {
+	if err := os.Mkdir(path, perm); err != nil {
+		return fmt.Errorf("Failed to make directory %v: %v", path, err)
+	}
+
+	if err := os.Chown(path, uid, gid); err != nil {
+		return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err)
+	}
+
+	return nil
+}
+
+func getUid(u *user.User) (int, error) {
+	uid, err := strconv.Atoi(u.Uid)
+	if err != nil {
+		return 0, fmt.Errorf("Unable to convert Uid to an int: %v", err)
+	}
+
+	return uid, nil
+}
+
+func getGid(u *user.User) (int, error) {
+	gid, err := strconv.Atoi(u.Gid)
+	if err != nil {
+		return 0, fmt.Errorf("Unable to convert Gid to an int: %v", err)
+	}
+
+	return gid, nil
+}

client/allocdir/alloc_dir_windows.go

@@ -0,0 +1,8 @@
+package allocdir
+
+import "github.com/hashicorp/nomad/nomad/structs"
+
+func (r *AllocRunner) Build(tasks []*structs.Task) error {
+	// TODO: Need to figure out how to do mounts on windows.
+	return nil
+}

client/client.go

@@ -143,6 +143,7 @@ func NewClient(cfg *config.Config) (*Client, error) {
 // needed before we begin starting its various components.
 func (c *Client) init() error {
 	// Ensure the alloc dir exists if we have one
+	// TODO(alex): Make a tmp directory if it doesn't?
 	if c.config.AllocDir != "" {
 		if err := os.MkdirAll(c.config.AllocDir, 0700); err != nil {
 			return fmt.Errorf("failed creating alloc dir: %s", err)
@@ -431,7 +432,7 @@ func (c *Client) fingerprint() error {
 // setupDrivers is used to find the available drivers
 func (c *Client) setupDrivers() error {
 	var avail []string
-	driverCtx := driver.NewDriverContext(c.config, c.config.Node, c.logger)
+	driverCtx := driver.NewDriverContext("", c.config, c.config.Node, c.logger)
 	for name := range driver.BuiltinDrivers {
 		d, err := driver.NewDriver(name, driverCtx)
 		if err != nil {

client/driver/driver.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -55,20 +56,22 @@ type Driver interface {
 // node attributes into a Driver without having to change the Driver interface
 // each time we do it. Used in conjection with Factory, above.
 type DriverContext struct {
-	config *config.Config
-	logger *log.Logger
-	node   *structs.Node
+	taskName string
+	config   *config.Config
+	logger   *log.Logger
+	node     *structs.Node
 }
 
 // NewDriverContext initializes a new DriverContext with the specified fields.
 // This enables other packages to create DriverContexts but keeps the fields
 // private to the driver. If we want to change this later we can gorename all of
 // the fields in DriverContext.
-func NewDriverContext(config *config.Config, node *structs.Node, logger *log.Logger) *DriverContext {
+func NewDriverContext(taskName string, config *config.Config, node *structs.Node, logger *log.Logger) *DriverContext {
 	return &DriverContext{
-		config: config,
-		node:   node,
-		logger: logger,
+		taskName: taskName,
+		config:   config,
+		node:     node,
+		logger:   logger,
 	}
 }
 
@@ -92,9 +95,8 @@ type DriverHandle interface {
 type ExecContext struct {
 	sync.Mutex
 
-	// AllocDir is the directory used for storing any state
-	// of this allocation. It will be purged on alloc destroy.
-	AllocDir string
+	// AllocDir contains information about the alloc directory structure.
+	AllocDir allocdir.AllocDirBuilder
 }
 
 // NewExecContext is used to create a new execution context

client/driver/java.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/executor"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -104,7 +105,15 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("Error downloading source for Java driver: %s", err)
 	}
 
-	fPath := filepath.Join(ctx.AllocDir, path.Base(source))
+	// Get the tasks local directory.
+	taskDir, err := ctx.AllocDir.TaskDir(d.DriverContext.taskName)
+	if err != nil {
+		return nil, err
+	}
+	taskLocal := filepath.Join(taskDir, allocdir.TaskLocal)
+
+	// Create a location to download the binary.
+	fPath := filepath.Join(taskLocal, path.Base(source))
 	f, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666)
 	if err != nil {
 		return nil, fmt.Errorf("Error opening file to download to: %s", err)
@@ -113,7 +122,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	defer f.Close()
 	defer resp.Body.Close()
 
-	// Copy remote file to local AllocDir for execution
+	// Copy remote file to local directory for execution
 	// TODO: a retry of sort if io.Copy fails, for large binaries
 	_, ioErr := io.Copy(f, resp.Body)
 	if ioErr != nil {

client/driver/qemu.go

@@ -19,6 +19,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
@@ -101,10 +102,17 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("Error downloading source for Qemu driver: %s", err)
 	}
 
-	// Create a location in the AllocDir to download and store the image.
+	// Get the tasks local directory.
+	taskDir, err := ctx.AllocDir.TaskDir(d.DriverContext.taskName)
+	if err != nil {
+		return nil, err
+	}
+	taskLocal := filepath.Join(taskDir, allocdir.TaskLocal)
+
+	// Create a location in the local directory to download and store the image.
 	// TODO: Caching
 	vmID := fmt.Sprintf("qemu-vm-%s-%s", structs.GenerateUUID(), filepath.Base(source))
-	fPath := filepath.Join(ctx.AllocDir, vmID)
+	fPath := filepath.Join(taskLocal, vmID)
 	vmPath, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666)
 	if err != nil {
 		return nil, fmt.Errorf("Error opening file to download to: %s", err)

client/task_runner.go

@@ -129,7 +129,7 @@ func (r *TaskRunner) setStatus(status, desc string) {
 
 // createDriver makes a driver for the task
 func (r *TaskRunner) createDriver() (driver.Driver, error) {
-	driverCtx := driver.NewDriverContext(r.config, r.config.Node, r.logger)
+	driverCtx := driver.NewDriverContext(r.task.Name, r.config, r.config.Node, r.logger)
 	driver, err := driver.NewDriver(r.task.Driver, driverCtx)
 	if err != nil {
 		err = fmt.Errorf("failed to create driver '%s' for alloc %s: %v",