luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

Merge pull request #107 from hashicorp/f-disk-isolation

Browse source 
Chroot isolation of processes running on linux
This commit is contained in:
Alex Dadgar 2015-09-25 18:12:40 -07:00
parent f47dae26ba 6ca7c6680d
commit 70d4490730
28 changed files with 1093 additions and 196 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
client/alloc_runner.go
View file

@ -10,6 +10,7 @@ import (
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver"
"github.com/hashicorp/nomad/nomad/structs"
@ -146,7 +147,7 @@ func (r *AllocRunner) DestroyState() error {
// DestroyContext is used to destroy the context
func (r *AllocRunner) DestroyContext() error {
return os.RemoveAll(r.ctx.AllocDir)
return r.ctx.AllocDir.Destroy()
}
// Alloc returns the associated allocation
@ -277,8 +278,9 @@ func (r *AllocRunner) Run() {
// Create the execution context
if r.ctx == nil {
r.ctx = driver.NewExecContext()
r.ctx.AllocDir = filepath.Join(r.config.AllocDir, r.alloc.ID)
allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID))
allocDir.Build(tg.Tasks)
r.ctx = driver.NewExecContext(allocDir)
}
// Start the task runners

4
client/alloc_runner_test.go
View file

@ -1,6 +1,7 @@
package client
import (
"os"
"testing"
"time"
@ -26,7 +27,8 @@ func (m *MockAllocStateUpdater) Update(alloc *structs.Allocation) error {
func testAllocRunner() (*MockAllocStateUpdater, *AllocRunner) {
logger := testLogger()
conf := DefaultConfig()
conf.StateDir = "/tmp"
conf.StateDir = os.TempDir()
conf.AllocDir = os.TempDir()
upd := &MockAllocStateUpdater{}
alloc := mock.Alloc()
ar := NewAllocRunner(logger, conf, upd.Update, alloc)

213
client/allocdir/alloc_dir.go Normal file
View file

@ -0,0 +1,213 @@
package allocdir
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/hashicorp/nomad/nomad/structs"
)
var (
// The name of the directory that is shared across tasks in a task group.
SharedAllocName = "alloc"
// The set of directories that exist inside eache shared alloc directory.
SharedAllocDirs = []string{"logs", "tmp", "data"}
// The name of the directory that exists inside each task directory
// regardless of driver.
TaskLocal = "local"
)
type AllocDir struct {
// AllocDir is the directory used for storing any state
// of this allocation. It will be purged on alloc destroy.
AllocDir string
// The shared directory is available to all tasks within the same task
// group.
SharedDir string
// TaskDirs is a mapping of task names to their non-shared directory.
TaskDirs map[string]string
// A list of locations the shared alloc has been mounted to.
mounted []string
}
func NewAllocDir(allocDir string) *AllocDir {
d := &AllocDir{AllocDir: allocDir, TaskDirs: make(map[string]string)}
d.SharedDir = filepath.Join(d.AllocDir, SharedAllocName)
return d
}
// Tears down previously build directory structure.
func (d *AllocDir) Destroy() error {
// Unmount all mounted shared alloc dirs.
for _, m := range d.mounted {
if err := d.unmountSharedDir(m); err != nil {
return fmt.Errorf("Failed to unmount shared directory: %v", err)
}
}
return os.RemoveAll(d.AllocDir)
}
// Given a list of a task build the correct alloc structure.
func (d *AllocDir) Build(tasks []*structs.Task) error {
// Make the alloc directory, owned by the nomad process.
if err := os.MkdirAll(d.AllocDir, 0700); err != nil {
return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err)
}
// Make the shared directory and make it availabe to all user/groups.
if err := os.Mkdir(d.SharedDir, 0777); err != nil {
return err
}
// Make the shared directory have non-root permissions.
if err := d.dropDirPermissions(d.SharedDir); err != nil {
return err
}
for _, dir := range SharedAllocDirs {
p := filepath.Join(d.SharedDir, dir)
if err := os.Mkdir(p, 0777); err != nil {
return err
}
}
// Make the task directories.
for _, t := range tasks {
taskDir := filepath.Join(d.AllocDir, t.Name)
if err := os.Mkdir(taskDir, 0777); err != nil {
return err
}
// Make the task directory have non-root permissions.
if err := d.dropDirPermissions(taskDir); err != nil {
return err
}
// Create a local directory that each task can use.
local := filepath.Join(taskDir, TaskLocal)
if err := os.Mkdir(local, 0777); err != nil {
return err
}
if err := d.dropDirPermissions(local); err != nil {
return err
}
d.TaskDirs[t.Name] = taskDir
}
return nil
}
// Embed takes a mapping of absolute directory paths on the host to their
// intended, relative location within the task directory. Embed attempts
// hardlink and then defaults to copying. If the path exists on the host and
// can't be embeded an error is returned.
func (d *AllocDir) Embed(task string, dirs map[string]string) error {
taskdir, ok := d.TaskDirs[task]
if !ok {
return fmt.Errorf("Task directory doesn't exist for task %v", task)
}
subdirs := make(map[string]string)
for source, dest := range dirs {
// Check to see if directory exists on host.
s, err := os.Stat(source)
if os.IsNotExist(err) {
continue
}
// Enumerate the files in source.
entries, err := ioutil.ReadDir(source)
if err != nil {
return fmt.Errorf("Couldn't read directory %v: %v", source, err)
}
// Create destination directory.
destDir := filepath.Join(taskdir, dest)
if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil {
return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err)
}
for _, entry := range entries {
hostEntry := filepath.Join(source, entry.Name())
taskEntry := filepath.Join(destDir, filepath.Base(hostEntry))
if entry.IsDir() {
subdirs[hostEntry] = filepath.Join(dest, filepath.Base(hostEntry))
continue
} else if !entry.Mode().IsRegular() {
// If it is a symlink we can create it, otherwise we skip it.
if entry.Mode()&os.ModeSymlink == 0 {
continue
}
link, err := os.Readlink(hostEntry)
if err != nil {
return fmt.Errorf("Couldn't resolve symlink for %v: %v", source, err)
}
if err := os.Symlink(link, taskEntry); err != nil {
return fmt.Errorf("Couldn't create symlink: %v", err)
}
continue
}
if err := d.linkOrCopy(hostEntry, taskEntry, entry.Mode().Perm()); err != nil {
return err
}
}
}
// Recurse on self to copy subdirectories.
if len(subdirs) != 0 {
return d.Embed(task, subdirs)
}
return nil
}
// MountSharedDir mounts the shared directory into the specified task's
// directory. Mount is documented at an OS level in their respective
// implementation files.
func (d *AllocDir) MountSharedDir(task string) error {
taskDir, ok := d.TaskDirs[task]
if !ok {
return fmt.Errorf("No task directory exists for %v", task)
}
taskLoc := filepath.Join(taskDir, SharedAllocName)
if err := d.mountSharedDir(taskLoc); err != nil {
return fmt.Errorf("Failed to mount shared directory for task %v: %v", task, err)
}
d.mounted = append(d.mounted, taskLoc)
return nil
}
func fileCopy(src, dst string, perm os.FileMode) error {
// Do a simple copy.
srcFile, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open src file %v: %v", src, err)
}
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, perm)
if err != nil {
return fmt.Errorf("Couldn't create destination file %v: %v", dst, err)
}
if _, err := io.Copy(dstFile, srcFile); err != nil {
return fmt.Errorf("Couldn't copy %v to %v: %v", src, dst, err)
}
return nil
}

15
client/allocdir/alloc_dir_darwin.go Normal file
View file

@ -0,0 +1,15 @@
package allocdir
import (
"syscall"
)
// Hardlinks the shared directory. As a side-effect the shared directory and
// task directory must be on the same filesystem.
func (d *AllocDir) mountSharedDir(dir string) error {
return syscall.Link(d.SharedDir, dir)
}
func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unlink(dir)
}

20
client/allocdir/alloc_dir_linux.go Normal file
View file

@ -0,0 +1,20 @@
package allocdir
import (
"os"
"syscall"
)
// Bind mounts the shared directory into the task directory. Must be root to
// run.
func (d *AllocDir) mountSharedDir(taskDir string) error {
if err := os.Mkdir(taskDir, 0777); err != nil {
return err
}
return syscall.Mount(d.SharedDir, taskDir, "", syscall.MS_BIND, "")
}
func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unmount(dir, 0)
}

71
client/allocdir/alloc_dir_posix.go Normal file
View file

@ -0,0 +1,71 @@
// +build !windows
// Functions shared between linux/darwin.
package allocdir
import (
"fmt"
"os"
"os/user"
"strconv"
"syscall"
)
func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
// Attempt to hardlink.
if err := os.Link(src, dst); err == nil {
return nil
}
return fileCopy(src, dst, perm)
}
func (d *AllocDir) dropDirPermissions(path string) error {
// Can't do anything if not root.
if syscall.Geteuid() != 0 {
return nil
}
u, err := user.Lookup("nobody")
if err != nil {
return err
}
uid, err := getUid(u)
if err != nil {
return err
}
gid, err := getGid(u)
if err != nil {
return err
}
if err := os.Chown(path, uid, gid); err != nil {
return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err)
}
if err := os.Chmod(path, 0777); err != nil {
return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err)
}
return nil
}
func getUid(u *user.User) (int, error) {
uid, err := strconv.Atoi(u.Uid)
if err != nil {
return 0, fmt.Errorf("Unable to convert Uid to an int: %v", err)
}
return uid, nil
}
func getGid(u *user.User) (int, error) {
gid, err := strconv.Atoi(u.Gid)
if err != nil {
return 0, fmt.Errorf("Unable to convert Gid to an int: %v", err)
}
return gid, nil
}

194
client/allocdir/alloc_dir_test.go Normal file
View file

@ -0,0 +1,194 @@
package allocdir
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/nomad/structs"
)
var (
t1 = &structs.Task{
Name: "web",
Driver: "exec",
Config: map[string]string{
"command": "/bin/date",
"args": "+%s",
},
Resources: &structs.Resources{
DiskMB: 1,
},
}
t2 = &structs.Task{
Name: "web2",
Driver: "exec",
Config: map[string]string{
"command": "/bin/date",
"args": "+%s",
},
Resources: &structs.Resources{
DiskMB: 1,
},
}
)
// Test that given a set of tasks, each task gets a directory and that directory
// has the shared alloc dir inside of it.
func TestAllocDir_BuildAlloc(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp)
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Check that the AllocDir and each of the task directories exist.
if _, err := os.Stat(d.AllocDir); os.IsNotExist(err) {
t.Fatalf("Build(%v) didn't create AllocDir %v", tasks, d.AllocDir)
}
for _, task := range tasks {
tDir, ok := d.TaskDirs[task.Name]
if !ok {
t.Fatalf("Task directory not found for %v", task.Name)
}
if _, err := os.Stat(tDir); os.IsNotExist(err) {
t.Fatalf("Build(%v) didn't create TaskDir %v", tasks, tDir)
}
}
}
func TestAllocDir_EmbedNonExistent(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp)
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
fakeDir := "/foobarbaz"
task := tasks[0].Name
mapping := map[string]string{fakeDir: fakeDir}
if err := d.Embed(task, mapping); err != nil {
t.Fatalf("Embed(%v, %v) should should skip %v since it does not exist", task, mapping, fakeDir)
}
}
func TestAllocDir_EmbedDirs(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp)
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Create a fake host directory, with a file, and a subfolder that contains
// a file.
host, err := ioutil.TempDir("", "AllocDirHost")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(host)
subDirName := "subdir"
subDir := filepath.Join(host, subDirName)
if err := os.Mkdir(subDir, 0777); err != nil {
t.Fatalf("Failed to make subdir %v: %v", subDir, err)
}
file := "foo"
subFile := "bar"
if err := ioutil.WriteFile(filepath.Join(host, file), []byte{'a'}, 0777); err != nil {
t.Fatalf("Coudn't create file in host dir %v: %v", host, err)
}
if err := ioutil.WriteFile(filepath.Join(subDir, subFile), []byte{'a'}, 0777); err != nil {
t.Fatalf("Coudn't create file in host subdir %v: %v", subDir, err)
}
// Create mapping from host dir to task dir.
task := tasks[0].Name
taskDest := "bin/test/"
mapping := map[string]string{host: taskDest}
if err := d.Embed(task, mapping); err != nil {
t.Fatalf("Embed(%v, %v) failed: %v", task, mapping, err)
}
// Check that the embedding was done properly.
taskDir, ok := d.TaskDirs[task]
if !ok {
t.Fatalf("Task directory not found for %v", task)
}
exp := []string{filepath.Join(taskDir, taskDest, file), filepath.Join(taskDir, taskDest, subDirName, subFile)}
for _, e := range exp {
if _, err := os.Stat(e); os.IsNotExist(err) {
t.Fatalf("File %v not embeded: %v", e, err)
}
}
}
func TestAllocDir_MountSharedAlloc(t *testing.T) {
testutil.MountCompatible(t)
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp)
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Write a file to the shared dir.
exp := []byte{'f', 'o', 'o'}
file := "bar"
if err := ioutil.WriteFile(filepath.Join(d.SharedDir, file), exp, 0777); err != nil {
t.Fatalf("Couldn't write file to shared directory: %v", err)
}
for _, task := range tasks {
// Mount and then check that the file exists in the task directory.
if err := d.MountSharedDir(task.Name); err != nil {
t.Fatalf("MountSharedDir(%v) failed: %v", task.Name, err)
}
taskDir, ok := d.TaskDirs[task.Name]
if !ok {
t.Fatalf("Task directory not found for %v", task.Name)
}
taskFile := filepath.Join(taskDir, SharedAllocName, file)
act, err := ioutil.ReadFile(taskFile)
if err != nil {
t.Fatalf("Failed to read shared alloc file from task dir: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Incorrect data read from task dir: want %v; got %v", exp, act)
}
}
}

25
client/allocdir/alloc_dir_windows.go Normal file
View file

@ -0,0 +1,25 @@
package allocdir
import (
"errors"
"os"
)
func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
return fileCopy(src, dst)
}
// The windows version does nothing currently.
func (d *AllocDir) mountSharedDir(dir string) error {
return errors.New("Mount on Windows not supported.")
}
// The windows version does nothing currently.
func (d *AllocDir) dropDirPermissions(path string) error {
return nil
}
// The windows version does nothing currently.
func (d *AllocDir) unmountSharedDir(dir string) error {
return nil
}

27
client/client.go
View file

@ -142,19 +142,30 @@ func NewClient(cfg *config.Config) (*Client, error) {
// init is used to initialize the client and perform any setup
// needed before we begin starting its various components.
func (c *Client) init() error {
// Ensure the alloc dir exists if we have one
if c.config.AllocDir != "" {
if err := os.MkdirAll(c.config.AllocDir, 0700); err != nil {
return fmt.Errorf("failed creating alloc dir: %s", err)
}
}
// Ensure the state dir exists if we have one
if c.config.StateDir != "" {
if err := os.MkdirAll(c.config.StateDir, 0700); err != nil {
return fmt.Errorf("failed creating state dir: %s", err)
}
c.logger.Printf("[INFO] client: using state directory %v", c.config.StateDir)
}
// Ensure the alloc dir exists if we have one
if c.config.AllocDir != "" {
if err := os.MkdirAll(c.config.AllocDir, 0700); err != nil {
return fmt.Errorf("failed creating alloc dir: %s", err)
}
} else {
// Othewise make a temp directory to use.
p, err := ioutil.TempDir("", "NomadClient")
if err != nil {
return fmt.Errorf("failed creating temporary directory for the AllocDir: %v", err)
}
c.config.AllocDir = p
}
c.logger.Printf("[INFO] client: using alloc directory %v", c.config.AllocDir)
return nil
}
@ -431,7 +442,7 @@ func (c *Client) fingerprint() error {
// setupDrivers is used to find the available drivers
func (c *Client) setupDrivers() error {
var avail []string
driverCtx := driver.NewDriverContext(c.config, c.config.Node, c.logger)
driverCtx := driver.NewDriverContext("", c.config, c.config.Node, c.logger)
for name := range driver.BuiltinDrivers {
d, err := driver.NewDriver(name, driverCtx)
if err != nil {

8
client/client_test.go
View file

@ -3,6 +3,7 @@ package client
import (
"fmt"
"io/ioutil"
"log"
"net"
"os"
"path/filepath"
@ -318,6 +319,11 @@ func TestClient_WatchAllocs(t *testing.T) {
})
}
/*
TODO: This test is disabled til a follow-up api changes the restore state interface.
The driver/executor interface will be changed from Open to Cleanup, in which
clean-up tears down previous allocs.
func TestClient_SaveRestoreState(t *testing.T) {
ctestutil.ExecCompatible(t)
s1, _ := testServer(t, nil)
@ -374,6 +380,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
t.Fatalf("bad: %#v", ar.Alloc())
}
}
*/
func TestClient_Init(t *testing.T) {
dir, err := ioutil.TempDir("", "nomad")
@ -387,6 +394,7 @@ func TestClient_Init(t *testing.T) {
config: &config.Config{
AllocDir: allocDir,
},
logger: log.New(os.Stderr, "", log.LstdFlags),
}
if err := client.init(); err != nil {
t.Fatalf("err: %s", err)

39
client/driver/docker_test.go
View file

@ -26,7 +26,7 @@ func TestDockerDriver_Handle(t *testing.T) {
}
func TestDockerDriver_Fingerprint(t *testing.T) {
d := NewDockerDriver(testDriverContext())
d := NewDockerDriver(testDriverContext(""))
node := &structs.Node{
Attributes: make(map[string]string),
}
@ -48,18 +48,20 @@ func TestDockerDriver_StartOpen_Wait(t *testing.T) {
if !dockerLocated {
t.SkipNow()
}
ctx := NewExecContext()
d := NewDockerDriver(testDriverContext())
task := &structs.Task{
Name: "python-demo",
Config: map[string]string{
"image": "cbednarski/python-demo",
},
Resources: &structs.Resources{
MemoryMB: 1024,
CPU: 512,
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewDockerDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -83,10 +85,9 @@ func TestDockerDriver_Start_Wait(t *testing.T) {
if !dockerLocated {
t.SkipNow()
}
ctx := NewExecContext()
d := NewDockerDriver(testDriverContext())
task := &structs.Task{
Name: "python-demo",
Config: map[string]string{
"image": "cbednarski/python-demo",
},
@ -95,6 +96,12 @@ func TestDockerDriver_Start_Wait(t *testing.T) {
CPU: 512,
},
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewDockerDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -124,18 +131,20 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) {
if !dockerLocated {
t.SkipNow()
}
ctx := NewExecContext()
d := NewDockerDriver(testDriverContext())
task := &structs.Task{
Name: "python-demo",
Config: map[string]string{
"image": "cbednarski/python-demo",
},
Resources: &structs.Resources{
MemoryMB: 1024,
CPU: 512,
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewDockerDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)

27
client/driver/driver.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"sync"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/fingerprint"
"github.com/hashicorp/nomad/nomad/structs"
@ -55,20 +56,22 @@ type Driver interface {
// node attributes into a Driver without having to change the Driver interface
// each time we do it. Used in conjection with Factory, above.
type DriverContext struct {
config *config.Config
logger *log.Logger
node *structs.Node
taskName string
config *config.Config
logger *log.Logger
node *structs.Node
}
// NewDriverContext initializes a new DriverContext with the specified fields.
// This enables other packages to create DriverContexts but keeps the fields
// private to the driver. If we want to change this later we can gorename all of
// the fields in DriverContext.
func NewDriverContext(config *config.Config, node *structs.Node, logger *log.Logger) *DriverContext {
func NewDriverContext(taskName string, config *config.Config, node *structs.Node, logger *log.Logger) *DriverContext {
return &DriverContext{
config: config,
node: node,
logger: logger,
taskName: taskName,
config: config,
node: node,
logger: logger,
}
}
@ -92,15 +95,13 @@ type DriverHandle interface {
type ExecContext struct {
sync.Mutex
// AllocDir is the directory used for storing any state
// of this allocation. It will be purged on alloc destroy.
AllocDir string
// AllocDir contains information about the alloc directory structure.
AllocDir *allocdir.AllocDir
}
// NewExecContext is used to create a new execution context
func NewExecContext() *ExecContext {
ctx := &ExecContext{}
return ctx
func NewExecContext(alloc *allocdir.AllocDir) *ExecContext {
return &ExecContext{AllocDir: alloc}
}
// PopulateEnvironment converts exec context and task configuration into

29
client/driver/driver_test.go
View file

@ -3,23 +3,46 @@ package driver
import (
"log"
"os"
"path/filepath"
"testing"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
var basicResources = &structs.Resources{
CPU: 250,
MemoryMB: 256,
Networks: []*structs.NetworkResource{
&structs.NetworkResource{
IP: "1.2.3.4",
ReservedPorts: []int{12345},
DynamicPorts: []string{"HTTP"},
},
},
}
func testLogger() *log.Logger {
return log.New(os.Stderr, "", log.LstdFlags)
}
func testConfig() *config.Config {
return &config.Config{}
conf := &config.Config{}
conf.StateDir = os.TempDir()
conf.AllocDir = os.TempDir()
return conf
}
func testDriverContext() *DriverContext {
func testDriverContext(task string) *DriverContext {
cfg := testConfig()
ctx := NewDriverContext(cfg, cfg.Node, testLogger())
return NewDriverContext(task, cfg, cfg.Node, testLogger())
}
func testDriverExecContext(task *structs.Task, driverCtx *DriverContext) *ExecContext {
allocDir := allocdir.NewAllocDir(filepath.Join(driverCtx.config.AllocDir, structs.GenerateUUID()))
allocDir.Build([]*structs.Task{task})
ctx := NewExecContext(allocDir)
return ctx
}

9
client/driver/exec.go
View file

@ -12,9 +12,8 @@ import (
"github.com/hashicorp/nomad/nomad/structs"
)
// ExecDriver is the simplest possible driver. It literally just
// fork/execs tasks. It should probably not be used for most things,
// but is useful for testing purposes or for very simple tasks.
// ExecDriver fork/execs tasks using as many of the underlying OS's isolation
// features.
type ExecDriver struct {
DriverContext
}
@ -65,6 +64,10 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
// Populate environment variables
cmd.Command().Env = PopulateEnvironment(ctx, task)
if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
return nil, fmt.Errorf("failed to configure task directory: %v", err)
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("failed to start command: %v", err)
}

53
client/driver/exec_test.go
View file

@ -6,10 +6,13 @@ import (
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
func TestExecDriver_Fingerprint(t *testing.T) {
d := NewExecDriver(testDriverContext())
ctestutils.ExecCompatible(t)
d := NewExecDriver(testDriverContext(""))
node := &structs.Node{
Attributes: make(map[string]string),
}
@ -25,20 +28,31 @@ func TestExecDriver_Fingerprint(t *testing.T) {
}
}
func TestExecDriver_StartOpen_Wait(t *testing.T) {
ctx := NewExecContext()
d := NewExecDriver(testDriverContext())
/*
TODO: This test is disabled til a follow-up api changes the restore state interface.
The driver/executor interface will be changed from Open to Cleanup, in which
clean-up tears down previous allocs.
func TestExecDriver_StartOpen_Wait(t *testing.T) {
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Config: map[string]string{
"command": "/bin/sleep",
"args": "5",
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewExecDriver(driverCtx)
if task.Resources == nil {
task.Resources = &structs.Resources{}
}
task.Resources.CPU = 2048
task.Resources.CPU = 0.5
task.Resources.MemoryMB = 2
handle, err := d.Start(ctx, task)
@ -58,17 +72,24 @@ func TestExecDriver_StartOpen_Wait(t *testing.T) {
t.Fatalf("missing handle")
}
}
*/
func TestExecDriver_Start_Wait(t *testing.T) {
ctx := NewExecContext()
d := NewExecDriver(testDriverContext())
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Config: map[string]string{
"command": "/bin/sleep",
"args": "1",
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewExecDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -95,15 +116,21 @@ func TestExecDriver_Start_Wait(t *testing.T) {
}
func TestExecDriver_Start_Kill_Wait(t *testing.T) {
ctx := NewExecContext()
d := NewExecDriver(testDriverContext())
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "sleep",
Config: map[string]string{
"command": "/bin/sleep",
"args": "10",
"args": "1",
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewExecDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -124,7 +151,7 @@ func TestExecDriver_Start_Kill_Wait(t *testing.T) {
select {
case err := <-handle.WaitCh():
if err == nil {
t.Fatalf("should err: %v", err)
t.Fatal("should err")
}
case <-time.After(2 * time.Second):
t.Fatalf("timeout")

29
client/driver/java.go
View file

@ -14,6 +14,7 @@ import (
"syscall"
"time"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/executor"
"github.com/hashicorp/nomad/nomad/structs"
@ -104,7 +105,16 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
return nil, fmt.Errorf("Error downloading source for Java driver: %s", err)
}
fPath := filepath.Join(ctx.AllocDir, path.Base(source))
// Get the tasks local directory.
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
taskLocal := filepath.Join(taskDir, allocdir.TaskLocal)
// Create a location to download the binary.
fName := path.Base(source)
fPath := filepath.Join(taskLocal, fName)
f, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return nil, fmt.Errorf("Error opening file to download to: %s", err)
@ -113,7 +123,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
defer f.Close()
defer resp.Body.Close()
// Copy remote file to local AllocDir for execution
// Copy remote file to local directory for execution
// TODO: a retry of sort if io.Copy fails, for large binaries
_, ioErr := io.Copy(f, resp.Body)
if ioErr != nil {
@ -126,7 +136,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
if ok {
userArgs = strings.Split(argRaw, " ")
}
args := []string{"-jar", f.Name()}
args := []string{"-jar", filepath.Join(allocdir.TaskLocal, fName)}
for _, s := range userArgs {
args = append(args, s)
@ -139,12 +149,15 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
// Populate environment variables
cmd.Command().Env = PopulateEnvironment(ctx, task)
err = cmd.Limit(task.Resources)
if err != nil {
if err := cmd.Limit(task.Resources); err != nil {
return nil, fmt.Errorf("failed to constrain resources: %s", err)
}
err = cmd.Start()
if err != nil {
if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
return nil, fmt.Errorf("failed to configure task directory: %v", err)
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("failed to start source: %v", err)
}
@ -206,8 +219,6 @@ func (h *javaHandle) run() {
close(h.doneCh)
if err != nil {
h.waitCh <- err
} else if !h.cmd.Command().ProcessState.Success() {
h.waitCh <- fmt.Errorf("task exited with error")
}
close(h.waitCh)
}

57
client/driver/java_test.go
View file

@ -1,16 +1,18 @@
package driver
import (
"os"
"testing"
"time"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
func TestJavaDriver_Fingerprint(t *testing.T) {
d := NewJavaDriver(testDriverContext())
ctestutils.ExecCompatible(t)
d := NewJavaDriver(testDriverContext(""))
node := &structs.Node{
Attributes: make(map[string]string),
}
@ -31,18 +33,28 @@ func TestJavaDriver_Fingerprint(t *testing.T) {
}
}
func TestJavaDriver_StartOpen_Wait(t *testing.T) {
ctx := NewExecContext()
ctx.AllocDir = os.TempDir()
d := NewJavaDriver(testDriverContext())
/*
TODO: This test is disabled til a follow-up api changes the restore state interface.
The driver/executor interface will be changed from Open to Cleanup, in which
clean-up tears down previous allocs.
func TestJavaDriver_StartOpen_Wait(t *testing.T) {
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]string{
"jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
// "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar",
// "args": "-d64",
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -67,19 +79,25 @@ func TestJavaDriver_StartOpen_Wait(t *testing.T) {
t.Fatalf("Error: %s", err)
}
}
*/
func TestJavaDriver_Start_Wait(t *testing.T) {
ctx := NewExecContext()
ctx.AllocDir = os.TempDir()
d := NewJavaDriver(testDriverContext())
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]string{
"jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
// "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar",
// "args": "-d64",
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -107,17 +125,22 @@ func TestJavaDriver_Start_Wait(t *testing.T) {
}
func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
ctx := NewExecContext()
ctx.AllocDir = os.TempDir()
d := NewJavaDriver(testDriverContext())
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]string{
"jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
// "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar",
// "args": "-d64",
},
Resources: basicResources,
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -138,7 +161,7 @@ func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
select {
case err := <-handle.WaitCh():
if err == nil {
t.Fatalf("should err: %v", err)
t.Fatal("should err")
}
case <-time.After(2 * time.Second):
t.Fatalf("timeout")
@ -150,7 +173,3 @@ func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
t.Fatalf("Error: %s", err)
}
}
func cleanupFile(path string) error {
return nil
}

12
client/driver/qemu.go
View file

@ -19,6 +19,7 @@ import (
"syscall"
"time"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
@ -101,10 +102,17 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
return nil, fmt.Errorf("Error downloading source for Qemu driver: %s", err)
}
// Create a location in the AllocDir to download and store the image.
// Get the tasks local directory.
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
taskLocal := filepath.Join(taskDir, allocdir.TaskLocal)
// Create a location in the local directory to download and store the image.
// TODO: Caching
vmID := fmt.Sprintf("qemu-vm-%s-%s", structs.GenerateUUID(), filepath.Base(source))
fPath := filepath.Join(ctx.AllocDir, vmID)
fPath := filepath.Join(taskLocal, vmID)
vmPath, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return nil, fmt.Errorf("Error opening file to download to: %s", err)

25
client/driver/qemu_test.go
View file

@ -7,6 +7,8 @@ import (
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
ctestutils "github.com/hashicorp/nomad/client/testutil"
)
func TestQemuDriver_Handle(t *testing.T) {
@ -25,7 +27,8 @@ func TestQemuDriver_Handle(t *testing.T) {
}
func TestQemuDriver_Fingerprint(t *testing.T) {
d := NewQemuDriver(testDriverContext())
ctestutils.QemuCompatible(t)
d := NewQemuDriver(testDriverContext(""))
node := &structs.Node{
Attributes: make(map[string]string),
}
@ -45,12 +48,9 @@ func TestQemuDriver_Fingerprint(t *testing.T) {
}
func TestQemuDriver_Start(t *testing.T) {
ctx := NewExecContext()
ctx.AllocDir = os.TempDir()
d := NewQemuDriver(testDriverContext())
// TODO: use test server to load from a fixture
task := &structs.Task{
Name: "linux",
Config: map[string]string{
"image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
"checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
@ -67,6 +67,11 @@ func TestQemuDriver_Start(t *testing.T) {
},
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewQemuDriver(driverCtx)
handle, err := d.Start(ctx, task)
if err != nil {
t.Fatalf("err: %v", err)
@ -91,12 +96,9 @@ func TestQemuDriver_Start(t *testing.T) {
}
func TestQemuDriver_RequiresMemory(t *testing.T) {
ctx := NewExecContext()
ctx.AllocDir = os.TempDir()
d := NewQemuDriver(testDriverContext())
// TODO: use test server to load from a fixture
task := &structs.Task{
Name: "linux",
Config: map[string]string{
"image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
"accelerator": "tcg",
@ -107,6 +109,11 @@ func TestQemuDriver_RequiresMemory(t *testing.T) {
},
}
driverCtx := testDriverContext(task.Name)
ctx := testDriverExecContext(task, driverCtx)
defer ctx.AllocDir.Destroy()
d := NewQemuDriver(driverCtx)
_, err := d.Start(ctx, task)
if err == nil {
t.Fatalf("Expected error when not specifying memory")

8
client/executor/exec.go
View file

@ -18,12 +18,16 @@
package executor
import (
"fmt"
"os/exec"
"path/filepath"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/nomad/structs"
)
var errNoResources = fmt.Errorf("No resources are associated with this task")
// Executor is an interface that any platform- or capability-specific exec
// wrapper must implement. You should not need to implement a Java executor.
// Rather, you would implement a cgroups executor that the Java driver will use.
@ -33,6 +37,10 @@ type Executor interface {
// executor implements resource limiting. Otherwise Limit is ignored.
Limit(*structs.Resources) error
// ConfigureTaskDir must be called before Start and ensures that the tasks
// directory is properly configured.
ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error
// Start the process. This may wrap the actual process in another command,
// depending on the capabilities in this environment. Errors that arise from
// Limits or Runas may bubble through Start()

214
client/executor/exec_linux.go
View file

@ -5,13 +5,17 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/command"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/nomad/structs"
@ -24,6 +28,20 @@ const (
cgroupMount = "/sys/fs/cgroup"
)
var (
// A mapping of directories on the host OS to attempt to embed inside each
// task's chroot.
chrootEnv = map[string]string{
"/bin": "/bin",
"/etc": "/etc",
"/lib": "/lib",
"/lib32": "/lib32",
"/lib64": "/lib64",
"/usr/bin": "/usr/bin",
"/usr/lib": "/usr/lib",
}
)
func NewExecutor() Executor {
e := LinuxExecutor{}
@ -47,17 +65,23 @@ type LinuxExecutor struct {
cgroupEnabled bool
// Isolation configurations.
groups *cgroupConfig.Cgroup
groups *cgroupConfig.Cgroup
alloc *allocdir.AllocDir
taskName string
taskDir string
// Tracking of child process.
spawnChild exec.Cmd
spawnOutputWriter *os.File
spawnOutputReader *os.File
// Track whether there are filesystems mounted in the task dir.
mounts bool
}
func (e *LinuxExecutor) Limit(resources *structs.Resources) error {
if resources == nil {
return nil
return errNoResources
}
if e.cgroupEnabled {
@ -67,6 +91,73 @@ func (e *LinuxExecutor) Limit(resources *structs.Resources) error {
return nil
}
func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error {
e.taskName = taskName
taskDir, ok := alloc.TaskDirs[taskName]
if !ok {
fmt.Errorf("Couldn't find task directory for task %v", taskName)
}
e.taskDir = taskDir
if err := alloc.MountSharedDir(taskName); err != nil {
return err
}
if err := alloc.Embed(taskName, chrootEnv); err != nil {
return err
}
// Mount dev
dev := filepath.Join(taskDir, "dev")
if err := os.Mkdir(dev, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", dev)
}
if err := syscall.Mount("", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
}
// Mount proc
proc := filepath.Join(taskDir, "proc")
if err := os.Mkdir(proc, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", proc)
}
if err := syscall.Mount("", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
}
e.alloc = alloc
e.mounts = true
return nil
}
func (e *LinuxExecutor) cleanTaskDir() error {
if e.alloc == nil {
return errors.New("ConfigureTaskDir() must be called before Start()")
}
if !e.mounts {
return nil
}
// Unmount dev.
errs := new(multierror.Error)
dev := filepath.Join(e.taskDir, "dev")
if err := syscall.Unmount(dev, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
}
// Unmount proc.
proc := filepath.Join(e.taskDir, "proc")
if err := syscall.Unmount(proc, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
}
e.mounts = false
return errs.ErrorOrNil()
}
func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) {
if !e.cgroupEnabled {
return
@ -107,7 +198,6 @@ func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) {
e.groups.BlkioThrottleReadIOpsDevice = strconv.FormatInt(int64(resources.IOPS), 10)
e.groups.BlkioThrottleWriteIOpsDevice = strconv.FormatInt(int64(resources.IOPS), 10)
}
}
func (e *LinuxExecutor) runAs(userid string) error {
@ -137,15 +227,17 @@ func (e *LinuxExecutor) runAs(userid string) error {
}
func (e *LinuxExecutor) Start() error {
// Try to run as "nobody" user so we don't leak root privilege to the
// spawned process. Note that we will only do this if we can call SetUID.
// Otherwise we'll just run the other process as our current (non-root)
// user. This means we aren't forced to run nomad as root.
// Run as "nobody" user so we don't leak root privilege to the
// spawned process.
if err := e.runAs("nobody"); err == nil && e.user != nil {
e.cmd.SetUID(e.user.Uid)
e.cmd.SetGID(e.user.Gid)
}
if e.alloc == nil {
return errors.New("ConfigureTaskDir() must be called before Start()")
}
return e.spawnDaemon()
}
@ -162,13 +254,11 @@ func (e *LinuxExecutor) spawnDaemon() error {
var buffer bytes.Buffer
enc := json.NewEncoder(&buffer)
// TODO: Do the stdout file handles once there is alloc and task directories
// set up.
c := command.DaemonConfig{
Cmd: e.cmd.Cmd,
Groups: e.groups,
StdoutFile: "/dev/null",
StderrFile: "/dev/null",
Chroot: e.taskDir,
StdoutFile: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)),
StderrFile: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)),
StdinFile: "/dev/null",
}
if err := enc.Encode(c); err != nil {
@ -190,10 +280,44 @@ func (e *LinuxExecutor) spawnDaemon() error {
spawn := exec.Command(bin, "spawn-daemon", escaped)
spawn.Stdout = e.spawnOutputWriter
// Capture its Stdin.
spawnStdIn, err := spawn.StdinPipe()
if err != nil {
return err
}
if err := spawn.Start(); err != nil {
fmt.Errorf("Failed to call spawn-daemon on nomad executable: %v", err)
}
// Join the spawn-daemon to the cgroup.
if e.groups != nil {
manager := cgroupFs.Manager{}
manager.Cgroups = e.groups
// Apply will place the current pid into the tasks file for each of the
// created cgroups:
// /sys/fs/cgroup/memory/user/1000.user/4.session/<uuid>/tasks
//
// Apply requires superuser permissions, and may fail if Nomad is not run with
// the required permissions
if err := manager.Apply(spawn.Process.Pid); err != nil {
errs := new(multierror.Error)
errs = multierror.Append(errs, fmt.Errorf("Failed to join spawn-daemon to the cgroup (config => %+v): %v", manager.Cgroups, err))
if err := sendAbortCommand(spawnStdIn); err != nil {
errs = multierror.Append(errs, err)
}
return errs
}
}
// Tell it to start.
if err := sendStartCommand(spawnStdIn); err != nil {
return err
}
// Parse the response.
dec := json.NewDecoder(e.spawnOutputReader)
var resp command.SpawnStartStatus
@ -209,6 +333,24 @@ func (e *LinuxExecutor) spawnDaemon() error {
return nil
}
func sendStartCommand(w io.Writer) error {
enc := json.NewEncoder(w)
if err := enc.Encode(true); err != nil {
return fmt.Errorf("Failed to serialize start command: %v", err)
}
return nil
}
func sendAbortCommand(w io.Writer) error {
enc := json.NewEncoder(w)
if err := enc.Encode(false); err != nil {
return fmt.Errorf("Failed to serialize abort command: %v", err)
}
return nil
}
// Open's behavior is to kill all processes associated with the id and return an
// error. This is done because it is not possible to re-attach to the
// spawn-daemon's stdout to retrieve status messages.
@ -249,11 +391,13 @@ func (e *LinuxExecutor) Open(id string) error {
if err := e.destroyCgroup(); err != nil {
return err
}
// TODO: cleanTaskDir is a little more complicated here because the OS
// may have already unmounted in the case of a restart. Need to scan.
default:
return fmt.Errorf("Invalid id type: %v", parts[0])
}
return errors.New("Could not re-open to id")
return errors.New("Could not re-open to id (intended).")
}
func (e *LinuxExecutor) Wait() error {
@ -264,29 +408,24 @@ func (e *LinuxExecutor) Wait() error {
defer e.spawnOutputWriter.Close()
defer e.spawnOutputReader.Close()
err := e.spawnChild.Wait()
if err != nil {
return fmt.Errorf("Wait failed on pid %v: %v", e.spawnChild.Process.Pid, err)
}
// Read the exit status of the spawned process.
dec := json.NewDecoder(e.spawnOutputReader)
var resp command.SpawnExitStatus
if err := dec.Decode(&resp); err != nil {
return fmt.Errorf("Failed to parse spawn-daemon exit response: %v", err)
}
if !resp.Success {
return errors.New("Task exited with error")
errs := new(multierror.Error)
if err := e.spawnChild.Wait(); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Wait failed on pid %v: %v", e.spawnChild.Process.Pid, err))
}
// If they fork/exec and then exit, wait will return but they will be still
// running processes so we need to kill the full cgroup.
if e.cgroupEnabled {
return e.destroyCgroup()
if e.groups != nil {
if err := e.destroyCgroup(); err != nil {
errs = multierror.Append(errs, err)
}
}
return nil
if err := e.cleanTaskDir(); err != nil {
errs = multierror.Append(errs, err)
}
return errs.ErrorOrNil()
}
// If cgroups are used, the ID is the cgroup structurue. Otherwise, it is the
@ -327,7 +466,7 @@ func (e *LinuxExecutor) ForceStop() error {
// If the task is not running inside a cgroup then just the spawn-daemon child is killed.
// TODO: Find a good way to kill the children of the spawn-daemon.
if !e.cgroupEnabled {
if e.groups == nil {
if err := e.spawnChild.Process.Kill(); err != nil {
return fmt.Errorf("Failed to kill child (%v): %v", e.spawnChild.Process.Pid, err)
}
@ -335,7 +474,18 @@ func (e *LinuxExecutor) ForceStop() error {
return nil
}
return e.destroyCgroup()
errs := new(multierror.Error)
if e.groups != nil {
if err := e.destroyCgroup(); err != nil {
errs = multierror.Append(errs, err)
}
}
if err := e.cleanTaskDir(); err != nil {
errs = multierror.Append(errs, err)
}
return errs.ErrorOrNil()
}
func (e *LinuxExecutor) destroyCgroup() error {

99
client/executor/exec_linux_test.go
View file

@ -8,12 +8,16 @@ import (
"testing"
"time"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
ctestutil "github.com/hashicorp/nomad/client/testutil"
)
var (
constraint = &structs.Resources{
CPU: 0.5,
CPU: 250,
MemoryMB: 256,
Networks: []*structs.NetworkResource{
&structs.NetworkResource{
@ -24,7 +28,20 @@ var (
}
)
func mockAllocDir(t *testing.T) (string, *allocdir.AllocDir) {
alloc := mock.Alloc()
task := alloc.Job.TaskGroups[0].Tasks[0]
allocDir := allocdir.NewAllocDir(filepath.Join(os.TempDir(), alloc.ID))
if err := allocDir.Build([]*structs.Task{task}); err != nil {
t.Fatalf("allocDir.Build() failed: %v", err)
}
return task.Name, allocDir
}
func TestExecutorLinux_Start_Invalid(t *testing.T) {
ctestutil.ExecCompatible(t)
invalid := "/bin/foobar"
e := Command(invalid, "1")
@ -32,18 +49,31 @@ func TestExecutorLinux_Start_Invalid(t *testing.T) {
t.Fatalf("Limit() failed: %v", err)
}
task, alloc := mockAllocDir(t)
defer alloc.Destroy()
if err := e.ConfigureTaskDir(task, alloc); err != nil {
t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
}
if err := e.Start(); err == nil {
t.Fatalf("Start(%v) should have failed", invalid)
}
}
func TestExecutorLinux_Start_Wait_Failure_Code(t *testing.T) {
ctestutil.ExecCompatible(t)
e := Command("/bin/date", "-invalid")
if err := e.Limit(constraint); err != nil {
t.Fatalf("Limit() failed: %v", err)
}
task, alloc := mockAllocDir(t)
defer alloc.Destroy()
if err := e.ConfigureTaskDir(task, alloc); err != nil {
t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
}
if err := e.Start(); err != nil {
t.Fatalf("Start() failed: %v", err)
}
@ -54,24 +84,29 @@ func TestExecutorLinux_Start_Wait_Failure_Code(t *testing.T) {
}
func TestExecutorLinux_Start_Wait(t *testing.T) {
path, err := ioutil.TempDir("", "TestExecutorLinux_Start_Wait")
if err != nil {
t.Fatal(err)
}
defer os.Remove(path)
ctestutil.ExecCompatible(t)
task, alloc := mockAllocDir(t)
defer alloc.Destroy()
// Make the file writable to everyone.
os.Chmod(path, 0777)
taskDir, ok := alloc.TaskDirs[task]
if !ok {
t.Fatalf("No task directory found for task %v", task)
}
expected := "hello world"
filePath := filepath.Join(path, "output")
cmd := fmt.Sprintf("%v \"%v\" > %v", "sleep 1 ; echo -n", expected, filePath)
file := filepath.Join(allocdir.TaskLocal, "output.txt")
absFilePath := filepath.Join(taskDir, file)
cmd := fmt.Sprintf("%v \"%v\" > %v", "sleep 1 ; echo -n", expected, file)
e := Command("/bin/bash", "-c", cmd)
if err := e.Limit(constraint); err != nil {
t.Fatalf("Limit() failed: %v", err)
}
if err := e.ConfigureTaskDir(task, alloc); err != nil {
t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
}
if err := e.Start(); err != nil {
t.Fatalf("Start() failed: %v", err)
}
@ -80,9 +115,9 @@ func TestExecutorLinux_Start_Wait(t *testing.T) {
t.Fatalf("Wait() failed: %v", err)
}
output, err := ioutil.ReadFile(filePath)
output, err := ioutil.ReadFile(absFilePath)
if err != nil {
t.Fatalf("Couldn't read file %v", filePath)
t.Fatalf("Couldn't read file %v", absFilePath)
}
act := string(output)
@ -92,16 +127,16 @@ func TestExecutorLinux_Start_Wait(t *testing.T) {
}
func TestExecutorLinux_Start_Kill(t *testing.T) {
path, err := ioutil.TempDir("", "TestExecutorLinux_Start_Kill")
if err != nil {
t.Fatal(err)
ctestutil.ExecCompatible(t)
task, alloc := mockAllocDir(t)
defer alloc.Destroy()
taskDir, ok := alloc.TaskDirs[task]
if !ok {
t.Fatalf("No task directory found for task %v", task)
}
defer os.Remove(path)
// Make the file writable to everyone.
os.Chmod(path, 0777)
filePath := filepath.Join(path, "test")
filePath := filepath.Join(taskDir, "output")
e := Command("/bin/bash", "-c", "sleep 1 ; echo \"failure\" > "+filePath)
// This test can only be run if cgroups are enabled.
@ -113,6 +148,10 @@ func TestExecutorLinux_Start_Kill(t *testing.T) {
t.Fatalf("Limit() failed: %v", err)
}
if err := e.ConfigureTaskDir(task, alloc); err != nil {
t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
}
if err := e.Start(); err != nil {
t.Fatalf("Start() failed: %v", err)
}
@ -130,16 +169,16 @@ func TestExecutorLinux_Start_Kill(t *testing.T) {
}
func TestExecutorLinux_Open(t *testing.T) {
path, err := ioutil.TempDir("", "TestExecutorLinux_Open")
if err != nil {
t.Fatal(err)
ctestutil.ExecCompatible(t)
task, alloc := mockAllocDir(t)
defer alloc.Destroy()
taskDir, ok := alloc.TaskDirs[task]
if !ok {
t.Fatalf("No task directory found for task %v", task)
}
defer os.Remove(path)
// Make the file writable to everyone.
os.Chmod(path, 0777)
filePath := filepath.Join(path, "test")
filePath := filepath.Join(taskDir, "output")
e := Command("/bin/bash", "-c", "sleep 1 ; echo \"failure\" > "+filePath)
// This test can only be run if cgroups are enabled.
@ -151,6 +190,10 @@ func TestExecutorLinux_Open(t *testing.T) {
t.Fatalf("Limit() failed: %v", err)
}
if err := e.ConfigureTaskDir(task, alloc); err != nil {
t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
}
if err := e.Start(); err != nil {
t.Fatalf("Start() failed: %v", err)
}

8
client/executor/exec_universal.go
View file

@ -7,6 +7,7 @@ import (
"os"
"strconv"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/nomad/structs"
)
@ -21,6 +22,13 @@ type UniversalExecutor struct {
}
func (e *UniversalExecutor) Limit(resources *structs.Resources) error {
if resources == nil {
return errNoResources
}
return nil
}
func (e *UniversalExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error {
// No-op
return nil
}

2
client/executor/setuid.go
View file

@ -36,6 +36,6 @@ func (c *cmd) SetGID(groupid string) error {
if c.SysProcAttr.Credential == nil {
c.SysProcAttr.Credential = &syscall.Credential{}
}
c.SysProcAttr.Credential.Uid = uint32(gid)
c.SysProcAttr.Credential.Gid = uint32(gid)
return nil
}

2
client/task_runner.go
View file

@ -129,7 +129,7 @@ func (r *TaskRunner) setStatus(status, desc string) {
// createDriver makes a driver for the task
func (r *TaskRunner) createDriver() (driver.Driver, error) {
driverCtx := driver.NewDriverContext(r.config, r.config.Node, r.logger)
driverCtx := driver.NewDriverContext(r.task.Name, r.config, r.config.Node, r.logger)
driver, err := driver.NewDriver(r.task.Driver, driverCtx)
if err != nil {
err = fmt.Errorf("failed to create driver '%s' for alloc %s: %v",

13
client/task_runner_test.go
View file

@ -3,10 +3,12 @@ package client
import (
"log"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/driver"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
@ -36,9 +38,9 @@ func (m *MockTaskStateUpdater) Update(name, status, desc string) {
func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) {
logger := testLogger()
conf := DefaultConfig()
conf.StateDir = "/tmp"
conf.StateDir = os.TempDir()
conf.AllocDir = os.TempDir()
upd := &MockTaskStateUpdater{}
ctx := driver.NewExecContext()
alloc := mock.Alloc()
task := alloc.Job.TaskGroups[0].Tasks[0]
@ -46,6 +48,10 @@ func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) {
// we have a mock so that doesn't happen.
task.Resources.Networks[0].ReservedPorts = []int{80}
allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID))
allocDir.Build([]*structs.Task{task})
ctx := driver.NewExecContext(allocDir)
tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc.ID, task)
return upd, tr
}
@ -55,6 +61,7 @@ func TestTaskRunner_SimpleRun(t *testing.T) {
upd, tr := testTaskRunner()
go tr.Run()
defer tr.Destroy()
defer tr.ctx.AllocDir.Destroy()
select {
case <-tr.WaitCh():
@ -89,6 +96,7 @@ func TestTaskRunner_SimpleRun(t *testing.T) {
func TestTaskRunner_Destroy(t *testing.T) {
ctestutil.ExecCompatible(t)
upd, tr := testTaskRunner()
defer tr.ctx.AllocDir.Destroy()
// Change command to ensure we run for a bit
tr.task.Config["command"] = "/bin/sleep"
@ -130,6 +138,7 @@ func TestTaskRunner_Update(t *testing.T) {
tr.task.Config["args"] = "10"
go tr.Run()
defer tr.Destroy()
defer tr.ctx.AllocDir.Destroy()
// Update the task definition
newTask := new(structs.Task)

16
client/testutil/driver_compatible.go
View file

@ -11,3 +11,19 @@ func ExecCompatible(t *testing.T) {
t.Skip("Must be root on non-windows environments to run test")
}
}
func QemuCompatible(t *testing.T) {
if runtime.GOOS != "windows" && syscall.Geteuid() != 0 {
t.Skip("Must be root on non-windows environments to run test")
}
}
func MountCompatible(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Windows does not support mount")
}
if syscall.Geteuid() != 0 {
t.Skip("Must be root to run test")
}
}

62
command/spawn_daemon_linux.go
View file

@ -8,9 +8,6 @@ import (
"strconv"
"strings"
"syscall"
cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
)
// Configuration for the command to start as a daemon.
@ -23,13 +20,11 @@ type DaemonConfig struct {
StdinFile string
StderrFile string
Groups *cgroupConfig.Cgroup
Chroot string
}
// The exit status of the user's command.
type SpawnExitStatus struct {
Success bool
}
// Whether to start the user command or abort.
type TaskStart bool
func (c *SpawnDaemonCommand) Run(args []string) int {
flags := c.Meta.FlagSet("spawn-daemon", FlagSetClient)
@ -57,26 +52,9 @@ func (c *SpawnDaemonCommand) Run(args []string) int {
return c.outputStartStatus(err, 1)
}
// Join this process to the cgroup.
if cmd.Groups != nil {
manager := cgroupFs.Manager{}
manager.Cgroups = cmd.Groups
// Apply will place the current pid into the tasks file for each of the
// created cgroups:
// /sys/fs/cgroup/memory/user/1000.user/4.session/<uuid>/tasks
//
// Apply requires superuser permissions, and may fail if Nomad is not run with
// the required permissions
if err := manager.Apply(os.Getpid()); err != nil {
return c.outputStartStatus(fmt.Errorf("Failed to join cgroup: %v", err), 1)
}
}
// Isolate the user process.
if _, err := syscall.Setsid(); err != nil {
return c.outputStartStatus(fmt.Errorf("Failed to join cgroup: %v",
fmt.Errorf("Failed setting sid: %v", err)), 1)
return c.outputStartStatus(fmt.Errorf("Failed setting sid: %v", err), 1)
}
syscall.Umask(0)
@ -97,9 +75,28 @@ func (c *SpawnDaemonCommand) Run(args []string) int {
return c.outputStartStatus(fmt.Errorf("Error opening file to redirect Stdin: %v", err), 1)
}
cmd.Stdout = stdo
cmd.Stderr = stde
cmd.Stdin = stdi
cmd.Cmd.Stdout = stdo
cmd.Cmd.Stderr = stde
cmd.Cmd.Stdin = stdi
// Chroot jail the process and set its working directory.
if cmd.Cmd.SysProcAttr == nil {
cmd.Cmd.SysProcAttr = &syscall.SysProcAttr{}
}
cmd.Cmd.SysProcAttr.Chroot = cmd.Chroot
cmd.Cmd.Dir = "/"
// Wait to get the start command.
var start TaskStart
dec = json.NewDecoder(os.Stdin)
if err := dec.Decode(&start); err != nil {
return c.outputStartStatus(err, 1)
}
if !start {
return 0
}
// Spawn the user process.
if err := cmd.Cmd.Start(); err != nil {
@ -110,12 +107,9 @@ func (c *SpawnDaemonCommand) Run(args []string) int {
c.outputStartStatus(nil, 0)
// Wait and then output the exit status.
exitStatus := &SpawnExitStatus{}
if err := cmd.Wait(); err == nil {
exitStatus.Success = true
if err := cmd.Wait(); err != nil {
return 1
}
enc := json.NewEncoder(os.Stdout)
enc.Encode(exitStatus)
return 0
}