Merge pull request #36 from hashicorp/f-executor

Added executor interface, includes drop privileges for linux

client/driver/exec.go

@@ -2,15 +2,12 @@ package driver
 
 import (
 	"fmt"
-	"os"
-	"os/exec"
 	"strconv"
 	"strings"
 	"time"
 
-	"golang.org/x/sys/unix"
-
 	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/executor"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -23,7 +20,7 @@ type ExecDriver struct {
 
 // execHandle is returned from Start/Open as a handle to the PID
 type execHandle struct {
-	proc   *os.Process
+	cmd    executor.Executor
 	waitCh chan error
 	doneCh chan struct{}
 }
@@ -54,15 +51,19 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	}
 
 	// Setup the command
-	cmd := exec.Command(command, args...)
-	err := cmd.Start()
+	cmd := executor.Command(command, args...)
+	err := cmd.Limit(task.Resources)
+	if err != nil {
+		return nil, fmt.Errorf("failed to constrain resources: %s", err)
+	}
+	err = cmd.Start()
 	if err != nil {
 		return nil, fmt.Errorf("failed to start command: %v", err)
 	}
 
 	// Return a driver handle
 	h := &execHandle{
-		proc:   cmd.Process,
+		cmd:    cmd,
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
 	}
@@ -79,14 +80,14 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 
 	// Find the process
-	proc, err := os.FindProcess(pid)
-	if proc == nil || err != nil {
+	cmd, err := executor.OpenPid(pid)
+	if err != nil {
 		return nil, fmt.Errorf("failed to find PID %d: %v", pid, err)
 	}
 
 	// Return a driver handle
 	h := &execHandle{
-		proc:   proc,
+		cmd:    cmd,
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
 	}
@@ -96,7 +97,8 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 func (h *execHandle) ID() string {
 	// Return a handle to the PID
-	return fmt.Sprintf("PID:%d", h.proc.Pid)
+	pid, _ := h.cmd.Pid()
+	return fmt.Sprintf("PID:%d", pid)
 }
 
 func (h *execHandle) WaitCh() chan error {
@@ -108,24 +110,22 @@ func (h *execHandle) Update(task *structs.Task) error {
 	return nil
 }
 
-// Kill is used to terminate the task. We send an Interrupt
-// and then provide a 5 second grace period before doing a Kill.
 func (h *execHandle) Kill() error {
-	h.proc.Signal(unix.SIGTERM)
+	h.cmd.Shutdown()
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(5 * time.Second):
-		return h.proc.Kill()
+		return h.cmd.ForceStop()
 	}
 }
 
 func (h *execHandle) run() {
-	ps, err := h.proc.Wait()
+	err := h.cmd.Wait()
 	close(h.doneCh)
 	if err != nil {
 		h.waitCh <- err
-	} else if !ps.Success() {
+	} else if !h.cmd.Command().ProcessState.Success() {
 		h.waitCh <- fmt.Errorf("task exited with error")
 	}
 	close(h.waitCh)

client/driver/java.go

@@ -13,9 +13,8 @@ import (
 	"strings"
 	"time"
 
-	"golang.org/x/sys/unix"
-
 	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/executor"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -27,7 +26,7 @@ type JavaDriver struct {
 
 // javaHandle is returned from Start/Open as a handle to the PID
 type javaHandle struct {
-	proc   *os.Process
+	cmd    executor.Executor
 	waitCh chan error
 	doneCh chan struct{}
 }
@@ -129,7 +128,11 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 	// Setup the command
 	// Assumes Java is in the $PATH, but could probably be detected
-	cmd := exec.Command("java", args...)
+	cmd := executor.Command("java", args...)
+	err = cmd.Limit(task.Resources)
+	if err != nil {
+		return nil, fmt.Errorf("failed to constrain resources: %s", err)
+	}
 	err = cmd.Start()
 	if err != nil {
 		return nil, fmt.Errorf("failed to start source: %v", err)
@@ -137,7 +140,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 	// Return a driver handle
 	h := &javaHandle{
-		proc:   cmd.Process,
+		cmd:    cmd,
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
 	}
@@ -155,14 +158,14 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 
 	// Find the process
-	proc, err := os.FindProcess(pid)
-	if proc == nil || err != nil {
+	cmd, err := executor.OpenPid(pid)
+	if err != nil {
 		return nil, fmt.Errorf("failed to find PID %d: %v", pid, err)
 	}
 
 	// Return a driver handle
 	h := &javaHandle{
-		proc:   proc,
+		cmd:    cmd,
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
 	}
@@ -173,7 +176,8 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 func (h *javaHandle) ID() string {
 	// Return a handle to the PID
-	return fmt.Sprintf("PID:%d", h.proc.Pid)
+	pid, _ := h.cmd.Pid()
+	return fmt.Sprintf("PID:%d", pid)
 }
 
 func (h *javaHandle) WaitCh() chan error {
@@ -185,24 +189,22 @@ func (h *javaHandle) Update(task *structs.Task) error {
 	return nil
 }
 
-// Kill is used to terminate the task. We send an Interrupt
-// and then provide a 5 second grace period before doing a Kill.
 func (h *javaHandle) Kill() error {
-	h.proc.Signal(unix.SIGTERM)
+	h.cmd.Shutdown()
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(5 * time.Second):
-		return h.proc.Kill()
+		return h.cmd.ForceStop()
 	}
 }
 
 func (h *javaHandle) run() {
-	ps, err := h.proc.Wait()
+	err := h.cmd.Wait()
 	close(h.doneCh)
 	if err != nil {
 		h.waitCh <- err
-	} else if !ps.Success() {
+	} else if !h.cmd.Command().ProcessState.Success() {
 		h.waitCh <- fmt.Errorf("task exited with error")
 	}
 	close(h.waitCh)

client/executor/exec.go

@@ -0,0 +1,146 @@
+// Package exec is used to invoke child processes across various platforms to
+// provide the following features:
+//
+// - Least privilege
+// - Resource constraints
+// - Process isolation
+//
+// A "platform" may be defined as coarsely as "Windows" or as specifically as
+// "linux 3.20 with systemd". This allows Nomad to use best-effort, best-
+// available capabilities of each platform to provide resource constraints,
+// process isolation, and security features, or otherwise take advantage of
+// features that are unique to that platform.
+//
+// The `semantics of any particular instance are left up to the implementation.
+// However, these should be completely transparent to the calling context. In
+// other words, the Java driver should be able to call exec for any platform and
+// just work.
+package executor
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// Executor is an interface that any platform- or capability-specific exec
+// wrapper must implement. You should not need to implement a Java executor.
+// Rather, you would implement a cgroups executor that the Java driver will use.
+type Executor interface {
+	// Limit must be called before Start and restricts the amount of resources
+	// the process can use. Note that an error may be returned ONLY IF the
+	// executor implements resource limiting. Otherwise Limit is ignored.
+	Limit(*structs.Resources) error
+
+	// RunAs sets the user we should use to run this command. This may be set as
+	// a username, uid, or other identifier. The implementation will decide what
+	// to do with it, if anything. Note that an error may be returned ONLY IF
+	// the executor implements user lookups. Otherwise RunAs is ignored.
+	RunAs(string) error
+
+	// Start the process. This may wrap the actual process in another command,
+	// depending on the capabilities in this environment. Errors that arise from
+	// Limits or Runas may bubble through Start()
+	Start() error
+
+	// Open should be called to restore a previous pid. This might be needed if
+	// nomad is restarted. This sets os.Process internally.
+	Open(int) error
+
+	// This is a convenience wrapper around Command().Wait()
+	Wait() error
+
+	// This is a convenience wrapper around Command().Process.Pid
+	Pid() (int, error)
+
+	// Shutdown should use a graceful stop mechanism so the application can
+	// perform checkpointing or cleanup, if such a mechanism is available.
+	// If such a mechanism is not available, Shutdown() should call ForceStop().
+	Shutdown() error
+
+	// ForceStop will terminate the process without waiting for cleanup. Every
+	// implementations must provide this.
+	ForceStop() error
+
+	// Command provides access the underlying Cmd struct in case the Executor
+	// interface doesn't expose the functionality you need.
+	Command() *cmd
+}
+
+// Command is a mirror of exec.Command that returns a platform-specific Executor
+func Command(name string, arg ...string) Executor {
+	executor := NewExecutor()
+	cmd := executor.Command()
+	cmd.Path = name
+	cmd.Args = append([]string{name}, arg...)
+
+	if filepath.Base(name) == name {
+		if lp, err := exec.LookPath(name); err != nil {
+			// cmd.lookPathErr = err
+		} else {
+			cmd.Path = lp
+		}
+	}
+	return executor
+}
+
+// OpenPid is similar to executor.Command but will initialize executor.Cmd with
+// the Pid set to the one specified.
+func OpenPid(pid int) (Executor, error) {
+	executor := NewExecutor()
+	err := executor.Open(pid)
+	if err != nil {
+		return nil, err
+	}
+	return executor, nil
+}
+
+// Cmd is an extension of exec.Cmd that incorporates functionality for
+// re-attaching to processes, dropping priviledges, etc., based on platform-
+// specific implementations.
+type cmd struct {
+	exec.Cmd
+
+	// Resources is used to limit CPU and RAM used by the process, by way of
+	// cgroups or a similar mechanism.
+	Resources structs.Resources
+
+	// RunAs may be a username or Uid. The implementation will decide how to use it.
+	RunAs string
+}
+
+// SetUID changes the Uid for this command (must be set before starting)
+func (c *cmd) SetUID(userid string) error {
+	uid, err := strconv.ParseUint(userid, 10, 32)
+	if err != nil {
+		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
+	}
+	if c.SysProcAttr == nil {
+		c.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	if c.SysProcAttr.Credential == nil {
+		c.SysProcAttr.Credential = &syscall.Credential{}
+	}
+	c.SysProcAttr.Credential.Uid = uint32(uid)
+	return nil
+}
+
+// SetGID changes the Gid for this command (must be set before starting)
+func (c *cmd) SetGID(groupid string) error {
+	gid, err := strconv.ParseUint(groupid, 10, 32)
+	if err != nil {
+		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
+	}
+	if c.SysProcAttr == nil {
+		c.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	if c.SysProcAttr.Credential == nil {
+		c.SysProcAttr.Credential = &syscall.Credential{}
+	}
+	c.SysProcAttr.Credential.Uid = uint32(gid)
+	return nil
+}

client/executor/exec_linux.go

@@ -0,0 +1,137 @@
+package executor
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"syscall"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+func NewExecutor() Executor {
+	return &LinuxExecutor{}
+}
+
+// Linux executor is designed to run on linux kernel 2.8+.
+type LinuxExecutor struct {
+	cmd
+	user *user.User
+}
+
+func (e *LinuxExecutor) Limit(resources *structs.Resources) error {
+	// TODO limit some things
+	return nil
+}
+
+func (e *LinuxExecutor) RunAs(userid string) error {
+	errs := new(multierror.Error)
+
+	// First, try to lookup the user by uid
+	u, err := user.LookupId(userid)
+	if err == nil {
+		e.user = u
+		return nil
+	} else {
+		errs = multierror.Append(errs, err)
+	}
+
+	// Lookup failed, so try by username instead
+	u, err = user.Lookup(userid)
+	if err == nil {
+		e.user = u
+		return nil
+	} else {
+		errs = multierror.Append(errs, err)
+	}
+
+	// If we got here we failed to lookup based on id and username, so we'll
+	// return those errors.
+	return fmt.Errorf("Failed to identify user to run as: %s", errs)
+}
+
+func (e *LinuxExecutor) Start() error {
+	// If no user has been specified, try to run as "nobody" user so we don't
+	// leak root privilege to the spawned process. Note that we will only do
+	// this if we can call SetUID. Otherwise we'll just run the other process
+	// as our current (non-root) user. This makes testing easier and also means
+	// we aren't forced to run nomad as root.
+	if e.user == nil && canSetUID() {
+		e.RunAs("nobody")
+	}
+
+	// Set the user and group this process should run as. If RunAs was called
+	// but we are not root this will cause Start to fail. This is intentional.
+	if e.user != nil {
+		e.cmd.SetUID(e.user.Uid)
+		e.cmd.SetGID(e.user.Gid)
+	}
+
+	// We don't want to call ourself. We want to call Start on our embedded Cmd.
+	return e.cmd.Start()
+}
+
+func (e *LinuxExecutor) Open(pid int) error {
+	process, err := os.FindProcess(pid)
+	// FindProcess doesn't do any checking against the process table so it's
+	// unlikely we'll ever see this error.
+	if err != nil {
+		return fmt.Errorf("Failed to reopen pid %d: %s", pid, err)
+	}
+
+	// On linux FindProcess() will return a pid but doesn't actually check to
+	// see whether that process is running. We'll send signal 0 to see if the
+	// process is alive.
+	err = process.Signal(syscall.Signal(0))
+	if err != nil {
+		return fmt.Errorf("Unable to signal pid %d: %s", err)
+	}
+	e.Process = process
+	return nil
+}
+
+func (e *LinuxExecutor) Wait() error {
+	// We don't want to call ourself. We want to call Start on our embedded Cmd
+	return e.cmd.Wait()
+}
+
+func (e *LinuxExecutor) Pid() (int, error) {
+	if e.cmd.Process != nil {
+		return e.cmd.Process.Pid, nil
+	} else {
+		return 0, fmt.Errorf("Process has finished or was never started")
+	}
+}
+
+func (e *LinuxExecutor) Shutdown() error {
+	return e.ForceStop()
+}
+
+func (e *LinuxExecutor) ForceStop() error {
+	return e.Process.Kill()
+}
+
+func (e *LinuxExecutor) Command() *cmd {
+	return &e.cmd
+}
+
+// canSetUID will tell us whether we're capable of using SetUID. If we are not
+// rootish this command will fail. In that case we'll just run the forked
+// process under our own user.
+func canSetUID() bool {
+	checkroot := Command("true")
+	u, err := user.Current()
+	if err != nil {
+		return false
+	}
+
+	// Make sure RunAs is explicitly set so we don't cause infinite recursion.
+	checkroot.RunAs(u.Uid)
+
+	err = checkroot.Start()
+	if err != nil {
+		return false
+	}
+	return true
+}

client/executor/exec_universal.go

@@ -0,0 +1,69 @@
+// +build !linux
+
+package executor
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+func NewExecutor() Executor {
+	return &UniversalExecutor{}
+}
+
+// UniversalExecutor should work everywhere, and as a result does not include
+// any resource restrictions or runas capabilities.
+type UniversalExecutor struct {
+	cmd
+}
+
+func (e *UniversalExecutor) Limit(resources *structs.Resources) error {
+	// No-op
+	return nil
+}
+
+func (e *UniversalExecutor) RunAs(userid string) error {
+	// No-op
+	return nil
+}
+
+func (e *UniversalExecutor) Start() error {
+	// We don't want to call ourself. We want to call Start on our embedded Cmd
+	return e.cmd.Start()
+}
+
+func (e *UniversalExecutor) Open(pid int) error {
+	process, err := os.FindProcess(pid)
+	if err != nil {
+		return fmt.Errorf("Failed to reopen pid %d: %s", pid, err)
+	}
+	e.Process = process
+	return nil
+}
+
+func (e *UniversalExecutor) Wait() error {
+	// We don't want to call ourself. We want to call Start on our embedded Cmd
+	return e.cmd.Wait()
+}
+
+func (e *UniversalExecutor) Pid() (int, error) {
+	if e.cmd.Process != nil {
+		return e.cmd.Process.Pid, nil
+	} else {
+		return 0, fmt.Errorf("Process has finished or was never started")
+	}
+}
+
+func (e *UniversalExecutor) Shutdown() error {
+	return e.ForceStop()
+}
+
+func (e *UniversalExecutor) ForceStop() error {
+	return e.Process.Kill()
+}
+
+func (e *UniversalExecutor) Command() *cmd {
+	return &e.cmd
+}