diff --git a/client/driver/exec.go b/client/driver/exec.go index 5e4564923..a96f3121d 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -2,15 +2,12 @@ package driver import ( "fmt" - "os" - "os/exec" "strconv" "strings" "time" - "golang.org/x/sys/unix" - "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/executor" "github.com/hashicorp/nomad/nomad/structs" ) @@ -23,7 +20,7 @@ type ExecDriver struct { // execHandle is returned from Start/Open as a handle to the PID type execHandle struct { - proc *os.Process + cmd executor.Executor waitCh chan error doneCh chan struct{} } @@ -54,15 +51,19 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } // Setup the command - cmd := exec.Command(command, args...) - err := cmd.Start() + cmd := executor.Command(command, args...) + err := cmd.Limit(task.Resources) + if err != nil { + return nil, fmt.Errorf("failed to constrain resources: %s", err) + } + err = cmd.Start() if err != nil { return nil, fmt.Errorf("failed to start command: %v", err) } // Return a driver handle h := &execHandle{ - proc: cmd.Process, + cmd: cmd, doneCh: make(chan struct{}), waitCh: make(chan error, 1), } @@ -79,14 +80,14 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro } // Find the process - proc, err := os.FindProcess(pid) - if proc == nil || err != nil { + cmd, err := executor.OpenPid(pid) + if err != nil { return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) } // Return a driver handle h := &execHandle{ - proc: proc, + cmd: cmd, doneCh: make(chan struct{}), waitCh: make(chan error, 1), } @@ -96,7 +97,8 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro func (h *execHandle) ID() string { // Return a handle to the PID - return fmt.Sprintf("PID:%d", h.proc.Pid) + pid, _ := h.cmd.Pid() + return fmt.Sprintf("PID:%d", pid) } func (h *execHandle) WaitCh() chan error { @@ -108,24 +110,22 @@ func (h *execHandle) Update(task *structs.Task) error { return nil } -// Kill is used to terminate the task. We send an Interrupt -// and then provide a 5 second grace period before doing a Kill. func (h *execHandle) Kill() error { - h.proc.Signal(unix.SIGTERM) + h.cmd.Shutdown() select { case <-h.doneCh: return nil case <-time.After(5 * time.Second): - return h.proc.Kill() + return h.cmd.ForceStop() } } func (h *execHandle) run() { - ps, err := h.proc.Wait() + err := h.cmd.Wait() close(h.doneCh) if err != nil { h.waitCh <- err - } else if !ps.Success() { + } else if !h.cmd.Command().ProcessState.Success() { h.waitCh <- fmt.Errorf("task exited with error") } close(h.waitCh) diff --git a/client/driver/java.go b/client/driver/java.go index b5e166205..ac183e166 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -13,9 +13,8 @@ import ( "strings" "time" - "golang.org/x/sys/unix" - "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/executor" "github.com/hashicorp/nomad/nomad/structs" ) @@ -27,7 +26,7 @@ type JavaDriver struct { // javaHandle is returned from Start/Open as a handle to the PID type javaHandle struct { - proc *os.Process + cmd executor.Executor waitCh chan error doneCh chan struct{} } @@ -129,7 +128,11 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, // Setup the command // Assumes Java is in the $PATH, but could probably be detected - cmd := exec.Command("java", args...) + cmd := executor.Command("java", args...) + err = cmd.Limit(task.Resources) + if err != nil { + return nil, fmt.Errorf("failed to constrain resources: %s", err) + } err = cmd.Start() if err != nil { return nil, fmt.Errorf("failed to start source: %v", err) @@ -137,7 +140,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, // Return a driver handle h := &javaHandle{ - proc: cmd.Process, + cmd: cmd, doneCh: make(chan struct{}), waitCh: make(chan error, 1), } @@ -155,14 +158,14 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro } // Find the process - proc, err := os.FindProcess(pid) - if proc == nil || err != nil { + cmd, err := executor.OpenPid(pid) + if err != nil { return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) } // Return a driver handle h := &javaHandle{ - proc: proc, + cmd: cmd, doneCh: make(chan struct{}), waitCh: make(chan error, 1), } @@ -173,7 +176,8 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro func (h *javaHandle) ID() string { // Return a handle to the PID - return fmt.Sprintf("PID:%d", h.proc.Pid) + pid, _ := h.cmd.Pid() + return fmt.Sprintf("PID:%d", pid) } func (h *javaHandle) WaitCh() chan error { @@ -185,24 +189,22 @@ func (h *javaHandle) Update(task *structs.Task) error { return nil } -// Kill is used to terminate the task. We send an Interrupt -// and then provide a 5 second grace period before doing a Kill. func (h *javaHandle) Kill() error { - h.proc.Signal(unix.SIGTERM) + h.cmd.Shutdown() select { case <-h.doneCh: return nil case <-time.After(5 * time.Second): - return h.proc.Kill() + return h.cmd.ForceStop() } } func (h *javaHandle) run() { - ps, err := h.proc.Wait() + err := h.cmd.Wait() close(h.doneCh) if err != nil { h.waitCh <- err - } else if !ps.Success() { + } else if !h.cmd.Command().ProcessState.Success() { h.waitCh <- fmt.Errorf("task exited with error") } close(h.waitCh) diff --git a/client/executor/exec.go b/client/executor/exec.go new file mode 100644 index 000000000..b62ec399e --- /dev/null +++ b/client/executor/exec.go @@ -0,0 +1,146 @@ +// Package exec is used to invoke child processes across various platforms to +// provide the following features: +// +// - Least privilege +// - Resource constraints +// - Process isolation +// +// A "platform" may be defined as coarsely as "Windows" or as specifically as +// "linux 3.20 with systemd". This allows Nomad to use best-effort, best- +// available capabilities of each platform to provide resource constraints, +// process isolation, and security features, or otherwise take advantage of +// features that are unique to that platform. +// +// The `semantics of any particular instance are left up to the implementation. +// However, these should be completely transparent to the calling context. In +// other words, the Java driver should be able to call exec for any platform and +// just work. +package executor + +import ( + "fmt" + "os/exec" + "path/filepath" + "strconv" + "syscall" + + "github.com/hashicorp/nomad/nomad/structs" +) + +// Executor is an interface that any platform- or capability-specific exec +// wrapper must implement. You should not need to implement a Java executor. +// Rather, you would implement a cgroups executor that the Java driver will use. +type Executor interface { + // Limit must be called before Start and restricts the amount of resources + // the process can use. Note that an error may be returned ONLY IF the + // executor implements resource limiting. Otherwise Limit is ignored. + Limit(*structs.Resources) error + + // RunAs sets the user we should use to run this command. This may be set as + // a username, uid, or other identifier. The implementation will decide what + // to do with it, if anything. Note that an error may be returned ONLY IF + // the executor implements user lookups. Otherwise RunAs is ignored. + RunAs(string) error + + // Start the process. This may wrap the actual process in another command, + // depending on the capabilities in this environment. Errors that arise from + // Limits or Runas may bubble through Start() + Start() error + + // Open should be called to restore a previous pid. This might be needed if + // nomad is restarted. This sets os.Process internally. + Open(int) error + + // This is a convenience wrapper around Command().Wait() + Wait() error + + // This is a convenience wrapper around Command().Process.Pid + Pid() (int, error) + + // Shutdown should use a graceful stop mechanism so the application can + // perform checkpointing or cleanup, if such a mechanism is available. + // If such a mechanism is not available, Shutdown() should call ForceStop(). + Shutdown() error + + // ForceStop will terminate the process without waiting for cleanup. Every + // implementations must provide this. + ForceStop() error + + // Command provides access the underlying Cmd struct in case the Executor + // interface doesn't expose the functionality you need. + Command() *cmd +} + +// Command is a mirror of exec.Command that returns a platform-specific Executor +func Command(name string, arg ...string) Executor { + executor := NewExecutor() + cmd := executor.Command() + cmd.Path = name + cmd.Args = append([]string{name}, arg...) + + if filepath.Base(name) == name { + if lp, err := exec.LookPath(name); err != nil { + // cmd.lookPathErr = err + } else { + cmd.Path = lp + } + } + return executor +} + +// OpenPid is similar to executor.Command but will initialize executor.Cmd with +// the Pid set to the one specified. +func OpenPid(pid int) (Executor, error) { + executor := NewExecutor() + err := executor.Open(pid) + if err != nil { + return nil, err + } + return executor, nil +} + +// Cmd is an extension of exec.Cmd that incorporates functionality for +// re-attaching to processes, dropping priviledges, etc., based on platform- +// specific implementations. +type cmd struct { + exec.Cmd + + // Resources is used to limit CPU and RAM used by the process, by way of + // cgroups or a similar mechanism. + Resources structs.Resources + + // RunAs may be a username or Uid. The implementation will decide how to use it. + RunAs string +} + +// SetUID changes the Uid for this command (must be set before starting) +func (c *cmd) SetUID(userid string) error { + uid, err := strconv.ParseUint(userid, 10, 32) + if err != nil { + return fmt.Errorf("Unable to convert userid to uint32: %s", err) + } + if c.SysProcAttr == nil { + c.SysProcAttr = &syscall.SysProcAttr{} + } + if c.SysProcAttr.Credential == nil { + c.SysProcAttr.Credential = &syscall.Credential{} + } + c.SysProcAttr.Credential.Uid = uint32(uid) + return nil +} + +// SetGID changes the Gid for this command (must be set before starting) +func (c *cmd) SetGID(groupid string) error { + gid, err := strconv.ParseUint(groupid, 10, 32) + if err != nil { + return fmt.Errorf("Unable to convert groupid to uint32: %s", err) + } + if c.SysProcAttr == nil { + c.SysProcAttr = &syscall.SysProcAttr{} + } + if c.SysProcAttr.Credential == nil { + c.SysProcAttr.Credential = &syscall.Credential{} + } + c.SysProcAttr.Credential.Uid = uint32(gid) + return nil +} diff --git a/client/executor/exec_linux.go b/client/executor/exec_linux.go new file mode 100644 index 000000000..be128b156 --- /dev/null +++ b/client/executor/exec_linux.go @@ -0,0 +1,137 @@ +package executor + +import ( + "fmt" + "os" + "os/user" + "syscall" + + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/nomad/structs" +) + +func NewExecutor() Executor { + return &LinuxExecutor{} +} + +// Linux executor is designed to run on linux kernel 2.8+. +type LinuxExecutor struct { + cmd + user *user.User +} + +func (e *LinuxExecutor) Limit(resources *structs.Resources) error { + // TODO limit some things + return nil +} + +func (e *LinuxExecutor) RunAs(userid string) error { + errs := new(multierror.Error) + + // First, try to lookup the user by uid + u, err := user.LookupId(userid) + if err == nil { + e.user = u + return nil + } else { + errs = multierror.Append(errs, err) + } + + // Lookup failed, so try by username instead + u, err = user.Lookup(userid) + if err == nil { + e.user = u + return nil + } else { + errs = multierror.Append(errs, err) + } + + // If we got here we failed to lookup based on id and username, so we'll + // return those errors. + return fmt.Errorf("Failed to identify user to run as: %s", errs) +} + +func (e *LinuxExecutor) Start() error { + // If no user has been specified, try to run as "nobody" user so we don't + // leak root privilege to the spawned process. Note that we will only do + // this if we can call SetUID. Otherwise we'll just run the other process + // as our current (non-root) user. This makes testing easier and also means + // we aren't forced to run nomad as root. + if e.user == nil && canSetUID() { + e.RunAs("nobody") + } + + // Set the user and group this process should run as. If RunAs was called + // but we are not root this will cause Start to fail. This is intentional. + if e.user != nil { + e.cmd.SetUID(e.user.Uid) + e.cmd.SetGID(e.user.Gid) + } + + // We don't want to call ourself. We want to call Start on our embedded Cmd. + return e.cmd.Start() +} + +func (e *LinuxExecutor) Open(pid int) error { + process, err := os.FindProcess(pid) + // FindProcess doesn't do any checking against the process table so it's + // unlikely we'll ever see this error. + if err != nil { + return fmt.Errorf("Failed to reopen pid %d: %s", pid, err) + } + + // On linux FindProcess() will return a pid but doesn't actually check to + // see whether that process is running. We'll send signal 0 to see if the + // process is alive. + err = process.Signal(syscall.Signal(0)) + if err != nil { + return fmt.Errorf("Unable to signal pid %d: %s", err) + } + e.Process = process + return nil +} + +func (e *LinuxExecutor) Wait() error { + // We don't want to call ourself. We want to call Start on our embedded Cmd + return e.cmd.Wait() +} + +func (e *LinuxExecutor) Pid() (int, error) { + if e.cmd.Process != nil { + return e.cmd.Process.Pid, nil + } else { + return 0, fmt.Errorf("Process has finished or was never started") + } +} + +func (e *LinuxExecutor) Shutdown() error { + return e.ForceStop() +} + +func (e *LinuxExecutor) ForceStop() error { + return e.Process.Kill() +} + +func (e *LinuxExecutor) Command() *cmd { + return &e.cmd +} + +// canSetUID will tell us whether we're capable of using SetUID. If we are not +// rootish this command will fail. In that case we'll just run the forked +// process under our own user. +func canSetUID() bool { + checkroot := Command("true") + u, err := user.Current() + if err != nil { + return false + } + + // Make sure RunAs is explicitly set so we don't cause infinite recursion. + checkroot.RunAs(u.Uid) + + err = checkroot.Start() + if err != nil { + return false + } + return true +} diff --git a/client/executor/exec_universal.go b/client/executor/exec_universal.go new file mode 100644 index 000000000..109fc86fd --- /dev/null +++ b/client/executor/exec_universal.go @@ -0,0 +1,69 @@ +// +build !linux + +package executor + +import ( + "fmt" + "os" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func NewExecutor() Executor { + return &UniversalExecutor{} +} + +// UniversalExecutor should work everywhere, and as a result does not include +// any resource restrictions or runas capabilities. +type UniversalExecutor struct { + cmd +} + +func (e *UniversalExecutor) Limit(resources *structs.Resources) error { + // No-op + return nil +} + +func (e *UniversalExecutor) RunAs(userid string) error { + // No-op + return nil +} + +func (e *UniversalExecutor) Start() error { + // We don't want to call ourself. We want to call Start on our embedded Cmd + return e.cmd.Start() +} + +func (e *UniversalExecutor) Open(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return fmt.Errorf("Failed to reopen pid %d: %s", pid, err) + } + e.Process = process + return nil +} + +func (e *UniversalExecutor) Wait() error { + // We don't want to call ourself. We want to call Start on our embedded Cmd + return e.cmd.Wait() +} + +func (e *UniversalExecutor) Pid() (int, error) { + if e.cmd.Process != nil { + return e.cmd.Process.Pid, nil + } else { + return 0, fmt.Errorf("Process has finished or was never started") + } +} + +func (e *UniversalExecutor) Shutdown() error { + return e.ForceStop() +} + +func (e *UniversalExecutor) ForceStop() error { + return e.Process.Kill() +} + +func (e *UniversalExecutor) Command() *cmd { + return &e.cmd +}