2016-02-04 21:34:14 +00:00
|
|
|
package driver
|
|
|
|
|
|
|
|
import (
|
2017-01-12 19:50:49 +00:00
|
|
|
"encoding/json"
|
2016-02-04 21:34:14 +00:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2016-02-04 21:53:30 +00:00
|
|
|
"os"
|
2016-03-16 02:22:40 +00:00
|
|
|
"os/exec"
|
|
|
|
"path/filepath"
|
2016-02-23 18:19:40 +00:00
|
|
|
"strings"
|
2016-03-03 17:21:21 +00:00
|
|
|
"time"
|
2016-02-04 21:34:14 +00:00
|
|
|
|
2017-12-06 17:16:12 +00:00
|
|
|
"github.com/hashicorp/consul-template/signals"
|
2018-09-24 18:37:45 +00:00
|
|
|
hclog "github.com/hashicorp/go-hclog"
|
2016-02-04 21:53:30 +00:00
|
|
|
"github.com/hashicorp/go-multierror"
|
2016-02-04 21:34:14 +00:00
|
|
|
"github.com/hashicorp/go-plugin"
|
2017-05-23 00:46:40 +00:00
|
|
|
"github.com/hashicorp/nomad/client/allocdir"
|
2016-02-08 21:29:53 +00:00
|
|
|
"github.com/hashicorp/nomad/client/config"
|
2017-05-23 00:46:40 +00:00
|
|
|
"github.com/hashicorp/nomad/client/driver/env"
|
2016-02-05 00:03:17 +00:00
|
|
|
"github.com/hashicorp/nomad/client/driver/executor"
|
2017-05-23 00:46:40 +00:00
|
|
|
dstructs "github.com/hashicorp/nomad/client/driver/structs"
|
|
|
|
cstructs "github.com/hashicorp/nomad/client/structs"
|
2017-01-12 19:50:49 +00:00
|
|
|
"github.com/hashicorp/nomad/helper/discover"
|
2016-03-23 11:57:31 +00:00
|
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
2016-02-04 21:34:14 +00:00
|
|
|
)
|
|
|
|
|
2017-02-21 03:35:51 +00:00
|
|
|
// cgroupsMounted returns true if the cgroups are mounted on a system otherwise
|
|
|
|
// returns false
|
|
|
|
func cgroupsMounted(node *structs.Node) bool {
|
|
|
|
_, ok := node.Attributes["unique.cgroup.mountpoint"]
|
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
2016-02-05 18:49:54 +00:00
|
|
|
// createExecutor launches an executor plugin and returns an instance of the
|
|
|
|
// Executor interface
|
2017-01-12 19:50:49 +00:00
|
|
|
func createExecutor(w io.Writer, clientConfig *config.Config,
|
2017-05-23 00:46:40 +00:00
|
|
|
executorConfig *dstructs.ExecutorConfig) (executor.Executor, *plugin.Client, error) {
|
2017-01-12 19:50:49 +00:00
|
|
|
|
|
|
|
c, err := json.Marshal(executorConfig)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("unable to create executor config: %v", err)
|
|
|
|
}
|
|
|
|
bin, err := discover.NomadExecutable()
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
config := &plugin.ClientConfig{
|
|
|
|
Cmd: exec.Command(bin, "executor", string(c)),
|
|
|
|
}
|
2016-02-05 00:03:17 +00:00
|
|
|
config.HandshakeConfig = HandshakeConfig
|
2018-09-24 18:37:45 +00:00
|
|
|
config.Plugins = GetPluginMap(w, hclog.LevelFromString(clientConfig.LogLevel), executorConfig.FSIsolation)
|
2016-02-08 21:29:53 +00:00
|
|
|
config.MaxPort = clientConfig.ClientMaxPort
|
|
|
|
config.MinPort = clientConfig.ClientMinPort
|
2016-02-09 00:27:31 +00:00
|
|
|
|
|
|
|
// setting the setsid of the plugin process so that it doesn't get signals sent to
|
|
|
|
// the nomad client.
|
2016-02-08 03:33:48 +00:00
|
|
|
if config.Cmd != nil {
|
|
|
|
isolateCommand(config.Cmd)
|
|
|
|
}
|
2016-02-09 00:27:31 +00:00
|
|
|
|
2016-02-04 21:34:14 +00:00
|
|
|
executorClient := plugin.NewClient(config)
|
|
|
|
rpcClient, err := executorClient.Client()
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
raw, err := rpcClient.Dispense("executor")
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
|
|
|
|
}
|
2016-02-05 00:03:17 +00:00
|
|
|
executorPlugin := raw.(executor.Executor)
|
2016-02-04 21:34:14 +00:00
|
|
|
return executorPlugin, executorClient, nil
|
|
|
|
}
|
2016-02-04 21:53:30 +00:00
|
|
|
|
2017-01-12 19:50:49 +00:00
|
|
|
func createExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
|
|
|
|
config.HandshakeConfig = HandshakeConfig
|
|
|
|
|
|
|
|
// Setting this to DEBUG since the log level at the executor server process
|
|
|
|
// is already set, and this effects only the executor client.
|
2018-09-24 18:37:45 +00:00
|
|
|
config.Plugins = GetPluginMap(w, hclog.Debug, false)
|
2017-01-12 19:50:49 +00:00
|
|
|
|
|
|
|
executorClient := plugin.NewClient(config)
|
|
|
|
rpcClient, err := executorClient.Client()
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
raw, err := rpcClient.Dispense("executor")
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
|
|
|
|
}
|
2017-02-01 00:43:57 +00:00
|
|
|
executorPlugin, ok := raw.(*ExecutorRPC)
|
|
|
|
if !ok {
|
|
|
|
return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw)
|
2016-03-24 22:39:10 +00:00
|
|
|
}
|
2017-02-01 00:43:57 +00:00
|
|
|
return executorPlugin, executorClient, nil
|
2016-03-23 19:19:19 +00:00
|
|
|
}
|
|
|
|
|
2016-02-05 18:49:54 +00:00
|
|
|
// killProcess kills a process with the given pid
|
2016-02-04 21:53:30 +00:00
|
|
|
func killProcess(pid int) error {
|
|
|
|
proc, err := os.FindProcess(pid)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return proc.Kill()
|
|
|
|
}
|
|
|
|
|
2016-02-05 18:49:54 +00:00
|
|
|
// destroyPlugin kills the plugin with the given pid and also kills the user
|
|
|
|
// process
|
2016-02-04 21:53:30 +00:00
|
|
|
func destroyPlugin(pluginPid int, userPid int) error {
|
|
|
|
var merr error
|
|
|
|
if err := killProcess(pluginPid); err != nil {
|
|
|
|
merr = multierror.Append(merr, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := killProcess(userPid); err != nil {
|
|
|
|
merr = multierror.Append(merr, err)
|
|
|
|
}
|
|
|
|
return merr
|
|
|
|
}
|
2016-02-23 18:19:40 +00:00
|
|
|
|
|
|
|
// validateCommand validates that the command only has a single value and
|
|
|
|
// returns a user friendly error message telling them to use the passed
|
|
|
|
// argField.
|
|
|
|
func validateCommand(command, argField string) error {
|
|
|
|
trimmed := strings.TrimSpace(command)
|
|
|
|
if len(trimmed) == 0 {
|
|
|
|
return fmt.Errorf("command empty: %q", command)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(trimmed) != len(command) {
|
|
|
|
return fmt.Errorf("command contains extra white space: %q", command)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2016-03-03 17:21:21 +00:00
|
|
|
|
|
|
|
// GetKillTimeout returns the kill timeout to use given the tasks desired kill
|
|
|
|
// timeout and the operator configured max kill timeout.
|
|
|
|
func GetKillTimeout(desired, max time.Duration) time.Duration {
|
|
|
|
maxNanos := max.Nanoseconds()
|
|
|
|
desiredNanos := desired.Nanoseconds()
|
|
|
|
|
|
|
|
// Make the minimum time between signal and kill, 1 second.
|
|
|
|
if desiredNanos <= 0 {
|
|
|
|
desiredNanos = (1 * time.Second).Nanoseconds()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Protect against max not being set properly.
|
|
|
|
if maxNanos <= 0 {
|
|
|
|
maxNanos = (10 * time.Second).Nanoseconds()
|
|
|
|
}
|
|
|
|
|
|
|
|
if desiredNanos < maxNanos {
|
|
|
|
return time.Duration(desiredNanos)
|
|
|
|
}
|
|
|
|
|
|
|
|
return max
|
|
|
|
}
|
2016-03-16 02:22:40 +00:00
|
|
|
|
|
|
|
// GetAbsolutePath returns the absolute path of the passed binary by resolving
|
|
|
|
// it in the path and following symlinks.
|
|
|
|
func GetAbsolutePath(bin string) (string, error) {
|
|
|
|
lp, err := exec.LookPath(bin)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return filepath.EvalSymlinks(lp)
|
|
|
|
}
|
2016-03-23 11:57:31 +00:00
|
|
|
|
|
|
|
// getExecutorUser returns the user of the task, defaulting to
|
2017-05-23 00:46:40 +00:00
|
|
|
// dstructs.DefaultUnprivilegedUser if none was given.
|
2016-03-23 11:57:31 +00:00
|
|
|
func getExecutorUser(task *structs.Task) string {
|
|
|
|
if task.User == "" {
|
2018-01-08 20:56:07 +00:00
|
|
|
return dstructs.DefaultUnprivilegedUser
|
2016-03-23 11:57:31 +00:00
|
|
|
}
|
|
|
|
return task.User
|
|
|
|
}
|
2017-05-23 00:46:40 +00:00
|
|
|
|
|
|
|
// SetEnvvars sets path and host env vars depending on the FS isolation used.
|
|
|
|
func SetEnvvars(envBuilder *env.Builder, fsi cstructs.FSIsolation, taskDir *allocdir.TaskDir, conf *config.Config) {
|
|
|
|
// Set driver-specific environment variables
|
|
|
|
switch fsi {
|
|
|
|
case cstructs.FSIsolationNone:
|
|
|
|
// Use host paths
|
|
|
|
envBuilder.SetAllocDir(taskDir.SharedAllocDir)
|
|
|
|
envBuilder.SetTaskLocalDir(taskDir.LocalDir)
|
|
|
|
envBuilder.SetSecretsDir(taskDir.SecretsDir)
|
|
|
|
default:
|
|
|
|
// filesystem isolation; use container paths
|
|
|
|
envBuilder.SetAllocDir(allocdir.SharedAllocContainerPath)
|
|
|
|
envBuilder.SetTaskLocalDir(allocdir.TaskLocalContainerPath)
|
|
|
|
envBuilder.SetSecretsDir(allocdir.TaskSecretsContainerPath)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the host environment variables for non-image based drivers
|
|
|
|
if fsi != cstructs.FSIsolationImage {
|
|
|
|
filter := strings.Split(conf.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
|
|
|
|
envBuilder.SetHostEnvvars(filter)
|
|
|
|
}
|
|
|
|
}
|
2017-12-06 17:16:12 +00:00
|
|
|
|
|
|
|
// getTaskKillSignal looks up the signal specified for the task if it has been
|
|
|
|
// specified. If it is not supported on the platform, returns an error.
|
|
|
|
func getTaskKillSignal(signal string) (os.Signal, error) {
|
|
|
|
if signal == "" {
|
2017-12-06 21:52:44 +00:00
|
|
|
return os.Interrupt, nil
|
2017-12-06 17:16:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
taskKillSignal := signals.SignalLookup[signal]
|
|
|
|
if taskKillSignal == nil {
|
|
|
|
return nil, fmt.Errorf("Signal %s is not supported", signal)
|
|
|
|
}
|
2017-12-06 21:52:44 +00:00
|
|
|
|
2017-12-06 17:16:12 +00:00
|
|
|
return taskKillSignal, nil
|
|
|
|
}
|