2018-10-31 14:30:13 +00:00
|
|
|
package java
|
|
|
|
|
|
|
|
import (
|
2018-11-13 01:09:27 +00:00
|
|
|
"context"
|
2018-10-31 14:30:13 +00:00
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"os/exec"
|
|
|
|
"path/filepath"
|
|
|
|
"runtime"
|
|
|
|
"time"
|
|
|
|
|
2021-04-08 05:03:58 +00:00
|
|
|
"github.com/hashicorp/nomad/client/lib/cgutil"
|
2021-05-15 16:55:44 +00:00
|
|
|
"github.com/hashicorp/nomad/drivers/shared/capabilities"
|
2021-04-08 05:03:58 +00:00
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
"github.com/hashicorp/consul-template/signals"
|
2019-01-09 23:06:58 +00:00
|
|
|
hclog "github.com/hashicorp/go-hclog"
|
2018-10-31 14:30:13 +00:00
|
|
|
"github.com/hashicorp/nomad/drivers/shared/eventer"
|
2018-12-07 01:54:14 +00:00
|
|
|
"github.com/hashicorp/nomad/drivers/shared/executor"
|
2020-04-28 03:11:06 +00:00
|
|
|
"github.com/hashicorp/nomad/drivers/shared/resolvconf"
|
2019-01-23 14:27:14 +00:00
|
|
|
"github.com/hashicorp/nomad/helper/pluginutils/loader"
|
2018-10-31 14:30:13 +00:00
|
|
|
"github.com/hashicorp/nomad/plugins/base"
|
|
|
|
"github.com/hashicorp/nomad/plugins/drivers"
|
2018-12-20 11:55:07 +00:00
|
|
|
"github.com/hashicorp/nomad/plugins/drivers/utils"
|
2018-10-31 14:30:13 +00:00
|
|
|
"github.com/hashicorp/nomad/plugins/shared/hclspec"
|
2018-11-21 00:30:39 +00:00
|
|
|
pstructs "github.com/hashicorp/nomad/plugins/shared/structs"
|
2018-10-31 14:30:13 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// pluginName is the name of the plugin
|
|
|
|
pluginName = "java"
|
|
|
|
|
|
|
|
// fingerprintPeriod is the interval at which the driver will send fingerprint responses
|
|
|
|
fingerprintPeriod = 30 * time.Second
|
|
|
|
|
|
|
|
// The key populated in Node Attributes to indicate presence of the Java driver
|
|
|
|
driverAttr = "driver.java"
|
|
|
|
driverVersionAttr = "driver.java.version"
|
2019-01-17 02:52:31 +00:00
|
|
|
|
|
|
|
// taskHandleVersion is the version of task handle which this driver sets
|
|
|
|
// and understands how to decode driver state
|
|
|
|
taskHandleVersion = 1
|
2018-10-31 14:30:13 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
// PluginID is the java plugin metadata registered in the plugin
|
|
|
|
// catalog.
|
|
|
|
PluginID = loader.PluginID{
|
|
|
|
Name: pluginName,
|
|
|
|
PluginType: base.PluginTypeDriver,
|
|
|
|
}
|
|
|
|
|
|
|
|
// PluginConfig is the java driver factory function registered in the
|
|
|
|
// plugin catalog.
|
|
|
|
PluginConfig = &loader.InternalPluginConfig{
|
|
|
|
Config: map[string]interface{}{},
|
2020-05-26 13:44:26 +00:00
|
|
|
Factory: func(ctx context.Context, l hclog.Logger) interface{} { return NewDriver(ctx, l) },
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// pluginInfo is the response returned for the PluginInfo RPC
|
|
|
|
pluginInfo = &base.PluginInfoResponse{
|
2018-12-18 00:40:58 +00:00
|
|
|
Type: base.PluginTypeDriver,
|
|
|
|
PluginApiVersions: []string{drivers.ApiVersion010},
|
|
|
|
PluginVersion: "0.1.0",
|
|
|
|
Name: pluginName,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// configSpec is the hcl specification returned by the ConfigSchema RPC
|
2021-02-04 19:01:51 +00:00
|
|
|
configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
|
|
|
|
"default_pid_mode": hclspec.NewDefault(
|
|
|
|
hclspec.NewAttr("default_pid_mode", "string", false),
|
|
|
|
hclspec.NewLiteral(`"private"`),
|
|
|
|
),
|
|
|
|
"default_ipc_mode": hclspec.NewDefault(
|
|
|
|
hclspec.NewAttr("default_ipc_mode", "string", false),
|
|
|
|
hclspec.NewLiteral(`"private"`),
|
|
|
|
),
|
2021-05-15 16:55:44 +00:00
|
|
|
"allow_caps": hclspec.NewDefault(
|
|
|
|
hclspec.NewAttr("allow_caps", "list(string)", false),
|
|
|
|
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
|
|
|
|
),
|
2021-02-04 19:01:51 +00:00
|
|
|
})
|
2018-10-31 14:30:13 +00:00
|
|
|
|
|
|
|
// taskConfigSpec is the hcl specification for the driver config section of
|
|
|
|
// a taskConfig within a job. It is returned in the TaskConfigSchema RPC
|
|
|
|
taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
|
|
|
|
// It's required for either `class` or `jar_path` to be set,
|
|
|
|
// but that's not expressable in hclspec. Marking both as optional
|
|
|
|
// and setting checking explicitly later
|
2018-11-08 02:00:34 +00:00
|
|
|
"class": hclspec.NewAttr("class", "string", false),
|
|
|
|
"class_path": hclspec.NewAttr("class_path", "string", false),
|
|
|
|
"jar_path": hclspec.NewAttr("jar_path", "string", false),
|
|
|
|
"jvm_options": hclspec.NewAttr("jvm_options", "list(string)", false),
|
|
|
|
"args": hclspec.NewAttr("args", "list(string)", false),
|
2021-02-08 16:36:11 +00:00
|
|
|
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
|
|
|
|
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
|
2021-05-15 16:55:44 +00:00
|
|
|
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
|
|
|
|
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
|
2018-10-31 14:30:13 +00:00
|
|
|
})
|
|
|
|
|
2021-05-15 16:55:44 +00:00
|
|
|
// driverCapabilities is returned by the Capabilities RPC and indicates what
|
2018-10-31 14:30:13 +00:00
|
|
|
// optional features this driver supports
|
2021-05-15 16:55:44 +00:00
|
|
|
driverCapabilities = &drivers.Capabilities{
|
2018-10-31 14:30:13 +00:00
|
|
|
SendSignals: false,
|
|
|
|
Exec: false,
|
2019-01-04 21:11:25 +00:00
|
|
|
FSIsolation: drivers.FSIsolationNone,
|
2019-09-25 15:48:14 +00:00
|
|
|
NetIsolationModes: []drivers.NetIsolationMode{
|
|
|
|
drivers.NetIsolationModeHost,
|
|
|
|
drivers.NetIsolationModeGroup,
|
|
|
|
},
|
2020-05-21 13:18:02 +00:00
|
|
|
MountConfigs: drivers.MountConfigSupportNone,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
_ drivers.DriverPlugin = (*Driver)(nil)
|
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
if runtime.GOOS == "linux" {
|
2021-05-15 16:55:44 +00:00
|
|
|
driverCapabilities.FSIsolation = drivers.FSIsolationChroot
|
|
|
|
driverCapabilities.MountConfigs = drivers.MountConfigSupportAll
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-04 19:01:51 +00:00
|
|
|
// Config is the driver configuration set by the SetConfig RPC call
|
|
|
|
type Config struct {
|
|
|
|
// DefaultModePID is the default PID isolation set for all tasks using
|
|
|
|
// exec-based task drivers.
|
|
|
|
DefaultModePID string `codec:"default_pid_mode"`
|
|
|
|
|
|
|
|
// DefaultModeIPC is the default IPC isolation set for all tasks using
|
|
|
|
// exec-based task drivers.
|
|
|
|
DefaultModeIPC string `codec:"default_ipc_mode"`
|
2021-05-15 16:55:44 +00:00
|
|
|
|
|
|
|
// AllowCaps configures which Linux Capabilities are enabled for tasks
|
|
|
|
// running on this node.
|
|
|
|
AllowCaps []string `codec:"allow_caps"`
|
2021-02-04 19:01:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Config) validate() error {
|
|
|
|
switch c.DefaultModePID {
|
2021-02-08 16:58:44 +00:00
|
|
|
case executor.IsolationModePrivate, executor.IsolationModeHost:
|
2021-02-04 19:01:51 +00:00
|
|
|
default:
|
2021-02-08 16:58:44 +00:00
|
|
|
return fmt.Errorf("default_pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModePID)
|
2021-02-04 19:01:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
switch c.DefaultModeIPC {
|
2021-02-08 16:58:44 +00:00
|
|
|
case executor.IsolationModePrivate, executor.IsolationModeHost:
|
2021-02-04 19:01:51 +00:00
|
|
|
default:
|
2021-02-08 16:58:44 +00:00
|
|
|
return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
|
2021-02-04 19:01:51 +00:00
|
|
|
}
|
|
|
|
|
2021-05-15 16:55:44 +00:00
|
|
|
badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps))
|
|
|
|
if !badCaps.Empty() {
|
|
|
|
return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps)
|
|
|
|
}
|
|
|
|
|
2021-02-04 19:01:51 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
// TaskConfig is the driver configuration of a taskConfig within a job
|
|
|
|
type TaskConfig struct {
|
2021-05-15 16:55:44 +00:00
|
|
|
// Class indicates which class contains the java entry point.
|
|
|
|
Class string `codec:"class"`
|
|
|
|
|
|
|
|
// ClassPath indicates where class files are found.
|
|
|
|
ClassPath string `codec:"class_path"`
|
|
|
|
|
|
|
|
// JarPath indicates where a jar file is found.
|
|
|
|
JarPath string `codec:"jar_path"`
|
|
|
|
|
|
|
|
// JvmOpts are arguments to pass to the JVM
|
|
|
|
JvmOpts []string `codec:"jvm_options"`
|
|
|
|
|
|
|
|
// Args are extra arguments to java executable
|
|
|
|
Args []string `codec:"args"`
|
|
|
|
|
|
|
|
// ModePID indicates whether PID namespace isolation is enabled for the task.
|
|
|
|
// Must be "private" or "host" if set.
|
|
|
|
ModePID string `codec:"pid_mode"`
|
|
|
|
|
|
|
|
// ModeIPC indicates whether IPC namespace isolation is enabled for the task.
|
|
|
|
// Must be "private" or "host" if set.
|
|
|
|
ModeIPC string `codec:"ipc_mode"`
|
|
|
|
|
|
|
|
// CapAdd is a set of linux capabilities to enable.
|
|
|
|
CapAdd []string `codec:"cap_add"`
|
|
|
|
|
|
|
|
// CapDrop is a set of linux capabilities to disable.
|
|
|
|
CapDrop []string `codec:"cap_drop"`
|
2021-02-08 16:36:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (tc *TaskConfig) validate() error {
|
|
|
|
switch tc.ModePID {
|
|
|
|
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
|
|
|
|
default:
|
|
|
|
return fmt.Errorf("pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModePID)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
switch tc.ModeIPC {
|
|
|
|
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
|
|
|
|
default:
|
|
|
|
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
|
|
|
|
}
|
|
|
|
|
2021-05-15 16:55:44 +00:00
|
|
|
supported := capabilities.Supported()
|
|
|
|
badAdds := supported.Difference(capabilities.New(tc.CapAdd))
|
|
|
|
if !badAdds.Empty() {
|
|
|
|
return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds)
|
|
|
|
}
|
|
|
|
badDrops := supported.Difference(capabilities.New(tc.CapDrop))
|
|
|
|
if !badDrops.Empty() {
|
|
|
|
return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
|
|
|
|
}
|
|
|
|
|
2021-02-08 16:36:11 +00:00
|
|
|
return nil
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// TaskState is the state which is encoded in the handle returned in
|
|
|
|
// StartTask. This information is needed to rebuild the taskConfig state and handler
|
|
|
|
// during recovery.
|
|
|
|
type TaskState struct {
|
2019-01-15 01:02:44 +00:00
|
|
|
ReattachConfig *pstructs.ReattachConfig
|
2018-10-31 14:30:13 +00:00
|
|
|
TaskConfig *drivers.TaskConfig
|
|
|
|
Pid int
|
|
|
|
StartedAt time.Time
|
|
|
|
}
|
|
|
|
|
|
|
|
// Driver is a driver for running images via Java
|
|
|
|
type Driver struct {
|
|
|
|
// eventer is used to handle multiplexing of TaskEvents calls such that an
|
|
|
|
// event can be broadcast to all callers
|
|
|
|
eventer *eventer.Eventer
|
|
|
|
|
2021-02-04 19:01:51 +00:00
|
|
|
// config is the driver configuration set by the SetConfig RPC
|
|
|
|
config Config
|
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
// tasks is the in memory datastore mapping taskIDs to taskHandle
|
|
|
|
tasks *taskStore
|
|
|
|
|
|
|
|
// ctx is the context for the driver. It is passed to other subsystems to
|
|
|
|
// coordinate shutdown
|
|
|
|
ctx context.Context
|
|
|
|
|
|
|
|
// nomadConf is the client agent's configuration
|
|
|
|
nomadConfig *base.ClientDriverConfig
|
|
|
|
|
2018-11-02 12:29:39 +00:00
|
|
|
// logger will log to the Nomad agent
|
2018-10-31 14:30:13 +00:00
|
|
|
logger hclog.Logger
|
|
|
|
}
|
|
|
|
|
2020-05-26 13:44:26 +00:00
|
|
|
func NewDriver(ctx context.Context, logger hclog.Logger) drivers.DriverPlugin {
|
2018-10-31 14:30:13 +00:00
|
|
|
logger = logger.Named(pluginName)
|
|
|
|
return &Driver{
|
2020-05-26 13:44:26 +00:00
|
|
|
eventer: eventer.NewEventer(ctx, logger),
|
|
|
|
tasks: newTaskStore(),
|
|
|
|
ctx: ctx,
|
|
|
|
logger: logger,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) {
|
|
|
|
return pluginInfo, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
|
|
|
|
return configSpec, nil
|
|
|
|
}
|
|
|
|
|
2018-12-18 00:40:58 +00:00
|
|
|
func (d *Driver) SetConfig(cfg *base.Config) error {
|
2021-02-04 19:01:51 +00:00
|
|
|
// unpack, validate, and set agent plugin config
|
|
|
|
var config Config
|
|
|
|
if len(cfg.PluginConfig) != 0 {
|
|
|
|
if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err := config.validate(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
d.config = config
|
|
|
|
|
2018-12-18 00:40:58 +00:00
|
|
|
if cfg != nil && cfg.AgentConfig != nil {
|
|
|
|
d.nomadConfig = cfg.AgentConfig.Driver
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
|
|
|
|
return taskConfigSpec, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
|
2021-05-15 16:55:44 +00:00
|
|
|
return driverCapabilities, nil
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
2018-11-06 20:49:29 +00:00
|
|
|
func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
|
2018-10-31 14:30:13 +00:00
|
|
|
ch := make(chan *drivers.Fingerprint)
|
2018-11-06 20:49:29 +00:00
|
|
|
go d.handleFingerprint(ctx, ch)
|
2018-10-31 14:30:13 +00:00
|
|
|
return ch, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) handleFingerprint(ctx context.Context, ch chan *drivers.Fingerprint) {
|
|
|
|
ticker := time.NewTimer(0)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case <-d.ctx.Done():
|
|
|
|
return
|
|
|
|
case <-ticker.C:
|
|
|
|
ticker.Reset(fingerprintPeriod)
|
|
|
|
ch <- d.buildFingerprint()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) buildFingerprint() *drivers.Fingerprint {
|
|
|
|
fp := &drivers.Fingerprint{
|
2018-11-21 00:30:39 +00:00
|
|
|
Attributes: map[string]*pstructs.Attribute{},
|
2018-10-31 14:30:13 +00:00
|
|
|
Health: drivers.HealthStateHealthy,
|
2019-01-07 04:04:15 +00:00
|
|
|
HealthDescription: drivers.DriverHealthy,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if runtime.GOOS == "linux" {
|
|
|
|
// Only enable if w are root and cgroups are mounted when running on linux system
|
2018-12-20 11:55:07 +00:00
|
|
|
if !utils.IsUnixRoot() {
|
|
|
|
fp.Health = drivers.HealthStateUndetected
|
|
|
|
fp.HealthDescription = drivers.DriverRequiresRootMessage
|
2018-10-31 14:30:13 +00:00
|
|
|
return fp
|
|
|
|
}
|
|
|
|
|
2021-04-08 05:03:58 +00:00
|
|
|
mount, err := cgutil.FindCgroupMountpointDir()
|
2018-10-31 14:30:13 +00:00
|
|
|
if err != nil {
|
|
|
|
fp.Health = drivers.HealthStateUnhealthy
|
2019-01-07 04:04:15 +00:00
|
|
|
fp.HealthDescription = drivers.NoCgroupMountMessage
|
2018-10-31 14:30:13 +00:00
|
|
|
d.logger.Warn(fp.HealthDescription, "error", err)
|
|
|
|
return fp
|
|
|
|
}
|
|
|
|
|
|
|
|
if mount == "" {
|
|
|
|
fp.Health = drivers.HealthStateUnhealthy
|
2019-01-07 04:04:15 +00:00
|
|
|
fp.HealthDescription = drivers.CgroupMountEmpty
|
2018-10-31 14:30:13 +00:00
|
|
|
return fp
|
|
|
|
}
|
|
|
|
}
|
2020-11-01 18:20:31 +00:00
|
|
|
if runtime.GOOS == "darwin" {
|
|
|
|
_, err := checkForMacJVM()
|
|
|
|
if err != nil {
|
|
|
|
// return no error, as it isn't an error to not find java, it just means we
|
|
|
|
// can't use it.
|
|
|
|
|
|
|
|
fp.Health = drivers.HealthStateUndetected
|
|
|
|
fp.HealthDescription = ""
|
|
|
|
d.logger.Trace("macOS jvm not found", "error", err)
|
|
|
|
return fp
|
|
|
|
}
|
|
|
|
}
|
2018-10-31 14:30:13 +00:00
|
|
|
|
2021-06-15 19:14:58 +00:00
|
|
|
version, jdkJRE, vm, err := javaVersionInfo()
|
2018-10-31 14:30:13 +00:00
|
|
|
if err != nil {
|
|
|
|
// return no error, as it isn't an error to not find java, it just means we
|
|
|
|
// can't use it.
|
|
|
|
fp.Health = drivers.HealthStateUndetected
|
|
|
|
fp.HealthDescription = ""
|
|
|
|
return fp
|
|
|
|
}
|
|
|
|
|
2018-11-21 00:30:39 +00:00
|
|
|
fp.Attributes[driverAttr] = pstructs.NewBoolAttribute(true)
|
|
|
|
fp.Attributes[driverVersionAttr] = pstructs.NewStringAttribute(version)
|
2021-06-15 19:14:58 +00:00
|
|
|
fp.Attributes["driver.java.runtime"] = pstructs.NewStringAttribute(jdkJRE)
|
2018-11-21 00:30:39 +00:00
|
|
|
fp.Attributes["driver.java.vm"] = pstructs.NewStringAttribute(vm)
|
2018-10-31 14:30:13 +00:00
|
|
|
|
|
|
|
return fp
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
|
|
|
|
if handle == nil {
|
2018-11-06 20:49:29 +00:00
|
|
|
return fmt.Errorf("handle cannot be nil")
|
|
|
|
}
|
|
|
|
|
|
|
|
// If already attached to handle there's nothing to recover.
|
|
|
|
if _, ok := d.tasks.Get(handle.Config.ID); ok {
|
|
|
|
d.logger.Debug("nothing to recover; task already exists",
|
|
|
|
"task_id", handle.Config.ID,
|
|
|
|
"task_name", handle.Config.Name,
|
|
|
|
)
|
|
|
|
return nil
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var taskState TaskState
|
|
|
|
if err := handle.GetDriverState(&taskState); err != nil {
|
|
|
|
d.logger.Error("failed to decode taskConfig state from handle", "error", err, "task_id", handle.Config.ID)
|
|
|
|
return fmt.Errorf("failed to decode taskConfig state from handle: %v", err)
|
|
|
|
}
|
|
|
|
|
2019-01-15 01:02:44 +00:00
|
|
|
plugRC, err := pstructs.ReattachConfigToGoPlugin(taskState.ReattachConfig)
|
2018-10-31 14:30:13 +00:00
|
|
|
if err != nil {
|
|
|
|
d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID)
|
|
|
|
return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err)
|
|
|
|
}
|
|
|
|
|
2019-01-14 17:25:59 +00:00
|
|
|
execImpl, pluginClient, err := executor.ReattachToExecutor(plugRC,
|
2019-01-09 23:06:58 +00:00
|
|
|
d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID))
|
2018-10-31 14:30:13 +00:00
|
|
|
if err != nil {
|
|
|
|
d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID)
|
|
|
|
return fmt.Errorf("failed to reattach to executor: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
h := &taskHandle{
|
|
|
|
exec: execImpl,
|
|
|
|
pid: taskState.Pid,
|
|
|
|
pluginClient: pluginClient,
|
|
|
|
taskConfig: taskState.TaskConfig,
|
|
|
|
procState: drivers.TaskStateRunning,
|
|
|
|
startedAt: taskState.StartedAt,
|
|
|
|
exitResult: &drivers.ExitResult{},
|
2019-11-22 15:44:59 +00:00
|
|
|
logger: d.logger,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
d.tasks.Set(taskState.TaskConfig.ID, h)
|
|
|
|
|
|
|
|
go h.run()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-01-04 23:01:35 +00:00
|
|
|
func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
|
2018-10-31 14:30:13 +00:00
|
|
|
if _, ok := d.tasks.Get(cfg.ID); ok {
|
|
|
|
return nil, nil, fmt.Errorf("task with ID %q already started", cfg.ID)
|
|
|
|
}
|
|
|
|
|
|
|
|
var driverConfig TaskConfig
|
|
|
|
if err := cfg.DecodeDriverConfig(&driverConfig); err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
|
|
|
|
}
|
|
|
|
|
2021-02-08 16:36:11 +00:00
|
|
|
if err := driverConfig.validate(); err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("failed driver config validation: %v", err)
|
|
|
|
}
|
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
if driverConfig.Class == "" && driverConfig.JarPath == "" {
|
|
|
|
return nil, nil, fmt.Errorf("jar_path or class must be specified")
|
|
|
|
}
|
|
|
|
|
|
|
|
absPath, err := GetAbsolutePath("java")
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("failed to find java binary: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
args := javaCmdArgs(driverConfig)
|
|
|
|
|
|
|
|
d.logger.Info("starting java task", "driver_cfg", hclog.Fmt("%+v", driverConfig), "args", args)
|
|
|
|
|
2019-01-17 02:52:31 +00:00
|
|
|
handle := drivers.NewTaskHandle(taskHandleVersion)
|
2018-10-31 14:30:13 +00:00
|
|
|
handle.Config = cfg
|
|
|
|
|
|
|
|
pluginLogFile := filepath.Join(cfg.TaskDir().Dir, "executor.out")
|
2018-12-07 02:13:45 +00:00
|
|
|
executorConfig := &executor.ExecutorConfig{
|
2019-01-10 02:29:53 +00:00
|
|
|
LogFile: pluginLogFile,
|
|
|
|
LogLevel: "debug",
|
2021-05-15 16:55:44 +00:00
|
|
|
FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
2019-01-09 23:06:58 +00:00
|
|
|
exec, pluginClient, err := executor.CreateExecutor(
|
|
|
|
d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID),
|
|
|
|
d.nomadConfig, executorConfig)
|
2018-10-31 14:30:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("failed to create executor: %v", err)
|
|
|
|
}
|
|
|
|
|
2019-01-03 17:36:22 +00:00
|
|
|
user := cfg.User
|
|
|
|
if user == "" {
|
|
|
|
user = "nobody"
|
|
|
|
}
|
|
|
|
|
2020-04-28 03:11:06 +00:00
|
|
|
if cfg.DNS != nil {
|
|
|
|
dnsMount, err := resolvconf.GenerateDNSMount(cfg.TaskDir().Dir, cfg.DNS)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err)
|
|
|
|
}
|
|
|
|
cfg.Mounts = append(cfg.Mounts, dnsMount)
|
|
|
|
}
|
|
|
|
|
2021-05-15 20:48:01 +00:00
|
|
|
caps, err := capabilities.Calculate(
|
|
|
|
capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
|
|
|
|
)
|
2021-05-15 16:55:44 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
2021-05-16 16:06:03 +00:00
|
|
|
d.logger.Debug("task capabilities", "capabilities", caps)
|
2021-05-15 16:55:44 +00:00
|
|
|
|
2018-12-07 01:54:14 +00:00
|
|
|
execCmd := &executor.ExecCommand{
|
2019-09-26 20:19:27 +00:00
|
|
|
Cmd: absPath,
|
|
|
|
Args: args,
|
|
|
|
Env: cfg.EnvList(),
|
|
|
|
User: user,
|
|
|
|
ResourceLimits: true,
|
|
|
|
Resources: cfg.Resources,
|
|
|
|
TaskDir: cfg.TaskDir().Dir,
|
|
|
|
StdoutPath: cfg.StdoutPath,
|
|
|
|
StderrPath: cfg.StderrPath,
|
|
|
|
Mounts: cfg.Mounts,
|
|
|
|
Devices: cfg.Devices,
|
|
|
|
NetworkIsolation: cfg.NetworkIsolation,
|
2021-02-08 16:36:11 +00:00
|
|
|
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
|
|
|
|
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
|
2021-05-15 16:55:44 +00:00
|
|
|
Capabilities: caps,
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ps, err := exec.Launch(execCmd)
|
|
|
|
if err != nil {
|
|
|
|
pluginClient.Kill()
|
|
|
|
return nil, nil, fmt.Errorf("failed to launch command with executor: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
h := &taskHandle{
|
|
|
|
exec: exec,
|
|
|
|
pid: ps.Pid,
|
|
|
|
pluginClient: pluginClient,
|
|
|
|
taskConfig: cfg,
|
|
|
|
procState: drivers.TaskStateRunning,
|
|
|
|
startedAt: time.Now().Round(time.Millisecond),
|
|
|
|
logger: d.logger,
|
|
|
|
}
|
|
|
|
|
|
|
|
driverState := TaskState{
|
2019-01-15 01:02:44 +00:00
|
|
|
ReattachConfig: pstructs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()),
|
2018-10-31 14:30:13 +00:00
|
|
|
Pid: ps.Pid,
|
|
|
|
TaskConfig: cfg,
|
|
|
|
StartedAt: h.startedAt,
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := handle.SetDriverState(&driverState); err != nil {
|
|
|
|
d.logger.Error("failed to start task, error setting driver state", "error", err)
|
|
|
|
exec.Shutdown("", 0)
|
|
|
|
pluginClient.Kill()
|
|
|
|
return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
d.tasks.Set(cfg.ID, h)
|
|
|
|
go h.run()
|
|
|
|
return handle, nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func javaCmdArgs(driverConfig TaskConfig) []string {
|
2021-05-15 16:55:44 +00:00
|
|
|
var args []string
|
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
// Look for jvm options
|
|
|
|
if len(driverConfig.JvmOpts) != 0 {
|
|
|
|
args = append(args, driverConfig.JvmOpts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the classpath
|
|
|
|
if driverConfig.ClassPath != "" {
|
|
|
|
args = append(args, "-cp", driverConfig.ClassPath)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the jar
|
|
|
|
if driverConfig.JarPath != "" {
|
|
|
|
args = append(args, "-jar", driverConfig.JarPath)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the class
|
|
|
|
if driverConfig.Class != "" {
|
|
|
|
args = append(args, driverConfig.Class)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add any args
|
|
|
|
if len(driverConfig.Args) != 0 {
|
|
|
|
args = append(args, driverConfig.Args...)
|
|
|
|
}
|
|
|
|
|
|
|
|
return args
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) {
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return nil, drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
|
|
|
ch := make(chan *drivers.ExitResult)
|
|
|
|
go d.handleWait(ctx, handle, ch)
|
|
|
|
|
|
|
|
return ch, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) {
|
|
|
|
defer close(ch)
|
|
|
|
var result *drivers.ExitResult
|
2018-12-05 16:04:18 +00:00
|
|
|
ps, err := handle.exec.Wait(ctx)
|
2018-10-31 14:30:13 +00:00
|
|
|
if err != nil {
|
|
|
|
result = &drivers.ExitResult{
|
|
|
|
Err: fmt.Errorf("executor: error waiting on process: %v", err),
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
result = &drivers.ExitResult{
|
|
|
|
ExitCode: ps.ExitCode,
|
|
|
|
Signal: ps.Signal,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case <-d.ctx.Done():
|
|
|
|
return
|
|
|
|
case ch <- result:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error {
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := handle.exec.Shutdown(signal, timeout); err != nil {
|
|
|
|
if handle.pluginClient.Exited() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return fmt.Errorf("executor Shutdown failed: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) DestroyTask(taskID string, force bool) error {
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
|
|
|
if handle.IsRunning() && !force {
|
|
|
|
return fmt.Errorf("cannot destroy running task")
|
|
|
|
}
|
|
|
|
|
|
|
|
if !handle.pluginClient.Exited() {
|
2019-11-15 14:33:46 +00:00
|
|
|
if err := handle.exec.Shutdown("", 0); err != nil {
|
|
|
|
handle.logger.Error("destroying executor failed", "err", err)
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
handle.pluginClient.Kill()
|
|
|
|
}
|
|
|
|
|
|
|
|
d.tasks.Delete(taskID)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) {
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return nil, drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
2018-11-02 12:29:39 +00:00
|
|
|
return handle.TaskStatus(), nil
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
2018-12-11 20:27:50 +00:00
|
|
|
func (d *Driver) TaskStats(ctx context.Context, taskID string, interval time.Duration) (<-chan *drivers.TaskResourceUsage, error) {
|
2018-10-31 14:30:13 +00:00
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return nil, drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
2018-12-11 20:27:50 +00:00
|
|
|
return handle.exec.Stats(ctx, interval)
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) {
|
|
|
|
return d.eventer.TaskEvents(ctx)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) SignalTask(taskID string, signal string) error {
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
|
|
|
sig := os.Interrupt
|
|
|
|
if s, ok := signals.SignalLookup[signal]; ok {
|
|
|
|
sig = s
|
2019-03-25 21:19:43 +00:00
|
|
|
} else {
|
|
|
|
d.logger.Warn("unknown signal to send to task, using SIGINT instead", "signal", signal, "task_id", handle.taskConfig.ID)
|
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
}
|
|
|
|
return handle.exec.Signal(sig)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) {
|
|
|
|
if len(cmd) == 0 {
|
|
|
|
return nil, fmt.Errorf("error cmd must have at least one value")
|
|
|
|
}
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return nil, drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
|
|
|
out, exitCode, err := handle.exec.Exec(time.Now().Add(timeout), cmd[0], cmd[1:])
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &drivers.ExecTaskResult{
|
|
|
|
Stdout: out,
|
|
|
|
ExitResult: &drivers.ExitResult{
|
|
|
|
ExitCode: exitCode,
|
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2019-04-28 21:31:02 +00:00
|
|
|
var _ drivers.ExecTaskStreamingRawDriver = (*Driver)(nil)
|
|
|
|
|
|
|
|
func (d *Driver) ExecTaskStreamingRaw(ctx context.Context,
|
|
|
|
taskID string,
|
|
|
|
command []string,
|
|
|
|
tty bool,
|
|
|
|
stream drivers.ExecTaskStream) error {
|
|
|
|
|
|
|
|
if len(command) == 0 {
|
2019-05-13 14:01:19 +00:00
|
|
|
return fmt.Errorf("error cmd must have at least one value")
|
2019-04-28 21:31:02 +00:00
|
|
|
}
|
|
|
|
handle, ok := d.tasks.Get(taskID)
|
|
|
|
if !ok {
|
|
|
|
return drivers.ErrTaskNotFound
|
|
|
|
}
|
|
|
|
|
|
|
|
return handle.exec.ExecStreaming(ctx, command, tty, stream)
|
|
|
|
}
|
|
|
|
|
2018-10-31 14:30:13 +00:00
|
|
|
// GetAbsolutePath returns the absolute path of the passed binary by resolving
|
|
|
|
// it in the path and following symlinks.
|
|
|
|
func GetAbsolutePath(bin string) (string, error) {
|
|
|
|
lp, err := exec.LookPath(bin)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return filepath.EvalSymlinks(lp)
|
|
|
|
}
|