From a8d69500078b4e06f35ba188547d3a7a5a7fb279 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 26 Feb 2020 22:16:41 -0500 Subject: [PATCH] Remove rkt as a built-in driver Rkt has been archived and is no longer an active project: * https://github.com/rkt/rkt * https://github.com/rkt/rkt/issues/4024 The rkt driver will continue to live as an external plugin. --- .circleci/config.yml | 2 +- .circleci/config/workflows/build-test.yml | 2 +- Vagrantfile | 4 - client/testutil/driver_compatible.go | 22 - drivers/rkt/api.go | 28 - drivers/rkt/driver.go | 1096 ------------------ drivers/rkt/driver_pre09.go | 46 - drivers/rkt/driver_test.go | 979 ---------------- drivers/rkt/handle.go | 80 -- drivers/rkt/state.go | 33 - helper/pluginutils/catalog/register_linux.go | 12 - scripts/travis-rkt.sh | 47 - scripts/vagrant-linux-priv-rkt.sh | 47 - 13 files changed, 2 insertions(+), 2396 deletions(-) delete mode 100644 drivers/rkt/api.go delete mode 100644 drivers/rkt/driver.go delete mode 100644 drivers/rkt/driver_pre09.go delete mode 100644 drivers/rkt/driver_test.go delete mode 100644 drivers/rkt/handle.go delete mode 100644 drivers/rkt/state.go delete mode 100644 helper/pluginutils/catalog/register_linux.go delete mode 100755 scripts/travis-rkt.sh delete mode 100755 scripts/vagrant-linux-priv-rkt.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 539567c9a..0e24b7fb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -483,7 +483,7 @@ jobs: - NOMAD_SLOW_TEST: 1 - PAGER: cat - GOTEST_PKGS: '' - - GOTEST_PKGS_EXCLUDE: ./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices + - GOTEST_PKGS_EXCLUDE: ./api|./client|./drivers/docker|./drivers/exec|./drivers/shared/executor|./nomad|./devices - GOTESTARCH: amd64 steps: - checkout diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml index 5260b6d21..eac10daa7 100644 --- a/.circleci/config/workflows/build-test.yml +++ b/.circleci/config/workflows/build-test.yml @@ -33,7 +33,7 @@ jobs: filters: *backend_branches_filter - test-machine: name: "test-other" - exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices" + exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/shared/executor|./nomad|./devices" filters: *backend_branches_filter - test-machine: name: "test-docker" diff --git a/Vagrantfile b/Vagrantfile index 3737ce96c..96d70fab9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -152,10 +152,6 @@ def configureLinuxProvisioners(vmCfg) privileged: true, path: './scripts/vagrant-linux-priv-vault.sh' - vmCfg.vm.provision "shell", - privileged: true, - path: './scripts/vagrant-linux-priv-rkt.sh' - vmCfg.vm.provision "shell", privileged: false, path: './scripts/vagrant-linux-unpriv-ui.sh' diff --git a/client/testutil/driver_compatible.go b/client/testutil/driver_compatible.go index 72307b6dc..7829b987d 100644 --- a/client/testutil/driver_compatible.go +++ b/client/testutil/driver_compatible.go @@ -3,7 +3,6 @@ package testutil import ( "os/exec" "runtime" - "sync" "syscall" "testing" @@ -57,27 +56,6 @@ func CgroupCompatible(t *testing.T) { } } -var rktExists bool -var rktOnce sync.Once - -func RktCompatible(t *testing.T) { - if runtime.GOOS != "linux" || syscall.Geteuid() != 0 { - t.Skip("Must be root on Linux to run test") - } - - // else see if rkt exists - rktOnce.Do(func() { - _, err := exec.Command("rkt", "version").CombinedOutput() - if err == nil { - rktExists = true - } - }) - - if !rktExists { - t.Skip("Must have rkt installed for rkt specific tests to run") - } -} - func MountCompatible(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows does not support mount") diff --git a/drivers/rkt/api.go b/drivers/rkt/api.go deleted file mode 100644 index 2a0eac467..000000000 --- a/drivers/rkt/api.go +++ /dev/null @@ -1,28 +0,0 @@ -package rkt - -import ( - "net" -) - -// This file contains the structrs used by this driver. -// Embedding structs here helps avoid depending on a linux only library - -// Pod is the pod object, as defined in -// https://github.com/rkt/rkt/blob/03285a7db960311faf887452538b2b8ae4304488/api/v1/json.go#L68-L88 -type Pod struct { - UUID string `json:"name"` - State string `json:"state"` - Networks []NetInfo `json:"networks,omitempty"` -} - -// A type and some structure to represent rkt's view of a *runtime* -// network instance. -// https://github.com/rkt/rkt/blob/4080b1743e0c46fa1645f4de64f1b75a980d82a3/networking/netinfo/netinfo.go#L29-L48 -type NetInfo struct { - NetName string `json:"netName"` - ConfPath string `json:"netConf"` - PluginPath string `json:"pluginPath"` - IfName string `json:"ifName"` - IP net.IP `json:"ip"` - Args string `json:"args"` -} diff --git a/drivers/rkt/driver.go b/drivers/rkt/driver.go deleted file mode 100644 index 6de9f4dfe..000000000 --- a/drivers/rkt/driver.go +++ /dev/null @@ -1,1096 +0,0 @@ -package rkt - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" - "net" - "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "syscall" - "time" - - appcschema "github.com/appc/spec/schema" - "github.com/hashicorp/consul-template/signals" - hclog "github.com/hashicorp/go-hclog" - version "github.com/hashicorp/go-version" - "github.com/hashicorp/nomad/client/config" - "github.com/hashicorp/nomad/client/taskenv" - "github.com/hashicorp/nomad/drivers/shared/eventer" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/helper" - "github.com/hashicorp/nomad/helper/pluginutils/hclutils" - "github.com/hashicorp/nomad/helper/pluginutils/loader" - "github.com/hashicorp/nomad/plugins/base" - "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/shared/hclspec" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -const ( - // pluginName is the name of the plugin - pluginName = "rkt" - - // fingerprintPeriod is the interval at which the driver will send fingerprint responses - fingerprintPeriod = 30 * time.Second - - // minRktVersion is the earliest supported version of rkt. rkt added support - // for CPU and memory isolators in 0.14.0. We cannot support an earlier - // version to maintain an uniform interface across all drivers - minRktVersion = "1.27.0" - - // rktCmd is the command rkt is installed as. - rktCmd = "rkt" - - // networkDeadline is how long to wait for container network - // information to become available. - networkDeadline = 1 * time.Minute - - // taskHandleVersion is the version of task handle which this driver sets - // and understands how to decode driver state - taskHandleVersion = 1 -) - -var ( - // PluginID is the rawexec plugin metadata registered in the plugin - // catalog. - PluginID = loader.PluginID{ - Name: pluginName, - PluginType: base.PluginTypeDriver, - } - - // PluginConfig is the rawexec factory function registered in the - // plugin catalog. - PluginConfig = &loader.InternalPluginConfig{ - Config: map[string]interface{}{}, - Factory: func(l hclog.Logger) interface{} { return NewRktDriver(l) }, - } -) - -// PluginLoader maps pre-0.9 client driver options to post-0.9 plugin options. -func PluginLoader(opts map[string]string) (map[string]interface{}, error) { - conf := map[string]interface{}{} - if v, err := strconv.ParseBool(opts["driver.rkt.volumes.enabled"]); err == nil { - conf["volumes_enabled"] = v - } - return conf, nil -} - -var ( - // pluginInfo is the response returned for the PluginInfo RPC - pluginInfo = &base.PluginInfoResponse{ - Type: base.PluginTypeDriver, - PluginApiVersions: []string{drivers.ApiVersion010}, - PluginVersion: "0.1.0", - Name: pluginName, - } - - // configSpec is the hcl specification returned by the ConfigSchema RPC - configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ - "volumes_enabled": hclspec.NewDefault( - hclspec.NewAttr("volumes_enabled", "bool", false), - hclspec.NewLiteral("true"), - ), - }) - - // taskConfigSpec is the hcl specification for the driver config section of - // a taskConfig within a job. It is returned in the TaskConfigSchema RPC - taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{ - "image": hclspec.NewAttr("image", "string", true), - "command": hclspec.NewAttr("command", "string", false), - "args": hclspec.NewAttr("args", "list(string)", false), - "trust_prefix": hclspec.NewAttr("trust_prefix", "string", false), - "dns_servers": hclspec.NewAttr("dns_servers", "list(string)", false), - "dns_search_domains": hclspec.NewAttr("dns_search_domains", "list(string)", false), - "net": hclspec.NewAttr("net", "list(string)", false), - "port_map": hclspec.NewAttr("port_map", "list(map(string))", false), - "volumes": hclspec.NewAttr("volumes", "list(string)", false), - "insecure_options": hclspec.NewAttr("insecure_options", "list(string)", false), - "no_overlay": hclspec.NewAttr("no_overlay", "bool", false), - "debug": hclspec.NewAttr("debug", "bool", false), - "group": hclspec.NewAttr("group", "string", false), - }) - - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ - SendSignals: true, - Exec: true, - FSIsolation: drivers.FSIsolationImage, - } - - reRktVersion = regexp.MustCompile(`rkt [vV]ersion[:]? (\d[.\d]+)`) - reAppcVersion = regexp.MustCompile(`appc [vV]ersion[:]? (\d[.\d]+)`) -) - -// Config is the client configuration for the driver -type Config struct { - // VolumesEnabled allows tasks to bind host paths (volumes) inside their - // container. Binding relative paths is always allowed and will be resolved - // relative to the allocation's directory. - VolumesEnabled bool `codec:"volumes_enabled"` -} - -// TaskConfig is the driver configuration of a taskConfig within a job -type TaskConfig struct { - ImageName string `codec:"image"` - Command string `codec:"command"` - Args []string `codec:"args"` - TrustPrefix string `codec:"trust_prefix"` - DNSServers []string `codec:"dns_servers"` // DNS Server for containers - DNSSearchDomains []string `codec:"dns_search_domains"` // DNS Search domains for containers - Net []string `codec:"net"` // Networks for the containers - PortMap hclutils.MapStrStr `codec:"port_map"` // A map of host port and the port name defined in the image manifest file - Volumes []string `codec:"volumes"` // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container[:readOnly] - InsecureOptions []string `codec:"insecure_options"` // list of args for --insecure-options - - NoOverlay bool `codec:"no_overlay"` // disable overlayfs for rkt run - Debug bool `codec:"debug"` // Enable debug option for rkt command - Group string `codec:"group"` // Group override for the container -} - -// TaskState is the state which is encoded in the handle returned in -// StartTask. This information is needed to rebuild the taskConfig state and handler -// during recovery. -type TaskState struct { - ReattachConfig *pstructs.ReattachConfig - TaskConfig *drivers.TaskConfig - Pid int - StartedAt time.Time - UUID string -} - -// Driver is a driver for running images via Rkt We attempt to chose sane -// defaults for now, with more configuration available planned in the future. -type Driver struct { - // eventer is used to handle multiplexing of TaskEvents calls such that an - // event can be broadcast to all callers - eventer *eventer.Eventer - - // config is the driver configuration set by the SetConfig RPC - config *Config - - // nomadConfig is the client config from nomad - nomadConfig *base.ClientDriverConfig - - // tasks is the in memory datastore mapping taskIDs to rktTaskHandles - tasks *taskStore - - // ctx is the context for the driver. It is passed to other subsystems to - // coordinate shutdown - ctx context.Context - - // signalShutdown is called when the driver is shutting down and cancels the - // ctx passed to any subsystems - signalShutdown context.CancelFunc - - // logger will log to the Nomad agent - logger hclog.Logger - - // A tri-state boolean to know if the fingerprinting has happened and - // whether it has been successful - fingerprintSuccess *bool - fingerprintLock sync.Mutex -} - -func NewRktDriver(logger hclog.Logger) drivers.DriverPlugin { - ctx, cancel := context.WithCancel(context.Background()) - logger = logger.Named(pluginName) - return &Driver{ - eventer: eventer.NewEventer(ctx, logger), - config: &Config{}, - tasks: newTaskStore(), - ctx: ctx, - signalShutdown: cancel, - logger: logger, - } -} - -func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) { - return pluginInfo, nil -} - -func (d *Driver) ConfigSchema() (*hclspec.Spec, error) { - return configSpec, nil -} - -func (d *Driver) SetConfig(cfg *base.Config) error { - var config Config - if len(cfg.PluginConfig) != 0 { - if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil { - return err - } - } - - d.config = &config - if cfg.AgentConfig != nil { - d.nomadConfig = cfg.AgentConfig.Driver - } - return nil -} - -func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { - return taskConfigSpec, nil -} - -func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil -} - -func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { - ch := make(chan *drivers.Fingerprint) - go d.handleFingerprint(ctx, ch) - return ch, nil -} - -func (d *Driver) handleFingerprint(ctx context.Context, ch chan *drivers.Fingerprint) { - defer close(ch) - ticker := time.NewTimer(0) - for { - select { - case <-ctx.Done(): - return - case <-d.ctx.Done(): - return - case <-ticker.C: - ticker.Reset(fingerprintPeriod) - ch <- d.buildFingerprint() - } - } -} - -// setFingerprintSuccess marks the driver as having fingerprinted successfully -func (d *Driver) setFingerprintSuccess() { - d.fingerprintLock.Lock() - d.fingerprintSuccess = helper.BoolToPtr(true) - d.fingerprintLock.Unlock() -} - -// setFingerprintFailure marks the driver as having failed fingerprinting -func (d *Driver) setFingerprintFailure() { - d.fingerprintLock.Lock() - d.fingerprintSuccess = helper.BoolToPtr(false) - d.fingerprintLock.Unlock() -} - -// fingerprintSuccessful returns true if the driver has -// never fingerprinted or has successfully fingerprinted -func (d *Driver) fingerprintSuccessful() bool { - d.fingerprintLock.Lock() - defer d.fingerprintLock.Unlock() - return d.fingerprintSuccess == nil || *d.fingerprintSuccess -} - -func (d *Driver) buildFingerprint() *drivers.Fingerprint { - fingerprint := &drivers.Fingerprint{ - Attributes: map[string]*pstructs.Attribute{}, - Health: drivers.HealthStateHealthy, - HealthDescription: drivers.DriverHealthy, - } - - // Only enable if we are root - if syscall.Geteuid() != 0 { - if d.fingerprintSuccessful() { - d.logger.Debug("must run as root user, disabling") - } - d.setFingerprintFailure() - fingerprint.Health = drivers.HealthStateUndetected - fingerprint.HealthDescription = drivers.DriverRequiresRootMessage - return fingerprint - } - - outBytes, err := exec.Command(rktCmd, "version").Output() - if err != nil { - fingerprint.Health = drivers.HealthStateUndetected - fingerprint.HealthDescription = fmt.Sprintf("Failed to execute %s version: %v", rktCmd, err) - d.setFingerprintFailure() - return fingerprint - } - out := strings.TrimSpace(string(outBytes)) - - rktMatches := reRktVersion.FindStringSubmatch(out) - appcMatches := reAppcVersion.FindStringSubmatch(out) - if len(rktMatches) != 2 || len(appcMatches) != 2 { - fingerprint.Health = drivers.HealthStateUndetected - fingerprint.HealthDescription = "Unable to parse rkt version string" - d.setFingerprintFailure() - return fingerprint - } - - minVersion, _ := version.NewVersion(minRktVersion) - currentVersion, _ := version.NewVersion(rktMatches[1]) - if currentVersion.LessThan(minVersion) { - // Do not allow ancient rkt versions - fingerprint.Health = drivers.HealthStateUndetected - fingerprint.HealthDescription = fmt.Sprintf("Unsuported rkt version %s", currentVersion) - if d.fingerprintSuccessful() { - d.logger.Warn("unsupported rkt version please upgrade to >= "+minVersion.String(), - "rkt_version", currentVersion) - } - d.setFingerprintFailure() - return fingerprint - } - - fingerprint.Attributes["driver.rkt"] = pstructs.NewBoolAttribute(true) - fingerprint.Attributes["driver.rkt.version"] = pstructs.NewStringAttribute(rktMatches[1]) - fingerprint.Attributes["driver.rkt.appc.version"] = pstructs.NewStringAttribute(appcMatches[1]) - if d.config.VolumesEnabled { - fingerprint.Attributes["driver.rkt.volumes.enabled"] = pstructs.NewBoolAttribute(true) - } - d.setFingerprintSuccess() - return fingerprint - -} - -func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { - if handle == nil { - return fmt.Errorf("error: handle cannot be nil") - } - - // COMPAT(0.10): pre 0.9 upgrade path check - if handle.Version == 0 { - return d.recoverPre09Task(handle) - } - - // If already attached to handle there's nothing to recover. - if _, ok := d.tasks.Get(handle.Config.ID); ok { - d.logger.Trace("nothing to recover; task already exists", - "task_id", handle.Config.ID, - "task_name", handle.Config.Name, - ) - return nil - } - - var taskState TaskState - if err := handle.GetDriverState(&taskState); err != nil { - d.logger.Error("failed to decode taskConfig state from handle", "error", err, "task_id", handle.Config.ID) - return fmt.Errorf("failed to decode taskConfig state from handle: %v", err) - } - - plugRC, err := pstructs.ReattachConfigToGoPlugin(taskState.ReattachConfig) - if err != nil { - d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID) - return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err) - } - - execImpl, pluginClient, err := executor.ReattachToExecutor(plugRC, - d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - // The taskConfig's environment is set via --set-env flags in Start, but the rkt - // command itself needs an environment with PATH set to find iptables. - // TODO (preetha) need to figure out how to read env.blacklist - eb := taskenv.NewEmptyBuilder() - filter := strings.Split(config.DefaultEnvBlacklist, ",") - rktEnv := eb.SetHostEnvvars(filter).Build() - - h := &taskHandle{ - exec: execImpl, - env: rktEnv, - pid: taskState.Pid, - uuid: taskState.UUID, - pluginClient: pluginClient, - taskConfig: taskState.TaskConfig, - procState: drivers.TaskStateRunning, - startedAt: taskState.StartedAt, - exitResult: &drivers.ExitResult{}, - logger: d.logger, - } - - d.tasks.Set(taskState.TaskConfig.ID, h) - - go h.run() - return nil -} - -func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) { - if _, ok := d.tasks.Get(cfg.ID); ok { - return nil, nil, fmt.Errorf("taskConfig with ID '%s' already started", cfg.ID) - } - - var driverConfig TaskConfig - - if err := cfg.DecodeDriverConfig(&driverConfig); err != nil { - return nil, nil, fmt.Errorf("failed to decode driver config: %v", err) - } - - handle := drivers.NewTaskHandle(taskHandleVersion) - handle.Config = cfg - - // todo(preetha) - port map in client v1 is a slice of maps that get merged. figure out if the caller will do this - //driverConfig.PortMap - - // ACI image - img := driverConfig.ImageName - - // Global arguments given to both prepare and run-prepared - globalArgs := make([]string, 0, 50) - - // Add debug option to rkt command. - debug := driverConfig.Debug - - // Add the given trust prefix - trustPrefix := driverConfig.TrustPrefix - insecure := false - if trustPrefix != "" { - var outBuf, errBuf bytes.Buffer - cmd := exec.Command(rktCmd, "trust", "--skip-fingerprint-review=true", fmt.Sprintf("--prefix=%s", trustPrefix), fmt.Sprintf("--debug=%t", debug)) - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - if err := cmd.Run(); err != nil { - return nil, nil, fmt.Errorf("Error running rkt trust: %s\n\nOutput: %s\n\nError: %s", - err, outBuf.String(), errBuf.String()) - } - d.logger.Debug("added trust prefix", "trust_prefix", trustPrefix, "task_name", cfg.Name) - } else { - // Disable signature verification if the trust command was not run. - insecure = true - } - - // if we have a selective insecure_options, prefer them - // insecure options are rkt's global argument, so we do this before the actual "run" - if len(driverConfig.InsecureOptions) > 0 { - globalArgs = append(globalArgs, fmt.Sprintf("--insecure-options=%s", strings.Join(driverConfig.InsecureOptions, ","))) - } else if insecure { - globalArgs = append(globalArgs, "--insecure-options=all") - } - - // debug is rkt's global argument, so add it before the actual "run" - globalArgs = append(globalArgs, fmt.Sprintf("--debug=%t", debug)) - - prepareArgs := make([]string, 0, 50) - runArgs := make([]string, 0, 50) - - prepareArgs = append(prepareArgs, globalArgs...) - prepareArgs = append(prepareArgs, "prepare") - runArgs = append(runArgs, globalArgs...) - runArgs = append(runArgs, "run-prepared") - - // disable overlayfs - if driverConfig.NoOverlay { - prepareArgs = append(prepareArgs, "--no-overlay=true") - } - - // Convert underscores to dashes in taskConfig names for use in volume names #2358 - sanitizedName := strings.Replace(cfg.Name, "_", "-", -1) - - // Mount /alloc - allocVolName := fmt.Sprintf("%s-%s-alloc", cfg.AllocID, sanitizedName) - prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", allocVolName, cfg.TaskDir().SharedAllocDir)) - prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", allocVolName, "/alloc")) - - // Mount /local - localVolName := fmt.Sprintf("%s-%s-local", cfg.AllocID, sanitizedName) - prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", localVolName, cfg.TaskDir().LocalDir)) - prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", localVolName, "/local")) - - // Mount /secrets - secretsVolName := fmt.Sprintf("%s-%s-secrets", cfg.AllocID, sanitizedName) - prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", secretsVolName, cfg.TaskDir().SecretsDir)) - prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", secretsVolName, "/secrets")) - - // Mount arbitrary volumes if enabled - if len(driverConfig.Volumes) > 0 { - if !d.config.VolumesEnabled { - return nil, nil, fmt.Errorf("volumes_enabled is false; cannot use rkt volumes: %+q", driverConfig.Volumes) - } - for i, rawvol := range driverConfig.Volumes { - parts := strings.Split(rawvol, ":") - readOnly := "false" - // job spec: - // volumes = ["/host/path:/container/path[:readOnly]"] - // the third parameter is optional, mount is read-write by default - if len(parts) == 3 { - if parts[2] == "readOnly" { - d.logger.Debug("mounting volume as readOnly", "volume", strings.Join(parts[:2], parts[1])) - readOnly = "true" - } else { - d.logger.Warn("unknown volume parameter ignored for mount", "parameter", parts[2], "mount", parts[0]) - } - } else if len(parts) != 2 { - return nil, nil, fmt.Errorf("invalid rkt volume: %q", rawvol) - } - volName := fmt.Sprintf("%s-%s-%d", cfg.AllocID, sanitizedName, i) - prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s,readOnly=%s", volName, parts[0], readOnly)) - prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", volName, parts[1])) - } - } - - // Mount task volumes, always do - for i, vol := range cfg.Mounts { - volName := fmt.Sprintf("%s-%s-taskmounts-%d", cfg.AllocID, sanitizedName, i) - prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s,readOnly=%v", volName, vol.HostPath, vol.Readonly)) - prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", volName, vol.TaskPath)) - } - - // Mount task devices, always do - for i, vol := range cfg.Devices { - volName := fmt.Sprintf("%s-%s-taskdevices-%d", cfg.AllocID, sanitizedName, i) - readOnly := !strings.Contains(vol.Permissions, "w") - prepareArgs = append(prepareArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s,readOnly=%v", volName, vol.HostPath, readOnly)) - prepareArgs = append(prepareArgs, fmt.Sprintf("--mount=volume=%s,target=%s", volName, vol.TaskPath)) - } - - // Inject environment variables - for k, v := range cfg.Env { - prepareArgs = append(prepareArgs, fmt.Sprintf("--set-env=%s=%s", k, v)) - } - - // Image is set here, because the commands that follow apply to it - prepareArgs = append(prepareArgs, img) - - // Check if the user has overridden the exec command. - if driverConfig.Command != "" { - prepareArgs = append(prepareArgs, fmt.Sprintf("--exec=%v", driverConfig.Command)) - } - - // Add memory isolator - prepareArgs = append(prepareArgs, fmt.Sprintf("--memory=%v", cfg.Resources.LinuxResources.MemoryLimitBytes)) - - // Add CPU isolator - prepareArgs = append(prepareArgs, fmt.Sprintf("--cpu=%v", cfg.Resources.LinuxResources.CPUShares)) - - // Add DNS servers - if len(driverConfig.DNSServers) == 1 && (driverConfig.DNSServers[0] == "host" || driverConfig.DNSServers[0] == "none") { - // Special case single item lists with the special values "host" or "none" - runArgs = append(runArgs, fmt.Sprintf("--dns=%s", driverConfig.DNSServers[0])) - } else { - for _, ip := range driverConfig.DNSServers { - if err := net.ParseIP(ip); err == nil { - wrappedErr := fmt.Errorf("invalid ip address for container dns server %q", ip) - d.logger.Debug("error parsing DNS server", "error", wrappedErr) - return nil, nil, wrappedErr - } - runArgs = append(runArgs, fmt.Sprintf("--dns=%s", ip)) - } - } - - // set DNS search domains - for _, domain := range driverConfig.DNSSearchDomains { - runArgs = append(runArgs, fmt.Sprintf("--dns-search=%s", domain)) - } - - // set network - network := strings.Join(driverConfig.Net, ",") - if network != "" { - runArgs = append(runArgs, fmt.Sprintf("--net=%s", network)) - } - - // Setup port mapping and exposed ports - if len(cfg.Resources.NomadResources.Networks) == 0 { - d.logger.Debug("no network interfaces are available") - if len(driverConfig.PortMap) > 0 { - return nil, nil, fmt.Errorf("Trying to map ports but no network interface is available") - } - } else if network == "host" { - // Port mapping is skipped when host networking is used. - d.logger.Debug("Ignoring port_map when using --net=host", "task_name", cfg.Name) - } else { - network := cfg.Resources.NomadResources.Networks[0] - for _, port := range network.ReservedPorts { - var containerPort string - - mapped, ok := driverConfig.PortMap[port.Label] - if !ok { - // If the user doesn't have a mapped port using port_map, driver stops running container. - return nil, nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.") - } - containerPort = mapped - - hostPortStr := strconv.Itoa(port.Value) - - d.logger.Debug("driver.rkt: exposed port", "containerPort", containerPort) - // Add port option to rkt run arguments. rkt allows multiple port args - prepareArgs = append(prepareArgs, fmt.Sprintf("--port=%s:%s", containerPort, hostPortStr)) - } - - for _, port := range network.DynamicPorts { - // By default we will map the allocated port 1:1 to the container - var containerPort string - - if mapped, ok := driverConfig.PortMap[port.Label]; ok { - containerPort = mapped - } else { - // If the user doesn't have mapped a port using port_map, driver stops running container. - return nil, nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.") - } - - hostPortStr := strconv.Itoa(port.Value) - - d.logger.Debug("exposed port", "containerPort", containerPort, "task_name", cfg.Name) - // Add port option to rkt run arguments. rkt allows multiple port args - prepareArgs = append(prepareArgs, fmt.Sprintf("--port=%s:%s", containerPort, hostPortStr)) - } - - } - - // If a user has been specified for the taskConfig, pass it through to the user - if cfg.User != "" { - prepareArgs = append(prepareArgs, fmt.Sprintf("--user=%s", cfg.User)) - } - - // There's no taskConfig-level parameter for groups so check the driver - // config for a custom group - if driverConfig.Group != "" { - prepareArgs = append(prepareArgs, fmt.Sprintf("--group=%s", driverConfig.Group)) - } - - // Add user passed arguments. - if len(driverConfig.Args) != 0 { - - // Need to start arguments with "--" - prepareArgs = append(prepareArgs, "--") - - for _, arg := range driverConfig.Args { - prepareArgs = append(prepareArgs, fmt.Sprintf("%v", arg)) - } - } - - pluginLogFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%s-executor.out", cfg.Name)) - executorConfig := &executor.ExecutorConfig{ - LogFile: pluginLogFile, - LogLevel: "debug", - } - - execImpl, pluginClient, err := executor.CreateExecutor( - d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID), - d.nomadConfig, executorConfig) - if err != nil { - return nil, nil, err - } - - absPath, err := GetAbsolutePath(rktCmd) - if err != nil { - return nil, nil, err - } - - var outBuf, errBuf bytes.Buffer - cmd := exec.Command(rktCmd, prepareArgs...) - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - d.logger.Debug("preparing taskConfig", "pod", img, "task_name", cfg.Name, "args", prepareArgs) - if err := cmd.Run(); err != nil { - return nil, nil, fmt.Errorf("Error preparing rkt pod: %s\n\nOutput: %s\n\nError: %s", - err, outBuf.String(), errBuf.String()) - } - uuid := strings.TrimSpace(outBuf.String()) - d.logger.Debug("taskConfig prepared", "pod", img, "task_name", cfg.Name, "uuid", uuid) - runArgs = append(runArgs, uuid) - - // The taskConfig's environment is set via --set-env flags above, but the rkt - // command itself needs an environment with PATH set to find iptables. - - // TODO (preetha) need to figure out how to pass env.blacklist from client config - eb := taskenv.NewEmptyBuilder() - filter := strings.Split(config.DefaultEnvBlacklist, ",") - rktEnv := eb.SetHostEnvvars(filter).Build() - - // Enable ResourceLimits to place the executor in a parent cgroup of - // the rkt container. This allows stats collection via the executor to - // work just like it does for exec. - execCmd := &executor.ExecCommand{ - Cmd: absPath, - Args: runArgs, - ResourceLimits: true, - Resources: cfg.Resources, - - // Use rktEnv, the environment needed for running rkt, not the task env - Env: rktEnv.List(), - - TaskDir: cfg.TaskDir().Dir, - StdoutPath: cfg.StdoutPath, - StderrPath: cfg.StderrPath, - } - ps, err := execImpl.Launch(execCmd) - if err != nil { - pluginClient.Kill() - return nil, nil, err - } - - d.logger.Debug("started taskConfig", "aci", img, "uuid", uuid, "task_name", cfg.Name, "args", runArgs) - h := &taskHandle{ - exec: execImpl, - env: rktEnv, - pid: ps.Pid, - uuid: uuid, - pluginClient: pluginClient, - taskConfig: cfg, - procState: drivers.TaskStateRunning, - startedAt: time.Now().Round(time.Millisecond), - logger: d.logger, - } - - rktDriverState := TaskState{ - ReattachConfig: pstructs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), - Pid: ps.Pid, - TaskConfig: cfg, - StartedAt: h.startedAt, - UUID: uuid, - } - - if err := handle.SetDriverState(&rktDriverState); err != nil { - d.logger.Error("failed to start task, error setting driver state", "error", err, "task_name", cfg.Name) - execImpl.Shutdown("", 0) - pluginClient.Kill() - return nil, nil, fmt.Errorf("failed to set driver state: %v", err) - } - - d.tasks.Set(cfg.ID, h) - go h.run() - - // Do not attempt to retrieve driver network if one won't exist: - // - "host" means the container itself has no networking metadata - // - "none" means no network is configured - // https://coreos.com/rkt/docs/latest/networking/overview.html#no-loopback-only-networking - var driverNetwork *drivers.DriverNetwork - if network != "host" && network != "none" { - d.logger.Debug("retrieving network information for pod", "pod", img, "UUID", uuid, "task_name", cfg.Name) - driverNetwork, err = rktGetDriverNetwork(uuid, driverConfig.PortMap, d.logger) - if err != nil && !pluginClient.Exited() { - d.logger.Warn("network status retrieval for pod failed", "pod", img, "UUID", uuid, "task_name", cfg.Name, "error", err) - - // If a portmap was given, this turns into a fatal error - if len(driverConfig.PortMap) != 0 { - pluginClient.Kill() - return nil, nil, fmt.Errorf("Trying to map ports but driver could not determine network information") - } - } - } - - return handle, driverNetwork, nil - -} - -func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) { - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - - ch := make(chan *drivers.ExitResult) - go d.handleWait(ctx, handle, ch) - - return ch, nil -} - -func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error { - handle, ok := d.tasks.Get(taskID) - if !ok { - return drivers.ErrTaskNotFound - } - - if err := handle.exec.Shutdown(signal, timeout); err != nil { - if handle.pluginClient.Exited() { - return nil - } - return fmt.Errorf("executor Shutdown failed: %v", err) - } - - return nil -} - -func (d *Driver) DestroyTask(taskID string, force bool) error { - handle, ok := d.tasks.Get(taskID) - if !ok { - return drivers.ErrTaskNotFound - } - - if handle.IsRunning() && !force { - return fmt.Errorf("cannot destroy running task") - } - - if !handle.pluginClient.Exited() { - if err := handle.exec.Shutdown("", 0); err != nil { - handle.logger.Error("destroying executor failed", "err", err) - } - - handle.pluginClient.Kill() - } - - d.tasks.Delete(taskID) - return nil -} - -func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) { - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - - return handle.TaskStatus(), nil -} - -func (d *Driver) TaskStats(ctx context.Context, taskID string, interval time.Duration) (<-chan *drivers.TaskResourceUsage, error) { - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - - return handle.exec.Stats(ctx, interval) -} - -func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) { - return d.eventer.TaskEvents(ctx) -} - -func (d *Driver) SignalTask(taskID string, signal string) error { - handle, ok := d.tasks.Get(taskID) - if !ok { - return drivers.ErrTaskNotFound - } - - sig := os.Interrupt - if s, ok := signals.SignalLookup[signal]; ok { - sig = s - } else { - d.logger.Warn("unknown signal to send to task, using SIGINT instead", "signal", signal, "task_id", handle.taskConfig.ID, "task_name", handle.taskConfig.Name) - - } - return handle.exec.Signal(sig) -} - -func (d *Driver) ExecTask(taskID string, cmdArgs []string, timeout time.Duration) (*drivers.ExecTaskResult, error) { - if len(cmdArgs) == 0 { - return nil, fmt.Errorf("error cmd must have at least one value") - } - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - // enter + UUID + cmd + args... - cmd := cmdArgs[0] - args := cmdArgs[1:] - enterArgs := make([]string, 3+len(args)) - enterArgs[0] = "enter" - enterArgs[1] = handle.uuid - enterArgs[2] = handle.env.ReplaceEnv(cmd) - copy(enterArgs[3:], handle.env.ParseAndReplace(args)) - out, exitCode, err := handle.exec.Exec(time.Now().Add(timeout), rktCmd, enterArgs) - if err != nil { - return nil, err - } - - return &drivers.ExecTaskResult{ - Stdout: out, - ExitResult: &drivers.ExitResult{ - ExitCode: exitCode, - }, - }, nil - -} - -var _ drivers.ExecTaskStreamingRawDriver = (*Driver)(nil) - -func (d *Driver) ExecTaskStreamingRaw(ctx context.Context, - taskID string, - command []string, - tty bool, - stream drivers.ExecTaskStream) error { - - if len(command) == 0 { - return fmt.Errorf("error cmd must have at least one value") - } - handle, ok := d.tasks.Get(taskID) - if !ok { - return drivers.ErrTaskNotFound - } - - enterCmd := []string{rktCmd, "enter", handle.uuid, handle.env.ReplaceEnv(command[0])} - enterCmd = append(enterCmd, handle.env.ParseAndReplace(command[1:])...) - - return handle.exec.ExecStreaming(ctx, enterCmd, tty, stream) -} - -// GetAbsolutePath returns the absolute path of the passed binary by resolving -// it in the path and following symlinks. -func GetAbsolutePath(bin string) (string, error) { - lp, err := exec.LookPath(bin) - if err != nil { - return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err) - } - - return filepath.EvalSymlinks(lp) -} - -func rktGetDriverNetwork(uuid string, driverConfigPortMap map[string]string, logger hclog.Logger) (*drivers.DriverNetwork, error) { - deadline := time.Now().Add(networkDeadline) - var lastErr error - try := 0 - - for time.Now().Before(deadline) { - try++ - if status, err := rktGetStatus(uuid, logger); err == nil { - for _, net := range status.Networks { - if !net.IP.IsGlobalUnicast() { - continue - } - - // Get the pod manifest so we can figure out which ports are exposed - var portmap map[string]int - manifest, err := rktGetManifest(uuid) - if err == nil { - portmap, err = rktManifestMakePortMap(manifest, driverConfigPortMap) - if err != nil { - lastErr = fmt.Errorf("could not create manifest-based portmap: %v", err) - return nil, lastErr - } - } else { - lastErr = fmt.Errorf("could not get pod manifest: %v", err) - return nil, lastErr - } - - // This is a successful landing; log if its not the first attempt. - if try > 1 { - logger.Debug("retrieved network info for pod", "uuid", uuid, "attempt", try) - } - return &drivers.DriverNetwork{ - PortMap: portmap, - IP: status.Networks[0].IP.String(), - }, nil - } - - if len(status.Networks) == 0 { - lastErr = fmt.Errorf("no networks found") - } else { - lastErr = fmt.Errorf("no good driver networks out of %d returned", len(status.Networks)) - } - } else { - lastErr = fmt.Errorf("getting status failed: %v", err) - } - - waitTime := getJitteredNetworkRetryTime() - logger.Debug("failed getting network info for pod, sleeping", "uuid", uuid, "attempt", try, "err", lastErr, "wait", waitTime) - time.Sleep(waitTime) - } - return nil, fmt.Errorf("timed out, last error: %v", lastErr) -} - -// Given a rkt/appc pod manifest and driver portmap configuration, create -// a driver portmap. -func rktManifestMakePortMap(manifest *appcschema.PodManifest, configPortMap map[string]string) (map[string]int, error) { - if len(manifest.Apps) == 0 { - return nil, fmt.Errorf("manifest has no apps") - } - if len(manifest.Apps) != 1 { - return nil, fmt.Errorf("manifest has multiple apps!") - } - app := manifest.Apps[0] - if app.App == nil { - return nil, fmt.Errorf("specified app has no App object") - } - - portMap := make(map[string]int) - for svc, name := range configPortMap { - for _, port := range app.App.Ports { - if port.Name.String() == name { - portMap[svc] = int(port.Port) - } - } - } - return portMap, nil -} - -// Retrieve pod status for the pod with the given UUID. -func rktGetStatus(uuid string, logger hclog.Logger) (*Pod, error) { - statusArgs := []string{ - "status", - "--format=json", - uuid, - } - var outBuf, errBuf bytes.Buffer - cmd := exec.Command(rktCmd, statusArgs...) - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - if err := cmd.Run(); err != nil { - if outBuf.Len() > 0 { - logger.Debug("status output for UUID", "uuid", uuid, "error", elide(outBuf)) - } - if errBuf.Len() == 0 { - return nil, err - } - logger.Debug("status error output", "uuid", uuid, "error", elide(errBuf)) - return nil, fmt.Errorf("%s. stderr: %q", err, elide(errBuf)) - } - var status Pod - if err := json.Unmarshal(outBuf.Bytes(), &status); err != nil { - return nil, err - } - return &status, nil -} - -// Retrieves a pod manifest -func rktGetManifest(uuid string) (*appcschema.PodManifest, error) { - statusArgs := []string{ - "cat-manifest", - uuid, - } - var outBuf bytes.Buffer - cmd := exec.Command(rktCmd, statusArgs...) - cmd.Stdout = &outBuf - cmd.Stderr = ioutil.Discard - if err := cmd.Run(); err != nil { - return nil, err - } - var manifest appcschema.PodManifest - if err := json.Unmarshal(outBuf.Bytes(), &manifest); err != nil { - return nil, err - } - return &manifest, nil -} - -// Create a time with a 0 to 100ms jitter for rktGetDriverNetwork retries -func getJitteredNetworkRetryTime() time.Duration { - return time.Duration(900+rand.Intn(100)) * time.Millisecond -} - -// Conditionally elide a buffer to an arbitrary length -func elideToLen(inBuf bytes.Buffer, length int) bytes.Buffer { - if inBuf.Len() > length { - inBuf.Truncate(length) - inBuf.WriteString("...") - } - return inBuf -} - -// Conditionally elide a buffer to an 80 character string -func elide(inBuf bytes.Buffer) string { - tempBuf := elideToLen(inBuf, 80) - return tempBuf.String() -} - -func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) { - defer close(ch) - var result *drivers.ExitResult - ps, err := handle.exec.Wait(ctx) - if err != nil { - result = &drivers.ExitResult{ - Err: fmt.Errorf("executor: error waiting on process: %v", err), - } - } else { - result = &drivers.ExitResult{ - ExitCode: ps.ExitCode, - Signal: ps.Signal, - } - } - - select { - case <-ctx.Done(): - case <-d.ctx.Done(): - case ch <- result: - } -} - -func (d *Driver) Shutdown() { - d.signalShutdown() -} diff --git a/drivers/rkt/driver_pre09.go b/drivers/rkt/driver_pre09.go deleted file mode 100644 index a30c80ad4..000000000 --- a/drivers/rkt/driver_pre09.go +++ /dev/null @@ -1,46 +0,0 @@ -package rkt - -import ( - "fmt" - "time" - - "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { - handle, err := state.UnmarshalPre09HandleID(h.DriverState) - if err != nil { - return fmt.Errorf("failed to decode pre09 driver handle: %v", err) - } - - reattach, err := pstructs.ReattachConfigToGoPlugin(handle.ReattachConfig()) - if err != nil { - return fmt.Errorf("failed to decode reattach config from pre09 handle: %v", err) - } - - exec, pluginClient, err := executor.ReattachToPre09Executor(reattach, - d.logger.With("task_name", h.Config.Name, "alloc_id", h.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_name", h.Config.Name) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - th := &taskHandle{ - exec: exec, - pid: reattach.Pid, - pluginClient: pluginClient, - taskConfig: h.Config, - procState: drivers.TaskStateRunning, - startedAt: time.Now(), - exitResult: &drivers.ExitResult{}, - logger: d.logger, - } - - d.tasks.Set(h.Config.ID, th) - - go th.run() - return nil -} diff --git a/drivers/rkt/driver_test.go b/drivers/rkt/driver_test.go deleted file mode 100644 index 1e49a3ec1..000000000 --- a/drivers/rkt/driver_test.go +++ /dev/null @@ -1,979 +0,0 @@ -package rkt - -import ( - "bytes" - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - "testing" - "time" - - ctestutil "github.com/hashicorp/nomad/client/testutil" - "github.com/hashicorp/nomad/helper/pluginutils/hclutils" - "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/helper/testtask" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/structs" - basePlug "github.com/hashicorp/nomad/plugins/base" - "github.com/hashicorp/nomad/plugins/drivers" - dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" - "github.com/hashicorp/nomad/testutil" - "github.com/stretchr/testify/require" - "golang.org/x/sys/unix" -) - -var _ drivers.DriverPlugin = (*Driver)(nil) - -func TestRktVersionRegex(t *testing.T) { - ctestutil.RktCompatible(t) - t.Parallel() - - inputRkt := "rkt version 0.8.1" - inputAppc := "appc version 1.2.0" - expectedRkt := "0.8.1" - expectedAppc := "1.2.0" - rktMatches := reRktVersion.FindStringSubmatch(inputRkt) - appcMatches := reAppcVersion.FindStringSubmatch(inputAppc) - if rktMatches[1] != expectedRkt { - fmt.Printf("Test failed; got %q; want %q\n", rktMatches[1], expectedRkt) - } - if appcMatches[1] != expectedAppc { - fmt.Printf("Test failed; got %q; want %q\n", appcMatches[1], expectedAppc) - } -} - -// Tests setting driver config options -func TestRktDriver_SetConfig(t *testing.T) { - t.Parallel() - require := require.New(t) - - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - // Enable Volumes - config := &Config{ - VolumesEnabled: true, - } - - var data []byte - require.NoError(basePlug.MsgPackEncode(&data, config)) - bconfig := &basePlug.Config{PluginConfig: data} - require.NoError(harness.SetConfig(bconfig)) - require.Exactly(config, d.(*Driver).config) - - config.VolumesEnabled = false - data = []byte{} - require.NoError(basePlug.MsgPackEncode(&data, config)) - bconfig = &basePlug.Config{PluginConfig: data} - require.NoError(harness.SetConfig(bconfig)) - require.Exactly(config, d.(*Driver).config) - -} - -// Verifies using a trust prefix and passing dns servers and search domains -// Also verifies sending sigterm correctly stops the driver instance -func TestRktDriver_Start_Wait_Stop_DNS(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - TrustPrefix: "coreos.com/etcd", - ImageName: "coreos.com/etcd:v2.0.4", - Command: "/etcd", - DNSServers: []string{"8.8.8.8", "8.8.4.4"}, - DNSSearchDomains: []string{"example.com", "example.org", "example.net"}, - Net: []string{"host"}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - handle, driverNet, err := harness.StartTask(task) - require.NoError(err) - require.Nil(driverNet) - - ch, err := harness.WaitTask(context.Background(), handle.Config.ID) - require.NoError(err) - - require.NoError(harness.WaitUntilStarted(task.ID, 1*time.Second)) - - go func() { - harness.StopTask(task.ID, 2*time.Second, "SIGTERM") - }() - - select { - case result := <-ch: - require.Equal(int(unix.SIGTERM), result.Signal) - case <-time.After(10 * time.Second): - require.Fail("timeout waiting for task to shutdown") - } - - // Ensure that the task is marked as dead, but account - // for WaitTask() closing channel before internal state is updated - testutil.WaitForResult(func() (bool, error) { - status, err := harness.InspectTask(task.ID) - if err != nil { - return false, fmt.Errorf("inspecting task failed: %v", err) - } - if status.State != drivers.TaskStateExited { - return false, fmt.Errorf("task hasn't exited yet; status: %v", status.State) - } - - return true, nil - }, func(err error) { - require.NoError(err) - }) - - require.NoError(harness.DestroyTask(task.ID, true)) -} - -// Verifies waiting on task to exit cleanly -func TestRktDriver_Start_Wait_Stop(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - TrustPrefix: "coreos.com/etcd", - ImageName: "coreos.com/etcd:v2.0.4", - Command: "/etcd", - Args: []string{"--version"}, - Net: []string{"none"}, - Debug: true, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - handle, _, err := harness.StartTask(task) - require.NoError(err) - - // Wait on the task, it should exit since we are only asking for etcd version here - ch, err := harness.WaitTask(context.Background(), handle.Config.ID) - require.NoError(err) - result := <-ch - require.Nil(result.Err) - - require.Zero(result.ExitCode) - - require.NoError(harness.DestroyTask(task.ID, true)) - -} - -// Verifies that skipping trust_prefix works -func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - ImageName: "coreos.com/etcd:v2.0.4", - Command: "/etcd", - Args: []string{"--version"}, - Net: []string{"none"}, - Debug: true, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - handle, _, err := harness.StartTask(task) - require.NoError(err) - - // Wait on the task, it should exit since we are only asking for etcd version here - ch, err := harness.WaitTask(context.Background(), handle.Config.ID) - require.NoError(err) - result := <-ch - require.Nil(result.Err) - require.Zero(result.ExitCode) - - require.NoError(harness.DestroyTask(task.ID, true)) - -} - -// Verifies that an invalid trust prefix returns expected error -func TestRktDriver_InvalidTrustPrefix(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - TrustPrefix: "example.com/invalid", - ImageName: "coreos.com/etcd:v2.0.4", - Command: "/etcd", - Args: []string{"--version"}, - Net: []string{"none"}, - Debug: true, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, _, err := harness.StartTask(task) - require.Error(err) - expectedErr := "Error running rkt trust" - require.Contains(err.Error(), expectedErr) - -} - -// Verifies reattaching to a running container -// This test manipulates the harness's internal state map -// to remove the task and then reattaches to it -func TestRktDriver_StartWaitRecoverWaitStop(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - ImageName: "coreos.com/etcd:v2.0.4", - Command: "/etcd", - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - handle, _, err := harness.StartTask(task) - require.NoError(err) - - ch, err := harness.WaitTask(context.Background(), task.ID) - require.NoError(err) - - var waitDone bool - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - result := <-ch - require.Error(result.Err) - waitDone = true - }() - - originalStatus, err := d.InspectTask(task.ID) - require.NoError(err) - - d.(*Driver).tasks.Delete(task.ID) - - wg.Wait() - require.True(waitDone) - _, err = d.InspectTask(task.ID) - require.Equal(drivers.ErrTaskNotFound, err) - - err = d.RecoverTask(handle) - require.NoError(err) - - status, err := d.InspectTask(task.ID) - require.NoError(err) - require.Exactly(originalStatus, status) - - ch, err = harness.WaitTask(context.Background(), task.ID) - require.NoError(err) - - require.NoError(d.StopTask(task.ID, 0, "SIGKILL")) - - select { - case result := <-ch: - require.NoError(result.Err) - require.NotZero(result.ExitCode) - - // when killing a task, signal might not propagate - // when executor proc.Wait() call gets "wait: no child processes" error - //require.Equal(9, result.Signal) - case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): - require.Fail("WaitTask timeout") - } - - require.NoError(d.DestroyTask(task.ID, false)) -} - -// Verifies mounting a volume from the host machine and writing -// some data to it from inside the container -func TestRktDriver_Start_Wait_Volume(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - // enable volumes - config := &Config{VolumesEnabled: true} - - var data []byte - require.NoError(basePlug.MsgPackEncode(&data, config)) - bconfig := &basePlug.Config{PluginConfig: data} - require.NoError(harness.SetConfig(bconfig)) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "rkttest_alpine", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - exp := []byte{'w', 'i', 'n'} - file := "output.txt" - tmpvol, err := ioutil.TempDir("", "nomadtest_rktdriver_volumes") - require.NoError(err) - defer os.RemoveAll(tmpvol) - hostpath := filepath.Join(tmpvol, file) - - tc := &TaskConfig{ - ImageName: "docker://redis:3.2-alpine", - Command: "/bin/sh", - Args: []string{ - "-c", - fmt.Sprintf("echo -n %s > /foo/%s", string(exp), file), - }, - Net: []string{"none"}, - Volumes: []string{fmt.Sprintf("%s:/foo", tmpvol)}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, _, err = harness.StartTask(task) - require.NoError(err) - - // Task should terminate quickly - waitCh, err := harness.WaitTask(context.Background(), task.ID) - require.NoError(err) - - select { - case res := <-waitCh: - require.NoError(res.Err) - require.True(res.Successful(), fmt.Sprintf("exit code %v", res.ExitCode)) - case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): - require.Fail("WaitTask timeout") - } - - // Check that data was written to the shared alloc directory. - act, err := ioutil.ReadFile(hostpath) - require.NoError(err) - require.Exactly(exp, act) - require.NoError(harness.DestroyTask(task.ID, true)) -} - -// Verifies mounting a task mount from the host machine and writing -// some data to it from inside the container -func TestRktDriver_Start_Wait_TaskMounts(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - // mounts through task config should be enabled regardless - config := &Config{VolumesEnabled: false} - - var data []byte - require.NoError(basePlug.MsgPackEncode(&data, config)) - bconfig := &basePlug.Config{PluginConfig: data} - require.NoError(harness.SetConfig(bconfig)) - - tmpvol, err := ioutil.TempDir("", "nomadtest_rktdriver_volumes") - require.NoError(err) - defer os.RemoveAll(tmpvol) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "rkttest_alpine", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - Mounts: []*drivers.MountConfig{ - {HostPath: tmpvol, TaskPath: "/foo", Readonly: false}, - }, - } - exp := []byte{'w', 'i', 'n'} - file := "output.txt" - hostpath := filepath.Join(tmpvol, file) - - tc := &TaskConfig{ - ImageName: "docker://redis:3.2-alpine", - Command: "/bin/sh", - Args: []string{ - "-c", - fmt.Sprintf("echo -n %s > /foo/%s", string(exp), file), - }, - Net: []string{"none"}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, _, err = harness.StartTask(task) - require.NoError(err) - - // Task should terminate quickly - waitCh, err := harness.WaitTask(context.Background(), task.ID) - require.NoError(err) - - select { - case res := <-waitCh: - require.NoError(res.Err) - require.True(res.Successful(), fmt.Sprintf("exit code %v", res.ExitCode)) - case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): - require.Fail("WaitTask timeout") - } - - // Check that data was written to the shared alloc directory. - act, err := ioutil.ReadFile(hostpath) - require.NoError(err) - require.Exactly(exp, act) - require.NoError(harness.DestroyTask(task.ID, true)) -} - -// Verifies port mapping -func TestRktDriver_PortMapping(t *testing.T) { - ctestutil.RktCompatible(t) - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "redis", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - Networks: []*structs.NetworkResource{ - { - IP: "127.0.0.1", - ReservedPorts: []structs.Port{{Label: "main", Value: 8080}}, - }, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - ImageName: "docker://redis:3.2-alpine", - PortMap: map[string]string{ - "main": "6379-tcp", - }, - Debug: true, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, driverNetwork, err := harness.StartTask(task) - require.NoError(err) - require.NotNil(driverNetwork) - require.NoError(harness.DestroyTask(task.ID, true)) -} - -// This test starts a redis container, setting user and group. -// It verifies that running ps inside the container shows the expected user and group -func TestRktDriver_UserGroup(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - User: "nobody", - Name: "rkttest_alpine", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - ImageName: "docker://redis:3.2-alpine", - Group: "nogroup", - Command: "sleep", - Args: []string{"9000"}, - Net: []string{"none"}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, _, err := harness.StartTask(task) - require.NoError(err) - - expected := []byte("\nnobody nogroup /bin/sleep 9000\n") - testutil.WaitForResult(func() (bool, error) { - res, err := d.ExecTask(task.ID, []string{"ps", "-o", "user,group,args"}, time.Second) - if err != nil { - return false, fmt.Errorf("failed to exec: %#v", err) - } - if !res.ExitResult.Successful() { - return false, fmt.Errorf("ps failed: %#v %#v", res.ExitResult, res) - } - raw := res.Stdout - return bytes.Contains(raw, expected), fmt.Errorf("expected %q but found:\n%s", expected, raw) - }, func(err error) { - require.NoError(err) - }) - - require.NoError(harness.DestroyTask(task.ID, true)) -} - -// Verifies executing both correct and incorrect commands inside the container -func TestRktDriver_Exec(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - TrustPrefix: "coreos.com/etcd", - ImageName: "coreos.com/etcd:v2.0.4", - Net: []string{"none"}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, _, err := harness.StartTask(task) - require.NoError(err) - - // Run command that should succeed - expected := []byte("etcd version") - testutil.WaitForResult(func() (bool, error) { - res, err := d.ExecTask(task.ID, []string{"/etcd", "--version"}, time.Second) - if err != nil { - return false, fmt.Errorf("failed to exec: %#v", err) - } - if !res.ExitResult.Successful() { - return false, fmt.Errorf("/etcd --version failed: %#v %#v", res.ExitResult, res) - } - raw := res.Stdout - return bytes.Contains(raw, expected), fmt.Errorf("expected %q but found:\n%s", expected, raw) - }, func(err error) { - require.NoError(err) - }) - - // Run command that should fail - expected = []byte("flag provided but not defined") - testutil.WaitForResult(func() (bool, error) { - res, err := d.ExecTask(task.ID, []string{"/etcd", "--cgdfgdfg"}, time.Second) - if err != nil { - return false, fmt.Errorf("failed to exec: %#v", err) - } - if res.ExitResult.Successful() { - return false, fmt.Errorf("/etcd --cgdfgdfg unexpected succeeded: %#v %#v", res.ExitResult, res) - } - raw := res.Stdout - return bytes.Contains(raw, expected), fmt.Errorf("expected %q but found:\n%s", expected, raw) - }, func(err error) { - require.NoError(err) - }) - - require.NoError(harness.DestroyTask(task.ID, true)) -} - -// Verifies getting resource usage stats -// TODO(preetha) figure out why stats are zero -func TestRktDriver_Stats(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - TrustPrefix: "coreos.com/etcd", - ImageName: "coreos.com/etcd:v2.0.4", - Command: "/etcd", - Net: []string{"none"}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - handle, _, err := harness.StartTask(task) - require.NoError(err) - - // Wait for task to start - _, err = harness.WaitTask(context.Background(), handle.Config.ID) - require.NoError(err) - - // Wait until task started - require.NoError(harness.WaitUntilStarted(task.ID, 1*time.Second)) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - statsCh, err := d.TaskStats(ctx, task.ID, time.Second*10) - require.Nil(err) - - select { - case ru := <-statsCh: - //TODO(preetha) why are these zero - fmt.Printf("pid map %v\n", ru.Pids) - fmt.Printf("CPU:%+v Memory:%+v", ru.ResourceUsage.CpuStats, ru.ResourceUsage.MemoryStats) - case <-time.After(time.Second): - require.Fail("timeout receiving stats from channel") - } - - require.NoError(harness.DestroyTask(task.ID, true)) - -} - -func TestConfig_ParseAllHCL(t *testing.T) { - cfgStr := ` -config { - image = "docker://redis:3.2" - command = "/bin/bash" - args = ["-c", "echo hi"] - trust_prefix = "coreos.com/etcd" - dns_servers = ["8.8.8.8"] - dns_search_domains = ["example.com"] - net = ["network1"] - port_map { - http = "80-tcp" - https = "443-tcp" - } - volumes = [ - "/host-path:/container-path", - ] - insecure_options = ["image", "tls", "ondisk"] - no_overlay = true - debug = true - group = "mygroup" -}` - - expected := &TaskConfig{ - ImageName: "docker://redis:3.2", - Command: "/bin/bash", - Args: []string{"-c", "echo hi"}, - TrustPrefix: "coreos.com/etcd", - DNSServers: []string{"8.8.8.8"}, - DNSSearchDomains: []string{"example.com"}, - Net: []string{"network1"}, - PortMap: map[string]string{ - "http": "80-tcp", - "https": "443-tcp", - }, - Volumes: []string{ - "/host-path:/container-path", - }, - InsecureOptions: []string{"image", "tls", "ondisk"}, - NoOverlay: true, - Debug: true, - Group: "mygroup", - } - - var tc *TaskConfig - hclutils.NewConfigParser(taskConfigSpec).ParseHCL(t, cfgStr, &tc) - - require.EqualValues(t, expected, tc) -} - -func TestRkt_ExecTaskStreaming(t *testing.T) { - ctestutil.RktCompatible(t) - if !testutil.IsCI() { - t.Parallel() - } - - require := require.New(t) - d := NewRktDriver(testlog.HCLogger(t)) - harness := dtestutil.NewDriverHarness(t, d) - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - AllocID: uuid.Generate(), - Name: "etcd", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 128, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 100, - }, - }, - LinuxResources: &drivers.LinuxResources{ - MemoryLimitBytes: 134217728, - CPUShares: 100, - }, - }, - } - - tc := &TaskConfig{ - ImageName: "docker://busybox:1.29.3", - Command: "/bin/sleep", - Args: []string{"1000"}, - Net: []string{"none"}, - } - require.NoError(task.EncodeConcreteDriverConfig(&tc)) - testtask.SetTaskConfigEnv(task) - - cleanup := harness.MkAllocDir(task, true) - defer cleanup() - - _, _, err := harness.StartTask(task) - require.NoError(err) - defer d.DestroyTask(task.ID, true) - - // wait for container to be up and executable - testutil.WaitForResult(func() (bool, error) { - res, err := d.ExecTask(task.ID, []string{"/bin/sh", "-c", "echo hi"}, time.Second) - if err != nil { - return false, fmt.Errorf("failed to exec: %#v", err) - } - if !res.ExitResult.Successful() { - return false, fmt.Errorf("ps failed: %#v %#v", res.ExitResult, res) - } - return true, nil - }, func(err error) { - require.NoError(err) - }) - - dtestutil.ExecTaskStreamingConformanceTests(t, harness, task.ID) - -} diff --git a/drivers/rkt/handle.go b/drivers/rkt/handle.go deleted file mode 100644 index ce234d7ea..000000000 --- a/drivers/rkt/handle.go +++ /dev/null @@ -1,80 +0,0 @@ -package rkt - -import ( - "context" - "strconv" - "sync" - "time" - - hclog "github.com/hashicorp/go-hclog" - plugin "github.com/hashicorp/go-plugin" - "github.com/hashicorp/nomad/client/taskenv" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" -) - -type taskHandle struct { - exec executor.Executor - env *taskenv.TaskEnv - uuid string - pid int - pluginClient *plugin.Client - logger hclog.Logger - - // stateLock syncs access to all fields below - stateLock sync.RWMutex - - taskConfig *drivers.TaskConfig - procState drivers.TaskState - startedAt time.Time - completedAt time.Time - exitResult *drivers.ExitResult -} - -func (h *taskHandle) TaskStatus() *drivers.TaskStatus { - h.stateLock.RLock() - defer h.stateLock.RUnlock() - - return &drivers.TaskStatus{ - ID: h.taskConfig.ID, - Name: h.taskConfig.Name, - State: h.procState, - StartedAt: h.startedAt, - CompletedAt: h.completedAt, - ExitResult: h.exitResult, - DriverAttributes: map[string]string{ - "pid": strconv.Itoa(h.pid), - }, - } -} - -func (h *taskHandle) IsRunning() bool { - h.stateLock.RLock() - defer h.stateLock.RUnlock() - return h.procState == drivers.TaskStateRunning -} - -func (h *taskHandle) run() { - h.stateLock.Lock() - if h.exitResult == nil { - h.exitResult = &drivers.ExitResult{} - } - h.stateLock.Unlock() - - ps, err := h.exec.Wait(context.Background()) - h.stateLock.Lock() - defer h.stateLock.Unlock() - - if err != nil { - h.exitResult.Err = err - h.procState = drivers.TaskStateUnknown - h.completedAt = time.Now() - return - } - h.procState = drivers.TaskStateExited - h.exitResult.ExitCode = ps.ExitCode - h.exitResult.Signal = ps.Signal - h.completedAt = ps.Time - - // TODO: detect if the taskConfig OOMed -} diff --git a/drivers/rkt/state.go b/drivers/rkt/state.go deleted file mode 100644 index b46c9292a..000000000 --- a/drivers/rkt/state.go +++ /dev/null @@ -1,33 +0,0 @@ -package rkt - -import ( - "sync" -) - -type taskStore struct { - store map[string]*taskHandle - lock sync.RWMutex -} - -func newTaskStore() *taskStore { - return &taskStore{store: map[string]*taskHandle{}} -} - -func (ts *taskStore) Set(id string, handle *taskHandle) { - ts.lock.Lock() - defer ts.lock.Unlock() - ts.store[id] = handle -} - -func (ts *taskStore) Get(id string) (*taskHandle, bool) { - ts.lock.RLock() - defer ts.lock.RUnlock() - t, ok := ts.store[id] - return t, ok -} - -func (ts *taskStore) Delete(id string) { - ts.lock.Lock() - defer ts.lock.Unlock() - delete(ts.store, id) -} diff --git a/helper/pluginutils/catalog/register_linux.go b/helper/pluginutils/catalog/register_linux.go deleted file mode 100644 index 42314cb57..000000000 --- a/helper/pluginutils/catalog/register_linux.go +++ /dev/null @@ -1,12 +0,0 @@ -package catalog - -import ( - "github.com/hashicorp/nomad/drivers/rkt" -) - -// This file is where all builtin plugins should be registered in the catalog. -// Plugins with build restrictions should be placed in the appropriate -// register_XXX.go file. -func init() { - RegisterDeferredConfig(rkt.PluginID, rkt.PluginConfig, rkt.PluginLoader) -} diff --git a/scripts/travis-rkt.sh b/scripts/travis-rkt.sh deleted file mode 100755 index 07375133f..000000000 --- a/scripts/travis-rkt.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -VERSION=1.27.0 -DOWNLOAD=https://github.com/coreos/rkt/releases/download/v${VERSION}/rkt-v${VERSION}.tar.gz - -function install_rkt() { - if [[ -e /usr/local/bin/rkt ]] ; then - if [ "rkt Version: ${VERSION}" == "$(rkt version | head -n1)" ] ; then - return - fi - fi - - wget -q -O /tmp/rkt.tar.gz "${DOWNLOAD}" - - tar -C /tmp -xvf /tmp/rkt.tar.gz - mv /tmp/rkt-v${VERSION}/rkt /usr/local/bin - mv /tmp/rkt-v${VERSION}/*.aci /usr/local/bin -} - -function configure_rkt_networking() { - if [[ -e /etc/rkt/net.d/99-network.conf ]] ; then - return - fi - - mkdir -p /etc/rkt/net.d - cat < /etc/rkt/net.d/99-network.conf -{ - "name": "default", - "type": "ptp", - "ipMasq": false, - "ipam": { - "type": "host-local", - "subnet": "172.16.28.0/24", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ] - } -} -EOT -} - -install_rkt -configure_rkt_networking diff --git a/scripts/vagrant-linux-priv-rkt.sh b/scripts/vagrant-linux-priv-rkt.sh deleted file mode 100755 index ad7c26af3..000000000 --- a/scripts/vagrant-linux-priv-rkt.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -VERSION=1.27.0 -DOWNLOAD=https://github.com/rkt/rkt/releases/download/v${VERSION}/rkt-v${VERSION}.tar.gz - -function install_rkt() { - if [[ -e /usr/local/bin/rkt ]] ; then - if [ "rkt Version: ${VERSION}" == "$(rkt version | head -n1)" ] ; then - return - fi - fi - - wget -q -O /tmp/rkt.tar.gz "${DOWNLOAD}" - - tar -C /tmp -xvf /tmp/rkt.tar.gz - mv /tmp/rkt-v${VERSION}/rkt /usr/local/bin - mv /tmp/rkt-v${VERSION}/*.aci /usr/local/bin -} - -function configure_rkt_networking() { - if [[ -e /etc/rkt/net.d/99-network.conf ]] ; then - return - fi - - mkdir -p /etc/rkt/net.d - cat < /etc/rkt/net.d/99-network.conf -{ - "name": "default", - "type": "ptp", - "ipMasq": false, - "ipam": { - "type": "host-local", - "subnet": "172.16.28.0/24", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ] - } -} -EOT -} - -install_rkt -configure_rkt_networking