diff --git a/.changelog/12791.txt b/.changelog/12791.txt new file mode 100644 index 000000000..e71742bf2 --- /dev/null +++ b/.changelog/12791.txt @@ -0,0 +1,3 @@ +```release-note:improvement +drivers: removed support for restoring tasks created before Nomad 0.9 +``` diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index d6e904204..f418434ac 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -185,11 +185,6 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return nil } - // COMPAT(0.10): pre 0.9 upgrade path check - if handle.Version == 0 { - return d.recoverPre09Task(handle) - } - var handleState taskHandleState if err := handle.GetDriverState(&handleState); err != nil { return fmt.Errorf("failed to decode driver task state: %v", err) diff --git a/drivers/docker/driver_pre09.go b/drivers/docker/driver_pre09.go deleted file mode 100644 index 7e361d32f..000000000 --- a/drivers/docker/driver_pre09.go +++ /dev/null @@ -1,78 +0,0 @@ -package docker - -import ( - "fmt" - - docker "github.com/fsouza/go-dockerclient" - "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/drivers/docker/docklog" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { - handle, err := state.UnmarshalPre09HandleID(h.DriverState) - if err != nil { - return fmt.Errorf("failed to decode pre09 driver handle: %v", err) - } - - reattach, err := pstructs.ReattachConfigToGoPlugin(handle.ReattachConfig()) - if err != nil { - return fmt.Errorf("failed to decode reattach config from pre09 handle: %v", err) - } - - exec, pluginClient, err := executor.ReattachToPre09Executor(reattach, - d.logger.With("task_name", h.Config.Name, "alloc_id", h.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_name", h.Config.Name) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - client, _, err := d.dockerClients() - if err != nil { - return fmt.Errorf("failed to get docker client: %v", err) - } - - container, err := client.InspectContainerWithOptions(docker.InspectContainerOptions{ - ID: handle.ContainerID, - }) - if err != nil { - return fmt.Errorf("failed to inspect container for id %q: %v", handle.ContainerID, err) - } - - th := &taskHandle{ - client: client, - waitClient: waitClient, - dlogger: &executorDockerLoggerShim{exec: exec}, - dloggerPluginClient: pluginClient, - logger: d.logger.With("container_id", container.ID), - task: h.Config, - containerID: container.ID, - containerImage: container.Image, - doneCh: make(chan bool), - waitCh: make(chan struct{}), - removeContainerOnExit: d.config.GC.Container, - } - - d.tasks.Set(h.Config.ID, th) - go th.run() - return nil -} - -// executorDockerLoggerShim is used by upgraded tasks as the docker logger. When -// the task exits, the Stop() func of the docker logger is called, this shim -// will proxy that call to the executor Shutdown() func which will stop the -// syslog server started by the pre09 executor -type executorDockerLoggerShim struct { - exec executor.Executor -} - -func (e *executorDockerLoggerShim) Start(*docklog.StartOpts) error { return nil } -func (e *executorDockerLoggerShim) Stop() error { - if err := e.exec.Shutdown("docker", 0); err != nil { - return err - } - - return nil -} diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 882e3ae62..706c677c5 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -375,11 +375,6 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("handle cannot be nil") } - // COMPAT(0.10): pre 0.9 upgrade path check - if handle.Version == 0 { - return d.recoverPre09Task(handle) - } - // If already attached to handle there's nothing to recover. if _, ok := d.tasks.Get(handle.Config.ID); ok { d.logger.Trace("nothing to recover; task already exists", diff --git a/drivers/exec/driver_pre09.go b/drivers/exec/driver_pre09.go deleted file mode 100644 index 607142d9e..000000000 --- a/drivers/exec/driver_pre09.go +++ /dev/null @@ -1,46 +0,0 @@ -package exec - -import ( - "fmt" - "time" - - "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { - handle, err := state.UnmarshalPre09HandleID(h.DriverState) - if err != nil { - return fmt.Errorf("failed to decode pre09 driver handle: %v", err) - } - - reattach, err := pstructs.ReattachConfigToGoPlugin(handle.ReattachConfig()) - if err != nil { - return fmt.Errorf("failed to decode reattach config from pre09 handle: %v", err) - } - - exec, pluginClient, err := executor.ReattachToPre09Executor(reattach, - d.logger.With("task_name", h.Config.Name, "alloc_id", h.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_name", h.Config.Name) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - th := &taskHandle{ - exec: exec, - pid: reattach.Pid, - pluginClient: pluginClient, - taskConfig: h.Config, - procState: drivers.TaskStateRunning, - startedAt: time.Now(), - exitResult: &drivers.ExitResult{}, - logger: d.logger, - } - - d.tasks.Set(h.Config.ID, th) - - go th.run() - return nil -} diff --git a/drivers/java/driver.go b/drivers/java/driver.go index ec286c547..7c18caf87 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -378,11 +378,6 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("handle cannot be nil") } - // COMPAT(0.10): pre 0.9 upgrade path check - if handle.Version == 0 { - return d.recoverPre09Task(handle) - } - // If already attached to handle there's nothing to recover. if _, ok := d.tasks.Get(handle.Config.ID); ok { d.logger.Debug("nothing to recover; task already exists", diff --git a/drivers/java/driver_pre09.go b/drivers/java/driver_pre09.go deleted file mode 100644 index 9f1b3f31f..000000000 --- a/drivers/java/driver_pre09.go +++ /dev/null @@ -1,46 +0,0 @@ -package java - -import ( - "fmt" - "time" - - "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { - handle, err := state.UnmarshalPre09HandleID(h.DriverState) - if err != nil { - return fmt.Errorf("failed to decode pre09 driver handle: %v", err) - } - - reattach, err := pstructs.ReattachConfigToGoPlugin(handle.ReattachConfig()) - if err != nil { - return fmt.Errorf("failed to decode reattach config from pre09 handle: %v", err) - } - - exec, pluginClient, err := executor.ReattachToPre09Executor(reattach, - d.logger.With("task_name", h.Config.Name, "alloc_id", h.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_name", h.Config.Name) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - th := &taskHandle{ - exec: exec, - pid: reattach.Pid, - pluginClient: pluginClient, - taskConfig: h.Config, - procState: drivers.TaskStateRunning, - startedAt: time.Now(), - exitResult: &drivers.ExitResult{}, - logger: d.logger, - } - - d.tasks.Set(h.Config.ID, th) - - go th.run() - return nil -} diff --git a/drivers/qemu/driver.go b/drivers/qemu/driver.go index 4a3db8082..12001b8c3 100644 --- a/drivers/qemu/driver.go +++ b/drivers/qemu/driver.go @@ -269,11 +269,6 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("error: handle cannot be nil") } - // COMPAT(0.10): pre 0.9 upgrade path check - if handle.Version == 0 { - return d.recoverPre09Task(handle) - } - // If already attached to handle there's nothing to recover. if _, ok := d.tasks.Get(handle.Config.ID); ok { d.logger.Trace("nothing to recover; task already exists", diff --git a/drivers/qemu/driver_pre09.go b/drivers/qemu/driver_pre09.go deleted file mode 100644 index eeef3d29f..000000000 --- a/drivers/qemu/driver_pre09.go +++ /dev/null @@ -1,46 +0,0 @@ -package qemu - -import ( - "fmt" - "time" - - "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { - handle, err := state.UnmarshalPre09HandleID(h.DriverState) - if err != nil { - return fmt.Errorf("failed to decode pre09 driver handle: %v", err) - } - - reattach, err := pstructs.ReattachConfigToGoPlugin(handle.ReattachConfig()) - if err != nil { - return fmt.Errorf("failed to decode reattach config from pre09 handle: %v", err) - } - - exec, pluginClient, err := executor.ReattachToPre09Executor(reattach, - d.logger.With("task_name", h.Config.Name, "alloc_id", h.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_name", h.Config.Name) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - th := &taskHandle{ - exec: exec, - pid: reattach.Pid, - pluginClient: pluginClient, - taskConfig: h.Config, - procState: drivers.TaskStateRunning, - startedAt: time.Now(), - exitResult: &drivers.ExitResult{}, - logger: d.logger, - } - - d.tasks.Set(h.Config.ID, th) - - go th.run() - return nil -} diff --git a/drivers/rawexec/driver.go b/drivers/rawexec/driver.go index 85005924c..e661a1994 100644 --- a/drivers/rawexec/driver.go +++ b/drivers/rawexec/driver.go @@ -246,11 +246,6 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("handle cannot be nil") } - // COMPAT(0.10): pre 0.9 upgrade path check - if handle.Version == 0 { - return d.recoverPre09Task(handle) - } - // If already attached to handle there's nothing to recover. if _, ok := d.tasks.Get(handle.Config.ID); ok { d.logger.Trace("nothing to recover; task already exists", diff --git a/drivers/rawexec/driver_pre09.go b/drivers/rawexec/driver_pre09.go deleted file mode 100644 index 90001df34..000000000 --- a/drivers/rawexec/driver_pre09.go +++ /dev/null @@ -1,47 +0,0 @@ -package rawexec - -import ( - "fmt" - "time" - - "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/drivers" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" -) - -func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { - handle, err := state.UnmarshalPre09HandleID(h.DriverState) - if err != nil { - return fmt.Errorf("failed to decode pre09 driver handle: %v", err) - } - - reattach, err := pstructs.ReattachConfigToGoPlugin(handle.ReattachConfig()) - if err != nil { - return fmt.Errorf("failed to decode reattach config from pre09 handle: %v", err) - } - - exec, pluginClient, err := executor.ReattachToPre09Executor(reattach, - d.logger.With("task_name", h.Config.Name, "alloc_id", h.Config.AllocID)) - if err != nil { - d.logger.Error("failed to reattach to executor", "error", err, "task_name", h.Config.Name) - return fmt.Errorf("failed to reattach to executor: %v", err) - } - - th := &taskHandle{ - exec: exec, - pid: reattach.Pid, - pluginClient: pluginClient, - taskConfig: h.Config, - procState: drivers.TaskStateRunning, - startedAt: time.Now(), - exitResult: &drivers.ExitResult{}, - logger: d.logger, - doneCh: make(chan struct{}), - } - - d.tasks.Set(h.Config.ID, th) - - go th.run() - return nil -} diff --git a/drivers/shared/executor/legacy_executor_wrapper.go b/drivers/shared/executor/legacy_executor_wrapper.go deleted file mode 100644 index 26c74559e..000000000 --- a/drivers/shared/executor/legacy_executor_wrapper.go +++ /dev/null @@ -1,221 +0,0 @@ -package executor - -import ( - "encoding/gob" - "fmt" - "net/rpc" - "os" - "syscall" - "time" - - hclog "github.com/hashicorp/go-hclog" - plugin "github.com/hashicorp/go-plugin" - cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/plugins/drivers" - "golang.org/x/net/context" -) - -const ( - // pre09DockerSignal is used in executor.Shutdown to know if it should - // call the ShutDown RPC on the pre09 executor - pre09DockerSignal = "docker" -) - -// Registering these types since we have to serialize and de-serialize the Task -// structs over the wire between drivers and the executor. -func init() { - gob.Register([]interface{}{}) - gob.Register(map[string]interface{}{}) - gob.Register([]map[string]string{}) - gob.Register([]map[string]int{}) - gob.Register(syscall.Signal(0x1)) -} - -type legacyExecutorWrapper struct { - client *pre09ExecutorRPC - logger hclog.Logger -} - -// validate that legacyExecutorWrapper is an executor -var _ Executor = (*legacyExecutorWrapper)(nil) - -func (l *legacyExecutorWrapper) Launch(launchCmd *ExecCommand) (*ProcessState, error) { - return nil, fmt.Errorf("operation not supported for legacy exec wrapper") -} - -func (l *legacyExecutorWrapper) Wait(ctx context.Context) (*ProcessState, error) { - ps, err := l.client.Wait() - if err != nil { - return nil, err - } - - return &ProcessState{ - Pid: ps.Pid, - ExitCode: ps.ExitCode, - Signal: ps.Signal, - Time: ps.Time, - }, nil -} - -func (l *legacyExecutorWrapper) Shutdown(signal string, gracePeriod time.Duration) error { - // The legacy docker driver only used the executor to start a syslog server - // for logging. Thus calling ShutDown for docker will always return an error - // because it never started a process through the executor. If signal is set - // to 'docker' then we'll skip the ShutDown RPC and just call Exit. - // - // This is painful to look at but will only be around a few releases - if signal != pre09DockerSignal { - if err := l.client.ShutDown(); err != nil { - return err - } - } - - if err := l.client.Exit(); err != nil { - return err - } - return nil -} - -func (l *legacyExecutorWrapper) UpdateResources(*drivers.Resources) error { - return fmt.Errorf("operation not supported for legacy exec wrapper") -} - -func (l *legacyExecutorWrapper) Version() (*ExecutorVersion, error) { - v, err := l.client.Version() - if err != nil { - return nil, err - } - - return &ExecutorVersion{ - Version: v.Version, - }, nil -} - -func (l *legacyExecutorWrapper) Stats(ctx context.Context, interval time.Duration) (<-chan *cstructs.TaskResourceUsage, error) { - ch := make(chan *cstructs.TaskResourceUsage, 1) - stats, err := l.client.Stats() - if err != nil { - close(ch) - return nil, err - } - select { - case ch <- stats: - default: - } - go l.handleStats(ctx, interval, ch) - return ch, nil -} - -func (l *legacyExecutorWrapper) handleStats(ctx context.Context, interval time.Duration, ch chan *cstructs.TaskResourceUsage) { - defer close(ch) - ticker := time.NewTicker(interval) - for range ticker.C { - stats, err := l.client.Stats() - if err != nil { - if err == rpc.ErrShutdown { - return - } - l.logger.Warn("stats collection from legacy executor failed, waiting for next interval", "error", err) - continue - } - if stats != nil { - select { - case ch <- stats: - default: - } - } - - } -} - -func (l *legacyExecutorWrapper) Signal(s os.Signal) error { - return l.client.Signal(s) -} - -func (l *legacyExecutorWrapper) Exec(deadline time.Time, cmd string, args []string) ([]byte, int, error) { - return l.client.Exec(deadline, cmd, args) -} - -func (l *legacyExecutorWrapper) ExecStreaming(ctx context.Context, cmd []string, tty bool, - stream drivers.ExecTaskStream) error { - return fmt.Errorf("operation not supported for legacy exec wrapper") -} - -type pre09ExecutorRPC struct { - client *rpc.Client - logger hclog.Logger -} - -type pre09ExecCmdArgs struct { - Deadline time.Time - Name string - Args []string -} - -type pre09ExecCmdReturn struct { - Output []byte - Code int -} - -func (e *pre09ExecutorRPC) Wait() (*ProcessState, error) { - var ps ProcessState - err := e.client.Call("Plugin.Wait", new(interface{}), &ps) - return &ps, err -} - -func (e *pre09ExecutorRPC) ShutDown() error { - return e.client.Call("Plugin.ShutDown", new(interface{}), new(interface{})) -} - -func (e *pre09ExecutorRPC) Exit() error { - return e.client.Call("Plugin.Exit", new(interface{}), new(interface{})) -} - -func (e *pre09ExecutorRPC) Version() (*ExecutorVersion, error) { - var version ExecutorVersion - err := e.client.Call("Plugin.Version", new(interface{}), &version) - return &version, err -} - -func (e *pre09ExecutorRPC) Stats() (*cstructs.TaskResourceUsage, error) { - var resourceUsage cstructs.TaskResourceUsage - err := e.client.Call("Plugin.Stats", new(interface{}), &resourceUsage) - return &resourceUsage, err -} - -func (e *pre09ExecutorRPC) Signal(s os.Signal) error { - return e.client.Call("Plugin.Signal", &s, new(interface{})) -} - -func (e *pre09ExecutorRPC) Exec(deadline time.Time, name string, args []string) ([]byte, int, error) { - req := pre09ExecCmdArgs{ - Deadline: deadline, - Name: name, - Args: args, - } - var resp *pre09ExecCmdReturn - err := e.client.Call("Plugin.Exec", req, &resp) - if resp == nil { - return nil, 0, err - } - return resp.Output, resp.Code, err -} - -type pre09ExecutorPlugin struct { - logger hclog.Logger -} - -func newPre09ExecutorPlugin(logger hclog.Logger) plugin.Plugin { - return &pre09ExecutorPlugin{logger: logger} -} - -func (p *pre09ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) { - return &legacyExecutorWrapper{ - client: &pre09ExecutorRPC{client: c, logger: p.logger}, - logger: p.logger, - }, nil -} - -func (p *pre09ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) { - return nil, fmt.Errorf("client only supported") -} diff --git a/drivers/shared/executor/plugins.go b/drivers/shared/executor/plugins.go index a572355be..0e3b977e9 100644 --- a/drivers/shared/executor/plugins.go +++ b/drivers/shared/executor/plugins.go @@ -30,12 +30,6 @@ func GetPluginMap(logger hclog.Logger, fsIsolation bool) map[string]plugin.Plugi } } -func GetPre09PluginMap(logger hclog.Logger, fsIsolation bool) map[string]plugin.Plugin { - return map[string]plugin.Plugin{ - "executor": newPre09ExecutorPlugin(logger), - } -} - // ExecutorReattachConfig is the config that we serialize and de-serialize and // store in disk type PluginReattachConfig struct { diff --git a/drivers/shared/executor/utils.go b/drivers/shared/executor/utils.go index 1e3cb74b9..237152a8b 100644 --- a/drivers/shared/executor/utils.go +++ b/drivers/shared/executor/utils.go @@ -80,19 +80,6 @@ func ReattachToExecutor(reattachConfig *plugin.ReattachConfig, logger hclog.Logg return newExecutorClient(config, logger) } -// ReattachToPre09Executor creates a plugin client that reattaches to an existing -// pre 0.9 Nomad executor -func ReattachToPre09Executor(reattachConfig *plugin.ReattachConfig, logger hclog.Logger) (Executor, *plugin.Client, error) { - config := &plugin.ClientConfig{ - HandshakeConfig: base.Handshake, - Reattach: reattachConfig, - Plugins: GetPre09PluginMap(logger, false), - AllowedProtocols: []plugin.Protocol{plugin.ProtocolNetRPC}, - Logger: logger.Named("executor"), - } - return newExecutorClient(config, logger) -} - func newExecutorClient(config *plugin.ClientConfig, logger hclog.Logger) (Executor, *plugin.Client, error) { executorClient := plugin.NewClient(config) rpcClient, err := executorClient.Client() diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index ecdacb4cc..5aeb8492d 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -23,7 +23,6 @@ import ( _ "github.com/hashicorp/nomad/e2e/namespaces" _ "github.com/hashicorp/nomad/e2e/networking" _ "github.com/hashicorp/nomad/e2e/nodedrain" - _ "github.com/hashicorp/nomad/e2e/nomad09upgrade" _ "github.com/hashicorp/nomad/e2e/nomadexec" _ "github.com/hashicorp/nomad/e2e/oversubscription" _ "github.com/hashicorp/nomad/e2e/parameterized" diff --git a/e2e/nomad09upgrade/docker.nomad b/e2e/nomad09upgrade/docker.nomad deleted file mode 100644 index 5d0bf10a7..000000000 --- a/e2e/nomad09upgrade/docker.nomad +++ /dev/null @@ -1,23 +0,0 @@ -job "sleep" { - datacenters = ["dc1"] - - constraint { - attribute = "${attr.kernel.name}" - value = "linux" - } - - group "sleep" { - task "sleep" { - driver = "docker" - - config { - image = "redis:5.0" - } - - resources { - cpu = 100 - memory = 32 - } - } - } -} diff --git a/e2e/nomad09upgrade/exec.nomad b/e2e/nomad09upgrade/exec.nomad deleted file mode 100644 index f3526177d..000000000 --- a/e2e/nomad09upgrade/exec.nomad +++ /dev/null @@ -1,24 +0,0 @@ -job "sleep" { - datacenters = ["dc1"] - - constraint { - attribute = "${attr.kernel.name}" - value = "linux" - } - - group "sleep" { - task "sleep" { - driver = "exec" - - config { - command = "sleep" - args = ["10000"] - } - - resources { - cpu = 100 - memory = 32 - } - } - } -} diff --git a/e2e/nomad09upgrade/rawexec.nomad b/e2e/nomad09upgrade/rawexec.nomad deleted file mode 100644 index cfdaa6bac..000000000 --- a/e2e/nomad09upgrade/rawexec.nomad +++ /dev/null @@ -1,24 +0,0 @@ -job "sleep" { - datacenters = ["dc1"] - - constraint { - attribute = "${attr.kernel.name}" - value = "linux" - } - - group "sleep" { - task "sleep" { - driver = "raw_exec" - - config { - command = "sleep" - args = ["10000"] - } - - resources { - cpu = 100 - memory = 32 - } - } - } -} diff --git a/e2e/nomad09upgrade/upgrade.go b/e2e/nomad09upgrade/upgrade.go deleted file mode 100644 index a428512b2..000000000 --- a/e2e/nomad09upgrade/upgrade.go +++ /dev/null @@ -1,232 +0,0 @@ -//go:build linux -// +build linux - -package nomad09upgrade - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - "time" - - getter "github.com/hashicorp/go-getter" - "github.com/hashicorp/nomad/ci" - "github.com/hashicorp/nomad/e2e/e2eutil" - "github.com/hashicorp/nomad/e2e/execagent" - "github.com/hashicorp/nomad/e2e/framework" - "github.com/hashicorp/nomad/helper/discover" - "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/testutil" - "github.com/stretchr/testify/require" -) - -func init() { - framework.AddSuites(&framework.TestSuite{ - Component: "nomad09upgrade", - CanRunLocal: true, - Cases: []framework.TestCase{ - &UpgradePathTC{}, - }, - }) -} - -var ( - nomadVersions = [...]string{ - "0.8.7", - "0.8.6", - "0.8.5", - "0.8.4", - "0.8.3", - "0.8.2", - "0.8.1", - "0.8.0", - } -) - -type UpgradePathTC struct { - framework.TC - - binDir string - bin string -} - -type upgradeAgents struct { - origAgent *execagent.NomadAgent - targetAgent *execagent.NomadAgent -} - -func (tc *UpgradePathTC) newNomadServer(t *testing.T, ver string) (*upgradeAgents, error) { - binPath := filepath.Join(tc.binDir, ver, "nomad") - srv, err := execagent.NewMixedAgent(binPath) - if err != nil { - return nil, err - } - - w := testlog.NewWriter(t) - srv.Cmd.Stdout = w - srv.Cmd.Stderr = w - - // Target should copy everything but the binary to target - target, err := execagent.NewMixedAgent(tc.bin) - if err != nil { - return nil, err - } - target.Cmd.Stdout = w - target.Cmd.Stderr = w - - agents := &upgradeAgents{ - origAgent: srv, - targetAgent: target, - } - return agents, nil -} - -// BeforeAll downloads all of the desired nomad versions to test against -func (tc *UpgradePathTC) BeforeAll(f *framework.F) { - // Upgrade tests currently fail because the nomad binary isn't found by - // discover.NomadExecutable(). Ensure that nomad binary is available - // and discoverable and enable this test - f.T().Skip("upgrade tests are expected to fail. TODO: Fix") - - bin, err := discover.NomadExecutable() - f.NoError(err) - tc.bin = bin - dir, err := ioutil.TempDir("", "") - f.NoError(err) - - tc.binDir = dir - for _, ver := range nomadVersions { - verBin := filepath.Join(tc.binDir, ver) - f.NoError(os.Mkdir(verBin, 0755)) - f.NoError( - getter.Get(verBin, fmt.Sprintf( - "https://releases.hashicorp.com/nomad/%s/nomad_%s_linux_amd64.zip", - ver, ver, - ))) - f.T().Logf("downloaded nomad version %s to %s", ver, verBin) - } -} - -// AfterAll cleans up the downloaded nomad binaries -func (tc *UpgradePathTC) AfterAll(f *framework.F) { - os.RemoveAll(tc.binDir) -} - -func (tc *UpgradePathTC) TestRawExecTaskUpgrade(f *framework.F) { - for _, ver := range nomadVersions { - ver := ver - f.T().Run(ver, func(t *testing.T) { - ci.Parallel(t) - tc.testUpgradeForJob(t, ver, "nomad09upgrade/rawexec.nomad") - }) - } -} - -func (tc *UpgradePathTC) TestExecTaskUpgrade(f *framework.F) { - for _, ver := range nomadVersions { - ver := ver - f.T().Run(ver, func(t *testing.T) { - ci.Parallel(t) - tc.testUpgradeForJob(t, ver, "nomad09upgrade/exec.nomad") - }) - } -} - -func (tc *UpgradePathTC) TestDockerTaskUpgrade(f *framework.F) { - for _, ver := range nomadVersions { - ver := ver - f.T().Run(ver, func(t *testing.T) { - ci.Parallel(t) - tc.testUpgradeForJob(t, ver, "nomad09upgrade/docker.nomad") - }) - } -} - -func (tc *UpgradePathTC) testUpgradeForJob(t *testing.T, ver string, jobfile string) { - require := require.New(t) - // Start a nomad agent for the given version - agents, err := tc.newNomadServer(t, ver) - require.NoError(err) - t.Logf("launching v%s nomad agent", ver) - require.NoError(agents.origAgent.Start()) - - // Wait for the agent to be ready - client, err := agents.origAgent.Client() - require.NoError(err) - e2eutil.WaitForNodesReady(t, client, 1) - - // Register a sleep job - jobID := "sleep-" + uuid.Generate()[:8] - t.Logf("registering exec job with id %s", jobID) - e2eutil.RegisterAndWaitForAllocs(t, client, jobfile, jobID, "") - allocs, _, err := client.Jobs().Allocations(jobID, false, nil) - require.NoError(err) - require.Len(allocs, 1) - - // Wait for sleep job to transition to running - id := allocs[0].ID - e2eutil.WaitForAllocRunning(t, client, id) - - // Stop the agent, leaving the sleep job running - require.NoError(agents.origAgent.Stop()) - - // Start a nomad agent with the to be tested nomad binary - t.Logf("launching test nomad agent") - require.NoError(agents.targetAgent.Start()) - - // Wait for the agent to be ready - e2eutil.WaitForNodesReady(t, client, 1) - - // Make sure the same allocation still exists - alloc, _, err := client.Allocations().Info(id, nil) - require.NoError(err) - // Pull stats from the allocation, testing that new code can interface with - // the old stats driver apis - testutil.WaitForResult(func() (bool, error) { - stats, err := client.Allocations().Stats(alloc, nil) - if err != nil { - return false, err - } - - return stats.ResourceUsage.MemoryStats.RSS > 0, fmt.Errorf("RSS for task should be greater than 0") - }, func(err error) { - require.NoError(err) - }) - - // Deregister the job. This tests that the new code can properly tear down - // upgraded allocs - _, _, err = client.Jobs().Deregister(alloc.JobID, true, nil) - require.NoError(err) - testutil.WaitForResult(func() (bool, error) { - j, _, _ := client.Jobs().Info(jobID, nil) - if j == nil { - return true, nil - } - time.Sleep(time.Millisecond * 100) - return false, fmt.Errorf("job with id %q should be purged", jobID) - }, func(err error) { - require.NoError(err) - }) - - // Check that the task dir mounts have been removed - testutil.WaitForResult(func() (bool, error) { - defer client.System().GarbageCollect() - time.Sleep(time.Millisecond * 100) - data, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return false, err - } - - return !strings.Contains(string(data), id), fmt.Errorf("taskdir mounts should be cleaned up, but found mount for id %q:\n%s", id, string(data)) - }, func(err error) { - require.NoError(err) - }) - - // Cleanup - agents.targetAgent.Stop() - agents.targetAgent.Destroy() -} diff --git a/e2e/nomad09upgrade/upgrade_basic.go b/e2e/nomad09upgrade/upgrade_basic.go deleted file mode 100644 index e77783e58..000000000 --- a/e2e/nomad09upgrade/upgrade_basic.go +++ /dev/null @@ -1,4 +0,0 @@ -//go:build !linux -// +build !linux - -package nomad09upgrade diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index d6c1ab6c1..57a42fe75 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -181,6 +181,16 @@ The new cgroup file system layout will look like the following: ├── 8b8da4cf-8ebf-b578-0bcf-77190749abf3.redis.scope └── a8c8e495-83c8-311b-4657-e6e3127e98bc.example.scope ``` +#### Support for pre-0.9 Tasks Removed + +Running tasks that were created on clusters from Nomad version 0.9 or +earlier will fail to restore after upgrading a cluster to Nomad +1.3.0. To safely upgrade without unplanned interruptions, force these +tasks to be rescheduled by `nomad alloc stop` before upgrading. Note +this only applies to tasks that have been running continuously from +before 0.9 without rescheduling. Jobs that were created before 0.9 but +have had tasks replaced over time after 0.9 will operate normally +during the upgrade. ## Nomad 1.2.6, 1.1.12, and 1.0.18