Add Valid command to spawner and make executors check when opening

client/driver/executor/exec_basic.go

@@ -89,7 +89,7 @@ func (e *BasicExecutor) Open(id string) error {
 
 	// Setup the executor.
 	e.spawn = &spawn
-	return nil
+	return e.spawn.Valid()
 }
 
 func (e *BasicExecutor) Wait() error {

client/driver/executor/exec_linux.go

@@ -95,8 +95,7 @@ func (e *LinuxExecutor) Open(id string) error {
 	e.groups = execID.Groups
 	e.spawn = execID.Spawn
 	e.taskDir = execID.TaskDir
-
+	return e.spawn.Valid()
-	return nil
 }
 
 func (e *LinuxExecutor) ID() (string, error) {

client/driver/executor/test_harness.go

@@ -229,3 +229,33 @@ func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func() Ex
 		log.Panicf("Command output incorrectly: want %v; got %v", expected, act)
 	}
 }
+
+func Executor_Open_Invalid(t *testing.T, command buildExecCommand, newExecutor func() Executor) {
+	task, alloc := mockAllocDir(t)
+	e := command("echo", "foo")
+
+	if err := e.Limit(constraint); err != nil {
+		log.Panicf("Limit() failed: %v", err)
+	}
+
+	if err := e.ConfigureTaskDir(task, alloc); err != nil {
+		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
+	}
+
+	if err := e.Start(); err != nil {
+		log.Panicf("Start() failed: %v", err)
+	}
+
+	id, err := e.ID()
+	if err != nil {
+		log.Panicf("ID() failed: %v", err)
+	}
+
+	// Destroy the allocdir which removes the exit code.
+	alloc.Destroy()
+
+	e2 := newExecutor()
+	if err := e2.Open(id); err == nil {
+		log.Panicf("Open(%v) should have failed", id)
+	}
+}

client/driver/spawn/spawn.go

@@ -285,3 +285,18 @@ func (s *Spawner) readExitCode() (int, error) {
 
 	return exitStatus.ExitCode, nil
 }
+
+// Valid checks that the state of the Spawner is valid and that a subsequent
+// Wait could be called. This is useful to call when reopening a Spawner
+// throught client restarts. If Valid a nil error is returned.
+func (s *Spawner) Valid() error {
+	if s.Alive() {
+		return nil
+	}
+
+	if _, err := s.readExitCode(); err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("Spawner not alive and exit code not written")
+}

client/driver/spawn/spawn_posix.go

@@ -2,12 +2,18 @@
 
 package spawn
 
-import "syscall"
+import (
+	"os"
+	"syscall"
+)
 
 func (s *Spawner) Alive() bool {
 	if s.spawn == nil {
+		var err error
+		if s.spawn, err = os.FindProcess(s.SpawnPid); err != nil {
 			return false
 		}
+	}
 
 	err := s.spawn.Signal(syscall.Signal(0))
 	return err == nil

client/driver/spawn/spawn_test.go

@@ -298,3 +298,71 @@ func TestSpawn_DeadSpawnDaemon_NonParent(t *testing.T) {
 		t.Fatalf("Wait() should have failed: %v", err)
 	}
 }
+
+func TestSpawn_Valid_TaskRunning(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("TempFile() failed")
+	}
+	defer os.Remove(f.Name())
+
+	spawn := NewSpawner(f.Name())
+	spawn.SetCommand(exec.Command("sleep", "2"))
+	if err := spawn.Spawn(nil); err != nil {
+		t.Fatalf("Spawn() failed %v", err)
+	}
+
+	if err := spawn.Valid(); err != nil {
+		t.Fatalf("Valid() failed: %v", err)
+	}
+
+	if _, err := spawn.Wait(); err != nil {
+		t.Fatalf("Wait() failed %v", err)
+	}
+}
+
+func TestSpawn_Valid_TaskExit_ExitCode(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("TempFile() failed")
+	}
+	defer os.Remove(f.Name())
+
+	spawn := NewSpawner(f.Name())
+	spawn.SetCommand(exec.Command("echo", "foo"))
+	if err := spawn.Spawn(nil); err != nil {
+		t.Fatalf("Spawn() failed %v", err)
+	}
+
+	if _, err := spawn.Wait(); err != nil {
+		t.Fatalf("Wait() failed %v", err)
+	}
+
+	if err := spawn.Valid(); err != nil {
+		t.Fatalf("Valid() failed: %v", err)
+	}
+}
+
+func TestSpawn_Valid_TaskExit_NoExitCode(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("TempFile() failed")
+	}
+
+	spawn := NewSpawner(f.Name())
+	spawn.SetCommand(exec.Command("echo", "foo"))
+	if err := spawn.Spawn(nil); err != nil {
+		t.Fatalf("Spawn() failed %v", err)
+	}
+
+	if _, err := spawn.Wait(); err != nil {
+		t.Fatalf("Wait() failed %v", err)
+	}
+
+	// Delete the file so that it can't find the exit code.
+	os.Remove(f.Name())
+
+	if err := spawn.Valid(); err == nil {
+		t.Fatalf("Valid() should have failed")
+	}
+}