Merge pull request #497 from hashicorp/f-windows-executor-tests

Fix tests for nomad/client/driver/executor package to work on Windows.

client/allocdir/alloc_dir.go

@@ -108,24 +108,40 @@ func (d *AllocDir) Build(tasks []*structs.Task) error {
 	return nil
 }
 
-// Embed takes a mapping of absolute directory paths on the host to their
+// Embed takes a mapping of absolute directory or file paths on the host to
-// intended, relative location within the task directory. Embed attempts
+// their intended, relative location within the task directory. Embed attempts
 // hardlink and then defaults to copying. If the path exists on the host and
 // can't be embeded an error is returned.
-func (d *AllocDir) Embed(task string, dirs map[string]string) error {
+func (d *AllocDir) Embed(task string, entries map[string]string) error {
 	taskdir, ok := d.TaskDirs[task]
 	if !ok {
 		return fmt.Errorf("Task directory doesn't exist for task %v", task)
 	}
 
 	subdirs := make(map[string]string)
-	for source, dest := range dirs {
+	for source, dest := range entries {
 		// Check to see if directory exists on host.
 		s, err := os.Stat(source)
 		if os.IsNotExist(err) {
 			continue
 		}
 
+		// Embedding a single file
+		if !s.IsDir() {
+			destDir := filepath.Join(taskdir, filepath.Dir(dest))
+			if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil {
+				return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err)
+			}
+
+			// Copy the file.
+			taskEntry := filepath.Join(destDir, filepath.Base(dest))
+			if err := d.linkOrCopy(source, taskEntry, s.Mode().Perm()); err != nil {
+				return err
+			}
+
+			continue
+		}
+
 		// Create destination directory.
 		destDir := filepath.Join(taskdir, dest)
 		if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil {
@@ -133,12 +149,12 @@ func (d *AllocDir) Embed(task string, dirs map[string]string) error {
 		}
 
 		// Enumerate the files in source.
-		entries, err := ioutil.ReadDir(source)
+		dirEntries, err := ioutil.ReadDir(source)
 		if err != nil {
 			return fmt.Errorf("Couldn't read directory %v: %v", source, err)
 		}
 
-		for _, entry := range entries {
+		for _, entry := range dirEntries {
 			hostEntry := filepath.Join(source, entry.Name())
 			taskEntry := filepath.Join(destDir, filepath.Base(hostEntry))
 			if entry.IsDir() {

client/driver/executor/exec.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"os/exec"
 	"path/filepath"
+	"strings"
 
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -33,6 +34,11 @@ import (
 
 var errNoResources = fmt.Errorf("No resources are associated with this task")
 
+// If testModeEnvVar is set in a process's environment variables, the
+// LinuxExecutor will detect it and inject the current binary into the chroot.
+// This enables using the test binary in tests to provide portability.
+var testModeEnvVar = "NOMAD_EXECUTOR_TEST_ONLY_13871827980214"
+
 // Executor is an interface that any platform- or capability-specific exec
 // wrapper must implement. You should not need to implement a Java executor.
 // Rather, you would implement a cgroups executor that the Java driver will use.
@@ -106,3 +112,18 @@ func OpenId(id string) (Executor, error) {
 	}
 	return executor, nil
 }
+
+// isTest returns whether the cmd is a test binary.
+func isTest(cmd *exec.Cmd) bool {
+	if cmd == nil {
+		return false
+	}
+
+	for _, env := range cmd.Env {
+		if strings.HasPrefix(env, testModeEnvVar) {
+			return true
+		}
+	}
+
+	return false
+}

client/driver/executor/exec_linux.go

@@ -252,6 +252,13 @@ func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocD
 		return err
 	}
 
+	// Embed ourselves if this is a test. This needs to be done so the test
+	// binary is inside the chroot.
+	if isTest(&e.cmd) {
+		bin := e.cmd.Args[0]
+		alloc.Embed(taskName, map[string]string{bin: bin})
+	}
+
 	if err := alloc.Embed(taskName, chrootEnv); err != nil {
 		return err
 	}

client/driver/executor/test_harness_test.go

@@ -5,7 +5,9 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -14,6 +16,96 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
+// testBinary is the path to the running test binary
+var testBinary = func() string {
+	abs, err := filepath.Abs(os.Args[0])
+	if err != nil {
+		return err.Error()
+	}
+
+	return abs
+}()
+
+func TestMain(m *testing.M) {
+	// The tests in this package recursively execute the test binary produced
+	// by go test. The TEST_MAIN environment variable controls the recursive
+	// execution.
+	switch tm := os.Getenv(testModeEnvVar); tm {
+	case "":
+		os.Exit(m.Run())
+	case "app":
+		appMain()
+	default:
+		fmt.Fprintf(os.Stderr,
+			"Unexpected value for test mode environment variable, %q\n", tm)
+		os.Exit(1)
+	}
+}
+
+// setTestAppEnv sets the environement of cmd for a recursive call into
+// TestMain.
+func setTestAppEnv(cmd *exec.Cmd) {
+	cmd.Env = append(os.Environ(), fmt.Sprintf("%v=app", testModeEnvVar))
+}
+
+func appMain() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "no command provided")
+		os.Exit(1)
+	}
+
+	args := os.Args[1:]
+
+	// popArg removes the first argument from args and returns it.
+	popArg := func() string {
+		s := args[0]
+		args = args[1:]
+		return s
+	}
+
+	// execute a sequence of operations from args
+	for len(args) > 0 {
+		switch cmd := popArg(); cmd {
+
+		case "sleep":
+			// sleep <dur>: sleep for a duration indicated by the first
+			// argument
+			if len(args) < 1 {
+				fmt.Fprintln(os.Stderr, "expected arg for sleep")
+				os.Exit(1)
+			}
+			dur, err := time.ParseDuration(popArg())
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "could not parse sleep time: %v", err)
+				os.Exit(1)
+			}
+			time.Sleep(dur)
+
+		case "echo":
+			// echo <msg ...>: write the remaining arguments to stdout each
+			// separated by a single space and followed by a newline.
+			fmt.Println(strings.Join(args, " "))
+			args = args[:0]
+
+		case "write":
+			// write <msg> <file>: write a message to a file. The first
+			// argument is the msg. The second argument is the path to the
+			// target file.
+			if len(args) < 2 {
+				fmt.Fprintln(os.Stderr, "expected two args for write")
+				os.Exit(1)
+			}
+			msg := popArg()
+			file := popArg()
+			ioutil.WriteFile(file, []byte(msg), 0666)
+
+		default:
+			fmt.Fprintln(os.Stderr, "unknown command:", cmd)
+			os.Exit(1)
+		}
+	}
+}
+
 var (
 	constraint = &structs.Resources{
 		CPU:      250,
@@ -45,9 +137,10 @@ func testExecutor(t *testing.T, buildExecutor func() Executor, compatible func(*
 	}
 
 	command := func(name string, args ...string) Executor {
-		b := buildExecutor()
+		e := buildExecutor()
-		SetCommand(b, name, args)
+		SetCommand(e, name, args)
-		return b
+		setTestAppEnv(e.Command())
+		return e
 	}
 
 	Executor_Start_Invalid(t, command)
@@ -55,6 +148,7 @@ func testExecutor(t *testing.T, buildExecutor func() Executor, compatible func(*
 	Executor_Start_Wait(t, command)
 	Executor_Start_Kill(t, command)
 	Executor_Open(t, command, buildExecutor)
+	Executor_Open_Invalid(t, command, buildExecutor)
 }
 
 type buildExecCommand func(name string, args ...string) Executor
@@ -79,7 +173,7 @@ func Executor_Start_Invalid(t *testing.T, command buildExecCommand) {
 }
 
 func Executor_Start_Wait_Failure_Code(t *testing.T, command buildExecCommand) {
-	e := command("/bin/date", "-invalid")
+	e := command(testBinary, "fail")
 
 	if err := e.Limit(constraint); err != nil {
 		log.Panicf("Limit() failed: %v", err)
@@ -112,8 +206,7 @@ func Executor_Start_Wait(t *testing.T, command buildExecCommand) {
 	expected := "hello world"
 	file := filepath.Join(allocdir.TaskLocal, "output.txt")
 	absFilePath := filepath.Join(taskDir, file)
-	cmd := fmt.Sprintf(`/bin/sleep 1 ; echo -n %v > %v`, expected, file)
+	e := command(testBinary, "sleep", "1s", "write", expected, file)
-	e := command("/bin/bash", "-c", cmd)
 
 	if err := e.Limit(constraint); err != nil {
 		log.Panicf("Limit() failed: %v", err)
@@ -152,7 +245,7 @@ func Executor_Start_Kill(t *testing.T, command buildExecCommand) {
 	}
 
 	filePath := filepath.Join(taskDir, "output")
-	e := command("/bin/bash", "-c", "sleep 1 ; echo \"failure\" > "+filePath)
+	e := command(testBinary, "sleep", "1s", "write", "failure", filePath)
 
 	if err := e.Limit(constraint); err != nil {
 		log.Panicf("Limit() failed: %v", err)
@@ -190,8 +283,7 @@ func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func() Ex
 	expected := "hello world"
 	file := filepath.Join(allocdir.TaskLocal, "output.txt")
 	absFilePath := filepath.Join(taskDir, file)
-	cmd := fmt.Sprintf(`/bin/sleep 1 ; echo -n %v > %v`, expected, file)
+	e := command(testBinary, "sleep", "1s", "write", expected, file)
-	e := command("/bin/bash", "-c", cmd)
 
 	if err := e.Limit(constraint); err != nil {
 		log.Panicf("Limit() failed: %v", err)
@@ -232,7 +324,7 @@ func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func() Ex
 
 func Executor_Open_Invalid(t *testing.T, command buildExecCommand, newExecutor func() Executor) {
 	task, alloc := mockAllocDir(t)
-	e := command("echo", "foo")
+	e := command(testBinary, "echo", "foo")
 
 	if err := e.Limit(constraint); err != nil {
 		log.Panicf("Limit() failed: %v", err)
@@ -251,8 +343,19 @@ func Executor_Open_Invalid(t *testing.T, command buildExecCommand, newExecutor f
 		log.Panicf("ID() failed: %v", err)
 	}
 
+	// Kill the task because some OSes (windows) will not let us destroy the
+	// alloc (below) if the task is still running.
+	if err := e.ForceStop(); err != nil {
+		log.Panicf("e.ForceStop() failed: %v", err)
+	}
+
+	// Wait until process is actually gone, we don't care what the result was.
+	e.Wait()
+
 	// Destroy the allocdir which removes the exit code.
-	alloc.Destroy()
+	if err := alloc.Destroy(); err != nil {
+		log.Panicf("alloc.Destroy() failed: %v", err)
+	}
 
 	e2 := newExecutor()
 	if err := e2.Open(id); err == nil {