diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 2eacf4398..7fe318a6b 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -108,24 +108,40 @@ func (d *AllocDir) Build(tasks []*structs.Task) error { return nil } -// Embed takes a mapping of absolute directory paths on the host to their -// intended, relative location within the task directory. Embed attempts +// Embed takes a mapping of absolute directory or file paths on the host to +// their intended, relative location within the task directory. Embed attempts // hardlink and then defaults to copying. If the path exists on the host and // can't be embeded an error is returned. -func (d *AllocDir) Embed(task string, dirs map[string]string) error { +func (d *AllocDir) Embed(task string, entries map[string]string) error { taskdir, ok := d.TaskDirs[task] if !ok { return fmt.Errorf("Task directory doesn't exist for task %v", task) } subdirs := make(map[string]string) - for source, dest := range dirs { + for source, dest := range entries { // Check to see if directory exists on host. s, err := os.Stat(source) if os.IsNotExist(err) { continue } + // Embedding a single file + if !s.IsDir() { + destDir := filepath.Join(taskdir, filepath.Dir(dest)) + if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil { + return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err) + } + + // Copy the file. + taskEntry := filepath.Join(destDir, filepath.Base(dest)) + if err := d.linkOrCopy(source, taskEntry, s.Mode().Perm()); err != nil { + return err + } + + continue + } + // Create destination directory. destDir := filepath.Join(taskdir, dest) if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil { @@ -133,12 +149,12 @@ func (d *AllocDir) Embed(task string, dirs map[string]string) error { } // Enumerate the files in source. - entries, err := ioutil.ReadDir(source) + dirEntries, err := ioutil.ReadDir(source) if err != nil { return fmt.Errorf("Couldn't read directory %v: %v", source, err) } - for _, entry := range entries { + for _, entry := range dirEntries { hostEntry := filepath.Join(source, entry.Name()) taskEntry := filepath.Join(destDir, filepath.Base(hostEntry)) if entry.IsDir() { diff --git a/client/driver/executor/exec.go b/client/driver/executor/exec.go index c514890ef..fd8122367 100644 --- a/client/driver/executor/exec.go +++ b/client/driver/executor/exec.go @@ -24,6 +24,7 @@ import ( "fmt" "os/exec" "path/filepath" + "strings" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/nomad/structs" @@ -33,6 +34,11 @@ import ( var errNoResources = fmt.Errorf("No resources are associated with this task") +// If testModeEnvVar is set in a process's environment variables, the +// LinuxExecutor will detect it and inject the current binary into the chroot. +// This enables using the test binary in tests to provide portability. +var testModeEnvVar = "NOMAD_EXECUTOR_TEST_ONLY_13871827980214" + // Executor is an interface that any platform- or capability-specific exec // wrapper must implement. You should not need to implement a Java executor. // Rather, you would implement a cgroups executor that the Java driver will use. @@ -106,3 +112,18 @@ func OpenId(id string) (Executor, error) { } return executor, nil } + +// isTest returns whether the cmd is a test binary. +func isTest(cmd *exec.Cmd) bool { + if cmd == nil { + return false + } + + for _, env := range cmd.Env { + if strings.HasPrefix(env, testModeEnvVar) { + return true + } + } + + return false +} diff --git a/client/driver/executor/exec_linux.go b/client/driver/executor/exec_linux.go index 43c9270e0..4e1753f61 100644 --- a/client/driver/executor/exec_linux.go +++ b/client/driver/executor/exec_linux.go @@ -252,6 +252,13 @@ func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocD return err } + // Embed ourselves if this is a test. This needs to be done so the test + // binary is inside the chroot. + if isTest(&e.cmd) { + bin := e.cmd.Args[0] + alloc.Embed(taskName, map[string]string{bin: bin}) + } + if err := alloc.Embed(taskName, chrootEnv); err != nil { return err } diff --git a/client/driver/executor/test_harness_test.go b/client/driver/executor/test_harness_test.go index b5413700c..10f16969e 100644 --- a/client/driver/executor/test_harness_test.go +++ b/client/driver/executor/test_harness_test.go @@ -5,7 +5,9 @@ import ( "io/ioutil" "log" "os" + "os/exec" "path/filepath" + "strings" "testing" "time" @@ -14,6 +16,96 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) +// testBinary is the path to the running test binary +var testBinary = func() string { + abs, err := filepath.Abs(os.Args[0]) + if err != nil { + return err.Error() + } + + return abs +}() + +func TestMain(m *testing.M) { + // The tests in this package recursively execute the test binary produced + // by go test. The TEST_MAIN environment variable controls the recursive + // execution. + switch tm := os.Getenv(testModeEnvVar); tm { + case "": + os.Exit(m.Run()) + case "app": + appMain() + default: + fmt.Fprintf(os.Stderr, + "Unexpected value for test mode environment variable, %q\n", tm) + os.Exit(1) + } +} + +// setTestAppEnv sets the environement of cmd for a recursive call into +// TestMain. +func setTestAppEnv(cmd *exec.Cmd) { + cmd.Env = append(os.Environ(), fmt.Sprintf("%v=app", testModeEnvVar)) +} + +func appMain() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "no command provided") + os.Exit(1) + } + + args := os.Args[1:] + + // popArg removes the first argument from args and returns it. + popArg := func() string { + s := args[0] + args = args[1:] + return s + } + + // execute a sequence of operations from args + for len(args) > 0 { + switch cmd := popArg(); cmd { + + case "sleep": + // sleep : sleep for a duration indicated by the first + // argument + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "expected arg for sleep") + os.Exit(1) + } + dur, err := time.ParseDuration(popArg()) + if err != nil { + fmt.Fprintf(os.Stderr, "could not parse sleep time: %v", err) + os.Exit(1) + } + time.Sleep(dur) + + case "echo": + // echo : write the remaining arguments to stdout each + // separated by a single space and followed by a newline. + fmt.Println(strings.Join(args, " ")) + args = args[:0] + + case "write": + // write : write a message to a file. The first + // argument is the msg. The second argument is the path to the + // target file. + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "expected two args for write") + os.Exit(1) + } + msg := popArg() + file := popArg() + ioutil.WriteFile(file, []byte(msg), 0666) + + default: + fmt.Fprintln(os.Stderr, "unknown command:", cmd) + os.Exit(1) + } + } +} + var ( constraint = &structs.Resources{ CPU: 250, @@ -45,9 +137,10 @@ func testExecutor(t *testing.T, buildExecutor func() Executor, compatible func(* } command := func(name string, args ...string) Executor { - b := buildExecutor() - SetCommand(b, name, args) - return b + e := buildExecutor() + SetCommand(e, name, args) + setTestAppEnv(e.Command()) + return e } Executor_Start_Invalid(t, command) @@ -55,6 +148,7 @@ func testExecutor(t *testing.T, buildExecutor func() Executor, compatible func(* Executor_Start_Wait(t, command) Executor_Start_Kill(t, command) Executor_Open(t, command, buildExecutor) + Executor_Open_Invalid(t, command, buildExecutor) } type buildExecCommand func(name string, args ...string) Executor @@ -79,7 +173,7 @@ func Executor_Start_Invalid(t *testing.T, command buildExecCommand) { } func Executor_Start_Wait_Failure_Code(t *testing.T, command buildExecCommand) { - e := command("/bin/date", "-invalid") + e := command(testBinary, "fail") if err := e.Limit(constraint); err != nil { log.Panicf("Limit() failed: %v", err) @@ -112,8 +206,7 @@ func Executor_Start_Wait(t *testing.T, command buildExecCommand) { expected := "hello world" file := filepath.Join(allocdir.TaskLocal, "output.txt") absFilePath := filepath.Join(taskDir, file) - cmd := fmt.Sprintf(`/bin/sleep 1 ; echo -n %v > %v`, expected, file) - e := command("/bin/bash", "-c", cmd) + e := command(testBinary, "sleep", "1s", "write", expected, file) if err := e.Limit(constraint); err != nil { log.Panicf("Limit() failed: %v", err) @@ -152,7 +245,7 @@ func Executor_Start_Kill(t *testing.T, command buildExecCommand) { } filePath := filepath.Join(taskDir, "output") - e := command("/bin/bash", "-c", "sleep 1 ; echo \"failure\" > "+filePath) + e := command(testBinary, "sleep", "1s", "write", "failure", filePath) if err := e.Limit(constraint); err != nil { log.Panicf("Limit() failed: %v", err) @@ -190,8 +283,7 @@ func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func() Ex expected := "hello world" file := filepath.Join(allocdir.TaskLocal, "output.txt") absFilePath := filepath.Join(taskDir, file) - cmd := fmt.Sprintf(`/bin/sleep 1 ; echo -n %v > %v`, expected, file) - e := command("/bin/bash", "-c", cmd) + e := command(testBinary, "sleep", "1s", "write", expected, file) if err := e.Limit(constraint); err != nil { log.Panicf("Limit() failed: %v", err) @@ -232,7 +324,7 @@ func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func() Ex func Executor_Open_Invalid(t *testing.T, command buildExecCommand, newExecutor func() Executor) { task, alloc := mockAllocDir(t) - e := command("echo", "foo") + e := command(testBinary, "echo", "foo") if err := e.Limit(constraint); err != nil { log.Panicf("Limit() failed: %v", err) @@ -251,8 +343,19 @@ func Executor_Open_Invalid(t *testing.T, command buildExecCommand, newExecutor f log.Panicf("ID() failed: %v", err) } + // Kill the task because some OSes (windows) will not let us destroy the + // alloc (below) if the task is still running. + if err := e.ForceStop(); err != nil { + log.Panicf("e.ForceStop() failed: %v", err) + } + + // Wait until process is actually gone, we don't care what the result was. + e.Wait() + // Destroy the allocdir which removes the exit code. - alloc.Destroy() + if err := alloc.Destroy(); err != nil { + log.Panicf("alloc.Destroy() failed: %v", err) + } e2 := newExecutor() if err := e2.Open(id); err == nil {