open-nomad/client/driver/driver_test.go

package driver

import (
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"testing"
	"time"

	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/env"
	"github.com/hashicorp/nomad/client/logmon"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/helper/testtask"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
)

var basicResources = &structs.Resources{
	CPU:      250,
	MemoryMB: 256,
	DiskMB:   20,
}

func init() {
	rand.Seed(49875)
}

func TestMain(m *testing.M) {
	if !testtask.Run() {
		os.Exit(m.Run())
	}
}

// copyFile moves an existing file to the destination
func copyFile(src, dst string, t *testing.T) {
	in, err := os.Open(src)
	if err != nil {
		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
	}
	defer func() {
		if err := out.Close(); err != nil {
			t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
	}
	if err := out.Sync(); err != nil {
		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
	}
}

func testConfig(t *testing.T) *config.Config {
	conf := config.DefaultConfig()

	// Evaluate the symlinks so that the temp directory resolves correctly on
	// Mac OS.
	d1, err := ioutil.TempDir("", "TestStateDir")
	if err != nil {
		t.Fatal(err)
	}
	d2, err := ioutil.TempDir("", "TestAllocDir")
	if err != nil {
		t.Fatal(err)
	}

	p1, err := filepath.EvalSymlinks(d1)
	if err != nil {
		t.Fatal(err)
	}
	p2, err := filepath.EvalSymlinks(d2)
	if err != nil {
		t.Fatal(err)
	}

	// Give the directories access to everyone
	if err := os.Chmod(p1, 0777); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(p2, 0777); err != nil {
		t.Fatal(err)
	}

	conf.StateDir = p1
	conf.AllocDir = p2
	conf.MaxKillTimeout = 10 * time.Second
	conf.Region = "global"
	conf.Node = mock.Node()
	conf.LogLevel = "DEBUG"
	conf.LogOutput = testlog.NewWriter(t)
	return conf
}

type testContext struct {
	AllocDir     *allocdir.AllocDir
	DriverCtx    *DriverContext
	ExecCtx      *ExecContext
	EnvBuilder   *env.Builder
	logmon       logmon.LogMon
	logmonPlugin *plugin.Client
}

func (ctx *testContext) Destroy() {
	ctx.AllocDir.Destroy()
	ctx.logmon.Stop()
	ctx.logmonPlugin.Kill()
}

// testDriverContext sets up an alloc dir, task dir, DriverContext, and ExecContext.
//
// It is up to the caller to call Destroy to cleanup.
func testDriverContexts(t *testing.T, task *structs.Task) *testContext {
	cfg := testConfig(t)
	cfg.Node = mock.Node()
	alloc := mock.Alloc()
	alloc.NodeID = cfg.Node.ID

	allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(cfg.AllocDir, alloc.ID))
	if err := allocDir.Build(); err != nil {
		t.Fatalf("AllocDir.Build() failed: %v", err)
	}

	// Build a temp driver so we can call FSIsolation and build the task dir
	tmpdrv, err := NewDriver(task.Driver, NewEmptyDriverContext())
	if err != nil {
		allocDir.Destroy()
		t.Fatalf("NewDriver(%q, nil) failed: %v", task.Driver, err)
		return nil
	}

	// Build the task dir
	td := allocDir.NewTaskDir(task.Name)
	if err := td.Build(false, config.DefaultChrootEnv, tmpdrv.FSIsolation()); err != nil {
		allocDir.Destroy()
		t.Fatalf("TaskDir.Build(%#v, %q) failed: %v", config.DefaultChrootEnv, tmpdrv.FSIsolation(), err)
		return nil
	}
	eb := env.NewBuilder(cfg.Node, alloc, task, cfg.Region)
	SetEnvvars(eb, tmpdrv.FSIsolation(), td, cfg)
	execCtx := NewExecContext(td, eb.Build())

	logger := testlog.Logger(t)
	hcLogger := testlog.HCLogger(t)
	emitter := func(m string, args ...interface{}) {
		hcLogger.Info(fmt.Sprintf("[EVENT] "+m, args...))
	}
	driverCtx := NewDriverContext(alloc.Job.Name, alloc.TaskGroup, task.Name, alloc.ID, cfg, cfg.Node, logger, emitter)
	l, c, err := logmon.LaunchLogMon(hcLogger)
	if err != nil {
		allocDir.Destroy()
		t.Fatalf("LaunchLogMon() failed: %v", err)
	}

	var stdoutFifo, stderrFifo string
	if runtime.GOOS == "windows" {
		id := uuid.Generate()[:8]
		stdoutFifo = fmt.Sprintf("//./pipe/%s.stdout.%s", id, task.Name)
		stderrFifo = fmt.Sprintf("//./pipe/%s.stderr.%s", id, task.Name)
	} else {
		stdoutFifo = filepath.Join(td.LogDir, fmt.Sprintf("%s.stdout", task.Name))
		stderrFifo = filepath.Join(td.LogDir, fmt.Sprintf("%s.stderr", task.Name))
	}

	err = l.Start(&logmon.LogConfig{
		LogDir:        td.LogDir,
		StdoutLogFile: fmt.Sprintf("%s.stdout", task.Name),
		StderrLogFile: fmt.Sprintf("%s.stderr", task.Name),
		StdoutFifo:    stdoutFifo,
		StderrFifo:    stderrFifo,
		MaxFiles:      10,
		MaxFileSizeMB: 10,
	})
	if err != nil {
		allocDir.Destroy()
		t.Fatalf("LogMon.Start() failed: %v", err)
	}

	execCtx.StdoutFifo = stdoutFifo
	execCtx.StderrFifo = stderrFifo

	return &testContext{allocDir, driverCtx, execCtx, eb, l, c}
}

// setupTaskEnv creates a test env for GetTaskEnv testing. Returns task dir,
// expected env, and actual env.
func setupTaskEnv(t *testing.T, driver string) (*allocdir.TaskDir, map[string]string, map[string]string) {
	task := &structs.Task{
		Name:   "Foo",
		Driver: driver,
		Env: map[string]string{
			"HELLO": "world",
			"lorem": "ipsum",
		},
		Resources: &structs.Resources{
			CPU:      1000,
			MemoryMB: 500,
			Networks: []*structs.NetworkResource{
				{
					IP:            "1.2.3.4",
					ReservedPorts: []structs.Port{{Label: "one", Value: 80}, {Label: "two", Value: 443}},
					DynamicPorts:  []structs.Port{{Label: "admin", Value: 8081}, {Label: "web", Value: 8086}},
				},
			},
		},
		Meta: map[string]string{
			"chocolate":  "cake",
			"strawberry": "icecream",
		},
	}

	alloc := mock.Alloc()
	alloc.Job.TaskGroups[0].Tasks[0] = task
	alloc.Name = "Bar"
	alloc.AllocatedResources.Tasks["web"].Networks[0].DynamicPorts[0].Value = 2000
	conf := testConfig(t)
	allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(conf.AllocDir, alloc.ID))
	taskDir := allocDir.NewTaskDir(task.Name)
	eb := env.NewBuilder(conf.Node, alloc, task, conf.Region)
	tmpDriver, err := NewDriver(driver, NewEmptyDriverContext())
	if err != nil {
		t.Fatalf("unable to create driver %q: %v", driver, err)
	}
	SetEnvvars(eb, tmpDriver.FSIsolation(), taskDir, conf)
	exp := map[string]string{
		"NOMAD_CPU_LIMIT":               "1000",
		"NOMAD_MEMORY_LIMIT":            "500",
		"NOMAD_ADDR_one":                "1.2.3.4:80",
		"NOMAD_IP_one":                  "1.2.3.4",
		"NOMAD_PORT_one":                "80",
		"NOMAD_HOST_PORT_one":           "80",
		"NOMAD_ADDR_two":                "1.2.3.4:443",
		"NOMAD_IP_two":                  "1.2.3.4",
		"NOMAD_PORT_two":                "443",
		"NOMAD_HOST_PORT_two":           "443",
		"NOMAD_ADDR_admin":              "1.2.3.4:8081",
		"NOMAD_ADDR_web_admin":          "192.168.0.100:5000",
		"NOMAD_ADDR_web_http":           "192.168.0.100:2000",
		"NOMAD_IP_web_admin":            "192.168.0.100",
		"NOMAD_IP_web_http":             "192.168.0.100",
		"NOMAD_PORT_web_http":           "2000",
		"NOMAD_PORT_web_admin":          "5000",
		"NOMAD_IP_admin":                "1.2.3.4",
		"NOMAD_PORT_admin":              "8081",
		"NOMAD_HOST_PORT_admin":         "8081",
		"NOMAD_ADDR_web":                "1.2.3.4:8086",
		"NOMAD_IP_web":                  "1.2.3.4",
		"NOMAD_PORT_web":                "8086",
		"NOMAD_HOST_PORT_web":           "8086",
		"NOMAD_META_CHOCOLATE":          "cake",
		"NOMAD_META_STRAWBERRY":         "icecream",
		"NOMAD_META_ELB_CHECK_INTERVAL": "30s",
		"NOMAD_META_ELB_CHECK_TYPE":     "http",
		"NOMAD_META_ELB_CHECK_MIN":      "3",
		"NOMAD_META_OWNER":              "armon",
		"NOMAD_META_chocolate":          "cake",
		"NOMAD_META_strawberry":         "icecream",
		"NOMAD_META_elb_check_interval": "30s",
		"NOMAD_META_elb_check_type":     "http",
		"NOMAD_META_elb_check_min":      "3",
		"NOMAD_META_owner":              "armon",
		"HELLO":                         "world",
		"lorem":                         "ipsum",
		"NOMAD_ALLOC_ID":                alloc.ID,
		"NOMAD_ALLOC_INDEX":             "0",
		"NOMAD_ALLOC_NAME":              alloc.Name,
		"NOMAD_TASK_NAME":               task.Name,
		"NOMAD_GROUP_NAME":              alloc.TaskGroup,
		"NOMAD_JOB_NAME":                alloc.Job.Name,
		"NOMAD_DC":                      "dc1",
		"NOMAD_REGION":                  "global",
	}

	act := eb.Build().Map()
	return taskDir, exp, act
}

func TestDriver_GetTaskEnv_None(t *testing.T) {
	t.Parallel()
	taskDir, exp, act := setupTaskEnv(t, "raw_exec")

	// raw_exec should use host alloc dir path
	exp[env.AllocDir] = taskDir.SharedAllocDir
	exp[env.TaskLocalDir] = taskDir.LocalDir
	exp[env.SecretsDir] = taskDir.SecretsDir

	// Since host env vars are included only ensure expected env vars are present
	for expk, expv := range exp {
		v, ok := act[expk]
		if !ok {
			t.Errorf("%q not found in task env", expk)
			continue
		}
		if v != expv {
			t.Errorf("Expected %s=%q but found %q", expk, expv, v)
		}
	}

	// Make sure common host env vars are included.
	for _, envvar := range [...]string{"PATH", "HOME", "USER"} {
		if exp := os.Getenv(envvar); act[envvar] != exp {
			t.Errorf("Expected envvar %s=%q  !=  %q", envvar, exp, act[envvar])
		}
	}
}

func TestDriver_GetTaskEnv_Chroot(t *testing.T) {
	t.Parallel()
	_, exp, act := setupTaskEnv(t, "exec")

	exp[env.AllocDir] = allocdir.SharedAllocContainerPath
	exp[env.TaskLocalDir] = allocdir.TaskLocalContainerPath
	exp[env.SecretsDir] = allocdir.TaskSecretsContainerPath

	// Since host env vars are included only ensure expected env vars are present
	for expk, expv := range exp {
		v, ok := act[expk]
		if !ok {
			t.Errorf("%q not found in task env", expk)
			continue
		}
		if v != expv {
			t.Errorf("Expected %s=%q but found %q", expk, expv, v)
		}
	}

	// Make sure common host env vars are included.
	for _, envvar := range [...]string{"PATH", "HOME", "USER"} {
		if exp := os.Getenv(envvar); act[envvar] != exp {
			t.Errorf("Expected envvar %s=%q  !=  %q", envvar, exp, act[envvar])
		}
	}
}

// TestDriver_TaskEnv_Image ensures host environment variables are not set
// for image based drivers. See #2211
func TestDriver_TaskEnv_Image(t *testing.T) {
	t.Parallel()
	_, exp, act := setupTaskEnv(t, "docker")

	exp[env.AllocDir] = allocdir.SharedAllocContainerPath
	exp[env.TaskLocalDir] = allocdir.TaskLocalContainerPath
	exp[env.SecretsDir] = allocdir.TaskSecretsContainerPath

	// Since host env vars are excluded expected and actual maps should be equal
	for expk, expv := range exp {
		v, ok := act[expk]
		delete(act, expk)
		if !ok {
			t.Errorf("Env var %s missing. Expected %s=%q", expk, expk, expv)
			continue
		}
		if v != expv {
			t.Errorf("Env var %s=%q -- Expected %q", expk, v, expk)
		}
	}
	// Any remaining env vars are unexpected
	for actk, actv := range act {
		t.Errorf("Env var %s=%q is unexpected", actk, actv)
	}
}

func TestMapMergeStrStr(t *testing.T) {
	t.Parallel()
	a := map[string]string{
		"cake":   "chocolate",
		"cookie": "caramel",
	}

	b := map[string]string{
		"cake": "strawberry",
		"pie":  "apple",
	}

	c := mapMergeStrStr(a, b)

	d := map[string]string{
		"cake":   "strawberry",
		"cookie": "caramel",
		"pie":    "apple",
	}

	if !reflect.DeepEqual(c, d) {
		t.Errorf("\nExpected\n%+v\nGot\n%+v\n", d, c)
	}
}

func TestCreatedResources_AddMerge(t *testing.T) {
	t.Parallel()
	res1 := NewCreatedResources()
	res1.Add("k1", "v1")
	res1.Add("k1", "v2")
	res1.Add("k1", "v1")
	res1.Add("k2", "v1")

	expected := map[string][]string{
		"k1": {"v1", "v2"},
		"k2": {"v1"},
	}
	if !reflect.DeepEqual(expected, res1.Resources) {
		t.Fatalf("1.  %#v != expected %#v", res1.Resources, expected)
	}

	// Make sure merging nil works
	var res2 *CreatedResources
	res1.Merge(res2)
	if !reflect.DeepEqual(expected, res1.Resources) {
		t.Fatalf("2.  %#v != expected %#v", res1.Resources, expected)
	}

	// Make sure a normal merge works
	res2 = NewCreatedResources()
	res2.Add("k1", "v3")
	res2.Add("k2", "v1")
	res2.Add("k3", "v3")
	res1.Merge(res2)

	expected = map[string][]string{
		"k1": {"v1", "v2", "v3"},
		"k2": {"v1"},
		"k3": {"v3"},
	}
	if !reflect.DeepEqual(expected, res1.Resources) {
		t.Fatalf("3.  %#v != expected %#v", res1.Resources, expected)
	}
}

func TestCreatedResources_CopyRemove(t *testing.T) {
	t.Parallel()
	res1 := NewCreatedResources()
	res1.Add("k1", "v1")
	res1.Add("k1", "v2")
	res1.Add("k1", "v3")
	res1.Add("k2", "v1")

	// Assert Copy creates a deep copy
	res2 := res1.Copy()

	if !reflect.DeepEqual(res1, res2) {
		t.Fatalf("%#v != %#v", res1, res2)
	}

	// Assert removing v1 from k1 returns true and updates Resources slice
	if removed := res2.Remove("k1", "v1"); !removed {
		t.Fatalf("expected v1 to be removed: %#v", res2)
	}

	if expected := []string{"v2", "v3"}; !reflect.DeepEqual(expected, res2.Resources["k1"]) {
		t.Fatalf("unexpected list for k1: %#v", res2.Resources["k1"])
	}

	// Assert removing the only value from a key removes the key
	if removed := res2.Remove("k2", "v1"); !removed {
		t.Fatalf("expected v1 to be removed from k2: %#v", res2.Resources)
	}

	if _, found := res2.Resources["k2"]; found {
		t.Fatalf("k2 should have been removed from Resources: %#v", res2.Resources)
	}

	// Make sure res1 wasn't updated
	if reflect.DeepEqual(res1, res2) {
		t.Fatalf("res1 should not equal res2: #%v", res1)
	}
}