552 lines
13 KiB
Go
552 lines
13 KiB
Go
package driver
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
docker "github.com/fsouza/go-dockerclient"
|
|
"github.com/hashicorp/nomad/client/config"
|
|
"github.com/hashicorp/nomad/client/driver/environment"
|
|
cstructs "github.com/hashicorp/nomad/client/driver/structs"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
)
|
|
|
|
func testDockerDriverContext(task string) *DriverContext {
|
|
cfg := testConfig()
|
|
cfg.DevMode = true
|
|
return NewDriverContext(task, cfg, cfg.Node, testLogger())
|
|
}
|
|
|
|
// dockerIsConnected checks to see if a docker daemon is available (local or remote)
|
|
func dockerIsConnected(t *testing.T) bool {
|
|
client, err := docker.NewClientFromEnv()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
// Creating a client doesn't actually connect, so make sure we do something
|
|
// like call Version() on it.
|
|
env, err := client.Version()
|
|
if err != nil {
|
|
t.Logf("Failed to connect to docker daemon: %s", err)
|
|
return false
|
|
}
|
|
|
|
t.Logf("Successfully connected to docker daemon running version %s", env.Get("Version"))
|
|
return true
|
|
}
|
|
|
|
func dockerIsRemote(t *testing.T) bool {
|
|
client, err := docker.NewClientFromEnv()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
// Technically this could be a local tcp socket but for testing purposes
|
|
// we'll just assume that tcp is only used for remote connections.
|
|
if client.Endpoint()[0:3] == "tcp" {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestDockerDriver_Handle(t *testing.T) {
|
|
h := &dockerHandle{
|
|
imageID: "imageid",
|
|
containerID: "containerid",
|
|
doneCh: make(chan struct{}),
|
|
waitCh: make(chan *cstructs.WaitResult, 1),
|
|
}
|
|
|
|
actual := h.ID()
|
|
expected := `DOCKER:{"ImageID":"imageid","ContainerID":"containerid"}`
|
|
if actual != expected {
|
|
t.Errorf("Expected `%s`, found `%s`", expected, actual)
|
|
}
|
|
}
|
|
|
|
// This test should always pass, even if docker daemon is not available
|
|
func TestDockerDriver_Fingerprint(t *testing.T) {
|
|
d := NewDockerDriver(testDockerDriverContext(""))
|
|
node := &structs.Node{
|
|
Attributes: make(map[string]string),
|
|
}
|
|
apply, err := d.Fingerprint(&config.Config{}, node)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if apply != dockerIsConnected(t) {
|
|
t.Fatalf("Fingerprinter should detect when docker is available")
|
|
}
|
|
if node.Attributes["driver.docker"] != "1" {
|
|
t.Log("Docker daemon not available. The remainder of the docker tests will be skipped.")
|
|
}
|
|
t.Logf("Found docker version %s", node.Attributes["driver.docker.version"])
|
|
}
|
|
|
|
func TestDockerDriver_StartOpen_Wait(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task := &structs.Task{
|
|
Name: "redis-demo",
|
|
Config: map[string]interface{}{
|
|
"image": "redis",
|
|
},
|
|
Resources: basicResources,
|
|
}
|
|
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
|
|
// Attempt to open
|
|
handle2, err := d.Open(ctx, handle.ID())
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle2 == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
}
|
|
|
|
func TestDockerDriver_Start_Wait(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task := &structs.Task{
|
|
Name: "redis-demo",
|
|
Config: map[string]interface{}{
|
|
"image": "redis",
|
|
"command": "redis-server",
|
|
"args": []string{"-v"},
|
|
},
|
|
Resources: &structs.Resources{
|
|
MemoryMB: 256,
|
|
CPU: 512,
|
|
},
|
|
}
|
|
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
|
|
// Update should be a no-op
|
|
err = handle.Update(task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
select {
|
|
case res := <-handle.WaitCh():
|
|
if !res.Successful() {
|
|
t.Fatalf("err: %v", res)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatalf("timeout")
|
|
}
|
|
}
|
|
|
|
func TestDockerDriver_Start_Wait_AllocDir(t *testing.T) {
|
|
// This test requires that the alloc dir be mounted into docker as a volume.
|
|
// Because this cannot happen when docker is run remotely, e.g. when running
|
|
// docker in a VM, we skip this when we detect Docker is being run remotely.
|
|
if !dockerIsConnected(t) || dockerIsRemote(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
exp := []byte{'w', 'i', 'n'}
|
|
file := "output.txt"
|
|
task := &structs.Task{
|
|
Name: "redis-demo",
|
|
Config: map[string]interface{}{
|
|
"image": "redis",
|
|
"command": "/bin/bash",
|
|
"args": []string{
|
|
"-c",
|
|
fmt.Sprintf(`sleep 1; echo -n %s > $%s/%s`,
|
|
string(exp), environment.AllocDir, file),
|
|
},
|
|
},
|
|
Resources: &structs.Resources{
|
|
MemoryMB: 256,
|
|
CPU: 512,
|
|
},
|
|
}
|
|
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
|
|
select {
|
|
case res := <-handle.WaitCh():
|
|
if !res.Successful() {
|
|
t.Fatalf("err: %v", res)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatalf("timeout")
|
|
}
|
|
|
|
// Check that data was written to the shared alloc directory.
|
|
outputFile := filepath.Join(ctx.AllocDir.SharedDir, file)
|
|
act, err := ioutil.ReadFile(outputFile)
|
|
if err != nil {
|
|
t.Fatalf("Couldn't read expected output: %v", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(act, exp) {
|
|
t.Fatalf("Command outputted %v; want %v", act, exp)
|
|
}
|
|
}
|
|
|
|
func TestDockerDriver_Start_Kill_Wait(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task := &structs.Task{
|
|
Name: "redis-demo",
|
|
Config: map[string]interface{}{
|
|
"image": "redis",
|
|
"command": "/bin/sleep",
|
|
"args": []string{"10"},
|
|
},
|
|
Resources: basicResources,
|
|
}
|
|
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
|
|
go func() {
|
|
time.Sleep(100 * time.Millisecond)
|
|
err := handle.Kill()
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
}()
|
|
|
|
select {
|
|
case res := <-handle.WaitCh():
|
|
if res.Successful() {
|
|
t.Fatalf("should err: %v", res)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("timeout")
|
|
}
|
|
}
|
|
|
|
func taskTemplate() *structs.Task {
|
|
return &structs.Task{
|
|
Name: "redis-demo",
|
|
Config: map[string]interface{}{
|
|
"image": "redis",
|
|
},
|
|
Resources: &structs.Resources{
|
|
MemoryMB: 256,
|
|
CPU: 512,
|
|
Networks: []*structs.NetworkResource{
|
|
&structs.NetworkResource{
|
|
IP: "127.0.0.1",
|
|
ReservedPorts: []structs.Port{{"main", 11110}},
|
|
DynamicPorts: []structs.Port{{"REDIS", 43330}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestDocker_StartN(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task1 := taskTemplate()
|
|
task1.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 11110}
|
|
task1.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43331}
|
|
|
|
task2 := taskTemplate()
|
|
task2.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 22222}
|
|
task2.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43332}
|
|
|
|
task3 := taskTemplate()
|
|
task3.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 33333}
|
|
task3.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43333}
|
|
|
|
taskList := []*structs.Task{task1, task2, task3}
|
|
|
|
handles := make([]DriverHandle, len(taskList))
|
|
|
|
t.Logf("==> Starting %d tasks", len(taskList))
|
|
|
|
// Let's spin up a bunch of things
|
|
var err error
|
|
for idx, task := range taskList {
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handles[idx], err = d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Errorf("Failed starting task #%d: %s", idx+1, err)
|
|
}
|
|
}
|
|
|
|
t.Log("==> All tasks are started. Terminating...")
|
|
|
|
for idx, handle := range handles {
|
|
if handle == nil {
|
|
t.Errorf("Bad handle for task #%d", idx+1)
|
|
continue
|
|
}
|
|
|
|
err := handle.Kill()
|
|
if err != nil {
|
|
t.Errorf("Failed stopping task #%d: %s", idx+1, err)
|
|
}
|
|
}
|
|
|
|
t.Log("==> Test complete!")
|
|
}
|
|
|
|
func TestDocker_StartNVersions(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task1 := taskTemplate()
|
|
task1.Config["image"] = "redis"
|
|
task1.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 11110}
|
|
task1.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43331}
|
|
|
|
task2 := taskTemplate()
|
|
task2.Config["image"] = "redis:latest"
|
|
task2.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 22222}
|
|
task2.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43332}
|
|
|
|
task3 := taskTemplate()
|
|
task3.Config["image"] = "redis:3.0"
|
|
task3.Resources.Networks[0].ReservedPorts[0] = structs.Port{"main", 33333}
|
|
task3.Resources.Networks[0].DynamicPorts[0] = structs.Port{"REDIS", 43333}
|
|
|
|
taskList := []*structs.Task{task1, task2, task3}
|
|
|
|
handles := make([]DriverHandle, len(taskList))
|
|
|
|
t.Logf("==> Starting %d tasks", len(taskList))
|
|
|
|
// Let's spin up a bunch of things
|
|
var err error
|
|
for idx, task := range taskList {
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handles[idx], err = d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Errorf("Failed starting task #%d: %s", idx+1, err)
|
|
}
|
|
}
|
|
|
|
t.Log("==> All tasks are started. Terminating...")
|
|
|
|
for idx, handle := range handles {
|
|
if handle == nil {
|
|
t.Errorf("Bad handle for task #%d", idx+1)
|
|
continue
|
|
}
|
|
|
|
err := handle.Kill()
|
|
if err != nil {
|
|
t.Errorf("Failed stopping task #%d: %s", idx+1, err)
|
|
}
|
|
}
|
|
|
|
t.Log("==> Test complete!")
|
|
}
|
|
|
|
func TestDockerHostNet(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task := &structs.Task{
|
|
Name: "redis-demo",
|
|
Config: map[string]interface{}{
|
|
"image": "redis",
|
|
"network_mode": "host",
|
|
},
|
|
Resources: &structs.Resources{
|
|
MemoryMB: 256,
|
|
CPU: 512,
|
|
},
|
|
}
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
}
|
|
|
|
func TestDockerLabels(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task := taskTemplate()
|
|
task.Config["labels"] = []map[string]string{
|
|
map[string]string{
|
|
"label1": "value1",
|
|
"label2": "value2",
|
|
},
|
|
}
|
|
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
|
|
client, err := docker.NewClientFromEnv()
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// don't know if is queriable in a clean way
|
|
parts := strings.SplitN(handle.ID(), ":", 2)
|
|
var pid dockerPID
|
|
err = json.Unmarshal([]byte(parts[1]), &pid)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
container, err := client.InspectContainer(pid.ContainerID)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if want, got := 2, len(container.Config.Labels); want != got {
|
|
t.Errorf("Wrong labels count for docker job. Expect: %d, got: %d", want, got)
|
|
}
|
|
|
|
if want, got := "value1", container.Config.Labels["label1"]; want != got {
|
|
t.Errorf("Wrong label value docker job. Expect: %s, got: %s", want, got)
|
|
}
|
|
}
|
|
|
|
func TestDockerDNS(t *testing.T) {
|
|
if !dockerIsConnected(t) {
|
|
t.SkipNow()
|
|
}
|
|
|
|
task := taskTemplate()
|
|
task.Config["dns_servers"] = []string{"8.8.8.8", "8.8.4.4"}
|
|
task.Config["dns_search_domains"] = []string{"example.com", "example.org", "example.net"}
|
|
|
|
driverCtx := testDockerDriverContext(task.Name)
|
|
ctx := testDriverExecContext(task, driverCtx)
|
|
defer ctx.AllocDir.Destroy()
|
|
d := NewDockerDriver(driverCtx)
|
|
|
|
handle, err := d.Start(ctx, task)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if handle == nil {
|
|
t.Fatalf("missing handle")
|
|
}
|
|
defer handle.Kill()
|
|
|
|
client, err := docker.NewClientFromEnv()
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// don't know if is queriable in a clean way
|
|
parts := strings.SplitN(handle.ID(), ":", 2)
|
|
var pid dockerPID
|
|
err = json.Unmarshal([]byte(parts[1]), &pid)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
container, err := client.InspectContainer(pid.ContainerID)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(task.Config["dns_servers"], container.HostConfig.DNS) {
|
|
t.Errorf("DNS Servers don't match.\nExpected:\n%s\nGot:\n%s\n", task.Config["dns_servers"], container.HostConfig.DNS)
|
|
}
|
|
|
|
if !reflect.DeepEqual(task.Config["dns_search_domains"], container.HostConfig.DNSSearch) {
|
|
t.Errorf("DNS Servers don't match.\nExpected:\n%s\nGot:\n%s\n", task.Config["dns_search_domains"], container.HostConfig.DNSSearch)
|
|
}
|
|
}
|