open-nomad/client/driver/qemu_test.go

package driver

import (
	"bytes"
	"fmt"
	"net"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"testing"
	"time"

	"github.com/hashicorp/consul/lib/freeport"
	"github.com/hashicorp/nomad/client/config"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"

	ctestutils "github.com/hashicorp/nomad/client/testutil"
)

// The fingerprinter test should always pass, even if QEMU is not installed.
func TestQemuDriver_Fingerprint(t *testing.T) {
	if !testutil.IsTravis() {
		t.Parallel()
	}
	ctestutils.QemuCompatible(t)
	task := &structs.Task{
		Name:      "foo",
		Driver:    "qemu",
		Resources: structs.DefaultResources(),
	}
	ctx := testDriverContexts(t, task)
	defer ctx.Destroy()
	d := NewQemuDriver(ctx.DriverCtx)

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: node}
	var response cstructs.FingerprintResponse
	err := d.Fingerprint(request, &response)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	if !response.Detected {
		t.Fatalf("expected response to be applicable")
	}

	attributes := response.Attributes
	if attributes == nil {
		t.Fatalf("attributes should not be nil")
	}

	if attributes[qemuDriverAttr] == "" {
		t.Fatalf("Missing Qemu driver")
	}

	if attributes[qemuDriverVersionAttr] == "" {
		t.Fatalf("Missing Qemu driver version")
	}
}

func TestQemuDriver_StartOpen_Wait(t *testing.T) {
	logger := testlog.Logger(t)
	if !testutil.IsTravis() {
		t.Parallel()
	}
	ctestutils.QemuCompatible(t)
	task := &structs.Task{
		Name:   "linux",
		Driver: "qemu",
		Config: map[string]interface{}{
			"image_path":        "linux-0.2.img",
			"accelerator":       "tcg",
			"graceful_shutdown": false,
			"port_map": []map[string]int{{
				"main": 22,
				"web":  8080,
			}},
			"args": []string{"-nodefconfig", "-nodefaults"},
		},
		LogConfig: &structs.LogConfig{
			MaxFiles:      10,
			MaxFileSizeMB: 10,
		},
		Resources: &structs.Resources{
			CPU:      500,
			MemoryMB: 512,
			Networks: []*structs.NetworkResource{
				{
					ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
				},
			},
		},
	}

	ctx := testDriverContexts(t, task)
	defer ctx.Destroy()
	d := NewQemuDriver(ctx.DriverCtx)

	// Copy the test image into the task's directory
	dst := ctx.ExecCtx.TaskDir.Dir

	copyFile("./test-resources/qemu/linux-0.2.img", filepath.Join(dst, "linux-0.2.img"), t)

	if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
		t.Fatalf("Prestart failed: %v", err)
	}

	resp, err := d.Start(ctx.ExecCtx, task)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Ensure that sending a Signal returns an error
	if err := resp.Handle.Signal(syscall.SIGINT); err == nil {
		t.Fatalf("Expect an error when signalling")
	}

	// Attempt to open
	handle2, err := d.Open(ctx.ExecCtx, resp.Handle.ID())
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if handle2 == nil {
		t.Fatalf("missing handle")
	}

	// Clean up
	if err := resp.Handle.Kill(); err != nil {
		logger.Printf("Error killing Qemu test: %s", err)
	}
}

func TestQemuDriver_GracefulShutdown(t *testing.T) {
	testutil.SkipSlow(t)
	if !testutil.IsTravis() {
		t.Parallel()
	}
	ctestutils.QemuCompatible(t)

	logger := testlog.Logger(t)

	// Graceful shutdown may be really slow unfortunately
	killTimeout := 3 * time.Minute

	// Grab a free port so we can tell when the image has started
	port := freeport.GetT(t, 1)[0]

	task := &structs.Task{
		Name:   "alpine-shutdown-test",
		Driver: "qemu",
		Config: map[string]interface{}{
			"image_path":        "alpine.qcow2",
			"graceful_shutdown": true,
			"args":              []string{"-nodefconfig", "-nodefaults"},
			"port_map": []map[string]int{{
				"ssh": 22,
			}},
		},
		LogConfig: &structs.LogConfig{
			MaxFiles:      10,
			MaxFileSizeMB: 10,
		},
		Resources: &structs.Resources{
			CPU:      1000,
			MemoryMB: 256,
			Networks: []*structs.NetworkResource{
				{
					ReservedPorts: []structs.Port{{Label: "ssh", Value: port}},
				},
			},
		},
		KillTimeout: killTimeout,
	}

	ctx := testDriverContexts(t, task)
	ctx.DriverCtx.config.MaxKillTimeout = killTimeout
	defer ctx.Destroy()
	d := NewQemuDriver(ctx.DriverCtx)

	request := &cstructs.FingerprintRequest{Config: &config.Config{}, Node: ctx.DriverCtx.node}
	var response cstructs.FingerprintResponse
	err := d.Fingerprint(request, &response)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	for name, value := range response.Attributes {
		ctx.DriverCtx.node.Attributes[name] = value
	}

	dst := ctx.ExecCtx.TaskDir.Dir

	copyFile("./test-resources/qemu/alpine.qcow2", filepath.Join(dst, "alpine.qcow2"), t)

	if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
		t.Fatalf("Prestart failed: %v", err)
	}

	resp, err := d.Start(ctx.ExecCtx, task)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Clean up
	defer func() {
		select {
		case <-resp.Handle.WaitCh():
			// Already exited
			return
		default:
		}

		if err := resp.Handle.Kill(); err != nil {
			logger.Printf("[TEST] Error killing Qemu test: %s", err)
		}
	}()

	// Wait until sshd starts before attempting to do a graceful shutdown
	testutil.WaitForResult(func() (bool, error) {
		conn, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
		if err != nil {
			return false, err
		}

		// Since the connection will be accepted by the QEMU process
		// before sshd actually starts, we need to block until we can
		// read the "SSH" magic bytes
		header := make([]byte, 3)
		conn.SetReadDeadline(time.Now().Add(10 * time.Second))
		_, err = conn.Read(header)
		if err != nil {
			return false, err
		}
		if !bytes.Equal(header, []byte{'S', 'S', 'H'}) {
			return false, fmt.Errorf("expected 'SSH' but received: %q %v", string(header), header)
		}

		logger.Printf("[TEST] connected to sshd in VM")
		conn.Close()
		return true, nil
	}, func(err error) {
		t.Fatalf("failed to connect to sshd in VM: %v", err)
	})

	monitorPath := filepath.Join(ctx.AllocDir.AllocDir, task.Name, qemuMonitorSocketName)

	// userPid supplied in sendQemuShutdown calls is bogus (it's used only
	// for log output)
	if err := sendQemuShutdown(ctx.DriverCtx.logger, "", 0); err == nil {
		t.Fatalf("sendQemuShutdown should return an error if monitorPath parameter is empty")
	}

	if err := sendQemuShutdown(ctx.DriverCtx.logger, "/path/that/does/not/exist", 0); err == nil {
		t.Fatalf("sendQemuShutdown should return an error if file does not exist at monitorPath")
	}

	if err := sendQemuShutdown(ctx.DriverCtx.logger, monitorPath, 0); err != nil {
		t.Fatalf("unexpected error from sendQemuShutdown: %s", err)
	}

	select {
	case <-resp.Handle.WaitCh():
		logger.Printf("[TEST] VM exited gracefully as expected")
	case <-time.After(killTimeout):
		t.Fatalf("VM did not exit gracefully exit before timeout: %s", killTimeout)
	}
}

func TestQemuDriverUser(t *testing.T) {
	if !testutil.IsTravis() {
		t.Parallel()
	}
	ctestutils.QemuCompatible(t)
	tasks := []*structs.Task{
		{
			Name:   "linux",
			Driver: "qemu",
			User:   "alice",
			Config: map[string]interface{}{
				"image_path":        "linux-0.2.img",
				"accelerator":       "tcg",
				"graceful_shutdown": false,
				"port_map": []map[string]int{{
					"main": 22,
					"web":  8080,
				}},
				"args": []string{"-nodefconfig", "-nodefaults"},
				"msg":  "unknown user alice",
			},
			LogConfig: &structs.LogConfig{
				MaxFiles:      10,
				MaxFileSizeMB: 10,
			},
			Resources: &structs.Resources{
				CPU:      500,
				MemoryMB: 512,
				Networks: []*structs.NetworkResource{
					{
						ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
					},
				},
			},
		},
		{
			Name:   "linux",
			Driver: "qemu",
			User:   "alice",
			Config: map[string]interface{}{
				"image_path":  "linux-0.2.img",
				"accelerator": "tcg",
				"port_map": []map[string]int{{
					"main": 22,
					"web":  8080,
				}},
				"args": []string{"-nodefconfig", "-nodefaults"},
				"msg":  "Qemu memory assignment out of bounds",
			},
			LogConfig: &structs.LogConfig{
				MaxFiles:      10,
				MaxFileSizeMB: 10,
			},
			Resources: &structs.Resources{
				CPU:      500,
				MemoryMB: -1,
				Networks: []*structs.NetworkResource{
					{
						ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
					},
				},
			},
		},
	}

	for _, task := range tasks {
		ctx := testDriverContexts(t, task)
		defer ctx.Destroy()
		d := NewQemuDriver(ctx.DriverCtx)

		if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
			t.Fatalf("Prestart faild: %v", err)
		}

		resp, err := d.Start(ctx.ExecCtx, task)
		if err == nil {
			resp.Handle.Kill()
			t.Fatalf("Should've failed")
		}

		msg := task.Config["msg"].(string)
		if !strings.Contains(err.Error(), msg) {
			t.Fatalf("Expecting '%v' in '%v'", msg, err)
		}
	}
}

func TestQemuDriverGetMonitorPathOldQemu(t *testing.T) {
	task := &structs.Task{
		Name:   "linux",
		Driver: "qemu",
		Config: map[string]interface{}{
			"image_path":        "linux-0.2.img",
			"accelerator":       "tcg",
			"graceful_shutdown": true,
			"port_map": []map[string]int{{
				"main": 22,
				"web":  8080,
			}},
			"args": []string{"-nodefconfig", "-nodefaults"},
		},
		KillTimeout: time.Duration(1 * time.Second),
		LogConfig: &structs.LogConfig{
			MaxFiles:      10,
			MaxFileSizeMB: 10,
		},
		Resources: &structs.Resources{
			CPU:      500,
			MemoryMB: 512,
			Networks: []*structs.NetworkResource{
				{
					ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
				},
			},
		},
	}

	ctx := testDriverContexts(t, task)
	defer ctx.Destroy()

	// Simulate an older version of qemu which does not support long monitor socket paths
	ctx.DriverCtx.node.Attributes[qemuDriverVersionAttr] = "2.0.0"

	d := &QemuDriver{DriverContext: *ctx.DriverCtx}

	shortPath := strings.Repeat("x", 10)
	_, err := d.getMonitorPath(shortPath)
	if err != nil {
		t.Fatal("Should not have returned an error")
	}

	longPath := strings.Repeat("x", qemuLegacyMaxMonitorPathLen+100)
	_, err = d.getMonitorPath(longPath)
	if err == nil {
		t.Fatal("Should have returned an error")
	}

	// Max length includes the '/' separator and socket name
	maxLengthCount := qemuLegacyMaxMonitorPathLen - len(qemuMonitorSocketName) - 1
	maxLengthLegacyPath := strings.Repeat("x", maxLengthCount)
	_, err = d.getMonitorPath(maxLengthLegacyPath)
	if err != nil {
		t.Fatalf("Should not have returned an error: %s", err)
	}
}

func TestQemuDriverGetMonitorPathNewQemu(t *testing.T) {
	task := &structs.Task{
		Name:   "linux",
		Driver: "qemu",
		Config: map[string]interface{}{
			"image_path":        "linux-0.2.img",
			"accelerator":       "tcg",
			"graceful_shutdown": true,
			"port_map": []map[string]int{{
				"main": 22,
				"web":  8080,
			}},
			"args": []string{"-nodefconfig", "-nodefaults"},
		},
		KillTimeout: time.Duration(1 * time.Second),
		LogConfig: &structs.LogConfig{
			MaxFiles:      10,
			MaxFileSizeMB: 10,
		},
		Resources: &structs.Resources{
			CPU:      500,
			MemoryMB: 512,
			Networks: []*structs.NetworkResource{
				{
					ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
				},
			},
		},
	}

	ctx := testDriverContexts(t, task)
	defer ctx.Destroy()

	// Simulate a version of qemu which supports long monitor socket paths
	ctx.DriverCtx.node.Attributes[qemuDriverVersionAttr] = "2.99.99"

	d := &QemuDriver{DriverContext: *ctx.DriverCtx}

	shortPath := strings.Repeat("x", 10)
	_, err := d.getMonitorPath(shortPath)
	if err != nil {
		t.Fatal("Should not have returned an error")
	}

	longPath := strings.Repeat("x", qemuLegacyMaxMonitorPathLen+100)
	_, err = d.getMonitorPath(longPath)
	if err != nil {
		t.Fatal("Should not have returned an error")
	}

	maxLengthCount := qemuLegacyMaxMonitorPathLen - len(qemuMonitorSocketName) - 1
	maxLengthLegacyPath := strings.Repeat("x", maxLengthCount)
	_, err = d.getMonitorPath(maxLengthLegacyPath)
	if err != nil {
		t.Fatal("Should not have returned an error")
	}
}