Test graceful shutdown

Uses an Alpine image which supports ACPI poweroff signal handling.
Handling is only enabled after the VM has booted, so this test blocks
until sshd starts before issuing the command.

.gitattributes

@@ -1,3 +1,4 @@
+*.qcow2 filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
 *.class filter=lfs diff=lfs merge=lfs -text
 *.DS_Store filter=lfs diff=lfs merge=lfs -text
@@ -12,7 +13,6 @@
 *.pdf filter=lfs diff=lfs merge=lfs -text
 *.pfx filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
-*.qcow2 filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text

client/driver/qemu_test.go

@@ -1,14 +1,17 @@
 package driver
 
 import (
+	"bytes"
 	"fmt"
-	"os"
+	"net"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/lib/freeport"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/testutil"
@@ -127,40 +130,42 @@ func TestQemuDriver_GracefulShutdown(t *testing.T) {
 		t.Parallel()
 	}
 	ctestutils.QemuCompatible(t)
-	ctestutils.RequireRoot(t)
+
+	// Graceful shutdown may be really slow unfortunately
+	killTimeout := 3 * time.Minute
+
+	// Grab a free port so we can tell when the image has started
+	port := freeport.GetT(t, 1)[0]
+
 	task := &structs.Task{
-		Name:   "linux",
+		Name:   "alpine-shutdown-test",
 		Driver: "qemu",
 		Config: map[string]interface{}{
-			"image_path":        "linux-0.2.img",
-			"accelerator":       "tcg",
+			"image_path":        "alpine.qcow2",
 			"graceful_shutdown": true,
+			"args":              []string{"-nodefconfig", "-nodefaults"},
 			"port_map": []map[string]int{{
-				"main": 22,
-				"web":  8080,
+				"ssh": 22,
 			}},
-			"args": []string{"-nodefconfig", "-nodefaults"},
 		},
-		// With the use of tcg acceleration, it's very unlikely a qemu instance
-		// will boot (and gracefully halt) in a reasonable amount of time, so
-		// this timeout is kept low to reduce test execution time.
-		KillTimeout: time.Duration(1 * time.Second),
 		LogConfig: &structs.LogConfig{
 			MaxFiles:      10,
 			MaxFileSizeMB: 10,
 		},
 		Resources: &structs.Resources{
-			CPU:      500,
-			MemoryMB: 512,
+			CPU:      1000,
+			MemoryMB: 256,
 			Networks: []*structs.NetworkResource{
 				{
-					ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
+					ReservedPorts: []structs.Port{{Label: "ssh", Value: port}},
 				},
 			},
 		},
+		KillTimeout: killTimeout,
 	}
 
 	ctx := testDriverContexts(t, task)
+	ctx.DriverCtx.config.MaxKillTimeout = killTimeout
 	defer ctx.AllocDir.Destroy()
 	d := NewQemuDriver(ctx.DriverCtx)
 
@@ -174,7 +179,7 @@ func TestQemuDriver_GracefulShutdown(t *testing.T) {
 
 	dst := ctx.ExecCtx.TaskDir.Dir
 
-	copyFile("./test-resources/qemu/linux-0.2.img", filepath.Join(dst, "linux-0.2.img"), t)
+	copyFile("./test-resources/qemu/alpine.qcow2", filepath.Join(dst, "alpine.qcow2"), t)
 
 	if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
 		t.Fatalf("Prestart failed: %v", err)
@@ -187,26 +192,46 @@ func TestQemuDriver_GracefulShutdown(t *testing.T) {
 
 	// Clean up
 	defer func() {
+		select {
+		case <-resp.Handle.WaitCh():
+			// Already exited
+			return
+		default:
+		}
+
 		if err := resp.Handle.Kill(); err != nil {
-			logger.Printf("Error killing Qemu test: %s", err)
+			logger.Printf("[TEST] Error killing Qemu test: %s", err)
 		}
 	}()
 
-	// The monitor socket will not exist immediately, so we'll wait up to
-	// 5 seconds for it to become available.
-	monitorPath := fmt.Sprintf("%s/linux/%s", ctx.AllocDir.AllocDir, qemuMonitorSocketName)
-	monitorPathExists := false
-	for i := 0; i < 100; i++ {
-		if _, err := os.Stat(monitorPath); !os.IsNotExist(err) {
-			logger.Printf("monitor socket exists at %q\n", monitorPath)
-			monitorPathExists = true
-			break
+	// Wait until sshd starts before attempting to do a graceful shutdown
+	testutil.WaitForResult(func() (bool, error) {
+		conn, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
+		if err != nil {
+			return false, err
 		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	if monitorPathExists == false {
-		t.Fatalf("monitor socket did not exist after waiting 20 seconds")
-	}
+
+		// Since the connection will be accepted by the QEMU process
+		// before sshd actually starts, we need to block until we can
+		// read the "SSH" magic bytes
+		header := make([]byte, 3)
+		conn.SetReadDeadline(time.Now().Add(10 * time.Second))
+		_, err = conn.Read(header)
+		if err != nil {
+			return false, err
+		}
+		if !bytes.Equal(header, []byte{'S', 'S', 'H'}) {
+			return false, fmt.Errorf("expected 'SSH' but received: %q %v", string(header), header)
+		}
+
+		logger.Printf("[TEST] connected to sshd in VM")
+		conn.Close()
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("failed to connect to sshd in VM: %v", err)
+	})
+
+	monitorPath := filepath.Join(ctx.AllocDir.AllocDir, task.Name, qemuMonitorSocketName)
 
 	// userPid supplied in sendQemuShutdown calls is bogus (it's used only
 	// for log output)
@@ -221,6 +246,13 @@ func TestQemuDriver_GracefulShutdown(t *testing.T) {
 	if err := sendQemuShutdown(ctx.DriverCtx.logger, monitorPath, 0); err != nil {
 		t.Fatalf("unexpected error from sendQemuShutdown: %s", err)
 	}
+
+	select {
+	case <-resp.Handle.WaitCh():
+		// VM exited gracefully!
+	case <-time.After(killTimeout):
+		t.Fatalf("VM did not exit gracefully exit before timeout: %s", killTimeout)
+	}
 }
 
 func TestQemuDriverUser(t *testing.T) {

client/driver/test-resources/qemu/README.md

@@ -0,0 +1,21 @@
+# QEMU Test Images
+
+## `linux-0.2.img`
+
+via https://en.wikibooks.org/wiki/QEMU/Images
+
+Does not support graceful shutdown.
+
+## Alpine
+
+```
+qemu-img create -fmt qcow2 alpine.qcow2 8G
+
+# Download virtual x86_64 Alpine image https://alpinelinux.org/downloads/
+qemu-system-x86_64 -cdrom path/to/alpine.iso -hda alpine.qcow2 -boot d -net nic -net user -m 256 -localtime
+
+# In the guest run setup-alpine and exit when complete
+
+# Boot again with:
+qemu-system-x86_64 alpine.qcow2
+```