open-nomad/command/alloc_status_test.go

package command

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/nomad/command/agent"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/mitchellh/cli"
	"github.com/posener/complete"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestAllocStatusCommand_Implements(t *testing.T) {
	t.Parallel()
	var _ cli.Command = &AllocStatusCommand{}
}

func TestAllocStatusCommand_Fails(t *testing.T) {
	t.Parallel()
	srv, _, url := testServer(t, false, nil)
	defer srv.Shutdown()

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}

	// Fails on misuse
	if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) {
		t.Fatalf("expected help output, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails on connection failure
	if code := cmd.Run([]string{"-address=nope", "foobar"}); code != 1 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error querying allocation") {
		t.Fatalf("expected failed query error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails on missing alloc
	if code := cmd.Run([]string{"-address=" + url, "26470238-5CF2-438F-8772-DC67CFB0705C"}); code != 1 {
		t.Fatalf("expected exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "No allocation(s) with prefix or id") {
		t.Fatalf("expected not found error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fail on identifier with too few characters
	if code := cmd.Run([]string{"-address=" + url, "2"}); code != 1 {
		t.Fatalf("expected exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "must contain at least two characters.") {
		t.Fatalf("expected too few characters error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Identifiers with uneven length should produce a query result
	if code := cmd.Run([]string{"-address=" + url, "123"}); code != 1 {
		t.Fatalf("expected exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "No allocation(s) with prefix or id") {
		t.Fatalf("expected not found error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Failed on both -json and -t options are specified
	if code := cmd.Run([]string{"-address=" + url, "-json", "-t", "{{.ID}}"}); code != 1 {
		t.Fatalf("expected exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Both json and template formatting are not allowed") {
		t.Fatalf("expected getting formatter error, got: %s", out)
	}
}

func TestAllocStatusCommand_LifecycleInfo(t *testing.T) {
	t.Parallel()
	srv, client, url := testServer(t, true, nil)
	defer srv.Shutdown()

	// Wait for a node to be ready
	testutil.WaitForResult(func() (bool, error) {
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			return false, err
		}
		for _, node := range nodes {
			if node.Status == structs.NodeStatusReady {
				return true, nil
			}
		}
		return false, fmt.Errorf("no ready nodes")
	}, func(err error) {
		require.NoError(t, err)
	})

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
	state := srv.Agent.Server().State()

	a := mock.Alloc()
	a.Metrics = &structs.AllocMetric{}
	tg := a.Job.LookupTaskGroup(a.TaskGroup)

	initTask := tg.Tasks[0].Copy()
	initTask.Name = "init_task"
	initTask.Lifecycle = &structs.TaskLifecycleConfig{
		Hook: "prestart",
	}

	prestartSidecarTask := tg.Tasks[0].Copy()
	prestartSidecarTask.Name = "prestart_sidecar"
	prestartSidecarTask.Lifecycle = &structs.TaskLifecycleConfig{
		Hook:    "prestart",
		Sidecar: true,
	}

	tg.Tasks = append(tg.Tasks, initTask, prestartSidecarTask)
	a.TaskResources["init_task"] = a.TaskResources["web"]
	a.TaskResources["prestart_sidecar"] = a.TaskResources["web"]
	a.TaskStates = map[string]*structs.TaskState{
		"web":              &structs.TaskState{State: "pending"},
		"init_task":        &structs.TaskState{State: "running"},
		"prestart_sidecar": &structs.TaskState{State: "running"},
	}

	require.Nil(t, state.UpsertAllocs(1000, []*structs.Allocation{a}))

	if code := cmd.Run([]string{"-address=" + url, a.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out := ui.OutputWriter.String()

	require.Contains(t, out, `Task "init_task" (prestart) is "running"`)
	require.Contains(t, out, `Task "prestart_sidecar" (prestart sidecar) is "running"`)
	require.Contains(t, out, `Task "web" is "pending"`)
}

func TestAllocStatusCommand_Run(t *testing.T) {
	t.Parallel()
	srv, client, url := testServer(t, true, nil)
	defer srv.Shutdown()

	// Wait for a node to be ready
	testutil.WaitForResult(func() (bool, error) {
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			return false, err
		}
		for _, node := range nodes {
			if _, ok := node.Drivers["mock_driver"]; ok &&
				node.Status == structs.NodeStatusReady {
				return true, nil
			}
		}
		return false, fmt.Errorf("no ready nodes")
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}

	jobID := "job1_sfx"
	job1 := testJob(jobID)
	resp, _, err := client.Jobs().Register(job1, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := waitForSuccess(ui, client, fullId, t, resp.EvalID); code != 0 {
		t.Fatalf("status code non zero saw %d", code)
	}
	// get an alloc id
	allocId1 := ""
	nodeName := ""
	if allocs, _, err := client.Jobs().Allocations(jobID, false, nil); err == nil {
		if len(allocs) > 0 {
			allocId1 = allocs[0].ID
			nodeName = allocs[0].NodeName
		}
	}
	if allocId1 == "" {
		t.Fatal("unable to find an allocation")
	}

	if code := cmd.Run([]string{"-address=" + url, allocId1}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out := ui.OutputWriter.String()
	if !strings.Contains(out, "Created") {
		t.Fatalf("expected to have 'Created' but saw: %s", out)
	}

	if !strings.Contains(out, "Modified") {
		t.Fatalf("expected to have 'Modified' but saw: %s", out)
	}

	nodeNameRegexpStr := fmt.Sprintf(`\nNode Name\s+= %s\n`, regexp.QuoteMeta(nodeName))
	require.Regexp(t, regexp.MustCompile(nodeNameRegexpStr), out)

	ui.OutputWriter.Reset()

	if code := cmd.Run([]string{"-address=" + url, "-verbose", allocId1}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out = ui.OutputWriter.String()
	if !strings.Contains(out, allocId1) {
		t.Fatal("expected to find alloc id in output")
	}
	if !strings.Contains(out, "Created") {
		t.Fatalf("expected to have 'Created' but saw: %s", out)
	}
	ui.OutputWriter.Reset()

	// Try the query with an even prefix that includes the hyphen
	if code := cmd.Run([]string{"-address=" + url, allocId1[:13]}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out = ui.OutputWriter.String()
	if !strings.Contains(out, "Created") {
		t.Fatalf("expected to have 'Created' but saw: %s", out)
	}
	ui.OutputWriter.Reset()

	if code := cmd.Run([]string{"-address=" + url, "-verbose", allocId1}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out = ui.OutputWriter.String()
	if !strings.Contains(out, allocId1) {
		t.Fatal("expected to find alloc id in output")
	}
	ui.OutputWriter.Reset()

}

func TestAllocStatusCommand_RescheduleInfo(t *testing.T) {
	t.Parallel()
	srv, client, url := testServer(t, true, nil)
	defer srv.Shutdown()

	// Wait for a node to be ready
	testutil.WaitForResult(func() (bool, error) {
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			return false, err
		}
		for _, node := range nodes {
			if node.Status == structs.NodeStatusReady {
				return true, nil
			}
		}
		return false, fmt.Errorf("no ready nodes")
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
	// Test reschedule attempt info
	require := require.New(t)
	state := srv.Agent.Server().State()
	a := mock.Alloc()
	a.Metrics = &structs.AllocMetric{}
	nextAllocId := uuid.Generate()
	a.NextAllocation = nextAllocId
	a.RescheduleTracker = &structs.RescheduleTracker{
		Events: []*structs.RescheduleEvent{
			{
				RescheduleTime: time.Now().Add(-2 * time.Minute).UTC().UnixNano(),
				PrevAllocID:    uuid.Generate(),
				PrevNodeID:     uuid.Generate(),
			},
		},
	}
	require.Nil(state.UpsertAllocs(1000, []*structs.Allocation{a}))

	if code := cmd.Run([]string{"-address=" + url, a.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out := ui.OutputWriter.String()
	require.Contains(out, "Replacement Alloc ID")
	require.Regexp(regexp.MustCompile(".*Reschedule Attempts\\s*=\\s*1/2"), out)
}

func TestAllocStatusCommand_ScoreMetrics(t *testing.T) {
	t.Parallel()
	srv, client, url := testServer(t, true, nil)
	defer srv.Shutdown()

	// Wait for a node to be ready
	testutil.WaitForResult(func() (bool, error) {
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			return false, err
		}
		for _, node := range nodes {
			if node.Status == structs.NodeStatusReady {
				return true, nil
			}
		}
		return false, fmt.Errorf("no ready nodes")
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
	// Test node metrics
	require := require.New(t)
	state := srv.Agent.Server().State()
	a := mock.Alloc()
	mockNode1 := mock.Node()
	mockNode2 := mock.Node()
	a.Metrics = &structs.AllocMetric{
		ScoreMetaData: []*structs.NodeScoreMeta{
			{
				NodeID: mockNode1.ID,
				Scores: map[string]float64{
					"binpack":       0.77,
					"node-affinity": 0.5,
				},
			},
			{
				NodeID: mockNode2.ID,
				Scores: map[string]float64{
					"binpack":       0.75,
					"node-affinity": 0.33,
				},
			},
		},
	}
	require.Nil(state.UpsertAllocs(1000, []*structs.Allocation{a}))

	if code := cmd.Run([]string{"-address=" + url, "-verbose", a.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out := ui.OutputWriter.String()
	require.Contains(out, "Placement Metrics")
	require.Contains(out, mockNode1.ID)
	require.Contains(out, mockNode2.ID)

	// assert we sort headers alphabetically
	require.Contains(out, "binpack  node-affinity")
	require.Contains(out, "final score")
}

func TestAllocStatusCommand_AutocompleteArgs(t *testing.T) {
	assert := assert.New(t)
	t.Parallel()

	srv, _, url := testServer(t, true, nil)
	defer srv.Shutdown()

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}}

	// Create a fake alloc
	state := srv.Agent.Server().State()
	a := mock.Alloc()
	assert.Nil(state.UpsertAllocs(1000, []*structs.Allocation{a}))

	prefix := a.ID[:5]
	args := complete.Args{Last: prefix}
	predictor := cmd.AutocompleteArgs()

	res := predictor.Predict(args)
	assert.Equal(1, len(res))
	assert.Equal(a.ID, res[0])
}

func TestAllocStatusCommand_HostVolumes(t *testing.T) {
	t.Parallel()
	// We have to create a tempdir for the host volume even though we're
	// not going to use it b/c the server validates the config on startup
	tmpDir, err := ioutil.TempDir("", "vol0")
	if err != nil {
		t.Fatalf("unable to create tempdir for test: %v", err)
	}
	defer os.RemoveAll(tmpDir)

	vol0 := uuid.Generate()
	srv, _, url := testServer(t, true, func(c *agent.Config) {
		c.Client.HostVolumes = []*structs.ClientHostVolumeConfig{
			{
				Name:     vol0,
				Path:     tmpDir,
				ReadOnly: false,
			},
		}
	})
	defer srv.Shutdown()
	state := srv.Agent.Server().State()

	// Upsert the job and alloc
	node := mock.Node()
	alloc := mock.Alloc()
	alloc.Metrics = &structs.AllocMetric{}
	alloc.NodeID = node.ID
	job := alloc.Job
	job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{
		vol0: {
			Name:   vol0,
			Type:   structs.VolumeTypeHost,
			Source: tmpDir,
		},
	}
	job.TaskGroups[0].Tasks[0].VolumeMounts = []*structs.VolumeMount{
		{
			Volume:          vol0,
			Destination:     "/var/www",
			ReadOnly:        true,
			PropagationMode: "private",
		},
	}
	// fakes the placement enough so that we have something to iterate
	// on in 'nomad alloc status'
	alloc.TaskStates = map[string]*structs.TaskState{
		"web": &structs.TaskState{
			Events: []*structs.TaskEvent{
				structs.NewTaskEvent("test event").SetMessage("test msg"),
			},
		},
	}
	summary := mock.JobSummary(alloc.JobID)
	require.NoError(t, state.UpsertJobSummary(1004, summary))
	require.NoError(t, state.UpsertAllocs(1005, []*structs.Allocation{alloc}))

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
	if code := cmd.Run([]string{"-address=" + url, "-verbose", alloc.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out := ui.OutputWriter.String()
	require.Contains(t, out, "Host Volumes")
	require.Contains(t, out, fmt.Sprintf("%s  true", vol0))
	require.NotContains(t, out, "CSI Volumes")
}

func TestAllocStatusCommand_CSIVolumes(t *testing.T) {
	t.Parallel()
	srv, _, url := testServer(t, true, nil)
	defer srv.Shutdown()
	state := srv.Agent.Server().State()

	// Upsert the node, plugin, and volume
	vol0 := uuid.Generate()
	node := mock.Node()
	node.CSINodePlugins = map[string]*structs.CSIInfo{
		"minnie": {
			PluginID: "minnie",
			Healthy:  true,
			NodeInfo: &structs.CSINodeInfo{},
		},
	}
	err := state.UpsertNode(1001, node)
	require.NoError(t, err)

	vols := []*structs.CSIVolume{{
		ID:             vol0,
		Namespace:      structs.DefaultNamespace,
		PluginID:       "minnie",
		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
		Topologies: []*structs.CSITopology{{
			Segments: map[string]string{"foo": "bar"},
		}},
	}}
	err = state.CSIVolumeRegister(1002, vols)
	require.NoError(t, err)

	// Upsert the job and alloc
	alloc := mock.Alloc()
	alloc.Metrics = &structs.AllocMetric{}
	alloc.NodeID = node.ID
	job := alloc.Job
	job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{
		vol0: {
			Name:   vol0,
			Type:   structs.VolumeTypeCSI,
			Source: "/tmp/vol0",
		},
	}
	job.TaskGroups[0].Tasks[0].VolumeMounts = []*structs.VolumeMount{
		{
			Volume:          vol0,
			Destination:     "/var/www",
			ReadOnly:        true,
			PropagationMode: "private",
		},
	}
	// if we don't set a task state, there's nothing to iterate on alloc status
	alloc.TaskStates = map[string]*structs.TaskState{
		"web": &structs.TaskState{
			Events: []*structs.TaskEvent{
				structs.NewTaskEvent("test event").SetMessage("test msg"),
			},
		},
	}
	summary := mock.JobSummary(alloc.JobID)
	require.NoError(t, state.UpsertJobSummary(1004, summary))
	require.NoError(t, state.UpsertAllocs(1005, []*structs.Allocation{alloc}))

	ui := new(cli.MockUi)
	cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
	if code := cmd.Run([]string{"-address=" + url, "-verbose", alloc.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}
	out := ui.OutputWriter.String()
	require.Contains(t, out, "CSI Volumes")
	require.Contains(t, out, fmt.Sprintf("%s  minnie", vol0))
	require.NotContains(t, out, "Host Volumes")
}