cli: add recommendation commands.

Adds new CLI commands for applying, dismissing and detailing Nomad
recommendations under a new top level recommendation command.

command/commands.go

@@ -648,6 +648,32 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 			}, nil
 		},
 
+		"recommendation": func() (cli.Command, error) {
+			return &RecommendationCommand{
+				Meta: meta,
+			}, nil
+		},
+		"recommendation apply": func() (cli.Command, error) {
+			return &RecommendationApplyCommand{
+				Meta: meta,
+			}, nil
+		},
+		"recommendation dismiss": func() (cli.Command, error) {
+			return &RecommendationDismissCommand{
+				Meta: meta,
+			}, nil
+		},
+		"recommendation info": func() (cli.Command, error) {
+			return &RecommendationInfoCommand{
+				Meta: meta,
+			}, nil
+		},
+		"recommendation list": func() (cli.Command, error) {
+			return &RecommendationListCommand{
+				Meta: meta,
+			}, nil
+		},
+
 		"run": func() (cli.Command, error) {
 			return &JobRunCommand{
 				Meta: meta,

command/recommendation.go

@@ -0,0 +1,38 @@
+package command
+
+import (
+	"strings"
+
+	"github.com/mitchellh/cli"
+)
+
+// Ensure RecommendationCommand satisfies the cli.Command interface.
+var _ cli.Command = &RecommendationCommand{}
+
+// RecommendationCommand implements cli.Command.
+type RecommendationCommand struct {
+	Meta
+}
+
+// Help satisfies the cli.Command Help function.
+func (r *RecommendationCommand) Help() string {
+	helpText := `
+Usage: nomad recommendation <subcommand> [options]
+
+  This command groups subcommands for interacting with the recommendation API.
+
+  Please see the individual subcommand help for detailed usage information.
+`
+	return strings.TrimSpace(helpText)
+}
+
+// Synopsis satisfies the cli.Command Synopsis function.
+func (r *RecommendationCommand) Synopsis() string {
+	return "Interact with the Nomad recommendation endpoint"
+}
+
+// Name returns the name of this command.
+func (r *RecommendationCommand) Name() string { return "recommendation" }
+
+// Run satisfies the cli.Command Run function.
+func (r *RecommendationCommand) Run(_ []string) int { return cli.RunResultHelp }

command/recommendation_apply.go

@@ -0,0 +1,159 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+// Ensure RecommendationApplyCommand satisfies the cli.Command interface.
+var _ cli.Command = &RecommendationApplyCommand{}
+
+// RecommendationApplyCommand implements cli.Command.
+type RecommendationApplyCommand struct {
+	Meta
+}
+
+// Help satisfies the cli.Command Help function.
+func (r *RecommendationApplyCommand) Help() string {
+	helpText := `
+Usage: nomad recommendation apply [options] <recommendation_ids>
+
+  Apply one or more Nomad recommendations.
+
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Recommendation Apply Options:
+
+  -detach
+    Return immediately instead of entering monitor mode. After applying a
+    recommendation, the evaluation ID will be printed to the screen, which can
+    be used to examine the evaluation using the eval-status command. If applying
+    recommendations for multiple jobs, this value will always be true.
+
+  -policy-override
+    If set, any soft mandatory Sentinel policies will be overridden. This allows
+    a recommendation to be applied when it would be denied by a policy.
+
+  -verbose
+    Display full information.
+`
+	return strings.TrimSpace(helpText)
+}
+
+// Synopsis satisfies the cli.Command Synopsis function.
+func (r *RecommendationApplyCommand) Synopsis() string {
+	return "Apply one or more Nomad recommendations"
+}
+
+func (r *RecommendationApplyCommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(r.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-detach":          complete.PredictNothing,
+			"-policy-override": complete.PredictNothing,
+			"-verbose":         complete.PredictNothing,
+		})
+}
+
+// Name returns the name of this command.
+func (r *RecommendationApplyCommand) Name() string { return "recommendation apply" }
+
+// Run satisfies the cli.Command Run function.
+func (r *RecommendationApplyCommand) Run(args []string) int {
+	var detach, override, verbose bool
+
+	flags := r.Meta.FlagSet(r.Name(), FlagSetClient)
+	flags.Usage = func() { r.Ui.Output(r.Help()) }
+	flags.BoolVar(&override, "policy-override", false, "")
+	flags.BoolVar(&detach, "detach", false, "")
+	flags.BoolVar(&verbose, "verbose", false, "")
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	if args = flags.Args(); len(args) < 1 {
+		r.Ui.Error("This command takes at least one argument: <recommendation_id>")
+		r.Ui.Error(commandErrorText(r))
+		return 1
+	}
+
+	// Get the HTTP client.
+	client, err := r.Meta.Client()
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	// Create a list of recommendations to apply.
+	ids := make([]string, len(args))
+	for i, id := range args {
+		ids[i] = id
+	}
+
+	resp, _, err := client.Recommendations().Apply(ids, override)
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error applying recommendations: %v", err))
+		return 1
+	}
+
+	// If we should detach, or must because we applied multiple recommendations
+	// resulting in more than a single eval to monitor.
+	if detach || len(resp.Errors) > 0 || len(resp.UpdatedJobs) > 1 {
+
+		// If we had apply errors, output these at the top so they are easy to
+		// find. Always output the heading, this provides some consistency,
+		// even if just to show there are no errors.
+		r.Ui.Output(r.Colorize().Color("[bold]Errors[reset]"))
+		if len(resp.Errors) > 0 {
+			r.outputApplyErrors(resp.Errors)
+		} else {
+			r.Ui.Output("None\n")
+		}
+
+		// If we had apply results, output these.
+		if len(resp.UpdatedJobs) > 0 {
+			r.outputApplyResult(resp.UpdatedJobs)
+		}
+		return 0
+	}
+
+	// When would we ever reach this case? Probably never, but catch this just
+	// in case.
+	if len(resp.UpdatedJobs) < 1 {
+		return 0
+	}
+
+	// If we reached here, we should have a single entry to interrogate and
+	// monitor.
+	length := shortId
+	if verbose {
+		length = fullId
+	}
+	mon := newMonitor(r.Ui, client, length)
+	return mon.monitor(resp.UpdatedJobs[0].EvalID)
+}
+
+func (r *RecommendationApplyCommand) outputApplyErrors(errs []*api.SingleRecommendationApplyError) {
+	output := []string{"IDs|Job ID|Error"}
+	for _, err := range errs {
+		output = append(output, fmt.Sprintf("%s|%s|%s", err.Recommendations, err.JobID, err.Error))
+	}
+	r.Ui.Output(formatList(output))
+	r.Ui.Output("\n")
+}
+
+func (r *RecommendationApplyCommand) outputApplyResult(res []*api.SingleRecommendationApplyResult) {
+	output := []string{"IDs|Namespace|Job ID|Eval ID|Warnings"}
+	for _, r := range res {
+		output = append(output, fmt.Sprintf(
+			"%s|%s|%s|%s|%s",
+			strings.Join(r.Recommendations, ","), r.Namespace, r.JobID, r.EvalID, r.Warnings))
+	}
+	r.Ui.Output(r.Colorize().Color("[bold]Results[reset]"))
+	r.Ui.Output(formatList(output))
+}

command/recommendation_apply_test.go

@@ -0,0 +1,85 @@
+package command
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecommendationApplyCommand_Run(t *testing.T) {
+	require := require.New(t)
+	t.Parallel()
+	srv, client, url := testServer(t, true, nil)
+	defer srv.Shutdown()
+	testutil.WaitForResult(func() (bool, error) {
+		nodes, _, err := client.Nodes().List(nil)
+		if err != nil {
+			return false, err
+		}
+		if len(nodes) == 0 {
+			return false, fmt.Errorf("missing node")
+		}
+		if _, ok := nodes[0].Drivers["mock_driver"]; !ok {
+			return false, fmt.Errorf("mock_driver not ready")
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("err: %s", err)
+	})
+
+	ui := cli.NewMockUi()
+	cmd := &RecommendationApplyCommand{Meta: Meta{Ui: ui}}
+
+	// Register a test job to write a recommendation against.
+	testJob := testJob("recommendation_apply")
+	regResp, _, err := client.Jobs().Register(testJob, nil)
+	require.NoError(err)
+	registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID)
+	require.Equal(0, registerCode)
+
+	// Write a recommendation.
+	rec := api.Recommendation{
+		JobID:    *testJob.ID,
+		Group:    *testJob.TaskGroups[0].Name,
+		Task:     testJob.TaskGroups[0].Tasks[0].Name,
+		Resource: "CPU",
+		Value:    1,
+		Meta:     map[string]interface{}{"test-meta-entry": "test-meta-value"},
+		Stats:    map[string]float64{"p13": 1.13},
+	}
+	recResp, _, err := client.Recommendations().Upsert(&rec, nil)
+	if srv.Enterprise {
+		require.NoError(err)
+
+		// Read the recommendation out to ensure it is there as a control on
+		// later tests.
+		recInfo, _, err := client.Recommendations().Info(recResp.ID, nil)
+		require.NoError(err)
+		require.NotNil(recInfo)
+	} else {
+		require.Error(err, "Nomad Enterprise only endpoint")
+	}
+
+	// Only perform the call if we are running enterprise tests. Otherwise the
+	// recResp object will be nil.
+	if !srv.Enterprise {
+		return
+	}
+	code := cmd.Run([]string{"-address=" + url, recResp.ID})
+	require.Equal(0, code)
+
+	// Perform an info call on the recommendation which should return not
+	// found.
+	recInfo, _, err := client.Recommendations().Info(recResp.ID, nil)
+	require.Error(err, "not found")
+	require.Nil(recInfo)
+
+	// Check the new jobspec to see if the resource value has changed.
+	jobResp, _, err := client.Jobs().Info(*testJob.ID, nil)
+	require.NoError(err)
+	require.Equal(1, *jobResp.TaskGroups[0].Tasks[0].Resources.CPU)
+}

command/recommendation_dismiss.go

@@ -0,0 +1,85 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+// Ensure RecommendationDismissCommand satisfies the cli.Command interface.
+var _ cli.Command = &RecommendationDismissCommand{}
+
+// RecommendationDismissCommand implements cli.Command.
+type RecommendationDismissCommand struct {
+	Meta
+}
+
+// Help satisfies the cli.Command Help function.
+func (r *RecommendationDismissCommand) Help() string {
+	helpText := `
+Usage: nomad recommendation dismiss [options] <recommendation_ids>
+
+  Dismiss one or more Nomad recommendations.
+
+General Options:
+
+  ` + generalOptionsUsage()
+	return strings.TrimSpace(helpText)
+}
+
+// Synopsis satisfies the cli.Command Synopsis function.
+func (r *RecommendationDismissCommand) Synopsis() string {
+	return "Dismiss one or more Nomad recommendations"
+}
+
+func (r *RecommendationDismissCommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(r.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{})
+}
+
+// Name returns the name of this command.
+func (r *RecommendationDismissCommand) Name() string { return "recommendation dismiss" }
+
+// Run satisfies the cli.Command Run function.
+func (r *RecommendationDismissCommand) Run(args []string) int {
+
+	flags := r.Meta.FlagSet(r.Name(), FlagSetClient)
+	flags.Usage = func() { r.Ui.Output(r.Help()) }
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	if args = flags.Args(); len(args) < 1 {
+		r.Ui.Error("This command takes at least one argument: <recommendation_id>")
+		r.Ui.Error(commandErrorText(r))
+		return 1
+	}
+
+	// Get the HTTP client.
+	client, err := r.Meta.Client()
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	// Create a list of recommendations to dismiss.
+	ids := make([]string, len(args))
+	for i, id := range args {
+		ids[i] = id
+	}
+
+	_, err = client.Recommendations().Delete(ids, nil)
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error dismissing recommendations: %v", err))
+		return 1
+	}
+
+	verb := "recommendation"
+	if len(ids) > 1 {
+		verb += "s"
+	}
+	r.Ui.Output(fmt.Sprintf("Successfully dismissed %s", verb))
+	return 0
+}

command/recommendation_dismiss_test.go

@@ -0,0 +1,82 @@
+package command
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecommendationDismissCommand_Run(t *testing.T) {
+	require := require.New(t)
+	t.Parallel()
+	srv, client, url := testServer(t, true, nil)
+	defer srv.Shutdown()
+	testutil.WaitForResult(func() (bool, error) {
+		nodes, _, err := client.Nodes().List(nil)
+		if err != nil {
+			return false, err
+		}
+		if len(nodes) == 0 {
+			return false, fmt.Errorf("missing node")
+		}
+		if _, ok := nodes[0].Drivers["mock_driver"]; !ok {
+			return false, fmt.Errorf("mock_driver not ready")
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("err: %s", err)
+	})
+
+	ui := cli.NewMockUi()
+	cmd := &RecommendationDismissCommand{Meta: Meta{Ui: ui}}
+
+	// Register a test job to write a recommendation against.
+	testJob := testJob("recommendation_dismiss")
+	regResp, _, err := client.Jobs().Register(testJob, nil)
+	require.NoError(err)
+	registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID)
+	require.Equal(0, registerCode)
+
+	// Write a recommendation.
+	rec := api.Recommendation{
+		JobID:    *testJob.ID,
+		Group:    *testJob.TaskGroups[0].Name,
+		Task:     testJob.TaskGroups[0].Tasks[0].Name,
+		Resource: "CPU",
+		Value:    1050,
+		Meta:     map[string]interface{}{"test-meta-entry": "test-meta-value"},
+		Stats:    map[string]float64{"p13": 1.13},
+	}
+	recResp, _, err := client.Recommendations().Upsert(&rec, nil)
+	if srv.Enterprise {
+		require.NoError(err)
+
+		// Read the recommendation out to ensure it is there as a control on
+		// later tests.
+		recInfo, _, err := client.Recommendations().Info(recResp.ID, nil)
+		require.NoError(err)
+		require.NotNil(recInfo)
+	} else {
+		require.Error(err, "Nomad Enterprise only endpoint")
+	}
+
+	// Only perform the call if we are running enterprise tests. Otherwise the
+	// recResp object will be nil.
+	if !srv.Enterprise {
+		return
+	}
+	code := cmd.Run([]string{"-address=" + url, recResp.ID})
+	require.Equal(0, code)
+	out := ui.OutputWriter.String()
+	require.Contains(out, "Successfully dismissed recommendation")
+
+	// Perform an info call on the recommendation which should return not
+	// found.
+	recInfo, _, err := client.Recommendations().Info(recResp.ID, nil)
+	require.Error(err, "not found")
+	require.Nil(recInfo)
+}

command/recommendation_info.go

@@ -0,0 +1,169 @@
+package command
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+// Ensure RecommendationInfoCommand satisfies the cli.Command interface.
+var _ cli.Command = &RecommendationInfoCommand{}
+
+// RecommendationInfoCommand implements cli.Command.
+type RecommendationInfoCommand struct {
+	Meta
+}
+
+// Help satisfies the cli.Command Help function.
+func (r *RecommendationInfoCommand) Help() string {
+	helpText := `
+Usage: nomad recommendation info [options] <recommendation_id>
+
+  Info is used to read the specified recommendation.
+
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Recommendation Info Options:
+
+  -json
+    Output the recommendation in its JSON format.
+
+  -t
+    Format and display the recommendation using a Go template.
+`
+	return strings.TrimSpace(helpText)
+}
+
+// Synopsis satisfies the cli.Command Synopsis function.
+func (r *RecommendationInfoCommand) Synopsis() string {
+	return "Display an individual Nomad recommendation"
+}
+
+func (r *RecommendationInfoCommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(r.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-json": complete.PredictNothing,
+			"-t":    complete.PredictAnything,
+		})
+}
+
+// Name returns the name of this command.
+func (r *RecommendationInfoCommand) Name() string { return "recommendation info" }
+
+// Run satisfies the cli.Command Run function.
+func (r *RecommendationInfoCommand) Run(args []string) int {
+	var json bool
+	var tmpl string
+
+	flags := r.Meta.FlagSet(r.Name(), FlagSetClient)
+	flags.Usage = func() { r.Ui.Output(r.Help()) }
+	flags.BoolVar(&json, "json", false, "")
+	flags.StringVar(&tmpl, "t", "", "")
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	if args = flags.Args(); len(args) != 1 {
+		r.Ui.Error("This command takes one argument: <recommendation_id>")
+		r.Ui.Error(commandErrorText(r))
+		return 1
+	}
+
+	// Get the recommendation ID.
+	recID := args[0]
+
+	// Get the HTTP client.
+	client, err := r.Meta.Client()
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	rec, _, err := client.Recommendations().Info(recID, nil)
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error reading recommendation: %s", err))
+		return 1
+	}
+
+	// If the user has specified to output the recommendation as JSON or using
+	// a template then perform this action for the entire object and exit the
+	// command.
+	if json || len(tmpl) > 0 {
+		out, err := Format(json, tmpl, rec)
+		if err != nil {
+			r.Ui.Error(err.Error())
+			return 1
+		}
+		r.Ui.Output(out)
+		return 0
+	}
+
+	info := []string{
+		fmt.Sprintf("ID|%s", rec.ID),
+		fmt.Sprintf("Namespace|%s", rec.Namespace),
+		fmt.Sprintf("Job ID|%s", rec.JobID),
+		fmt.Sprintf("Task Group|%s", rec.Group),
+		fmt.Sprintf("Task|%s", rec.Task),
+		fmt.Sprintf("Resource|%s", rec.Resource),
+		fmt.Sprintf("Value|%v", rec.Value),
+		fmt.Sprintf("Current|%v", rec.Current),
+	}
+	r.Ui.Output(formatKV(info))
+
+	// If we have stats, format and output these.
+	if len(rec.Stats) > 0 {
+
+		// Sort the stats keys into an alphabetically ordered list to provide
+		// consistent outputs.
+		keys := []string{}
+
+		for k := range rec.Stats {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		// We will only need two rows; key:value.
+		output := make([]string, 2)
+
+		for _, stat := range keys {
+			output[0] += fmt.Sprintf("%s|", stat)
+			output[1] += fmt.Sprintf("%.2f|", rec.Stats[stat])
+		}
+
+		// Trim any trailing pipes so we can use the formatList function thus
+		// providing a nice clean output.
+		output[0] = strings.TrimRight(output[0], "|")
+		output[1] = strings.TrimRight(output[1], "|")
+
+		r.Ui.Output(r.Colorize().Color("\n[bold]Stats[reset]"))
+		r.Ui.Output(formatList(output))
+	}
+
+	// If we have meta, format and output the entries.
+	if len(rec.Meta) > 0 {
+
+		// Sort the meta keys into an alphabetically ordered list to provide
+		// consistent outputs.
+		keys := []string{}
+
+		for k := range rec.Meta {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		output := make([]string, len(rec.Meta))
+
+		for i, key := range keys {
+			output[i] = fmt.Sprintf("%s|%v", key, rec.Meta[key])
+		}
+		r.Ui.Output(r.Colorize().Color("\n[bold]Meta[reset]"))
+		r.Ui.Output(formatKV(output))
+	}
+
+	return 0
+}

command/recommendation_info_test.go

@@ -0,0 +1,83 @@
+package command
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecommendationInfoCommand_Run(t *testing.T) {
+	require := require.New(t)
+	t.Parallel()
+	srv, client, url := testServer(t, true, nil)
+	defer srv.Shutdown()
+	testutil.WaitForResult(func() (bool, error) {
+		nodes, _, err := client.Nodes().List(nil)
+		if err != nil {
+			return false, err
+		}
+		if len(nodes) == 0 {
+			return false, fmt.Errorf("missing node")
+		}
+		if _, ok := nodes[0].Drivers["mock_driver"]; !ok {
+			return false, fmt.Errorf("mock_driver not ready")
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("err: %s", err)
+	})
+
+	ui := cli.NewMockUi()
+	cmd := &RecommendationInfoCommand{Meta: Meta{Ui: ui}}
+
+	// Perform an initial call, which should return a not found error.
+	code := cmd.Run([]string{"-address=" + url, "2c13f001-f5b6-ce36-03a5-e37afe160df5"})
+	if srv.Enterprise {
+		require.Equal(1, code)
+		out := ui.ErrorWriter.String()
+		require.Contains(out, "Recommendation not found")
+	} else {
+		require.Equal(1, code)
+		require.Contains(ui.ErrorWriter.String(), "Nomad Enterprise only endpoint")
+	}
+
+	// Register a test job to write a recommendation against.
+	testJob := testJob("recommendation_info")
+	regResp, _, err := client.Jobs().Register(testJob, nil)
+	require.NoError(err)
+	registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID)
+	require.Equal(0, registerCode)
+
+	// Write a recommendation.
+	rec := api.Recommendation{
+		JobID:    *testJob.ID,
+		Group:    *testJob.TaskGroups[0].Name,
+		Task:     testJob.TaskGroups[0].Tasks[0].Name,
+		Resource: "CPU",
+		Value:    1050,
+		Meta:     map[string]interface{}{"test-meta-entry": "test-meta-value"},
+		Stats:    map[string]float64{"p13": 1.13},
+	}
+	recResp, _, err := client.Recommendations().Upsert(&rec, nil)
+	if srv.Enterprise {
+		require.NoError(err)
+	} else {
+		require.Error(err, "Nomad Enterprise only endpoint")
+	}
+
+	// Only perform the call if we are running enterprise tests. Otherwise the
+	// recResp object will be nil.
+	if srv.Enterprise {
+		code = cmd.Run([]string{"-address=" + url, recResp.ID})
+		require.Equal(0, code)
+		out := ui.OutputWriter.String()
+		require.Contains(out, "test-meta-entry")
+		require.Contains(out, "p13")
+		require.Contains(out, "1.13")
+		require.Contains(out, recResp.ID)
+	}
+}

command/recommendation_list.go

@@ -0,0 +1,202 @@
+package command
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+// Ensure RecommendationListCommand satisfies the cli.Command interface.
+var _ cli.Command = &RecommendationListCommand{}
+
+// RecommendationListCommand implements cli.Command.
+type RecommendationListCommand struct {
+	Meta
+}
+
+// Help satisfies the cli.Command Help function.
+func (r *RecommendationListCommand) Help() string {
+	helpText := `
+Usage: nomad recommendation list [options]
+
+  List is used to list the available recommendations.
+
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Recommendation List Options:
+
+  -job
+    Specifies the job ID to filter the recommendations list by.
+
+  -group
+    Specifies the task group name to filter within a job. If specified, the -job
+    flag must also be specified.
+
+  -task
+    Specifies the task name to filter within a job and task group. If specified,
+    the -job and -group flags must also be specified.
+
+  -json
+    Output the recommendations in JSON format.
+
+  -t
+    Format and display the recommendations using a Go template.
+`
+	return strings.TrimSpace(helpText)
+}
+
+// Synopsis satisfies the cli.Command Synopsis function.
+func (r *RecommendationListCommand) Synopsis() string {
+	return "Display all Nomad recommendations"
+}
+
+func (r *RecommendationListCommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(r.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-job":   complete.PredictNothing,
+			"-group": complete.PredictNothing,
+			"-task":  complete.PredictNothing,
+			"-json":  complete.PredictNothing,
+			"-t":     complete.PredictAnything,
+		})
+}
+
+// Name returns the name of this command.
+func (r *RecommendationListCommand) Name() string { return "recommendation list" }
+
+// Run satisfies the cli.Command Run function.
+func (r *RecommendationListCommand) Run(args []string) int {
+	var json bool
+	var tmpl, job, group, task string
+
+	flags := r.Meta.FlagSet(r.Name(), FlagSetClient)
+	flags.Usage = func() { r.Ui.Output(r.Help()) }
+	flags.BoolVar(&json, "json", false, "")
+	flags.StringVar(&tmpl, "t", "", "")
+	flags.StringVar(&job, "job", "", "")
+	flags.StringVar(&group, "group", "", "")
+	flags.StringVar(&task, "task", "", "")
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	if args = flags.Args(); len(args) > 0 {
+		r.Ui.Error("This command takes no arguments")
+		r.Ui.Error(commandErrorText(r))
+	}
+
+	// Get the HTTP client.
+	client, err := r.Meta.Client()
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	// Validate the input flags. This is done by the HTTP API anyway, but there
+	// is no harm doing it here to avoid calls that we know wont succeed.
+	if group != "" && job == "" {
+		r.Ui.Error("Job flag must be supplied when using group flag")
+		return 1
+
+	}
+	if task != "" && group == "" {
+		r.Ui.Error("Group flag must be supplied when using task flag")
+		return 1
+	}
+
+	// Setup the query params.
+	q := &api.QueryOptions{
+		Params: map[string]string{},
+	}
+	if job != "" {
+		q.Params["job"] = job
+	}
+	if group != "" {
+		q.Params["group"] = group
+	}
+	if task != "" {
+		q.Params["task"] = task
+	}
+
+	recommendations, _, err := client.Recommendations().List(q)
+	if err != nil {
+		r.Ui.Error(fmt.Sprintf("Error listing recommendations: %s", err))
+		return 1
+	}
+
+	if len(recommendations) == 0 {
+		r.Ui.Output("No recommendations found")
+		return 0
+	}
+
+	if json || len(tmpl) > 0 {
+		out, err := Format(json, tmpl, recommendations)
+		if err != nil {
+			r.Ui.Error(err.Error())
+			return 1
+		}
+		r.Ui.Output(out)
+		return 0
+	}
+
+	// Create the output table header.
+	output := []string{"ID|"}
+
+	// If the operator is using the namespace wildcard option, add this header.
+	if r.Meta.namespace == "*" {
+		output[0] += "Namespace|"
+	}
+	output[0] += "Job|Group|Task|Resource|Value"
+
+	// Sort the list of recommendations based on their job, group and task.
+	sortedRecs := recommendationList{r: recommendations}
+	sort.Sort(sortedRecs)
+
+	// Iterate the recommendations and add to the output.
+	for i, rec := range sortedRecs.r {
+
+		output = append(output, rec.ID)
+
+		if r.Meta.namespace == "*" {
+			output[i+1] += fmt.Sprintf("|%s", rec.Namespace)
+		}
+		output[i+1] += fmt.Sprintf("|%s|%s|%s|%s|%v", rec.JobID, rec.Group, rec.Task, rec.Resource, rec.Value)
+	}
+
+	// Output.
+	r.Ui.Output(formatList(output))
+	return 0
+}
+
+// recommendationList is a wrapper around []*api.Recommendation that lets us
+// sort the recommendations alphabetically based on their job, group and task.
+type recommendationList struct {
+	r []*api.Recommendation
+}
+
+// Len satisfies the Len function of the sort.Interface interface.
+func (r recommendationList) Len() int { return len(r.r) }
+
+// Swap satisfies the Swap function of the sort.Interface interface.
+func (r recommendationList) Swap(i, j int) {
+	r.r[i], r.r[j] = r.r[j], r.r[i]
+}
+
+// Less satisfies the Less function of the sort.Interface interface.
+func (r recommendationList) Less(i, j int) bool {
+	recI := r.stringFromResource(i)
+	recJ := r.stringFromResource(j)
+	stringList := []string{recI, recJ}
+	sort.Strings(stringList)
+	return stringList[0] == recI
+}
+
+func (r recommendationList) stringFromResource(i int) string {
+	return strings.Join([]string{r.r[i].Namespace, r.r[i].JobID, r.r[i].Group, r.r[i].Task, r.r[i].Resource}, ":")
+}

command/recommendation_list_test.go

@@ -0,0 +1,175 @@
+package command
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecommendationListCommand_Run(t *testing.T) {
+	require := require.New(t)
+	t.Parallel()
+	srv, client, url := testServer(t, true, nil)
+	defer srv.Shutdown()
+	testutil.WaitForResult(func() (bool, error) {
+		nodes, _, err := client.Nodes().List(nil)
+		if err != nil {
+			return false, err
+		}
+		if len(nodes) == 0 {
+			return false, fmt.Errorf("missing node")
+		}
+		if _, ok := nodes[0].Drivers["mock_driver"]; !ok {
+			return false, fmt.Errorf("mock_driver not ready")
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("err: %s", err)
+	})
+
+	ui := cli.NewMockUi()
+	cmd := &RecommendationListCommand{Meta: Meta{Ui: ui}}
+
+	// Perform an initial list, which should return zero results.
+	code := cmd.Run([]string{"-address=" + url})
+	if srv.Enterprise {
+		require.Equal(0, code)
+		out := ui.OutputWriter.String()
+		require.Contains(out, "No recommendations found")
+	} else {
+		require.Equal(1, code)
+		require.Contains(ui.ErrorWriter.String(), "Nomad Enterprise only endpoint")
+	}
+
+	// Register a test job to write a recommendation against.
+	testJob := testJob("recommendation_list")
+	regResp, _, err := client.Jobs().Register(testJob, nil)
+	require.NoError(err)
+	registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID)
+	require.Equal(0, registerCode)
+
+	// Write a recommendation.
+	rec := api.Recommendation{
+		JobID:    *testJob.ID,
+		Group:    *testJob.TaskGroups[0].Name,
+		Task:     testJob.TaskGroups[0].Tasks[0].Name,
+		Resource: "CPU",
+		Value:    1050,
+		Meta:     map[string]interface{}{"test-meta-entry": "test-meta-value"},
+		Stats:    map[string]float64{"p13": 1.13},
+	}
+	_, _, err = client.Recommendations().Upsert(&rec, nil)
+	if srv.Enterprise {
+		require.NoError(err)
+	} else {
+		require.Error(err, "Nomad Enterprise only endpoint")
+	}
+
+	// Perform a new list which should yield results.
+	code = cmd.Run([]string{"-address=" + url})
+	if srv.Enterprise {
+		require.Equal(0, code)
+		out := ui.OutputWriter.String()
+		require.Contains(out, "ID")
+		require.Contains(out, "Job")
+		require.Contains(out, "Group")
+		require.Contains(out, "Task")
+		require.Contains(out, "Resource")
+		require.Contains(out, "Value")
+		require.Contains(out, "CPU")
+	} else {
+		require.Equal(1, code)
+		require.Contains(ui.ErrorWriter.String(), "Nomad Enterprise only endpoint")
+	}
+}
+
+func TestRecommendationList_Sort(t *testing.T) {
+	testCases := []struct {
+		inputRecommendationList []*api.Recommendation
+		expectedOutputList      []*api.Recommendation
+		name                    string
+	}{
+		{
+			inputRecommendationList: []*api.Recommendation{
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+			},
+			expectedOutputList: []*api.Recommendation{
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+			},
+			name: "single job with both resources",
+		},
+		{
+			inputRecommendationList: []*api.Recommendation{
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "CPU"},
+			},
+			expectedOutputList: []*api.Recommendation{
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+			},
+			name: "single job with multiple groups",
+		},
+		{
+			inputRecommendationList: []*api.Recommendation{
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "CPU"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "CPU"},
+			},
+			expectedOutputList: []*api.Recommendation{
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+			},
+			name: "multiple jobs",
+		},
+		{
+			inputRecommendationList: []*api.Recommendation{
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "cefault", JobID: "distro", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "CPU"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "cefault", JobID: "distro", Group: "cache", Task: "redis", Resource: "CPU"},
+			},
+			expectedOutputList: []*api.Recommendation{
+				{Namespace: "cefault", JobID: "distro", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "cefault", JobID: "distro", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "distro", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "mongodb", Resource: "MemoryMB"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "CPU"},
+				{Namespace: "default", JobID: "example", Group: "cache", Task: "redis", Resource: "MemoryMB"},
+			},
+			name: "multiple namespaces",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			sortedRecs := recommendationList{r: tc.inputRecommendationList}
+			sort.Sort(sortedRecs)
+			assert.Equal(t, tc.expectedOutputList, sortedRecs.r, tc.name)
+		})
+	}
+}