luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Multiregion deploy status and job status CLI

This commit is contained in:
Drew Bailey 2020-06-15 10:05:31 -04:00 committed by Tim Gross
parent 473a0f1d44
commit 9263fcb0d3
14 changed files with 355 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/deployments.go
View File

@ -150,6 +150,9 @@ type Deployment struct {
// present the correct list of deployments for the job and not old ones.
JobCreateIndex uint64
// IsMultiregion specifies if this deployment is part of a multi-region deployment
IsMultiregion bool
// TaskGroups is the set of task groups effected by the deployment and their
// current deployment status.
TaskGroups map[string]*DeploymentState

2
command/agent/job_endpoint.go
View File

@ -772,7 +772,7 @@ func ApiJobToStructJob(job *api.Job) *structs.Job {
j.Multiregion.Regions = []*structs.MultiregionRegion{}
for _, region := range job.Multiregion.Regions {
r := &structs.MultiregionRegion{}
r.Name = *region.Name
r.Name = region.Name
r.Count = *region.Count
r.Datacenters = region.Datacenters
r.Meta = region.Meta

2
command/agent/job_endpoint_test.go
View File

@ -1582,7 +1582,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
},
Regions: []*api.MultiregionRegion{
{
Name: helper.StringToPtr("west"),
Name: "west",
Count: helper.IntToPtr(1),
Datacenters: []string{"dc1", "dc2"},
Meta: map[string]string{"region_code": "W"},

84
command/deployment_status.go
View File

@ -1,6 +1,7 @@
package command
import (
"errors"
"fmt"
"sort"
"strings"
@ -140,7 +141,7 @@ func (c *DeploymentStatusCommand) Run(args []string) int {
return 0
}
c.Ui.Output(c.Colorize().Color(formatDeployment(deploy, length)))
c.Ui.Output(c.Colorize().Color(formatDeployment(client, deploy, length)))
return 0
}
@ -182,7 +183,7 @@ func getDeployment(client *api.Deployments, dID string) (match *api.Deployment,
}
}
func formatDeployment(d *api.Deployment, uuidLength int) string {
func formatDeployment(c *api.Client, d *api.Deployment, uuidLength int) string {
if d == nil {
return "No deployment found"
}
@ -196,6 +197,18 @@ func formatDeployment(d *api.Deployment, uuidLength int) string {
}
base := formatKV(high)
// Fetch and Format Multi-region info
if d.IsMultiregion {
regions, err := fetchMultiRegionDeployments(c, d)
if err != nil {
base += "\n\nError fetching Multiregion deployments\n\n"
} else if len(regions) > 0 {
base += "\n\n[bold]Multiregion Deployment[reset]\n"
base += formatMultiregionDeployment(regions, uuidLength)
}
}
if len(d.TaskGroups) == 0 {
return base
}
@ -204,6 +217,73 @@ func formatDeployment(d *api.Deployment, uuidLength int) string {
return base
}
type regionResult struct {
region string
d *api.Deployment
err error
}
func fetchMultiRegionDeployments(c *api.Client, d *api.Deployment) (map[string]*api.Deployment, error) {
results := make(map[string]*api.Deployment)
job, _, err := c.Jobs().Info(d.JobID, &api.QueryOptions{})
if err != nil {
return nil, err
}
requests := make(chan regionResult, len(job.Multiregion.Regions))
for i := 0; i < cap(requests); i++ {
go func(itr int) {
region := job.Multiregion.Regions[itr]
d, err := fetchRegionDeployment(c, d, region)
requests <- regionResult{d: d, err: err, region: region.Name}
}(i)
}
for i := 0; i < cap(requests); i++ {
res := <-requests
if res.err != nil {
key := fmt.Sprintf("%s (error)", res.region)
results[key] = &api.Deployment{}
continue
}
results[res.region] = res.d
}
return results, nil
}
func fetchRegionDeployment(c *api.Client, d *api.Deployment, region *api.MultiregionRegion) (*api.Deployment, error) {
if region == nil {
return nil, errors.New("Region not found")
}
opts := &api.QueryOptions{Region: region.Name}
deploys, _, err := c.Jobs().Deployments(d.JobID, false, opts)
if err != nil {
return nil, err
}
for _, dep := range deploys {
if dep.JobVersion == d.JobVersion {
return dep, nil
}
}
return nil, fmt.Errorf("Could not find job version %d for region", d.JobVersion)
}
func formatMultiregionDeployment(regions map[string]*api.Deployment, uuidLength int) string {
rowString := "Region|ID|Status"
rows := make([]string, len(regions)+1)
rows[0] = rowString
i := 1
for k, v := range regions {
row := fmt.Sprintf("%s|%s|%s", k, limit(v.ID, uuidLength), v.Status)
rows[i] = row
i++
}
sort.Strings(rows)
return formatList(rows)
}
func formatDeploymentGroups(d *api.Deployment, uuidLength int) string {
// Detect if we need to add these columns
var canaries, autorevert, progressDeadline bool

104
command/deployment_status_test.go
View File

@ -1,9 +1,13 @@
package command
import (
"fmt"
"testing"
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/command/agent"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/testutil"
"github.com/mitchellh/cli"
"github.com/posener/complete"
"github.com/stretchr/testify/assert"
@ -64,3 +68,103 @@ func TestDeploymentStatusCommand_AutocompleteArgs(t *testing.T) {
assert.Equal(1, len(res))
assert.Equal(d.ID, res[0])
}
func TestDeploymentStatusCommand_Multiregion(t *testing.T) {
t.Parallel()
cbe := func(config *agent.Config) {
config.Region = "east"
config.Datacenter = "east-1"
}
cbw := func(config *agent.Config) {
config.Region = "west"
config.Datacenter = "west-1"
}
srv, clientEast, url := testServer(t, true, cbe)
defer srv.Shutdown()
srv2, clientWest, _ := testServer(t, true, cbw)
defer srv2.Shutdown()
// Join with srv1
addr1 := fmt.Sprintf("127.0.0.1:%d",
srv.Agent.Server().GetConfig().SerfConfig.MemberlistConfig.BindPort)
if _, err := srv2.Agent.Server().Join([]string{addr1}); err != nil {
t.Fatalf("Join err: %v", err)
}
// wait for client node
testutil.WaitForResult(func() (bool, error) {
nodes, _, err := clientEast.Nodes().List(nil)
if err != nil {
return false, err
}
if len(nodes) == 0 {
return false, fmt.Errorf("missing node")
}
if _, ok := nodes[0].Drivers["mock_driver"]; !ok {
return false, fmt.Errorf("mock_driver not ready")
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
ui := new(cli.MockUi)
cmd := &DeploymentStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}}
// Register multiregion job in east
jobEast := testMultiRegionJob("job1_sfxx", "east", "east-1")
resp, _, err := clientEast.Jobs().Register(jobEast, nil)
require.NoError(t, err)
if code := waitForSuccess(ui, clientEast, fullId, t, resp.EvalID); code != 0 {
t.Fatalf("status code non zero saw %d", code)
}
// Register multiregion job in west
jobWest := testMultiRegionJob("job1_sfxx", "west", "west-1")
resp2, _, err := clientWest.Jobs().Register(jobWest, &api.WriteOptions{Region: "west"})
require.NoError(t, err)
if code := waitForSuccess(ui, clientWest, fullId, t, resp2.EvalID); code != 0 {
t.Fatalf("status code non zero saw %d", code)
}
jobs, _, err := clientEast.Jobs().List(&api.QueryOptions{})
require.NoError(t, err)
require.Len(t, jobs, 1)
deploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{})
require.NoError(t, err)
require.Len(t, deploys, 1)
// Grab both deployments to verify output
eastDeploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "east"})
require.NoError(t, err)
require.Len(t, eastDeploys, 1)
westDeploys, _, err := clientWest.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "west"})
require.NoError(t, err)
require.Len(t, westDeploys, 1)
// Run command for specific deploy
if code := cmd.Run([]string{"-region=east", "-address=" + url, deploys[0].ID}); code != 0 {
t.Fatalf("expected exit 0, got: %d", code)
}
// Verify Multi-region Deployment info populated
out := ui.OutputWriter.String()
require.Contains(t, out, "Multiregion Deployment")
require.Contains(t, out, "Region")
require.Contains(t, out, "ID")
require.Contains(t, out, "Status")
require.Contains(t, out, "east")
require.Contains(t, out, eastDeploys[0].ID[0:7])
require.Contains(t, out, "west")
require.Contains(t, out, westDeploys[0].ID[0:7])
require.Contains(t, out, "running")
require.NotContains(t, out, "<none>")
}

2
command/job_deployments.go
View File

@ -148,7 +148,7 @@ func (c *JobDeploymentsCommand) Run(args []string) int {
return 0
}
c.Ui.Output(c.Colorize().Color(formatDeployment(deploy, length)))
c.Ui.Output(c.Colorize().Color(formatDeployment(client, deploy, length)))
return 0
}

15
command/job_status.go
View File

@ -381,7 +381,7 @@ func (c *JobStatusCommand) outputJobInfo(client *api.Client, job *api.Job) error
if latestDeployment != nil {
c.Ui.Output(c.Colorize().Color("\n[bold]Latest Deployment[reset]"))
c.Ui.Output(c.Colorize().Color(c.formatDeployment(latestDeployment)))
c.Ui.Output(c.Colorize().Color(c.formatDeployment(client, latestDeployment)))
}
// Format the allocs
@ -390,7 +390,7 @@ func (c *JobStatusCommand) outputJobInfo(client *api.Client, job *api.Job) error
return nil
}
func (c *JobStatusCommand) formatDeployment(d *api.Deployment) string {
func (c *JobStatusCommand) formatDeployment(client *api.Client, d *api.Deployment) string {
// Format the high-level elements
high := []string{
fmt.Sprintf("ID|%s", limit(d.ID, c.length)),
@ -399,6 +399,17 @@ func (c *JobStatusCommand) formatDeployment(d *api.Deployment) string {
}
base := formatKV(high)
if d.IsMultiregion {
regions, err := fetchMultiRegionDeployments(client, d)
if err != nil {
base += "\n\nError fetching Multiregion deployments\n\n"
} else if len(regions) > 0 {
base += "\n\n[bold]Multiregion Deployment[reset]\n"
base += formatMultiregionDeployment(regions, 8)
}
}
if len(d.TaskGroups) == 0 {
return base
}

101
command/job_status_test.go
View File

@ -385,6 +385,107 @@ func TestJobStatusCommand_RescheduleEvals(t *testing.T) {
require.Contains(out, e.ID[:8])
}
// TestJobStatusCommand_Multiregion tests multiregion deployment output
func TestJobStatusCommand_Multiregion(t *testing.T) {
t.Parallel()
cbe := func(config *agent.Config) {
config.Region = "east"
config.Datacenter = "east-1"
}
cbw := func(config *agent.Config) {
config.Region = "west"
config.Datacenter = "west-1"
}
srv, clientEast, url := testServer(t, true, cbe)
defer srv.Shutdown()
srv2, clientWest, _ := testServer(t, true, cbw)
defer srv2.Shutdown()
// Join with srv1
addr := fmt.Sprintf("127.0.0.1:%d",
srv.Agent.Server().GetConfig().SerfConfig.MemberlistConfig.BindPort)
if _, err := srv2.Agent.Server().Join([]string{addr}); err != nil {
t.Fatalf("Join err: %v", err)
}
// wait for client node
testutil.WaitForResult(func() (bool, error) {
nodes, _, err := clientEast.Nodes().List(nil)
if err != nil {
return false, err
}
if len(nodes) == 0 {
return false, fmt.Errorf("missing node")
}
if _, ok := nodes[0].Drivers["mock_driver"]; !ok {
return false, fmt.Errorf("mock_driver not ready")
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
ui := new(cli.MockUi)
cmd := &JobStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}}
// Register multiregion job
// Register multiregion job in east
jobEast := testMultiRegionJob("job1_sfxx", "east", "east-1")
resp, _, err := clientEast.Jobs().Register(jobEast, nil)
require.NoError(t, err)
if code := waitForSuccess(ui, clientEast, fullId, t, resp.EvalID); code != 0 {
t.Fatalf("status code non zero saw %d", code)
}
// Register multiregion job in west
jobWest := testMultiRegionJob("job1_sfxx", "west", "west-1")
resp2, _, err := clientWest.Jobs().Register(jobWest, &api.WriteOptions{Region: "west"})
require.NoError(t, err)
if code := waitForSuccess(ui, clientWest, fullId, t, resp2.EvalID); code != 0 {
t.Fatalf("status code non zero saw %d", code)
}
jobs, _, err := clientEast.Jobs().List(&api.QueryOptions{})
require.NoError(t, err)
require.Len(t, jobs, 1)
deploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{})
require.NoError(t, err)
require.Len(t, deploys, 1)
// Grab both deployments to verify output
eastDeploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "east"})
require.NoError(t, err)
require.Len(t, eastDeploys, 1)
westDeploys, _, err := clientWest.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "west"})
require.NoError(t, err)
// require.Len(t, westDeploys, 1)
// Run command for specific deploy
if code := cmd.Run([]string{"-address=" + url, jobs[0].ID}); code != 0 {
t.Fatalf("expected exit 0, got: %d", code)
}
// Verify Multi-region Deployment info populated
out := ui.OutputWriter.String()
require.Contains(t, out, "Multiregion Deployment")
require.Contains(t, out, "Region")
require.Contains(t, out, "ID")
require.Contains(t, out, "Status")
require.Contains(t, out, "east")
require.Contains(t, out, eastDeploys[0].ID[0:7])
require.Contains(t, out, "west")
require.Contains(t, out, westDeploys[0].ID[0:7])
require.Contains(t, out, "running")
require.NotContains(t, out, "<none>")
}
func waitForSuccess(ui cli.Ui, client *api.Client, length int, t *testing.T, evalId string) int {
mon := newMonitor(ui, client, length)
monErr := mon.monitor(evalId, false)

37
command/util_test.go
View File

@ -48,3 +48,40 @@ func testJob(jobID string) *api.Job {
return job
}
func testMultiRegionJob(jobID, region, datacenter string) *api.Job {
task := api.NewTask("task1", "mock_driver").
SetConfig("kill_after", "10s").
SetConfig("run_for", "15s").
SetConfig("exit_code", 0).
Require(&api.Resources{
MemoryMB: helper.IntToPtr(256),
CPU: helper.IntToPtr(100),
}).
SetLogConfig(&api.LogConfig{
MaxFiles: helper.IntToPtr(1),
MaxFileSizeMB: helper.IntToPtr(2),
})
group := api.NewTaskGroup("group1", 1).
AddTask(task).
RequireDisk(&api.EphemeralDisk{
SizeMB: helper.IntToPtr(20),
})
job := api.NewServiceJob(jobID, jobID, region, 1).AddDatacenter(datacenter).AddTaskGroup(group)
job.Multiregion = &api.Multiregion{
Regions: []*api.MultiregionRegion{
{
Name: "east",
Datacenters: []string{"east-1"},
},
{
Name: "west",
Datacenters: []string{"west-1"},
},
},
}
return job
}

2
jobspec/parse_multiregion.go
View File

@ -144,7 +144,7 @@ func parseMultiregionRegions(result *api.Multiregion, list *ast.ObjectList) erro
// Build the region with the basic decode
var r api.MultiregionRegion
r.Name = helper.StringToPtr(n)
r.Name = n
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
WeaklyTypedInput: true,
Result: &r,

4
jobspec/parse_test.go
View File

@ -1350,13 +1350,13 @@ func TestParse(t *testing.T) {
},
Regions: []*api.MultiregionRegion{
{
Name: helper.StringToPtr("west"),
Name: "west",
Count: helper.IntToPtr(2),
Datacenters: []string{"west-1"},
Meta: map[string]string{"region_code": "W"},
},
{
Name: helper.StringToPtr("east"),
Name: "east",
Count: helper.IntToPtr(1),
Datacenters: []string{"east-1", "east-2"},
Meta: map[string]string{"region_code": "E"},

4
nomad/structs/structs.go
View File

@ -7948,6 +7948,9 @@ type Deployment struct {
// present the correct list of deployments for the job and not old ones.
JobCreateIndex uint64
// Multiregion specifies if deployment is part of multiregion deployment
IsMultiregion bool
// TaskGroups is the set of task groups effected by the deployment and their
// current deployment status.
TaskGroups map[string]*DeploymentState
@ -7973,6 +7976,7 @@ func NewDeployment(job *Job) *Deployment {
JobModifyIndex: job.ModifyIndex,
JobSpecModifyIndex: job.JobModifyIndex,
JobCreateIndex: job.CreateIndex,
IsMultiregion: job.IsMultiregion(),
Status: DeploymentStatusRunning,
StatusDescription: DeploymentStatusDescriptionRunning,
TaskGroups: make(map[string]*DeploymentState, len(job.TaskGroups)),

3
vendor/github.com/hashicorp/nomad/api/deployments.go generated vendored
View File

@ -150,6 +150,9 @@ type Deployment struct {
// present the correct list of deployments for the job and not old ones.
JobCreateIndex uint64
// IsMultiregion specifies if this deployment is part of a multi-region deployment
IsMultiregion bool
// TaskGroups is the set of task groups effected by the deployment and their
// current deployment status.
TaskGroups map[string]*DeploymentState

7
vendor/github.com/hashicorp/nomad/api/jobs.go generated vendored
View File

@ -645,9 +645,6 @@ func (m *Multiregion) Canonicalize() {
m.Regions = []*MultiregionRegion{}
}
for _, region := range m.Regions {
if region.Name == nil {
region.Name = stringToPtr("")
}
if region.Count == nil {
region.Count = intToPtr(1)
}
@ -672,7 +669,7 @@ func (m *Multiregion) Copy() *Multiregion {
}
for _, region := range m.Regions {
copyRegion := new(MultiregionRegion)
copyRegion.Name = stringToPtr(*region.Name)
copyRegion.Name = region.Name
copyRegion.Count = intToPtr(*region.Count)
for _, dc := range region.Datacenters {
copyRegion.Datacenters = append(copyRegion.Datacenters, dc)
@ -691,7 +688,7 @@ type MultiregionStrategy struct {
}
type MultiregionRegion struct {
Name *string
Name string
Count *int
Datacenters []string
Meta map[string]string