Merge pull request #1235 from hashicorp/stats-cli
Rework the node-status and alloc-status cli
This commit is contained in:
Diptanu Choudhury
commit
38b3894667
|
|
@ -656,30 +656,38 @@ func (r *TaskRunner) Destroy() {
|
|||
// emitStats emits resource usage stats of tasks to remote metrics collector
|
||||
// sinks
|
||||
func (r *TaskRunner) emitStats(ru *cstructs.TaskResourceUsage) {
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "rss"}, float32(ru.ResourceUsage.MemoryStats.RSS))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "cache"}, float32(ru.ResourceUsage.MemoryStats.Cache))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "swap"}, float32(ru.ResourceUsage.MemoryStats.Swap))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "max_usage"}, float32(ru.ResourceUsage.MemoryStats.MaxUsage))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelUsage))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_max_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelMaxUsage))
|
||||
if ru.ResourceUsage.MemoryStats != nil {
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "rss"}, float32(ru.ResourceUsage.MemoryStats.RSS))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "cache"}, float32(ru.ResourceUsage.MemoryStats.Cache))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "swap"}, float32(ru.ResourceUsage.MemoryStats.Swap))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "max_usage"}, float32(ru.ResourceUsage.MemoryStats.MaxUsage))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelUsage))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_max_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelMaxUsage))
|
||||
}
|
||||
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "percent"}, float32(ru.ResourceUsage.CpuStats.Percent))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "system"}, float32(ru.ResourceUsage.CpuStats.SystemMode))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "user"}, float32(ru.ResourceUsage.CpuStats.UserMode))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_time"}, float32(ru.ResourceUsage.CpuStats.ThrottledTime))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_periods"}, float32(ru.ResourceUsage.CpuStats.ThrottledPeriods))
|
||||
if ru.ResourceUsage.CpuStats != nil {
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "percent"}, float32(ru.ResourceUsage.CpuStats.Percent))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "system"}, float32(ru.ResourceUsage.CpuStats.SystemMode))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "user"}, float32(ru.ResourceUsage.CpuStats.UserMode))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_time"}, float32(ru.ResourceUsage.CpuStats.ThrottledTime))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_periods"}, float32(ru.ResourceUsage.CpuStats.ThrottledPeriods))
|
||||
}
|
||||
|
||||
for pid, pidStats := range ru.Pids {
|
||||
// Not emitting max, kernel usages since we never get them on a per-pid
|
||||
// basis
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "rss"}, float32(pidStats.MemoryStats.RSS))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "cache"}, float32(pidStats.MemoryStats.Cache))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "swap"}, float32(pidStats.MemoryStats.Swap))
|
||||
if pidStats.MemoryStats != nil {
|
||||
// Not emitting max, kernel usages since we never get them on a per-pid
|
||||
// basis
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "rss"}, float32(pidStats.MemoryStats.RSS))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "cache"}, float32(pidStats.MemoryStats.Cache))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "swap"}, float32(pidStats.MemoryStats.Swap))
|
||||
}
|
||||
|
||||
// Not emitting throttled time and periods since we never get them on a
|
||||
// per pid basis
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "percent"}, float32(pidStats.CpuStats.Percent))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "system"}, float32(pidStats.CpuStats.SystemMode))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "user"}, float32(pidStats.CpuStats.UserMode))
|
||||
if pidStats.CpuStats != nil {
|
||||
// Not emitting throttled time and periods since we never get them on a
|
||||
// per pid basis
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "percent"}, float32(pidStats.CpuStats.Percent))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "system"}, float32(pidStats.CpuStats.SystemMode))
|
||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "user"}, float32(pidStats.CpuStats.UserMode))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,17 +2,22 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/mitchellh/colorstring"
|
||||
|
||||
"github.com/hashicorp/nomad/api"
|
||||
"github.com/hashicorp/nomad/client"
|
||||
)
|
||||
|
||||
type AllocStatusCommand struct {
|
||||
Meta
|
||||
color *colorstring.Colorize
|
||||
}
|
||||
|
||||
func (c *AllocStatusCommand) Help() string {
|
||||
|
|
@ -33,6 +38,9 @@ Alloc Status Options:
|
|||
-short
|
||||
Display short output. Shows only the most recent task event.
|
||||
|
||||
-stats
|
||||
Display detailed resource usage statistics
|
||||
|
||||
-verbose
|
||||
Show full information.
|
||||
`
|
||||
|
|
@ -45,12 +53,13 @@ func (c *AllocStatusCommand) Synopsis() string {
|
|||
}
|
||||
|
||||
func (c *AllocStatusCommand) Run(args []string) int {
|
||||
var short, verbose bool
|
||||
var short, displayStats, verbose bool
|
||||
|
||||
flags := c.Meta.FlagSet("alloc-status", FlagSetClient)
|
||||
flags.Usage = func() { c.Ui.Output(c.Help()) }
|
||||
flags.BoolVar(&short, "short", false, "")
|
||||
flags.BoolVar(&verbose, "verbose", false, "")
|
||||
flags.BoolVar(&displayStats, "stats", false, "")
|
||||
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
|
|
@ -147,7 +156,7 @@ func (c *AllocStatusCommand) Run(args []string) int {
|
|||
c.Ui.Output(formatKV(basic))
|
||||
|
||||
if !short {
|
||||
c.taskResources(alloc, stats)
|
||||
c.taskResources(alloc, stats, displayStats)
|
||||
}
|
||||
|
||||
// Print the state of each task.
|
||||
|
|
@ -309,7 +318,7 @@ func (c *AllocStatusCommand) allocResources(alloc *api.Allocation) {
|
|||
}
|
||||
|
||||
// taskResources prints out the tasks current resource usage
|
||||
func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[string]*api.TaskResourceUsage) {
|
||||
func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[string]*api.TaskResourceUsage, displayStats bool) {
|
||||
if len(alloc.TaskResources) == 0 {
|
||||
return
|
||||
}
|
||||
|
|
@ -326,12 +335,12 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
|||
for _, task := range tasks {
|
||||
resource := alloc.TaskResources[task]
|
||||
|
||||
header := fmt.Sprintf("\nTask: %q", task)
|
||||
header := fmt.Sprintf("\n[bold]Task: %q[reset]", task)
|
||||
if firstLine {
|
||||
header = fmt.Sprintf("Task: %q", task)
|
||||
header = fmt.Sprintf("[bold]Task: %q[reset]", task)
|
||||
firstLine = false
|
||||
}
|
||||
c.Ui.Output(header)
|
||||
c.Ui.Output(c.Colorize().Color(header))
|
||||
var addr []string
|
||||
for _, nw := range resource.Networks {
|
||||
ports := append(nw.DynamicPorts, nw.ReservedPorts...)
|
||||
|
|
@ -340,7 +349,7 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
|||
}
|
||||
}
|
||||
var resourcesOutput []string
|
||||
resourcesOutput = append(resourcesOutput, "CPU|Memory MB|Disk MB|IOPS|Addresses")
|
||||
resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|IOPS|Addresses")
|
||||
firstAddr := ""
|
||||
if len(addr) > 0 {
|
||||
firstAddr = addr[0]
|
||||
|
|
@ -348,12 +357,12 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
|||
cpuUsage := strconv.Itoa(resource.CPU)
|
||||
memUsage := strconv.Itoa(resource.MemoryMB)
|
||||
if ru, ok := stats[task]; ok && ru != nil && ru.ResourceUsage != nil {
|
||||
cpuStats := ru.ResourceUsage.CpuStats
|
||||
cpuTicksConsumed := (ru.ResourceUsage.CpuStats.Percent / 100) * float64(resource.CPU)
|
||||
memoryStats := ru.ResourceUsage.MemoryStats
|
||||
cpuUsage = fmt.Sprintf("%v/%v", (cpuStats.SystemMode + cpuStats.UserMode), resource.CPU)
|
||||
cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU)
|
||||
memUsage = fmt.Sprintf("%v/%v", memoryStats.RSS/(1024*1024), resource.MemoryMB)
|
||||
}
|
||||
resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v|%v|%v|%v",
|
||||
resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v MB|%v MB|%v|%v",
|
||||
cpuUsage,
|
||||
memUsage,
|
||||
resource.DiskMB,
|
||||
|
|
@ -363,5 +372,37 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
|||
resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i]))
|
||||
}
|
||||
c.Ui.Output(formatListWithSpaces(resourcesOutput))
|
||||
|
||||
if ru, ok := stats[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil {
|
||||
c.Ui.Output("")
|
||||
c.printTaskResourceUsage(task, ru.ResourceUsage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *AllocStatusCommand) printTaskResourceUsage(task string, resourceUsage *api.ResourceUsage) {
|
||||
memoryStats := resourceUsage.MemoryStats
|
||||
cpuStats := resourceUsage.CpuStats
|
||||
c.Ui.Output("Memory Stats")
|
||||
out := make([]string, 2)
|
||||
out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|Kernel Max Usage"
|
||||
out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v",
|
||||
humanize.Bytes(memoryStats.RSS),
|
||||
humanize.Bytes(memoryStats.Cache),
|
||||
humanize.Bytes(memoryStats.Swap),
|
||||
humanize.Bytes(memoryStats.MaxUsage),
|
||||
humanize.Bytes(memoryStats.KernelUsage),
|
||||
humanize.Bytes(memoryStats.KernelMaxUsage),
|
||||
)
|
||||
c.Ui.Output(formatList(out))
|
||||
|
||||
c.Ui.Output("")
|
||||
|
||||
c.Ui.Output("CPU Stats")
|
||||
out = make([]string, 2)
|
||||
out[0] = "Percent|Throttled Periods|Throttled Time"
|
||||
percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
|
||||
out[1] = fmt.Sprintf("%v %|%v|%v", percent,
|
||||
cpuStats.ThrottledPeriods, cpuStats.ThrottledTime)
|
||||
c.Ui.Output(formatList(out))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,12 +8,14 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/mitchellh/colorstring"
|
||||
|
||||
"github.com/hashicorp/nomad/api"
|
||||
)
|
||||
|
||||
type NodeStatusCommand struct {
|
||||
Meta
|
||||
color *colorstring.Colorize
|
||||
}
|
||||
|
||||
func (c *NodeStatusCommand) Help() string {
|
||||
|
|
@ -42,6 +44,9 @@ Node Status Options:
|
|||
-verbose
|
||||
Display full information.
|
||||
|
||||
-stats
|
||||
Display detailed resource usage statistics
|
||||
|
||||
-self
|
||||
Query the status of the local node.
|
||||
|
||||
|
|
@ -56,7 +61,7 @@ func (c *NodeStatusCommand) Synopsis() string {
|
|||
}
|
||||
|
||||
func (c *NodeStatusCommand) Run(args []string) int {
|
||||
var short, verbose, list_allocs, self bool
|
||||
var short, verbose, list_allocs, self, stats bool
|
||||
var hostStats *api.HostStats
|
||||
|
||||
flags := c.Meta.FlagSet("node-status", FlagSetClient)
|
||||
|
|
@ -65,6 +70,7 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
|||
flags.BoolVar(&verbose, "verbose", false, "")
|
||||
flags.BoolVar(&list_allocs, "allocs", false, "")
|
||||
flags.BoolVar(&self, "self", false, "")
|
||||
flags.BoolVar(&stats, "stats", false, "")
|
||||
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
|
|
@ -204,7 +210,7 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
|||
|
||||
// Format the output
|
||||
basic := []string{
|
||||
fmt.Sprintf("ID|%s", limit(node.ID, length)),
|
||||
fmt.Sprintf("[bold]Node ID[reset]|%s", limit(node.ID, length)),
|
||||
fmt.Sprintf("Name|%s", node.Name),
|
||||
fmt.Sprintf("Class|%s", node.NodeClass),
|
||||
fmt.Sprintf("DC|%s", node.Datacenter),
|
||||
|
|
@ -215,22 +221,29 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
|||
uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
|
||||
basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
|
||||
}
|
||||
c.Ui.Output(formatKV(basic))
|
||||
c.Ui.Output(c.Colorize().Color(formatKV(basic)))
|
||||
|
||||
if !short {
|
||||
resources, err := getResources(client, node)
|
||||
allocatedResources, err := getAllocatedResources(client, node)
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error querying node resources: %s", err))
|
||||
return 1
|
||||
}
|
||||
c.Ui.Output("\n==> Resource Utilization")
|
||||
c.Ui.Output(formatList(resources))
|
||||
if hostStats != nil {
|
||||
c.Ui.Output("\n===> Node CPU Stats")
|
||||
c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Allocated)[reset]"))
|
||||
c.Ui.Output(formatList(allocatedResources))
|
||||
|
||||
actualResources, err := getActualResources(hostStats, node)
|
||||
if err == nil {
|
||||
c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Actual)[reset]"))
|
||||
c.Ui.Output(formatList(actualResources))
|
||||
}
|
||||
|
||||
if hostStats != nil && stats {
|
||||
c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed CPU Stats[reset]"))
|
||||
c.printCpuStats(hostStats)
|
||||
c.Ui.Output("\n===> Node Memory Stats")
|
||||
c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Memory Stats[reset]"))
|
||||
c.printMemoryStats(hostStats)
|
||||
c.Ui.Output("\n===> Node Disk Stats")
|
||||
c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Disk Stats[reset]"))
|
||||
c.printDiskStats(hostStats)
|
||||
}
|
||||
|
||||
|
|
@ -271,9 +284,9 @@ func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) {
|
|||
for _, cpuStat := range hostStats.CPU {
|
||||
cpuStatsAttr := make([]string, 4)
|
||||
cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
|
||||
cpuStatsAttr[1] = fmt.Sprintf("User|%v", formatFloat64(cpuStat.User))
|
||||
cpuStatsAttr[2] = fmt.Sprintf("System|%v", formatFloat64(cpuStat.System))
|
||||
cpuStatsAttr[3] = fmt.Sprintf("Idle|%v", formatFloat64(cpuStat.Idle))
|
||||
cpuStatsAttr[1] = fmt.Sprintf("User|%v %%", formatFloat64(cpuStat.User))
|
||||
cpuStatsAttr[2] = fmt.Sprintf("System|%v %%", formatFloat64(cpuStat.System))
|
||||
cpuStatsAttr[3] = fmt.Sprintf("Idle|%v %%", formatFloat64(cpuStat.Idle))
|
||||
c.Ui.Output(formatKV(cpuStatsAttr))
|
||||
c.Ui.Output("")
|
||||
}
|
||||
|
|
@ -297,8 +310,8 @@ func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) {
|
|||
diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.Bytes(diskStat.Size))
|
||||
diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.Bytes(diskStat.Used))
|
||||
diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.Bytes(diskStat.Available))
|
||||
diskStatsAttr[5] = fmt.Sprintf("Used Percent|%s", formatFloat64(diskStat.UsedPercent))
|
||||
diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%s", formatFloat64(diskStat.InodesUsedPercent))
|
||||
diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v %%", formatFloat64(diskStat.UsedPercent))
|
||||
diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v %%", formatFloat64(diskStat.InodesUsedPercent))
|
||||
c.Ui.Output(formatKV(diskStatsAttr))
|
||||
c.Ui.Output("")
|
||||
}
|
||||
|
|
@ -339,8 +352,8 @@ func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error)
|
|||
return allocs, err
|
||||
}
|
||||
|
||||
// getResources returns the resource usage of the node.
|
||||
func getResources(client *api.Client, node *api.Node) ([]string, error) {
|
||||
// getAllocatedResources returns the resource usage of the node.
|
||||
func getAllocatedResources(client *api.Client, node *api.Node) ([]string, error) {
|
||||
var resources []string
|
||||
var cpu, mem, disk, iops int
|
||||
var totalCpu, totalMem, totalDisk, totalIops int
|
||||
|
|
@ -382,6 +395,43 @@ func getResources(client *api.Client, node *api.Node) ([]string, error) {
|
|||
return resources, err
|
||||
}
|
||||
|
||||
// getActualResources returns the actual resource usage of the node.
|
||||
func getActualResources(hostStats *api.HostStats, node *api.Node) ([]string, error) {
|
||||
if hostStats == nil {
|
||||
return nil, fmt.Errorf("actual resource usage not present")
|
||||
}
|
||||
var resources []string
|
||||
|
||||
// Calculate cpu usage
|
||||
usedCPUPercent := 0.0
|
||||
for _, cpu := range hostStats.CPU {
|
||||
usedCPUPercent += (cpu.User + cpu.System)
|
||||
}
|
||||
usedCPUTicks := (usedCPUPercent / 100) * float64(node.Resources.CPU)
|
||||
|
||||
// calculate disk usage
|
||||
storageDevice := node.Attributes["unique.storage.volume"]
|
||||
var diskUsed, diskSize uint64
|
||||
for _, disk := range hostStats.DiskStats {
|
||||
if disk.Device == storageDevice {
|
||||
diskUsed = disk.Used
|
||||
diskSize = disk.Size
|
||||
}
|
||||
}
|
||||
|
||||
resources = make([]string, 2)
|
||||
resources[0] = "CPU|Memory|Disk"
|
||||
resources[1] = fmt.Sprintf("%v/%v|%v/%v|%v/%v",
|
||||
int64(usedCPUTicks),
|
||||
node.Resources.CPU,
|
||||
humanize.Bytes(hostStats.Memory.Used),
|
||||
humanize.Bytes(hostStats.Memory.Total),
|
||||
humanize.Bytes(diskUsed),
|
||||
humanize.Bytes(diskSize),
|
||||
)
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
func formatFloat64(val float64) string {
|
||||
return strconv.FormatFloat(val, 'f', 2, 64)
|
||||
}
|
||||
|
|
|
|||
168
command/stats.go
168
command/stats.go
|
|
@ -1,168 +0,0 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/hashicorp/nomad/api"
|
||||
)
|
||||
|
||||
type StatsCommand struct {
|
||||
Meta
|
||||
}
|
||||
|
||||
func (f *StatsCommand) Help() string {
|
||||
helpText := `
|
||||
Usage: nomad stats [options] <alloc-id>
|
||||
|
||||
Displays statistics related to resource usage of tasks in an allocation. Use
|
||||
the -task flag to query statistics of an individual task running in an
|
||||
allocation.
|
||||
|
||||
General Options:
|
||||
|
||||
` + generalOptionsUsage() + `
|
||||
|
||||
Node Stats Options:
|
||||
|
||||
-task
|
||||
Display statistics for a specific task in an allocation.
|
||||
`
|
||||
|
||||
return strings.TrimSpace(helpText)
|
||||
}
|
||||
|
||||
func (f *StatsCommand) Synopsis() string {
|
||||
return "Dispalys resource usage stats of an allocation or a task running on a nomad client"
|
||||
}
|
||||
|
||||
func (f *StatsCommand) Run(args []string) int {
|
||||
var verbose bool
|
||||
var task string
|
||||
flags := f.Meta.FlagSet("stats", FlagSetClient)
|
||||
flags.BoolVar(&verbose, "verbose", false, "")
|
||||
flags.StringVar(&task, "task", "", "")
|
||||
flags.Usage = func() { f.Ui.Output(f.Help()) }
|
||||
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
args = flags.Args()
|
||||
if len(args) < 1 {
|
||||
f.Ui.Error("allocation id is a required parameter")
|
||||
return 1
|
||||
}
|
||||
client, err := f.Meta.Client()
|
||||
if err != nil {
|
||||
f.Ui.Error(fmt.Sprintf("Error initializing client: %v", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
var allocID string
|
||||
allocID = strings.TrimSpace(args[0])
|
||||
|
||||
// Truncate the id unless full length is requested
|
||||
length := shortId
|
||||
if verbose {
|
||||
length = fullId
|
||||
}
|
||||
|
||||
// Query the allocation info
|
||||
if len(allocID) == 1 {
|
||||
f.Ui.Error(fmt.Sprintf("Alloc ID must contain at least two characters."))
|
||||
return 1
|
||||
}
|
||||
if len(allocID)%2 == 1 {
|
||||
// Identifiers must be of even length, so we strip off the last byte
|
||||
// to provide a consistent user experience.
|
||||
allocID = allocID[:len(allocID)-1]
|
||||
}
|
||||
|
||||
allocs, _, err := client.Allocations().PrefixList(allocID)
|
||||
if err != nil {
|
||||
f.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err))
|
||||
return 1
|
||||
}
|
||||
if len(allocs) == 0 {
|
||||
f.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID))
|
||||
return 1
|
||||
}
|
||||
if len(allocs) > 1 {
|
||||
// Format the allocs
|
||||
out := make([]string, len(allocs)+1)
|
||||
out[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status"
|
||||
for i, alloc := range allocs {
|
||||
out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
|
||||
limit(alloc.ID, length),
|
||||
limit(alloc.EvalID, length),
|
||||
alloc.JobID,
|
||||
alloc.TaskGroup,
|
||||
alloc.DesiredStatus,
|
||||
alloc.ClientStatus,
|
||||
)
|
||||
}
|
||||
f.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", formatList(out)))
|
||||
return 0
|
||||
}
|
||||
// Prefix lookup matched a single allocation
|
||||
alloc, _, err := client.Allocations().Info(allocs[0].ID, nil)
|
||||
if err != nil {
|
||||
f.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
stats, err := client.Allocations().Stats(alloc, nil)
|
||||
if err != nil {
|
||||
f.Ui.Error(fmt.Sprintf("unable to get stats: %v", err))
|
||||
return 1
|
||||
}
|
||||
if task == "" {
|
||||
f.printAllocResourceUsage(alloc, stats)
|
||||
} else {
|
||||
f.printTaskResourceUsage(task, stats)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (f *StatsCommand) printTaskResourceUsage(task string, resourceUsage map[string]*api.TaskResourceUsage) {
|
||||
tu, ok := resourceUsage[task]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
memoryStats := tu.ResourceUsage.MemoryStats
|
||||
cpuStats := tu.ResourceUsage.CpuStats
|
||||
f.Ui.Output(fmt.Sprintf("===> Task: %q", task))
|
||||
f.Ui.Output("Memory Stats")
|
||||
out := make([]string, 2)
|
||||
out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|KernelMaxUsage"
|
||||
out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v",
|
||||
humanize.Bytes(memoryStats.RSS),
|
||||
humanize.Bytes(memoryStats.Cache),
|
||||
humanize.Bytes(memoryStats.Swap),
|
||||
humanize.Bytes(memoryStats.MaxUsage),
|
||||
humanize.Bytes(memoryStats.KernelUsage),
|
||||
humanize.Bytes(memoryStats.KernelMaxUsage),
|
||||
)
|
||||
f.Ui.Output(formatList(out))
|
||||
|
||||
f.Ui.Output("")
|
||||
|
||||
f.Ui.Output("CPU Stats")
|
||||
out = make([]string, 2)
|
||||
out[0] = "Percent|Throttled Periods|Throttled Time"
|
||||
percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
|
||||
out[1] = fmt.Sprintf("%v|%v|%v", percent,
|
||||
cpuStats.ThrottledPeriods, cpuStats.ThrottledTime)
|
||||
f.Ui.Output(formatList(out))
|
||||
}
|
||||
|
||||
func (f *StatsCommand) printAllocResourceUsage(alloc *api.Allocation, resourceUsage map[string]*api.TaskResourceUsage) {
|
||||
f.Ui.Output(fmt.Sprintf("Resource Usage of Tasks running in Allocation %q", alloc.ID))
|
||||
for task, _ := range alloc.TaskStates {
|
||||
f.printTaskResourceUsage(task, resourceUsage)
|
||||
}
|
||||
}
|
||||
|
|
@ -121,11 +121,6 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
|
|||
Meta: meta,
|
||||
}, nil
|
||||
},
|
||||
"stats": func() (cli.Command, error) {
|
||||
return &command.StatsCommand{
|
||||
Meta: meta,
|
||||
}, nil
|
||||
},
|
||||
"status": func() (cli.Command, error) {
|
||||
return &command.StatusCommand{
|
||||
Meta: meta,
|
||||
|
|
|
|||
Loading…
Reference in a new issue