Merge pull request #1235 from hashicorp/stats-cli
Rework the node-status and alloc-status cli
This commit is contained in:
commit
38b3894667
|
@ -656,30 +656,38 @@ func (r *TaskRunner) Destroy() {
|
||||||
// emitStats emits resource usage stats of tasks to remote metrics collector
|
// emitStats emits resource usage stats of tasks to remote metrics collector
|
||||||
// sinks
|
// sinks
|
||||||
func (r *TaskRunner) emitStats(ru *cstructs.TaskResourceUsage) {
|
func (r *TaskRunner) emitStats(ru *cstructs.TaskResourceUsage) {
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "rss"}, float32(ru.ResourceUsage.MemoryStats.RSS))
|
if ru.ResourceUsage.MemoryStats != nil {
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "cache"}, float32(ru.ResourceUsage.MemoryStats.Cache))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "rss"}, float32(ru.ResourceUsage.MemoryStats.RSS))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "swap"}, float32(ru.ResourceUsage.MemoryStats.Swap))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "cache"}, float32(ru.ResourceUsage.MemoryStats.Cache))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "max_usage"}, float32(ru.ResourceUsage.MemoryStats.MaxUsage))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "swap"}, float32(ru.ResourceUsage.MemoryStats.Swap))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelUsage))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "max_usage"}, float32(ru.ResourceUsage.MemoryStats.MaxUsage))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_max_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelMaxUsage))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelUsage))
|
||||||
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "memory", "kernel_max_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelMaxUsage))
|
||||||
|
}
|
||||||
|
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "percent"}, float32(ru.ResourceUsage.CpuStats.Percent))
|
if ru.ResourceUsage.CpuStats != nil {
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "system"}, float32(ru.ResourceUsage.CpuStats.SystemMode))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "percent"}, float32(ru.ResourceUsage.CpuStats.Percent))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "user"}, float32(ru.ResourceUsage.CpuStats.UserMode))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "system"}, float32(ru.ResourceUsage.CpuStats.SystemMode))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_time"}, float32(ru.ResourceUsage.CpuStats.ThrottledTime))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "user"}, float32(ru.ResourceUsage.CpuStats.UserMode))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_periods"}, float32(ru.ResourceUsage.CpuStats.ThrottledPeriods))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_time"}, float32(ru.ResourceUsage.CpuStats.ThrottledTime))
|
||||||
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, "cpu", "throttled_periods"}, float32(ru.ResourceUsage.CpuStats.ThrottledPeriods))
|
||||||
|
}
|
||||||
|
|
||||||
for pid, pidStats := range ru.Pids {
|
for pid, pidStats := range ru.Pids {
|
||||||
// Not emitting max, kernel usages since we never get them on a per-pid
|
if pidStats.MemoryStats != nil {
|
||||||
// basis
|
// Not emitting max, kernel usages since we never get them on a per-pid
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "rss"}, float32(pidStats.MemoryStats.RSS))
|
// basis
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "cache"}, float32(pidStats.MemoryStats.Cache))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "rss"}, float32(pidStats.MemoryStats.RSS))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "swap"}, float32(pidStats.MemoryStats.Swap))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "cache"}, float32(pidStats.MemoryStats.Cache))
|
||||||
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "memory", "swap"}, float32(pidStats.MemoryStats.Swap))
|
||||||
|
}
|
||||||
|
|
||||||
// Not emitting throttled time and periods since we never get them on a
|
if pidStats.CpuStats != nil {
|
||||||
// per pid basis
|
// Not emitting throttled time and periods since we never get them on a
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "percent"}, float32(pidStats.CpuStats.Percent))
|
// per pid basis
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "system"}, float32(pidStats.CpuStats.SystemMode))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "percent"}, float32(pidStats.CpuStats.Percent))
|
||||||
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "user"}, float32(pidStats.CpuStats.UserMode))
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "system"}, float32(pidStats.CpuStats.SystemMode))
|
||||||
|
metrics.EmitKey([]string{r.alloc.Job.Name, r.alloc.Name, r.alloc.ID, r.task.Name, pid, "cpu", "user"}, float32(pidStats.CpuStats.UserMode))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,17 +2,22 @@ package command
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/dustin/go-humanize"
|
||||||
|
"github.com/mitchellh/colorstring"
|
||||||
|
|
||||||
"github.com/hashicorp/nomad/api"
|
"github.com/hashicorp/nomad/api"
|
||||||
"github.com/hashicorp/nomad/client"
|
"github.com/hashicorp/nomad/client"
|
||||||
)
|
)
|
||||||
|
|
||||||
type AllocStatusCommand struct {
|
type AllocStatusCommand struct {
|
||||||
Meta
|
Meta
|
||||||
|
color *colorstring.Colorize
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *AllocStatusCommand) Help() string {
|
func (c *AllocStatusCommand) Help() string {
|
||||||
|
@ -33,6 +38,9 @@ Alloc Status Options:
|
||||||
-short
|
-short
|
||||||
Display short output. Shows only the most recent task event.
|
Display short output. Shows only the most recent task event.
|
||||||
|
|
||||||
|
-stats
|
||||||
|
Display detailed resource usage statistics
|
||||||
|
|
||||||
-verbose
|
-verbose
|
||||||
Show full information.
|
Show full information.
|
||||||
`
|
`
|
||||||
|
@ -45,12 +53,13 @@ func (c *AllocStatusCommand) Synopsis() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *AllocStatusCommand) Run(args []string) int {
|
func (c *AllocStatusCommand) Run(args []string) int {
|
||||||
var short, verbose bool
|
var short, displayStats, verbose bool
|
||||||
|
|
||||||
flags := c.Meta.FlagSet("alloc-status", FlagSetClient)
|
flags := c.Meta.FlagSet("alloc-status", FlagSetClient)
|
||||||
flags.Usage = func() { c.Ui.Output(c.Help()) }
|
flags.Usage = func() { c.Ui.Output(c.Help()) }
|
||||||
flags.BoolVar(&short, "short", false, "")
|
flags.BoolVar(&short, "short", false, "")
|
||||||
flags.BoolVar(&verbose, "verbose", false, "")
|
flags.BoolVar(&verbose, "verbose", false, "")
|
||||||
|
flags.BoolVar(&displayStats, "stats", false, "")
|
||||||
|
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
return 1
|
return 1
|
||||||
|
@ -147,7 +156,7 @@ func (c *AllocStatusCommand) Run(args []string) int {
|
||||||
c.Ui.Output(formatKV(basic))
|
c.Ui.Output(formatKV(basic))
|
||||||
|
|
||||||
if !short {
|
if !short {
|
||||||
c.taskResources(alloc, stats)
|
c.taskResources(alloc, stats, displayStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Print the state of each task.
|
// Print the state of each task.
|
||||||
|
@ -309,7 +318,7 @@ func (c *AllocStatusCommand) allocResources(alloc *api.Allocation) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// taskResources prints out the tasks current resource usage
|
// taskResources prints out the tasks current resource usage
|
||||||
func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[string]*api.TaskResourceUsage) {
|
func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[string]*api.TaskResourceUsage, displayStats bool) {
|
||||||
if len(alloc.TaskResources) == 0 {
|
if len(alloc.TaskResources) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -326,12 +335,12 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
||||||
for _, task := range tasks {
|
for _, task := range tasks {
|
||||||
resource := alloc.TaskResources[task]
|
resource := alloc.TaskResources[task]
|
||||||
|
|
||||||
header := fmt.Sprintf("\nTask: %q", task)
|
header := fmt.Sprintf("\n[bold]Task: %q[reset]", task)
|
||||||
if firstLine {
|
if firstLine {
|
||||||
header = fmt.Sprintf("Task: %q", task)
|
header = fmt.Sprintf("[bold]Task: %q[reset]", task)
|
||||||
firstLine = false
|
firstLine = false
|
||||||
}
|
}
|
||||||
c.Ui.Output(header)
|
c.Ui.Output(c.Colorize().Color(header))
|
||||||
var addr []string
|
var addr []string
|
||||||
for _, nw := range resource.Networks {
|
for _, nw := range resource.Networks {
|
||||||
ports := append(nw.DynamicPorts, nw.ReservedPorts...)
|
ports := append(nw.DynamicPorts, nw.ReservedPorts...)
|
||||||
|
@ -340,7 +349,7 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var resourcesOutput []string
|
var resourcesOutput []string
|
||||||
resourcesOutput = append(resourcesOutput, "CPU|Memory MB|Disk MB|IOPS|Addresses")
|
resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|IOPS|Addresses")
|
||||||
firstAddr := ""
|
firstAddr := ""
|
||||||
if len(addr) > 0 {
|
if len(addr) > 0 {
|
||||||
firstAddr = addr[0]
|
firstAddr = addr[0]
|
||||||
|
@ -348,12 +357,12 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
||||||
cpuUsage := strconv.Itoa(resource.CPU)
|
cpuUsage := strconv.Itoa(resource.CPU)
|
||||||
memUsage := strconv.Itoa(resource.MemoryMB)
|
memUsage := strconv.Itoa(resource.MemoryMB)
|
||||||
if ru, ok := stats[task]; ok && ru != nil && ru.ResourceUsage != nil {
|
if ru, ok := stats[task]; ok && ru != nil && ru.ResourceUsage != nil {
|
||||||
cpuStats := ru.ResourceUsage.CpuStats
|
cpuTicksConsumed := (ru.ResourceUsage.CpuStats.Percent / 100) * float64(resource.CPU)
|
||||||
memoryStats := ru.ResourceUsage.MemoryStats
|
memoryStats := ru.ResourceUsage.MemoryStats
|
||||||
cpuUsage = fmt.Sprintf("%v/%v", (cpuStats.SystemMode + cpuStats.UserMode), resource.CPU)
|
cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU)
|
||||||
memUsage = fmt.Sprintf("%v/%v", memoryStats.RSS/(1024*1024), resource.MemoryMB)
|
memUsage = fmt.Sprintf("%v/%v", memoryStats.RSS/(1024*1024), resource.MemoryMB)
|
||||||
}
|
}
|
||||||
resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v|%v|%v|%v",
|
resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v MB|%v MB|%v|%v",
|
||||||
cpuUsage,
|
cpuUsage,
|
||||||
memUsage,
|
memUsage,
|
||||||
resource.DiskMB,
|
resource.DiskMB,
|
||||||
|
@ -363,5 +372,37 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
|
||||||
resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i]))
|
resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i]))
|
||||||
}
|
}
|
||||||
c.Ui.Output(formatListWithSpaces(resourcesOutput))
|
c.Ui.Output(formatListWithSpaces(resourcesOutput))
|
||||||
|
|
||||||
|
if ru, ok := stats[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil {
|
||||||
|
c.Ui.Output("")
|
||||||
|
c.printTaskResourceUsage(task, ru.ResourceUsage)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *AllocStatusCommand) printTaskResourceUsage(task string, resourceUsage *api.ResourceUsage) {
|
||||||
|
memoryStats := resourceUsage.MemoryStats
|
||||||
|
cpuStats := resourceUsage.CpuStats
|
||||||
|
c.Ui.Output("Memory Stats")
|
||||||
|
out := make([]string, 2)
|
||||||
|
out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|Kernel Max Usage"
|
||||||
|
out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v",
|
||||||
|
humanize.Bytes(memoryStats.RSS),
|
||||||
|
humanize.Bytes(memoryStats.Cache),
|
||||||
|
humanize.Bytes(memoryStats.Swap),
|
||||||
|
humanize.Bytes(memoryStats.MaxUsage),
|
||||||
|
humanize.Bytes(memoryStats.KernelUsage),
|
||||||
|
humanize.Bytes(memoryStats.KernelMaxUsage),
|
||||||
|
)
|
||||||
|
c.Ui.Output(formatList(out))
|
||||||
|
|
||||||
|
c.Ui.Output("")
|
||||||
|
|
||||||
|
c.Ui.Output("CPU Stats")
|
||||||
|
out = make([]string, 2)
|
||||||
|
out[0] = "Percent|Throttled Periods|Throttled Time"
|
||||||
|
percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
|
||||||
|
out[1] = fmt.Sprintf("%v %|%v|%v", percent,
|
||||||
|
cpuStats.ThrottledPeriods, cpuStats.ThrottledTime)
|
||||||
|
c.Ui.Output(formatList(out))
|
||||||
|
}
|
||||||
|
|
|
@ -8,12 +8,14 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
|
"github.com/mitchellh/colorstring"
|
||||||
|
|
||||||
"github.com/hashicorp/nomad/api"
|
"github.com/hashicorp/nomad/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
type NodeStatusCommand struct {
|
type NodeStatusCommand struct {
|
||||||
Meta
|
Meta
|
||||||
|
color *colorstring.Colorize
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *NodeStatusCommand) Help() string {
|
func (c *NodeStatusCommand) Help() string {
|
||||||
|
@ -42,6 +44,9 @@ Node Status Options:
|
||||||
-verbose
|
-verbose
|
||||||
Display full information.
|
Display full information.
|
||||||
|
|
||||||
|
-stats
|
||||||
|
Display detailed resource usage statistics
|
||||||
|
|
||||||
-self
|
-self
|
||||||
Query the status of the local node.
|
Query the status of the local node.
|
||||||
|
|
||||||
|
@ -56,7 +61,7 @@ func (c *NodeStatusCommand) Synopsis() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *NodeStatusCommand) Run(args []string) int {
|
func (c *NodeStatusCommand) Run(args []string) int {
|
||||||
var short, verbose, list_allocs, self bool
|
var short, verbose, list_allocs, self, stats bool
|
||||||
var hostStats *api.HostStats
|
var hostStats *api.HostStats
|
||||||
|
|
||||||
flags := c.Meta.FlagSet("node-status", FlagSetClient)
|
flags := c.Meta.FlagSet("node-status", FlagSetClient)
|
||||||
|
@ -65,6 +70,7 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
||||||
flags.BoolVar(&verbose, "verbose", false, "")
|
flags.BoolVar(&verbose, "verbose", false, "")
|
||||||
flags.BoolVar(&list_allocs, "allocs", false, "")
|
flags.BoolVar(&list_allocs, "allocs", false, "")
|
||||||
flags.BoolVar(&self, "self", false, "")
|
flags.BoolVar(&self, "self", false, "")
|
||||||
|
flags.BoolVar(&stats, "stats", false, "")
|
||||||
|
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
return 1
|
return 1
|
||||||
|
@ -204,7 +210,7 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
||||||
|
|
||||||
// Format the output
|
// Format the output
|
||||||
basic := []string{
|
basic := []string{
|
||||||
fmt.Sprintf("ID|%s", limit(node.ID, length)),
|
fmt.Sprintf("[bold]Node ID[reset]|%s", limit(node.ID, length)),
|
||||||
fmt.Sprintf("Name|%s", node.Name),
|
fmt.Sprintf("Name|%s", node.Name),
|
||||||
fmt.Sprintf("Class|%s", node.NodeClass),
|
fmt.Sprintf("Class|%s", node.NodeClass),
|
||||||
fmt.Sprintf("DC|%s", node.Datacenter),
|
fmt.Sprintf("DC|%s", node.Datacenter),
|
||||||
|
@ -215,22 +221,29 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
||||||
uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
|
uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
|
||||||
basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
|
basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
|
||||||
}
|
}
|
||||||
c.Ui.Output(formatKV(basic))
|
c.Ui.Output(c.Colorize().Color(formatKV(basic)))
|
||||||
|
|
||||||
if !short {
|
if !short {
|
||||||
resources, err := getResources(client, node)
|
allocatedResources, err := getAllocatedResources(client, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.Ui.Error(fmt.Sprintf("Error querying node resources: %s", err))
|
c.Ui.Error(fmt.Sprintf("Error querying node resources: %s", err))
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
c.Ui.Output("\n==> Resource Utilization")
|
c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Allocated)[reset]"))
|
||||||
c.Ui.Output(formatList(resources))
|
c.Ui.Output(formatList(allocatedResources))
|
||||||
if hostStats != nil {
|
|
||||||
c.Ui.Output("\n===> Node CPU Stats")
|
actualResources, err := getActualResources(hostStats, node)
|
||||||
|
if err == nil {
|
||||||
|
c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Actual)[reset]"))
|
||||||
|
c.Ui.Output(formatList(actualResources))
|
||||||
|
}
|
||||||
|
|
||||||
|
if hostStats != nil && stats {
|
||||||
|
c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed CPU Stats[reset]"))
|
||||||
c.printCpuStats(hostStats)
|
c.printCpuStats(hostStats)
|
||||||
c.Ui.Output("\n===> Node Memory Stats")
|
c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Memory Stats[reset]"))
|
||||||
c.printMemoryStats(hostStats)
|
c.printMemoryStats(hostStats)
|
||||||
c.Ui.Output("\n===> Node Disk Stats")
|
c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Disk Stats[reset]"))
|
||||||
c.printDiskStats(hostStats)
|
c.printDiskStats(hostStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -271,9 +284,9 @@ func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) {
|
||||||
for _, cpuStat := range hostStats.CPU {
|
for _, cpuStat := range hostStats.CPU {
|
||||||
cpuStatsAttr := make([]string, 4)
|
cpuStatsAttr := make([]string, 4)
|
||||||
cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
|
cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
|
||||||
cpuStatsAttr[1] = fmt.Sprintf("User|%v", formatFloat64(cpuStat.User))
|
cpuStatsAttr[1] = fmt.Sprintf("User|%v %%", formatFloat64(cpuStat.User))
|
||||||
cpuStatsAttr[2] = fmt.Sprintf("System|%v", formatFloat64(cpuStat.System))
|
cpuStatsAttr[2] = fmt.Sprintf("System|%v %%", formatFloat64(cpuStat.System))
|
||||||
cpuStatsAttr[3] = fmt.Sprintf("Idle|%v", formatFloat64(cpuStat.Idle))
|
cpuStatsAttr[3] = fmt.Sprintf("Idle|%v %%", formatFloat64(cpuStat.Idle))
|
||||||
c.Ui.Output(formatKV(cpuStatsAttr))
|
c.Ui.Output(formatKV(cpuStatsAttr))
|
||||||
c.Ui.Output("")
|
c.Ui.Output("")
|
||||||
}
|
}
|
||||||
|
@ -297,8 +310,8 @@ func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) {
|
||||||
diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.Bytes(diskStat.Size))
|
diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.Bytes(diskStat.Size))
|
||||||
diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.Bytes(diskStat.Used))
|
diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.Bytes(diskStat.Used))
|
||||||
diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.Bytes(diskStat.Available))
|
diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.Bytes(diskStat.Available))
|
||||||
diskStatsAttr[5] = fmt.Sprintf("Used Percent|%s", formatFloat64(diskStat.UsedPercent))
|
diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v %%", formatFloat64(diskStat.UsedPercent))
|
||||||
diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%s", formatFloat64(diskStat.InodesUsedPercent))
|
diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v %%", formatFloat64(diskStat.InodesUsedPercent))
|
||||||
c.Ui.Output(formatKV(diskStatsAttr))
|
c.Ui.Output(formatKV(diskStatsAttr))
|
||||||
c.Ui.Output("")
|
c.Ui.Output("")
|
||||||
}
|
}
|
||||||
|
@ -339,8 +352,8 @@ func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error)
|
||||||
return allocs, err
|
return allocs, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// getResources returns the resource usage of the node.
|
// getAllocatedResources returns the resource usage of the node.
|
||||||
func getResources(client *api.Client, node *api.Node) ([]string, error) {
|
func getAllocatedResources(client *api.Client, node *api.Node) ([]string, error) {
|
||||||
var resources []string
|
var resources []string
|
||||||
var cpu, mem, disk, iops int
|
var cpu, mem, disk, iops int
|
||||||
var totalCpu, totalMem, totalDisk, totalIops int
|
var totalCpu, totalMem, totalDisk, totalIops int
|
||||||
|
@ -382,6 +395,43 @@ func getResources(client *api.Client, node *api.Node) ([]string, error) {
|
||||||
return resources, err
|
return resources, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getActualResources returns the actual resource usage of the node.
|
||||||
|
func getActualResources(hostStats *api.HostStats, node *api.Node) ([]string, error) {
|
||||||
|
if hostStats == nil {
|
||||||
|
return nil, fmt.Errorf("actual resource usage not present")
|
||||||
|
}
|
||||||
|
var resources []string
|
||||||
|
|
||||||
|
// Calculate cpu usage
|
||||||
|
usedCPUPercent := 0.0
|
||||||
|
for _, cpu := range hostStats.CPU {
|
||||||
|
usedCPUPercent += (cpu.User + cpu.System)
|
||||||
|
}
|
||||||
|
usedCPUTicks := (usedCPUPercent / 100) * float64(node.Resources.CPU)
|
||||||
|
|
||||||
|
// calculate disk usage
|
||||||
|
storageDevice := node.Attributes["unique.storage.volume"]
|
||||||
|
var diskUsed, diskSize uint64
|
||||||
|
for _, disk := range hostStats.DiskStats {
|
||||||
|
if disk.Device == storageDevice {
|
||||||
|
diskUsed = disk.Used
|
||||||
|
diskSize = disk.Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resources = make([]string, 2)
|
||||||
|
resources[0] = "CPU|Memory|Disk"
|
||||||
|
resources[1] = fmt.Sprintf("%v/%v|%v/%v|%v/%v",
|
||||||
|
int64(usedCPUTicks),
|
||||||
|
node.Resources.CPU,
|
||||||
|
humanize.Bytes(hostStats.Memory.Used),
|
||||||
|
humanize.Bytes(hostStats.Memory.Total),
|
||||||
|
humanize.Bytes(diskUsed),
|
||||||
|
humanize.Bytes(diskSize),
|
||||||
|
)
|
||||||
|
return resources, nil
|
||||||
|
}
|
||||||
|
|
||||||
func formatFloat64(val float64) string {
|
func formatFloat64(val float64) string {
|
||||||
return strconv.FormatFloat(val, 'f', 2, 64)
|
return strconv.FormatFloat(val, 'f', 2, 64)
|
||||||
}
|
}
|
||||||
|
|
168
command/stats.go
168
command/stats.go
|
@ -1,168 +0,0 @@
|
||||||
package command
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
|
|
||||||
"github.com/hashicorp/nomad/api"
|
|
||||||
)
|
|
||||||
|
|
||||||
type StatsCommand struct {
|
|
||||||
Meta
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *StatsCommand) Help() string {
|
|
||||||
helpText := `
|
|
||||||
Usage: nomad stats [options] <alloc-id>
|
|
||||||
|
|
||||||
Displays statistics related to resource usage of tasks in an allocation. Use
|
|
||||||
the -task flag to query statistics of an individual task running in an
|
|
||||||
allocation.
|
|
||||||
|
|
||||||
General Options:
|
|
||||||
|
|
||||||
` + generalOptionsUsage() + `
|
|
||||||
|
|
||||||
Node Stats Options:
|
|
||||||
|
|
||||||
-task
|
|
||||||
Display statistics for a specific task in an allocation.
|
|
||||||
`
|
|
||||||
|
|
||||||
return strings.TrimSpace(helpText)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *StatsCommand) Synopsis() string {
|
|
||||||
return "Dispalys resource usage stats of an allocation or a task running on a nomad client"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *StatsCommand) Run(args []string) int {
|
|
||||||
var verbose bool
|
|
||||||
var task string
|
|
||||||
flags := f.Meta.FlagSet("stats", FlagSetClient)
|
|
||||||
flags.BoolVar(&verbose, "verbose", false, "")
|
|
||||||
flags.StringVar(&task, "task", "", "")
|
|
||||||
flags.Usage = func() { f.Ui.Output(f.Help()) }
|
|
||||||
|
|
||||||
if err := flags.Parse(args); err != nil {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
args = flags.Args()
|
|
||||||
if len(args) < 1 {
|
|
||||||
f.Ui.Error("allocation id is a required parameter")
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
client, err := f.Meta.Client()
|
|
||||||
if err != nil {
|
|
||||||
f.Ui.Error(fmt.Sprintf("Error initializing client: %v", err))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
var allocID string
|
|
||||||
allocID = strings.TrimSpace(args[0])
|
|
||||||
|
|
||||||
// Truncate the id unless full length is requested
|
|
||||||
length := shortId
|
|
||||||
if verbose {
|
|
||||||
length = fullId
|
|
||||||
}
|
|
||||||
|
|
||||||
// Query the allocation info
|
|
||||||
if len(allocID) == 1 {
|
|
||||||
f.Ui.Error(fmt.Sprintf("Alloc ID must contain at least two characters."))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if len(allocID)%2 == 1 {
|
|
||||||
// Identifiers must be of even length, so we strip off the last byte
|
|
||||||
// to provide a consistent user experience.
|
|
||||||
allocID = allocID[:len(allocID)-1]
|
|
||||||
}
|
|
||||||
|
|
||||||
allocs, _, err := client.Allocations().PrefixList(allocID)
|
|
||||||
if err != nil {
|
|
||||||
f.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if len(allocs) == 0 {
|
|
||||||
f.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if len(allocs) > 1 {
|
|
||||||
// Format the allocs
|
|
||||||
out := make([]string, len(allocs)+1)
|
|
||||||
out[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status"
|
|
||||||
for i, alloc := range allocs {
|
|
||||||
out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
|
|
||||||
limit(alloc.ID, length),
|
|
||||||
limit(alloc.EvalID, length),
|
|
||||||
alloc.JobID,
|
|
||||||
alloc.TaskGroup,
|
|
||||||
alloc.DesiredStatus,
|
|
||||||
alloc.ClientStatus,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
f.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", formatList(out)))
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
// Prefix lookup matched a single allocation
|
|
||||||
alloc, _, err := client.Allocations().Info(allocs[0].ID, nil)
|
|
||||||
if err != nil {
|
|
||||||
f.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
stats, err := client.Allocations().Stats(alloc, nil)
|
|
||||||
if err != nil {
|
|
||||||
f.Ui.Error(fmt.Sprintf("unable to get stats: %v", err))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if task == "" {
|
|
||||||
f.printAllocResourceUsage(alloc, stats)
|
|
||||||
} else {
|
|
||||||
f.printTaskResourceUsage(task, stats)
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *StatsCommand) printTaskResourceUsage(task string, resourceUsage map[string]*api.TaskResourceUsage) {
|
|
||||||
tu, ok := resourceUsage[task]
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
memoryStats := tu.ResourceUsage.MemoryStats
|
|
||||||
cpuStats := tu.ResourceUsage.CpuStats
|
|
||||||
f.Ui.Output(fmt.Sprintf("===> Task: %q", task))
|
|
||||||
f.Ui.Output("Memory Stats")
|
|
||||||
out := make([]string, 2)
|
|
||||||
out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|KernelMaxUsage"
|
|
||||||
out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v",
|
|
||||||
humanize.Bytes(memoryStats.RSS),
|
|
||||||
humanize.Bytes(memoryStats.Cache),
|
|
||||||
humanize.Bytes(memoryStats.Swap),
|
|
||||||
humanize.Bytes(memoryStats.MaxUsage),
|
|
||||||
humanize.Bytes(memoryStats.KernelUsage),
|
|
||||||
humanize.Bytes(memoryStats.KernelMaxUsage),
|
|
||||||
)
|
|
||||||
f.Ui.Output(formatList(out))
|
|
||||||
|
|
||||||
f.Ui.Output("")
|
|
||||||
|
|
||||||
f.Ui.Output("CPU Stats")
|
|
||||||
out = make([]string, 2)
|
|
||||||
out[0] = "Percent|Throttled Periods|Throttled Time"
|
|
||||||
percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
|
|
||||||
out[1] = fmt.Sprintf("%v|%v|%v", percent,
|
|
||||||
cpuStats.ThrottledPeriods, cpuStats.ThrottledTime)
|
|
||||||
f.Ui.Output(formatList(out))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *StatsCommand) printAllocResourceUsage(alloc *api.Allocation, resourceUsage map[string]*api.TaskResourceUsage) {
|
|
||||||
f.Ui.Output(fmt.Sprintf("Resource Usage of Tasks running in Allocation %q", alloc.ID))
|
|
||||||
for task, _ := range alloc.TaskStates {
|
|
||||||
f.printTaskResourceUsage(task, resourceUsage)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -121,11 +121,6 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
|
||||||
Meta: meta,
|
Meta: meta,
|
||||||
}, nil
|
}, nil
|
||||||
},
|
},
|
||||||
"stats": func() (cli.Command, error) {
|
|
||||||
return &command.StatsCommand{
|
|
||||||
Meta: meta,
|
|
||||||
}, nil
|
|
||||||
},
|
|
||||||
"status": func() (cli.Command, error) {
|
"status": func() (cli.Command, error) {
|
||||||
return &command.StatusCommand{
|
return &command.StatusCommand{
|
||||||
Meta: meta,
|
Meta: meta,
|
||||||
|
|
Loading…
Reference in a new issue