Showing host resource usage stats
This commit is contained in:
Diptanu Choudhury
parent
d5a6d6533f
commit
73fa700a88
|
|
@ -124,8 +124,9 @@ type Client struct {
|
|||
|
||||
consulService *consul.ConsulService
|
||||
|
||||
resourceUsage *stats.RingBuff
|
||||
resourceUsageLock sync.RWMutex
|
||||
hostStatsCollector *stats.HostStatsCollector
|
||||
resourceUsage *stats.RingBuff
|
||||
resourceUsageLock sync.RWMutex
|
||||
|
||||
shutdown bool
|
||||
shutdownCh chan struct{}
|
||||
|
|
@ -141,17 +142,22 @@ func NewClient(cfg *config.Config) (*Client, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostStatsCollector, err := stats.NewHostStatsCollector()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create the client
|
||||
c := &Client{
|
||||
config: cfg,
|
||||
start: time.Now(),
|
||||
connPool: nomad.NewPool(cfg.LogOutput, clientRPCCache, clientMaxStreams, nil),
|
||||
logger: logger,
|
||||
resourceUsage: resourceUsage,
|
||||
allocs: make(map[string]*AllocRunner),
|
||||
allocUpdates: make(chan *structs.Allocation, 64),
|
||||
shutdownCh: make(chan struct{}),
|
||||
config: cfg,
|
||||
start: time.Now(),
|
||||
connPool: nomad.NewPool(cfg.LogOutput, clientRPCCache, clientMaxStreams, nil),
|
||||
logger: logger,
|
||||
hostStatsCollector: hostStatsCollector,
|
||||
resourceUsage: resourceUsage,
|
||||
allocs: make(map[string]*AllocRunner),
|
||||
allocUpdates: make(chan *structs.Allocation, 64),
|
||||
shutdownCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Initialize the client
|
||||
|
|
@ -1276,7 +1282,7 @@ func (c *Client) monitorUsage() {
|
|||
next := time.NewTimer(1 * time.Second)
|
||||
select {
|
||||
case <-next.C:
|
||||
ru, err := stats.CollectHostStats()
|
||||
ru, err := c.hostStatsCollector.Collect()
|
||||
if err != nil {
|
||||
c.logger.Printf("[DEBUG] client: error fetching stats of host: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,10 +25,29 @@ type CPUStats struct {
|
|||
User float64
|
||||
System float64
|
||||
Idle float64
|
||||
Total float64
|
||||
}
|
||||
|
||||
// CollectHostStats collects stats related to resource usage of a host
|
||||
func CollectHostStats() (*HostStats, error) {
|
||||
// HostStatsCollector collects host resource usage stats
|
||||
type HostStatsCollector struct {
|
||||
statsCalculator map[string]*HostCpuStatsCalculator
|
||||
}
|
||||
|
||||
// NewHostStatsCollector returns a HostStatsCollector
|
||||
func NewHostStatsCollector() (*HostStatsCollector, error) {
|
||||
times, err := cpu.Times(true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
statsCalculator := make(map[string]*HostCpuStatsCalculator)
|
||||
for _, time := range times {
|
||||
statsCalculator[time.CPU] = NewHostCpuStatsCalculator()
|
||||
}
|
||||
return &HostStatsCollector{statsCalculator: statsCalculator}, nil
|
||||
}
|
||||
|
||||
// Collect collects stats related to resource usage of a host
|
||||
func (h *HostStatsCollector) Collect() (*HostStats, error) {
|
||||
memStats, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -49,6 +68,15 @@ func CollectHostStats() (*HostStats, error) {
|
|||
System: cpuStat.System,
|
||||
Idle: cpuStat.Idle,
|
||||
}
|
||||
if percentCalculator, ok := h.statsCalculator[cpuStat.CPU]; ok {
|
||||
idle, user, system, total := percentCalculator.Calculate(cpuStat)
|
||||
cs[idx].Idle = idle
|
||||
cs[idx].System = system
|
||||
cs[idx].User = user
|
||||
cs[idx].Total = total
|
||||
} else {
|
||||
h.statsCalculator[cpuStat.CPU] = NewHostCpuStatsCalculator()
|
||||
}
|
||||
}
|
||||
|
||||
hs := &HostStats{
|
||||
|
|
@ -57,3 +85,43 @@ func CollectHostStats() (*HostStats, error) {
|
|||
}
|
||||
return hs, nil
|
||||
}
|
||||
|
||||
// HostCpuStatsCalculator calculates cpu usage percentages
|
||||
type HostCpuStatsCalculator struct {
|
||||
prevIdle float64
|
||||
prevUser float64
|
||||
prevSystem float64
|
||||
prevBusy float64
|
||||
prevTotal float64
|
||||
}
|
||||
|
||||
// NewHostCpuStatsCalculator returns a HostCpuStatsCalculator
|
||||
func NewHostCpuStatsCalculator() *HostCpuStatsCalculator {
|
||||
return &HostCpuStatsCalculator{}
|
||||
}
|
||||
|
||||
// Calculate calculates the current cpu usage percentages
|
||||
func (h *HostCpuStatsCalculator) Calculate(times cpu.TimesStat) (idle float64, user float64, system float64, total float64) {
|
||||
currentIdle := times.Idle
|
||||
currentUser := times.User
|
||||
currentSystem := times.System
|
||||
currentTotal := times.Total()
|
||||
|
||||
deltaTotal := currentTotal - h.prevTotal
|
||||
idle = ((currentIdle - h.prevIdle) / deltaTotal) * 100
|
||||
user = ((currentUser - h.prevUser) / deltaTotal) * 100
|
||||
system = ((currentSystem - h.prevSystem) / deltaTotal) * 100
|
||||
|
||||
currentBusy := times.User + times.System + times.Nice + times.Iowait + times.Irq +
|
||||
times.Softirq + times.Steal + times.Guest + times.GuestNice + times.Stolen
|
||||
|
||||
total = ((currentBusy - h.prevBusy) / deltaTotal) * 100
|
||||
|
||||
h.prevIdle = currentIdle
|
||||
h.prevUser = currentUser
|
||||
h.prevSystem = currentSystem
|
||||
h.prevTotal = currentTotal
|
||||
h.prevBusy = currentBusy
|
||||
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,11 @@ package command
|
|||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/hashicorp/nomad/api"
|
||||
)
|
||||
|
||||
|
|
@ -43,6 +46,9 @@ Node Status Options:
|
|||
|
||||
-allocs
|
||||
Display a count of running allocations for each node.
|
||||
|
||||
-stats
|
||||
Display the resource usage of the node.
|
||||
`
|
||||
return strings.TrimSpace(helpText)
|
||||
}
|
||||
|
|
@ -52,7 +58,7 @@ func (c *NodeStatusCommand) Synopsis() string {
|
|||
}
|
||||
|
||||
func (c *NodeStatusCommand) Run(args []string) int {
|
||||
var short, verbose, list_allocs, self bool
|
||||
var short, verbose, list_allocs, self, stats bool
|
||||
|
||||
flags := c.Meta.FlagSet("node-status", FlagSetClient)
|
||||
flags.Usage = func() { c.Ui.Output(c.Help()) }
|
||||
|
|
@ -60,6 +66,7 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
|||
flags.BoolVar(&verbose, "verbose", false, "")
|
||||
flags.BoolVar(&list_allocs, "allocs", false, "")
|
||||
flags.BoolVar(&self, "self", false, "")
|
||||
flags.BoolVar(&stats, "stats", false, "")
|
||||
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
|
|
@ -212,6 +219,16 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
|||
}
|
||||
c.Ui.Output("\n==> Resource Utilization")
|
||||
c.Ui.Output(formatList(resources))
|
||||
if stats {
|
||||
if hostStats, err := client.Nodes().Stats(node.ID, nil); err == nil {
|
||||
c.Ui.Output("\n===> Node CPU Stats")
|
||||
c.printCpuStats(hostStats)
|
||||
c.Ui.Output("\n===> Node Memory Stats")
|
||||
c.printMemoryStats(hostStats)
|
||||
} else {
|
||||
c.Ui.Output(fmt.Sprintf("error getting node stats", err))
|
||||
}
|
||||
}
|
||||
|
||||
allocs, err := getAllocs(client, node, length)
|
||||
if err != nil {
|
||||
|
|
@ -246,6 +263,28 @@ func (c *NodeStatusCommand) Run(args []string) int {
|
|||
return 0
|
||||
}
|
||||
|
||||
func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) {
|
||||
for _, cpuStat := range hostStats.CPU {
|
||||
cpuStatsAttr := make([]string, 4)
|
||||
cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
|
||||
cpuStatsAttr[1] = fmt.Sprintf("User|%v", formatFloat64(cpuStat.User))
|
||||
cpuStatsAttr[2] = fmt.Sprintf("System|%v", formatFloat64(cpuStat.System))
|
||||
cpuStatsAttr[3] = fmt.Sprintf("Idle|%v", formatFloat64(cpuStat.Idle))
|
||||
c.Ui.Output(formatKV(cpuStatsAttr))
|
||||
c.Ui.Output("")
|
||||
}
|
||||
}
|
||||
|
||||
func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) {
|
||||
memoryStat := hostStats.Memory
|
||||
memStatsAttr := make([]string, 4)
|
||||
memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.Bytes(memoryStat.Total))
|
||||
memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.Bytes(memoryStat.Available))
|
||||
memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.Bytes(memoryStat.Used))
|
||||
memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.Bytes(memoryStat.Free))
|
||||
c.Ui.Output(formatKV(memStatsAttr))
|
||||
}
|
||||
|
||||
// getRunningAllocs returns a slice of allocation id's running on the node
|
||||
func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) {
|
||||
var allocs []*api.Allocation
|
||||
|
|
@ -323,3 +362,7 @@ func getResources(client *api.Client, node *api.Node) ([]string, error) {
|
|||
|
||||
return resources, err
|
||||
}
|
||||
|
||||
func formatFloat64(val float64) string {
|
||||
return strconv.FormatFloat(val, 'f', 2, 64)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue