luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Alloc-status only shows measured statistics and fixes to CPU calculations

This commit is contained in:
Alex Dadgar 2016-06-10 10:38:29 -07:00
parent 3ab7d4a512
commit 3cf74e7fd8
5 changed files with 134 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
client/driver/docker.go
View File

@ -34,8 +34,8 @@ var (
client *docker.Client
// The statistics the Docker driver exposes
DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage"}
DockerMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"}
DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"}
DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
)
const (
@ -487,7 +487,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task,
d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
config.Cmd = cmd
} else if len(driverConfig.Args) != 0 {
d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified")
config.Cmd = parsedArgs
}
if len(driverConfig.Labels) > 0 {
@ -998,20 +998,22 @@ func (h *DockerHandle) collectStats() {
}
cs := &cstructs.CpuStats{
SystemMode: float64(s.CPUStats.CPUUsage.UsageInKernelmode),
UserMode: float64(s.CPUStats.CPUUsage.UsageInKernelmode),
ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime,
Measured: DockerMeasuredCpuStats,
}
// Calculate percentage
cs.Percent = 0.0
cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage) - float64(s.PreCPUStats.CPUUsage.TotalUsage)
systemDelta := float64(s.CPUStats.SystemCPUUsage) - float64(s.PreCPUStats.SystemCPUUsage)
if cpuDelta > 0.0 && systemDelta > 0.0 {
cs.Percent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0
}
cores := len(s.CPUStats.CPUUsage.PercpuUsage)
cs.Percent = calculatePercent(
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, cores)
cs.SystemMode = calculatePercent(
s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores)
cs.UserMode = calculatePercent(
s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores)
h.resourceUsageLock.Lock()
h.resourceUsage = &cstructs.TaskResourceUsage{
@ -1028,3 +1030,13 @@ func (h *DockerHandle) collectStats() {
}
}
}
func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 {
numerator := newSample - oldSample
denom := newTotal - oldTotal
if numerator <= 0 || denom <= 0 {
return 0.0
}
return (float64(numerator) / float64(denom)) * float64(cores) * 100.0
}

33
client/driver/executor/executor.go
View File

@ -34,12 +34,15 @@ const (
// tree for finding out the pids that the executor and it's child processes
// have forked
pidScanInterval = 5 * time.Second
// nanosecondsInSecond is the number of nanoseconds in a second.
nanosecondsInSecond float64 = 1000000000.0
)
var (
// The statistics the basic executor exposes
ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"}
ExecutorBasicMeasuredCpuStats = []string{"SystemMode", "UserMode", "Percent"}
ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
)
// Executor is the interface which allows a driver to launch and supervise
@ -188,18 +191,22 @@ type UniversalExecutor struct {
cgPaths map[string]string
cgLock sync.Mutex
consulService *consul.ConsulService
consulCtx *ConsulContext
cpuStats *stats.CpuStats
logger *log.Logger
consulService *consul.ConsulService
consulCtx *ConsulContext
totalCpuStats *stats.CpuStats
userCpuStats *stats.CpuStats
systemCpuStats *stats.CpuStats
logger *log.Logger
}
// NewExecutor returns an Executor
func NewExecutor(logger *log.Logger) Executor {
exec := &UniversalExecutor{
logger: logger,
processExited: make(chan interface{}),
cpuStats: stats.NewCpuStats(),
logger: logger,
processExited: make(chan interface{}),
totalCpuStats: stats.NewCpuStats(),
userCpuStats: stats.NewCpuStats(),
systemCpuStats: stats.NewCpuStats(),
}
return exec
@ -515,12 +522,12 @@ func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, erro
cs := &cstructs.CpuStats{}
if cpuStats, err := p.Times(); err == nil {
cs.SystemMode = pid.cpuStatsSys.Percent(cpuStats.System)
cs.UserMode = pid.cpuStatsUser.Percent(cpuStats.User)
cs.SystemMode = pid.cpuStatsSys.Percent(cpuStats.System * nanosecondsInSecond)
cs.UserMode = pid.cpuStatsUser.Percent(cpuStats.User * nanosecondsInSecond)
cs.Measured = ExecutorBasicMeasuredCpuStats
// calculate cpu usage percent
cs.Percent = pid.cpuStatsTotal.Percent(cpuStats.Total())
cs.Percent = pid.cpuStatsTotal.Percent(cpuStats.Total() * nanosecondsInSecond)
}
stats[strconv.Itoa(pid.pid)] = &cstructs.ResourceUsage{MemoryStats: ms, CpuStats: cs}
}
@ -781,13 +788,13 @@ func (e *UniversalExecutor) aggregatedResourceUsage(pidStats map[string]*cstruct
SystemMode: systemModeCPU,
UserMode: userModeCPU,
Percent: percent,
Measured: ExecutorBasicMeasuredMemStats,
Measured: ExecutorBasicMeasuredCpuStats,
}
totalMemory := &cstructs.MemoryStats{
RSS: totalRSS,
Swap: totalSwap,
Measured: ExecutorBasicMeasuredCpuStats,
Measured: ExecutorBasicMeasuredMemStats,
}
resourceUsage := cstructs.ResourceUsage{

25
client/driver/executor/executor_linux.go
View File

@ -40,12 +40,9 @@ var (
// clockTicks is the clocks per second of the machine
clockTicks = uint64(system.GetClockTicks())
// nanosecondsInSecond is the number of nanoseconds in a second.
nanosecondsInSecond = uint64(1000000000)
// The statistics the executor exposes when using cgroups
ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage", "KernelUsage", "KernelMaxUsage"}
ExecutorCgroupMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"}
ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
)
// configureIsolation configures chroot and creates cgroups
@ -165,23 +162,19 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
}
// CPU Related Stats
totalProcessCPUUsage := stats.CpuStats.CpuUsage.TotalUsage
userModeTime := stats.CpuStats.CpuUsage.UsageInUsermode
kernelModeTime := stats.CpuStats.CpuUsage.UsageInKernelmode
umTicks := (userModeTime * clockTicks) / nanosecondsInSecond
kmTicks := (kernelModeTime * clockTicks) / nanosecondsInSecond
totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)
cs := &cstructs.CpuStats{
SystemMode: float64(kmTicks),
UserMode: float64(umTicks),
SystemMode: e.systemCpuStats.Percent(kernelModeTime),
UserMode: e.userCpuStats.Percent(userModeTime),
Percent: e.totalCpuStats.Percent(totalProcessCPUUsage),
ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
ThrottledTime: stats.CpuStats.ThrottlingData.ThrottledTime,
Measured: ExecutorCgroupMeasuredCpuStats,
}
if e.cpuStats != nil {
cs.Percent = e.cpuStats.Percent(float64(totalProcessCPUUsage / nanosecondsInSecond))
}
taskResUsage := cstructs.TaskResourceUsage{
ResourceUsage: &cstructs.ResourceUsage{
MemoryStats: ms,

30
client/stats/cpu.go
View File

@ -7,8 +7,8 @@ import (
// CpuStats calculates cpu usage percentage
type CpuStats struct {
prevProcessUsage float64
prevTime time.Time
prevCpuTime float64
prevTime time.Time
totalCpus int
}
@ -20,30 +20,32 @@ func NewCpuStats() *CpuStats {
}
// Percent calculates the cpu usage percentage based on the current cpu usage
// and the previous cpu usage
func (c *CpuStats) Percent(currentProcessUsage float64) float64 {
// and the previous cpu usage where usage is given as time in nanoseconds spend
// in the cpu
func (c *CpuStats) Percent(cpuTime float64) float64 {
now := time.Now()
if c.prevProcessUsage == 0.0 {
if c.prevCpuTime == 0.0 {
// invoked first time
c.prevProcessUsage = currentProcessUsage
c.prevCpuTime = cpuTime
c.prevTime = now
return 0.0
}
delta := (now.Sub(c.prevTime).Seconds()) * float64(c.totalCpus)
ret := c.calculatePercent(c.prevProcessUsage, currentProcessUsage, delta)
c.prevProcessUsage = currentProcessUsage
timeDelta := now.Sub(c.prevTime).Nanoseconds()
ret := c.calculatePercent(c.prevCpuTime, cpuTime, timeDelta)
c.prevCpuTime = cpuTime
c.prevTime = now
return ret
}
func (c *CpuStats) calculatePercent(t1, t2 float64, delta float64) float64 {
if delta == 0 {
return 0
func (c *CpuStats) calculatePercent(t1, t2 float64, timeDelta int64) float64 {
vDelta := t2 - t1
if timeDelta <= 0 || vDelta <= 0.0 {
return 0.0
}
delta_proc := t2 - t1
overall_percent := ((delta_proc / delta) * 100) * float64(c.totalCpus)
overall_percent := (vDelta / float64(timeDelta)) * 100.0
return overall_percent
}

90
command/alloc_status.go
View File

@ -361,10 +361,13 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
cpuUsage := strconv.Itoa(resource.CPU)
memUsage := strconv.Itoa(resource.MemoryMB)
if ru, ok := stats[task]; ok && ru != nil && ru.ResourceUsage != nil {
cpuTicksConsumed := (ru.ResourceUsage.CpuStats.Percent / 100) * float64(resource.CPU)
memoryStats := ru.ResourceUsage.MemoryStats
cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU)
memUsage = fmt.Sprintf("%v/%v", memoryStats.RSS/(1024*1024), resource.MemoryMB)
if cs := ru.ResourceUsage.CpuStats; cs != nil {
cpuTicksConsumed := (cs.Percent / 100) * float64(resource.CPU)
cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU)
}
if ms := ru.ResourceUsage.MemoryStats; ms != nil {
memUsage = fmt.Sprintf("%v/%v", ms.RSS/(1024*1024), resource.MemoryMB)
}
}
resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v MB|%v MB|%v|%v",
cpuUsage,
@ -387,26 +390,65 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
func (c *AllocStatusCommand) printTaskResourceUsage(task string, resourceUsage *api.ResourceUsage) {
memoryStats := resourceUsage.MemoryStats
cpuStats := resourceUsage.CpuStats
c.Ui.Output("Memory Stats")
out := make([]string, 2)
out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|Kernel Max Usage"
out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v",
humanize.Bytes(memoryStats.RSS),
humanize.Bytes(memoryStats.Cache),
humanize.Bytes(memoryStats.Swap),
humanize.Bytes(memoryStats.MaxUsage),
humanize.Bytes(memoryStats.KernelUsage),
humanize.Bytes(memoryStats.KernelMaxUsage),
)
c.Ui.Output(formatList(out))
if memoryStats != nil && len(memoryStats.Measured) > 0 {
c.Ui.Output("Memory Stats")
c.Ui.Output("")
// Sort the measured stats
sort.Strings(memoryStats.Measured)
c.Ui.Output("CPU Stats")
out = make([]string, 2)
out[0] = "Percent|Throttled Periods|Throttled Time"
percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
out[1] = fmt.Sprintf("%v %%|%v|%v", percent,
cpuStats.ThrottledPeriods, cpuStats.ThrottledTime)
c.Ui.Output(formatList(out))
var measuredStats []string
for _, measured := range memoryStats.Measured {
switch measured {
case "RSS":
measuredStats = append(measuredStats, humanize.Bytes(memoryStats.RSS))
case "Cache":
measuredStats = append(measuredStats, humanize.Bytes(memoryStats.Cache))
case "Swap":
measuredStats = append(measuredStats, humanize.Bytes(memoryStats.Swap))
case "Max Usage":
measuredStats = append(measuredStats, humanize.Bytes(memoryStats.MaxUsage))
case "Kernel Usage":
measuredStats = append(measuredStats, humanize.Bytes(memoryStats.KernelUsage))
case "Kernel Max Usage":
measuredStats = append(measuredStats, humanize.Bytes(memoryStats.KernelMaxUsage))
}
}
out := make([]string, 2)
out[0] = strings.Join(memoryStats.Measured, "|")
out[1] = strings.Join(measuredStats, "|")
c.Ui.Output(formatList(out))
c.Ui.Output("")
}
if cpuStats != nil && len(cpuStats.Measured) > 0 {
c.Ui.Output("CPU Stats")
// Sort the measured stats
sort.Strings(cpuStats.Measured)
var measuredStats []string
for _, measured := range cpuStats.Measured {
switch measured {
case "Percent":
percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
case "Throttled Periods":
measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods))
case "Throttled Time":
measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime))
case "User Mode":
percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64)
measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
case "System Mode":
percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64)
measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
}
}
out := make([]string, 2)
out[0] = strings.Join(cpuStats.Measured, "|")
out[1] = strings.Join(measuredStats, "|")
c.Ui.Output(formatList(out))
}
}