Alloc-status only shows measured statistics and fixes to CPU calculations

client/driver/docker.go

@@ -34,8 +34,8 @@ var (
 	client       *docker.Client
 
 	// The statistics the Docker driver exposes
-	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage"}
+	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"}
-	DockerMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"}
+	DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
 )
 
 const (
@@ -487,7 +487,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task,
 		d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
 		config.Cmd = cmd
 	} else if len(driverConfig.Args) != 0 {
-		d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified")
+		config.Cmd = parsedArgs
 	}
 
 	if len(driverConfig.Labels) > 0 {
@@ -998,20 +998,22 @@ func (h *DockerHandle) collectStats() {
 				}
 
 				cs := &cstructs.CpuStats{
-					SystemMode:       float64(s.CPUStats.CPUUsage.UsageInKernelmode),
-					UserMode:         float64(s.CPUStats.CPUUsage.UsageInKernelmode),
 					ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
 					ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
 					Measured:         DockerMeasuredCpuStats,
 				}
 
 				// Calculate percentage
-				cs.Percent = 0.0
+				cores := len(s.CPUStats.CPUUsage.PercpuUsage)
-				cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage) - float64(s.PreCPUStats.CPUUsage.TotalUsage)
+				cs.Percent = calculatePercent(
-				systemDelta := float64(s.CPUStats.SystemCPUUsage) - float64(s.PreCPUStats.SystemCPUUsage)
+					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
-				if cpuDelta > 0.0 && systemDelta > 0.0 {
+					s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, cores)
-					cs.Percent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0
+				cs.SystemMode = calculatePercent(
-				}
+					s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
+					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores)
+				cs.UserMode = calculatePercent(
+					s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
+					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores)
 
 				h.resourceUsageLock.Lock()
 				h.resourceUsage = &cstructs.TaskResourceUsage{
@@ -1028,3 +1030,13 @@ func (h *DockerHandle) collectStats() {
 		}
 	}
 }
+
+func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 {
+	numerator := newSample - oldSample
+	denom := newTotal - oldTotal
+	if numerator <= 0 || denom <= 0 {
+		return 0.0
+	}
+
+	return (float64(numerator) / float64(denom)) * float64(cores) * 100.0
+}

client/driver/executor/executor.go

@@ -34,12 +34,15 @@ const (
 	// tree for finding out the pids that the executor and it's child processes
 	// have forked
 	pidScanInterval = 5 * time.Second
+
+	// nanosecondsInSecond is the number of nanoseconds in a second.
+	nanosecondsInSecond float64 = 1000000000.0
 )
 
 var (
 	// The statistics the basic executor exposes
 	ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"}
-	ExecutorBasicMeasuredCpuStats = []string{"SystemMode", "UserMode", "Percent"}
+	ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
 )
 
 // Executor is the interface which allows a driver to launch and supervise
@@ -188,18 +191,22 @@ type UniversalExecutor struct {
 	cgPaths map[string]string
 	cgLock  sync.Mutex
 
-	consulService *consul.ConsulService
+	consulService  *consul.ConsulService
-	consulCtx     *ConsulContext
+	consulCtx      *ConsulContext
-	cpuStats      *stats.CpuStats
+	totalCpuStats  *stats.CpuStats
-	logger        *log.Logger
+	userCpuStats   *stats.CpuStats
+	systemCpuStats *stats.CpuStats
+	logger         *log.Logger
 }
 
 // NewExecutor returns an Executor
 func NewExecutor(logger *log.Logger) Executor {
 	exec := &UniversalExecutor{
-		logger:        logger,
+		logger:         logger,
-		processExited: make(chan interface{}),
+		processExited:  make(chan interface{}),
-		cpuStats:      stats.NewCpuStats(),
+		totalCpuStats:  stats.NewCpuStats(),
+		userCpuStats:   stats.NewCpuStats(),
+		systemCpuStats: stats.NewCpuStats(),
 	}
 
 	return exec
@@ -515,12 +522,12 @@ func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, erro
 
 		cs := &cstructs.CpuStats{}
 		if cpuStats, err := p.Times(); err == nil {
-			cs.SystemMode = pid.cpuStatsSys.Percent(cpuStats.System)
+			cs.SystemMode = pid.cpuStatsSys.Percent(cpuStats.System * nanosecondsInSecond)
-			cs.UserMode = pid.cpuStatsUser.Percent(cpuStats.User)
+			cs.UserMode = pid.cpuStatsUser.Percent(cpuStats.User * nanosecondsInSecond)
 			cs.Measured = ExecutorBasicMeasuredCpuStats
 
 			// calculate cpu usage percent
-			cs.Percent = pid.cpuStatsTotal.Percent(cpuStats.Total())
+			cs.Percent = pid.cpuStatsTotal.Percent(cpuStats.Total() * nanosecondsInSecond)
 		}
 		stats[strconv.Itoa(pid.pid)] = &cstructs.ResourceUsage{MemoryStats: ms, CpuStats: cs}
 	}
@@ -781,13 +788,13 @@ func (e *UniversalExecutor) aggregatedResourceUsage(pidStats map[string]*cstruct
 		SystemMode: systemModeCPU,
 		UserMode:   userModeCPU,
 		Percent:    percent,
-		Measured:   ExecutorBasicMeasuredMemStats,
+		Measured:   ExecutorBasicMeasuredCpuStats,
 	}
 
 	totalMemory := &cstructs.MemoryStats{
 		RSS:      totalRSS,
 		Swap:     totalSwap,
-		Measured: ExecutorBasicMeasuredCpuStats,
+		Measured: ExecutorBasicMeasuredMemStats,
 	}
 
 	resourceUsage := cstructs.ResourceUsage{

client/driver/executor/executor_linux.go

@@ -40,12 +40,9 @@ var (
 	// clockTicks is the clocks per second of the machine
 	clockTicks = uint64(system.GetClockTicks())
 
-	// nanosecondsInSecond is the number of nanoseconds in a second.
-	nanosecondsInSecond = uint64(1000000000)
-
 	// The statistics the executor exposes when using cgroups
-	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage", "KernelUsage", "KernelMaxUsage"}
+	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
-	ExecutorCgroupMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"}
+	ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
 )
 
 // configureIsolation configures chroot and creates cgroups
@@ -165,23 +162,19 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
 	}
 
 	// CPU Related Stats
-	totalProcessCPUUsage := stats.CpuStats.CpuUsage.TotalUsage
+	totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
-	userModeTime := stats.CpuStats.CpuUsage.UsageInUsermode
+	userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
-	kernelModeTime := stats.CpuStats.CpuUsage.UsageInKernelmode
+	kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)
-
-	umTicks := (userModeTime * clockTicks) / nanosecondsInSecond
-	kmTicks := (kernelModeTime * clockTicks) / nanosecondsInSecond
 
 	cs := &cstructs.CpuStats{
-		SystemMode:       float64(kmTicks),
+		SystemMode:       e.systemCpuStats.Percent(kernelModeTime),
-		UserMode:         float64(umTicks),
+		UserMode:         e.userCpuStats.Percent(userModeTime),
+		Percent:          e.totalCpuStats.Percent(totalProcessCPUUsage),
 		ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
 		ThrottledTime:    stats.CpuStats.ThrottlingData.ThrottledTime,
 		Measured:         ExecutorCgroupMeasuredCpuStats,
 	}
-	if e.cpuStats != nil {
+
-		cs.Percent = e.cpuStats.Percent(float64(totalProcessCPUUsage / nanosecondsInSecond))
-	}
 	taskResUsage := cstructs.TaskResourceUsage{
 		ResourceUsage: &cstructs.ResourceUsage{
 			MemoryStats: ms,

client/stats/cpu.go

@@ -7,8 +7,8 @@ import (
 
 // CpuStats calculates cpu usage percentage
 type CpuStats struct {
-	prevProcessUsage float64
+	prevCpuTime float64
-	prevTime         time.Time
+	prevTime    time.Time
 
 	totalCpus int
 }
@@ -20,30 +20,32 @@ func NewCpuStats() *CpuStats {
 }
 
 // Percent calculates the cpu usage percentage based on the current cpu usage
-// and the previous cpu usage
+// and the previous cpu usage where usage is given as time in nanoseconds spend
-func (c *CpuStats) Percent(currentProcessUsage float64) float64 {
+// in the cpu
+func (c *CpuStats) Percent(cpuTime float64) float64 {
 	now := time.Now()
 
-	if c.prevProcessUsage == 0.0 {
+	if c.prevCpuTime == 0.0 {
 		// invoked first time
-		c.prevProcessUsage = currentProcessUsage
+		c.prevCpuTime = cpuTime
 		c.prevTime = now
 		return 0.0
 	}
 
-	delta := (now.Sub(c.prevTime).Seconds()) * float64(c.totalCpus)
+	timeDelta := now.Sub(c.prevTime).Nanoseconds()
-	ret := c.calculatePercent(c.prevProcessUsage, currentProcessUsage, delta)
+	ret := c.calculatePercent(c.prevCpuTime, cpuTime, timeDelta)
-	c.prevProcessUsage = currentProcessUsage
+	c.prevCpuTime = cpuTime
 	c.prevTime = now
 	return ret
 
 }
 
-func (c *CpuStats) calculatePercent(t1, t2 float64, delta float64) float64 {
+func (c *CpuStats) calculatePercent(t1, t2 float64, timeDelta int64) float64 {
-	if delta == 0 {
+	vDelta := t2 - t1
-		return 0
+	if timeDelta <= 0 || vDelta <= 0.0 {
+		return 0.0
 	}
-	delta_proc := t2 - t1
+
-	overall_percent := ((delta_proc / delta) * 100) * float64(c.totalCpus)
+	overall_percent := (vDelta / float64(timeDelta)) * 100.0
 	return overall_percent
 }

command/alloc_status.go

@@ -361,10 +361,13 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
 		cpuUsage := strconv.Itoa(resource.CPU)
 		memUsage := strconv.Itoa(resource.MemoryMB)
 		if ru, ok := stats[task]; ok && ru != nil && ru.ResourceUsage != nil {
-			cpuTicksConsumed := (ru.ResourceUsage.CpuStats.Percent / 100) * float64(resource.CPU)
+			if cs := ru.ResourceUsage.CpuStats; cs != nil {
-			memoryStats := ru.ResourceUsage.MemoryStats
+				cpuTicksConsumed := (cs.Percent / 100) * float64(resource.CPU)
-			cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU)
+				cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU)
-			memUsage = fmt.Sprintf("%v/%v", memoryStats.RSS/(1024*1024), resource.MemoryMB)
+			}
+			if ms := ru.ResourceUsage.MemoryStats; ms != nil {
+				memUsage = fmt.Sprintf("%v/%v", ms.RSS/(1024*1024), resource.MemoryMB)
+			}
 		}
 		resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v MB|%v MB|%v|%v",
 			cpuUsage,
@@ -387,26 +390,65 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri
 func (c *AllocStatusCommand) printTaskResourceUsage(task string, resourceUsage *api.ResourceUsage) {
 	memoryStats := resourceUsage.MemoryStats
 	cpuStats := resourceUsage.CpuStats
-	c.Ui.Output("Memory Stats")
+	if memoryStats != nil && len(memoryStats.Measured) > 0 {
-	out := make([]string, 2)
+		c.Ui.Output("Memory Stats")
-	out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|Kernel Max Usage"
-	out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v",
-		humanize.Bytes(memoryStats.RSS),
-		humanize.Bytes(memoryStats.Cache),
-		humanize.Bytes(memoryStats.Swap),
-		humanize.Bytes(memoryStats.MaxUsage),
-		humanize.Bytes(memoryStats.KernelUsage),
-		humanize.Bytes(memoryStats.KernelMaxUsage),
-	)
-	c.Ui.Output(formatList(out))
 
-	c.Ui.Output("")
+		// Sort the measured stats
+		sort.Strings(memoryStats.Measured)
 
-	c.Ui.Output("CPU Stats")
+		var measuredStats []string
-	out = make([]string, 2)
+		for _, measured := range memoryStats.Measured {
-	out[0] = "Percent|Throttled Periods|Throttled Time"
+			switch measured {
-	percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
+			case "RSS":
-	out[1] = fmt.Sprintf("%v %%|%v|%v", percent,
+				measuredStats = append(measuredStats, humanize.Bytes(memoryStats.RSS))
-		cpuStats.ThrottledPeriods, cpuStats.ThrottledTime)
+			case "Cache":
-	c.Ui.Output(formatList(out))
+				measuredStats = append(measuredStats, humanize.Bytes(memoryStats.Cache))
+			case "Swap":
+				measuredStats = append(measuredStats, humanize.Bytes(memoryStats.Swap))
+			case "Max Usage":
+				measuredStats = append(measuredStats, humanize.Bytes(memoryStats.MaxUsage))
+			case "Kernel Usage":
+				measuredStats = append(measuredStats, humanize.Bytes(memoryStats.KernelUsage))
+			case "Kernel Max Usage":
+				measuredStats = append(measuredStats, humanize.Bytes(memoryStats.KernelMaxUsage))
+			}
+		}
+
+		out := make([]string, 2)
+		out[0] = strings.Join(memoryStats.Measured, "|")
+		out[1] = strings.Join(measuredStats, "|")
+		c.Ui.Output(formatList(out))
+		c.Ui.Output("")
+	}
+
+	if cpuStats != nil && len(cpuStats.Measured) > 0 {
+		c.Ui.Output("CPU Stats")
+
+		// Sort the measured stats
+		sort.Strings(cpuStats.Measured)
+
+		var measuredStats []string
+		for _, measured := range cpuStats.Measured {
+			switch measured {
+			case "Percent":
+				percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64)
+				measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
+			case "Throttled Periods":
+				measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods))
+			case "Throttled Time":
+				measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime))
+			case "User Mode":
+				percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64)
+				measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
+			case "System Mode":
+				percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64)
+				measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent))
+			}
+		}
+
+		out := make([]string, 2)
+		out[0] = strings.Join(cpuStats.Measured, "|")
+		out[1] = strings.Join(measuredStats, "|")
+		c.Ui.Output(formatList(out))
+	}
 }