client/fingerprint: correctly fingerprint E/P cores of Apple Silicon chips (#16672)
* client/fingerprint: correctly fingerprint E/P cores of Apple Silicon chips This PR adds detection of asymetric core types (Power & Efficiency) (P/E) when running on M1/M2 Apple Silicon CPUs. This functionality is provided by shoenig/go-m1cpu which makes use of the Apple IOKit framework to read undocumented registers containing CPU performance data. Currently working on getting that functionality merged upstream into gopsutil, but gopsutil would still not support detecting P vs E cores like this PR does. Also refactors the CPUFingerprinter code to handle the mixed core types, now setting power vs efficiency cpu attributes. For now the scheduler is still unaware of mixed core types - on Apple platforms tasks cannot reserve cores anyway so it doesn't matter, but at least now the total CPU shares available will be correct. Future work should include adding support for detecting P/E cores on the latest and upcoming Intel chips, where computation of total cpu shares is currently incorrect. For that, we should also include updating the scheduler to be core-type aware, so that tasks of resources.cores on Linux platforms can be assigned the correct number of CPU shares for the core type(s) they have been assigned. node attributes before cpu.arch = arm64 cpu.modelname = Apple M2 Pro cpu.numcores = 12 cpu.reservablecores = 0 cpu.totalcompute = 1000 node attributes after cpu.arch = arm64 cpu.frequency.efficiency = 2424 cpu.frequency.power = 3504 cpu.modelname = Apple M2 Pro cpu.numcores.efficiency = 4 cpu.numcores.power = 8 cpu.reservablecores = 0 cpu.totalcompute = 37728 * fingerprint/cpu: follow up cr items
This commit is contained in:
parent
a18e480a57
commit
87f4b71df0
|
@ -0,0 +1,3 @@
|
||||||
|
```release-note:improvement
|
||||||
|
fingerprint/cpu: correctly fingerprint P/E cores of Apple Silicon chips
|
||||||
|
```
|
|
@ -11,7 +11,7 @@ require (
|
||||||
github.com/hashicorp/go-rootcerts v1.0.2
|
github.com/hashicorp/go-rootcerts v1.0.2
|
||||||
github.com/mitchellh/go-testing-interface v1.14.1
|
github.com/mitchellh/go-testing-interface v1.14.1
|
||||||
github.com/mitchellh/mapstructure v1.5.0
|
github.com/mitchellh/mapstructure v1.5.0
|
||||||
github.com/shoenig/test v0.6.2
|
github.com/shoenig/test v0.6.3
|
||||||
golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a
|
golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -25,8 +25,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
|
||||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/shoenig/test v0.6.2 h1:tdq+WGnznwE5xcOMXkqqXuudK75RkSGBazBGcP1lX6w=
|
github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c=
|
||||||
github.com/shoenig/test v0.6.2/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
|
|
||||||
"github.com/hashicorp/nomad/lib/cpuset"
|
"github.com/hashicorp/nomad/lib/cpuset"
|
||||||
|
|
||||||
log "github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
"github.com/hashicorp/nomad/helper/stats"
|
"github.com/hashicorp/nomad/helper/stats"
|
||||||
"github.com/hashicorp/nomad/nomad/structs"
|
"github.com/hashicorp/nomad/nomad/structs"
|
||||||
)
|
)
|
||||||
|
@ -22,85 +22,128 @@ const (
|
||||||
// CPUFingerprint is used to fingerprint the CPU
|
// CPUFingerprint is used to fingerprint the CPU
|
||||||
type CPUFingerprint struct {
|
type CPUFingerprint struct {
|
||||||
StaticFingerprinter
|
StaticFingerprinter
|
||||||
logger log.Logger
|
logger hclog.Logger
|
||||||
|
|
||||||
|
// accumulates result in these resource structs
|
||||||
|
resources *structs.Resources
|
||||||
|
nodeResources *structs.NodeResources
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCPUFingerprint is used to create a CPU fingerprint
|
// NewCPUFingerprint is used to create a CPU fingerprint
|
||||||
func NewCPUFingerprint(logger log.Logger) Fingerprint {
|
func NewCPUFingerprint(logger hclog.Logger) Fingerprint {
|
||||||
f := &CPUFingerprint{logger: logger.Named("cpu")}
|
return &CPUFingerprint{
|
||||||
return f
|
logger: logger.Named("cpu"),
|
||||||
|
resources: new(structs.Resources), // COMPAT (to be removed after 0.10)
|
||||||
|
nodeResources: new(structs.NodeResources),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *CPUFingerprint) Fingerprint(req *FingerprintRequest, resp *FingerprintResponse) error {
|
func (f *CPUFingerprint) Fingerprint(request *FingerprintRequest, response *FingerprintResponse) error {
|
||||||
cfg := req.Config
|
f.initialize()
|
||||||
setResourcesCPU := func(totalCompute int, totalCores uint16, reservableCores []uint16) {
|
|
||||||
// COMPAT(0.10): Remove in 0.10
|
|
||||||
resp.Resources = &structs.Resources{
|
|
||||||
CPU: totalCompute,
|
|
||||||
}
|
|
||||||
|
|
||||||
resp.NodeResources = &structs.NodeResources{
|
f.setModelName(response)
|
||||||
Cpu: structs.NodeCpuResources{
|
|
||||||
CpuShares: int64(totalCompute),
|
|
||||||
TotalCpuCores: totalCores,
|
|
||||||
ReservableCpuCores: reservableCores,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := stats.Init(); err != nil {
|
f.setFrequency(response)
|
||||||
f.logger.Warn("failed initializing stats collector", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if modelName := stats.CPUModelName(); modelName != "" {
|
f.setCoreCount(response)
|
||||||
resp.AddAttribute("cpu.modelname", modelName)
|
|
||||||
}
|
|
||||||
|
|
||||||
if mhz := stats.CPUMHzPerCore(); mhz > 0 {
|
f.setReservableCores(request, response)
|
||||||
resp.AddAttribute("cpu.frequency", fmt.Sprintf("%.0f", mhz))
|
|
||||||
f.logger.Debug("detected cpu frequency", "MHz", log.Fmt("%.0f", mhz))
|
|
||||||
}
|
|
||||||
|
|
||||||
var numCores int
|
f.setTotalCompute(request, response)
|
||||||
if numCores = stats.CPUNumCores(); numCores > 0 {
|
|
||||||
resp.AddAttribute("cpu.numcores", strconv.Itoa(numCores))
|
|
||||||
f.logger.Debug("detected core count", "cores", numCores)
|
|
||||||
}
|
|
||||||
|
|
||||||
var reservableCores []uint16
|
f.setResponseResources(response)
|
||||||
if req.Config.ReservableCores != nil {
|
|
||||||
reservableCores = req.Config.ReservableCores
|
|
||||||
f.logger.Debug("reservable cores set by config", "cpuset", reservableCores)
|
|
||||||
} else {
|
|
||||||
if cores, err := f.deriveReservableCores(req); err != nil {
|
|
||||||
f.logger.Warn("failed to detect set of reservable cores", "error", err)
|
|
||||||
} else {
|
|
||||||
if req.Node.ReservedResources != nil {
|
|
||||||
reservableCores = cpuset.New(cores...).Difference(cpuset.New(req.Node.ReservedResources.Cpu.ReservedCpuCores...)).ToSlice()
|
|
||||||
}
|
|
||||||
f.logger.Debug("detected reservable cores", "cpuset", reservableCores)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp.AddAttribute("cpu.reservablecores", strconv.Itoa(len(reservableCores)))
|
|
||||||
|
|
||||||
tt := int(stats.TotalTicksAvailable())
|
response.Detected = true
|
||||||
if cfg.CpuCompute > 0 {
|
|
||||||
f.logger.Debug("using user specified cpu compute", "cpu_compute", cfg.CpuCompute)
|
|
||||||
tt = cfg.CpuCompute
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we cannot detect the cpu total compute, fallback to a very low default
|
|
||||||
// value and log a message about configuring cpu_total_compute. This happens
|
|
||||||
// on Graviton instances where CPU information is unavailable. In that case,
|
|
||||||
// the env_aws fingerprinter updates the value with correct information.
|
|
||||||
if tt == 0 {
|
|
||||||
f.logger.Info("fallback to default cpu total compute, set client config option cpu_total_compute to override")
|
|
||||||
tt = defaultCPUTicks
|
|
||||||
}
|
|
||||||
|
|
||||||
resp.AddAttribute("cpu.totalcompute", fmt.Sprintf("%d", tt))
|
|
||||||
setResourcesCPU(tt, uint16(numCores), reservableCores)
|
|
||||||
resp.Detected = true
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) initialize() {
|
||||||
|
if err := stats.Init(); err != nil {
|
||||||
|
f.logger.Warn("failed initializing stats collector", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) setModelName(response *FingerprintResponse) {
|
||||||
|
if modelName := stats.CPUModelName(); modelName != "" {
|
||||||
|
response.AddAttribute("cpu.modelname", modelName)
|
||||||
|
f.logger.Debug("detected CPU model", "name", modelName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*CPUFingerprint) frequency(mhz uint64) string {
|
||||||
|
return fmt.Sprintf("%.0f", float64(mhz))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) setFrequency(response *FingerprintResponse) {
|
||||||
|
power, efficiency := stats.CPUMHzPerCore()
|
||||||
|
switch {
|
||||||
|
case efficiency > 0:
|
||||||
|
response.AddAttribute("cpu.frequency.efficiency", f.frequency(efficiency))
|
||||||
|
response.AddAttribute("cpu.frequency.power", f.frequency(power))
|
||||||
|
f.logger.Debug("detected CPU efficiency core speed", "mhz", efficiency)
|
||||||
|
f.logger.Debug("detected CPU power core speed", "mhz", power)
|
||||||
|
case power > 0:
|
||||||
|
response.AddAttribute("cpu.frequency", f.frequency(power))
|
||||||
|
f.logger.Debug("detected CPU frequency", "mhz", power)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*CPUFingerprint) cores(count int) string {
|
||||||
|
return strconv.Itoa(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) setCoreCount(response *FingerprintResponse) {
|
||||||
|
power, efficiency := stats.CPUNumCores()
|
||||||
|
switch {
|
||||||
|
case efficiency > 0:
|
||||||
|
response.AddAttribute("cpu.numcores.efficiency", f.cores(efficiency))
|
||||||
|
response.AddAttribute("cpu.numcores.power", f.cores(power))
|
||||||
|
f.logger.Debug("detected CPU efficiency core count", "cores", efficiency)
|
||||||
|
f.logger.Debug("detected CPU power core count", "cores", power)
|
||||||
|
case power > 0:
|
||||||
|
response.AddAttribute("cpu.numcores", f.cores(power))
|
||||||
|
f.logger.Debug("detected CPU core count", power)
|
||||||
|
}
|
||||||
|
f.nodeResources.Cpu.TotalCpuCores = uint16(power + efficiency)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) setReservableCores(request *FingerprintRequest, response *FingerprintResponse) {
|
||||||
|
reservable := request.Config.ReservableCores
|
||||||
|
if len(reservable) > 0 {
|
||||||
|
f.logger.Debug("reservable cores set by config", "cpuset", reservable)
|
||||||
|
} else {
|
||||||
|
cgroupParent := request.Config.CgroupParent
|
||||||
|
if reservable = f.deriveReservableCores(cgroupParent); reservable != nil {
|
||||||
|
if request.Node.ReservedResources != nil {
|
||||||
|
forNode := request.Node.ReservedResources.Cpu.ReservedCpuCores
|
||||||
|
reservable = cpuset.New(reservable...).Difference(cpuset.New(forNode...)).ToSlice()
|
||||||
|
f.logger.Debug("client configuration reserves these cores for node", "cores", forNode)
|
||||||
|
}
|
||||||
|
f.logger.Debug("set of reservable cores available for tasks", "cores", reservable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response.AddAttribute("cpu.reservablecores", strconv.Itoa(len(reservable)))
|
||||||
|
f.nodeResources.Cpu.ReservableCpuCores = reservable
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) setTotalCompute(request *FingerprintRequest, response *FingerprintResponse) {
|
||||||
|
var ticks uint64
|
||||||
|
switch {
|
||||||
|
case request.Config.CpuCompute > 0:
|
||||||
|
ticks = uint64(request.Config.CpuCompute)
|
||||||
|
case stats.TotalTicksAvailable() > 0:
|
||||||
|
ticks = stats.TotalTicksAvailable()
|
||||||
|
default:
|
||||||
|
ticks = defaultCPUTicks
|
||||||
|
}
|
||||||
|
response.AddAttribute("cpu.totalcompute", fmt.Sprintf("%d", ticks))
|
||||||
|
f.resources.CPU = int(ticks)
|
||||||
|
f.nodeResources.Cpu.CpuShares = int64(ticks)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *CPUFingerprint) setResponseResources(response *FingerprintResponse) {
|
||||||
|
response.Resources = f.resources
|
||||||
|
response.NodeResources = f.nodeResources
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
//go:build darwin && arm64 && cgo
|
||||||
|
|
||||||
|
package fingerprint
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/hashicorp/nomad/ci"
|
||||||
|
"github.com/hashicorp/nomad/client/config"
|
||||||
|
"github.com/hashicorp/nomad/helper/testlog"
|
||||||
|
"github.com/hashicorp/nomad/nomad/structs"
|
||||||
|
"github.com/shoenig/test/must"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCPUFingerprint_AppleSilicon(t *testing.T) {
|
||||||
|
ci.Parallel(t)
|
||||||
|
|
||||||
|
f := NewCPUFingerprint(testlog.HCLogger(t))
|
||||||
|
node := &structs.Node{Attributes: make(map[string]string)}
|
||||||
|
|
||||||
|
request := &FingerprintRequest{Config: new(config.Config), Node: node}
|
||||||
|
var response FingerprintResponse
|
||||||
|
|
||||||
|
err := f.Fingerprint(request, &response)
|
||||||
|
must.NoError(t, err)
|
||||||
|
|
||||||
|
must.True(t, response.Detected)
|
||||||
|
|
||||||
|
attributes := response.Attributes
|
||||||
|
must.NotNil(t, attributes)
|
||||||
|
must.MapContainsKey(t, attributes, "cpu.modelname")
|
||||||
|
must.MapContainsKey(t, attributes, "cpu.numcores.power")
|
||||||
|
must.MapContainsKey(t, attributes, "cpu.numcores.efficiency")
|
||||||
|
must.MapContainsKey(t, attributes, "cpu.frequency.power")
|
||||||
|
must.MapContainsKey(t, attributes, "cpu.frequency.efficiency")
|
||||||
|
must.MapContainsKey(t, attributes, "cpu.totalcompute")
|
||||||
|
must.Positive(t, response.Resources.CPU)
|
||||||
|
must.Positive(t, response.NodeResources.Cpu.CpuShares)
|
||||||
|
must.Positive(t, response.NodeResources.Cpu.SharesPerCore())
|
||||||
|
must.SliceEmpty(t, response.NodeResources.Cpu.ReservableCpuCores)
|
||||||
|
|
||||||
|
// not included for mixed core types (that we can detect)
|
||||||
|
must.MapNotContainsKey(t, attributes, "cpu.numcores")
|
||||||
|
must.MapNotContainsKey(t, attributes, "cpu.frequency")
|
||||||
|
}
|
|
@ -1,8 +1,7 @@
|
||||||
//go:build !linux
|
//go:build !linux
|
||||||
// +build !linux
|
|
||||||
|
|
||||||
package fingerprint
|
package fingerprint
|
||||||
|
|
||||||
func (f *CPUFingerprint) deriveReservableCores(req *FingerprintRequest) ([]uint16, error) {
|
func (_ *CPUFingerprint) deriveReservableCores(string) []uint16 {
|
||||||
return nil, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
//go:build !darwin || !arm64 || !cgo
|
||||||
|
|
||||||
package fingerprint
|
package fingerprint
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
@ -8,54 +10,36 @@ import (
|
||||||
"github.com/hashicorp/nomad/client/config"
|
"github.com/hashicorp/nomad/client/config"
|
||||||
"github.com/hashicorp/nomad/helper/testlog"
|
"github.com/hashicorp/nomad/helper/testlog"
|
||||||
"github.com/hashicorp/nomad/nomad/structs"
|
"github.com/hashicorp/nomad/nomad/structs"
|
||||||
|
"github.com/shoenig/test/must"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCPUFingerprint(t *testing.T) {
|
func TestCPUFingerprint_Classic(t *testing.T) {
|
||||||
ci.Parallel(t)
|
ci.Parallel(t)
|
||||||
|
|
||||||
f := NewCPUFingerprint(testlog.HCLogger(t))
|
f := NewCPUFingerprint(testlog.HCLogger(t))
|
||||||
node := &structs.Node{
|
node := &structs.Node{Attributes: make(map[string]string)}
|
||||||
Attributes: make(map[string]string),
|
|
||||||
}
|
|
||||||
|
|
||||||
request := &FingerprintRequest{Config: &config.Config{}, Node: node}
|
request := &FingerprintRequest{Config: &config.Config{}, Node: node}
|
||||||
var response FingerprintResponse
|
var response FingerprintResponse
|
||||||
|
|
||||||
err := f.Fingerprint(request, &response)
|
err := f.Fingerprint(request, &response)
|
||||||
if err != nil {
|
must.NoError(t, err)
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !response.Detected {
|
must.True(t, response.Detected)
|
||||||
t.Fatalf("expected response to be applicable")
|
|
||||||
}
|
|
||||||
|
|
||||||
// CPU info
|
|
||||||
attributes := response.Attributes
|
attributes := response.Attributes
|
||||||
if attributes == nil {
|
must.NotNil(t, attributes)
|
||||||
t.Fatalf("expected attributes to be initialized")
|
must.MapContainsKey(t, attributes, "cpu.numcores")
|
||||||
}
|
must.MapContainsKey(t, attributes, "cpu.modelname")
|
||||||
if attributes["cpu.numcores"] == "" {
|
must.MapContainsKey(t, attributes, "cpu.frequency")
|
||||||
t.Fatalf("Missing Num Cores")
|
must.MapContainsKey(t, attributes, "cpu.totalcompute")
|
||||||
}
|
must.Positive(t, response.Resources.CPU)
|
||||||
if attributes["cpu.modelname"] == "" {
|
must.Positive(t, response.NodeResources.Cpu.CpuShares)
|
||||||
t.Fatalf("Missing Model Name")
|
must.Positive(t, response.NodeResources.Cpu.SharesPerCore())
|
||||||
}
|
must.SliceNotEmpty(t, response.NodeResources.Cpu.ReservableCpuCores)
|
||||||
|
|
||||||
if attributes["cpu.frequency"] == "" {
|
// asymetric core detection currently only works with apple silicon
|
||||||
t.Fatalf("Missing CPU Frequency")
|
must.MapNotContainsKey(t, attributes, "cpu.numcores.power")
|
||||||
}
|
must.MapNotContainsKey(t, attributes, "cpu.numcores.efficiency")
|
||||||
if attributes["cpu.totalcompute"] == "" {
|
|
||||||
t.Fatalf("Missing CPU Total Compute")
|
|
||||||
}
|
|
||||||
|
|
||||||
// COMPAT(0.10): Remove in 0.10
|
|
||||||
if response.Resources == nil || response.Resources.CPU == 0 {
|
|
||||||
t.Fatalf("Expected to find CPU Resources")
|
|
||||||
}
|
|
||||||
|
|
||||||
if response.NodeResources == nil || response.NodeResources.Cpu.CpuShares == 0 {
|
|
||||||
t.Fatalf("Expected to find CPU Resources")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCPUFingerprint_OverrideCompute asserts that setting cpu_total_compute in
|
// TestCPUFingerprint_OverrideCompute asserts that setting cpu_total_compute in
|
|
@ -4,9 +4,14 @@ import (
|
||||||
"github.com/hashicorp/nomad/client/lib/cgutil"
|
"github.com/hashicorp/nomad/client/lib/cgutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (f *CPUFingerprint) deriveReservableCores(req *FingerprintRequest) ([]uint16, error) {
|
func (f *CPUFingerprint) deriveReservableCores(cgroupParent string) []uint16 {
|
||||||
// The cpuset cgroup manager is initialized (on linux), but not accessible
|
// The cpuset cgroup manager is initialized (on linux), but not accessible
|
||||||
// from the finger-printer. So we reach in and grab the information manually.
|
// from the finger-printer. So we reach in and grab the information manually.
|
||||||
// We may assume the hierarchy is already setup.
|
// We may assume the hierarchy is already setup.
|
||||||
return cgutil.GetCPUsFromCgroup(req.Config.CgroupParent)
|
cpuset, err := cgutil.GetCPUsFromCgroup(cgroupParent)
|
||||||
|
if err != nil {
|
||||||
|
f.logger.Warn("failed to detect set of reservable cores", "error", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return cpuset
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,7 +48,7 @@ func (c *CpuStats) Percent(cpuTime float64) float64 {
|
||||||
// TicksConsumed calculates the total ticks consumes by the process across all
|
// TicksConsumed calculates the total ticks consumes by the process across all
|
||||||
// cpu cores
|
// cpu cores
|
||||||
func (c *CpuStats) TicksConsumed(percent float64) float64 {
|
func (c *CpuStats) TicksConsumed(percent float64) float64 {
|
||||||
return (percent / 100) * shelpers.TotalTicksAvailable() / float64(c.totalCpus)
|
return (percent / 100) * float64(shelpers.TotalTicksAvailable()) / float64(c.totalCpus)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *CpuStats) calculatePercent(t1, t2 float64, timeDelta int64) float64 {
|
func (c *CpuStats) calculatePercent(t1, t2 float64, timeDelta int64) float64 {
|
||||||
|
@ -83,7 +83,7 @@ func (h *HostStatsCollector) collectCPUStats() (cpus []*CPUStats, totalTicks flo
|
||||||
Idle: idle,
|
Idle: idle,
|
||||||
Total: total,
|
Total: total,
|
||||||
}
|
}
|
||||||
ticksConsumed += (total / 100.0) * (shelpers.TotalTicksAvailable() / float64(len(cpuStats)))
|
ticksConsumed += (total / 100.0) * (float64(shelpers.TotalTicksAvailable()) / float64(len(cpuStats)))
|
||||||
}
|
}
|
||||||
|
|
||||||
return cs, ticksConsumed, nil
|
return cs, ticksConsumed, nil
|
||||||
|
|
|
@ -53,7 +53,7 @@ func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage
|
||||||
cs.UserMode = CalculateCPUPercent(
|
cs.UserMode = CalculateCPUPercent(
|
||||||
s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
|
s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
|
||||||
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU())
|
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU())
|
||||||
cs.TotalTicks = (cs.Percent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU())
|
cs.TotalTicks = (cs.Percent / 100) * float64(stats.TotalTicksAvailable()) / float64(runtime.NumCPU())
|
||||||
|
|
||||||
return &cstructs.TaskResourceUsage{
|
return &cstructs.TaskResourceUsage{
|
||||||
ResourceUsage: &cstructs.ResourceUsage{
|
ResourceUsage: &cstructs.ResourceUsage{
|
||||||
|
|
|
@ -42,7 +42,7 @@ func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage
|
||||||
ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
|
ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
|
||||||
ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime,
|
ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime,
|
||||||
Percent: cpuPercent,
|
Percent: cpuPercent,
|
||||||
TotalTicks: (cpuPercent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU()),
|
TotalTicks: (cpuPercent / 100) * float64(stats.TotalTicksAvailable()) / float64(runtime.NumCPU()),
|
||||||
Measured: DockerMeasuredCPUStats,
|
Measured: DockerMeasuredCPUStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
3
go.mod
3
go.mod
|
@ -108,7 +108,8 @@ require (
|
||||||
github.com/ryanuber/go-glob v1.0.0
|
github.com/ryanuber/go-glob v1.0.0
|
||||||
github.com/shirou/gopsutil/v3 v3.23.1
|
github.com/shirou/gopsutil/v3 v3.23.1
|
||||||
github.com/shoenig/go-landlock v0.1.5
|
github.com/shoenig/go-landlock v0.1.5
|
||||||
github.com/shoenig/test v0.6.2
|
github.com/shoenig/go-m1cpu v0.1.4
|
||||||
|
github.com/shoenig/test v0.6.3
|
||||||
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c
|
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c
|
||||||
github.com/stretchr/testify v1.8.1
|
github.com/stretchr/testify v1.8.1
|
||||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
|
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
|
||||||
|
|
6
go.sum
6
go.sum
|
@ -1310,8 +1310,10 @@ github.com/shirou/gopsutil/v3 v3.23.1 h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz
|
||||||
github.com/shirou/gopsutil/v3 v3.23.1/go.mod h1:NN6mnm5/0k8jw4cBfCnJtr5L7ErOTg18tMNpgFkn0hA=
|
github.com/shirou/gopsutil/v3 v3.23.1/go.mod h1:NN6mnm5/0k8jw4cBfCnJtr5L7ErOTg18tMNpgFkn0hA=
|
||||||
github.com/shoenig/go-landlock v0.1.5 h1:0a/YjKzqdbll7f/iztN/6pKRSHJEmm8olFWD8xSM86A=
|
github.com/shoenig/go-landlock v0.1.5 h1:0a/YjKzqdbll7f/iztN/6pKRSHJEmm8olFWD8xSM86A=
|
||||||
github.com/shoenig/go-landlock v0.1.5/go.mod h1:CxztF/8LRAUKUMguGxGTQBJIBiiawxx/BKYVGrHg1/0=
|
github.com/shoenig/go-landlock v0.1.5/go.mod h1:CxztF/8LRAUKUMguGxGTQBJIBiiawxx/BKYVGrHg1/0=
|
||||||
github.com/shoenig/test v0.6.2 h1:tdq+WGnznwE5xcOMXkqqXuudK75RkSGBazBGcP1lX6w=
|
github.com/shoenig/go-m1cpu v0.1.4 h1:SZPIgRM2sEF9NJy50mRHu9PKGwxyyTTJIWvCtgVbozs=
|
||||||
github.com/shoenig/test v0.6.2/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ=
|
||||||
|
github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c=
|
||||||
|
github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||||
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
|
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
|
||||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||||
|
|
|
@ -2,13 +2,13 @@ package stats
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
multierror "github.com/hashicorp/go-multierror"
|
|
||||||
"github.com/shirou/gopsutil/v3/cpu"
|
"github.com/shirou/gopsutil/v3/cpu"
|
||||||
|
"github.com/shoenig/go-m1cpu"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -19,63 +19,97 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
cpuMhzPerCore float64
|
cpuPowerCoreCount int
|
||||||
cpuModelName string
|
cpuPowerCoreMHz uint64
|
||||||
cpuNumCores int
|
cpuEfficiencyCoreCount int
|
||||||
cpuTotalTicks float64
|
cpuEfficiencyCoreMHz uint64
|
||||||
|
cpuTotalTicks uint64
|
||||||
|
cpuModelName string
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
initErr error
|
initErr error
|
||||||
onceLer sync.Once
|
onceLer sync.Once
|
||||||
)
|
)
|
||||||
|
|
||||||
func Init() error {
|
func Init() error {
|
||||||
onceLer.Do(func() {
|
onceLer.Do(func() {
|
||||||
var merrs *multierror.Error
|
switch {
|
||||||
var err error
|
case m1cpu.IsAppleSilicon():
|
||||||
if cpuNumCores, err = cpu.Counts(true); err != nil {
|
cpuModelName = m1cpu.ModelName()
|
||||||
merrs = multierror.Append(merrs, fmt.Errorf("Unable to determine the number of CPU cores available: %v", err))
|
cpuPowerCoreCount = m1cpu.PCoreCount()
|
||||||
|
cpuPowerCoreMHz = m1cpu.PCoreHz() / 1_000_000
|
||||||
|
cpuEfficiencyCoreCount = m1cpu.ECoreCount()
|
||||||
|
cpuEfficiencyCoreMHz = m1cpu.ECoreHz() / 1_000_000
|
||||||
|
bigTicks := uint64(cpuPowerCoreCount) * cpuPowerCoreMHz
|
||||||
|
littleTicks := uint64(cpuEfficiencyCoreCount) * cpuEfficiencyCoreMHz
|
||||||
|
cpuTotalTicks = bigTicks + littleTicks
|
||||||
|
default:
|
||||||
|
// for now, all other cpu types assume only power cores
|
||||||
|
// todo: this is already not true for Intel 13th generation
|
||||||
|
|
||||||
|
var err error
|
||||||
|
if cpuPowerCoreCount, err = cpu.Counts(true); err != nil {
|
||||||
|
initErr = errors.Join(initErr, fmt.Errorf("failed to detect number of CPU cores: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), cpuInfoTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var cpuInfoStats []cpu.InfoStat
|
||||||
|
if cpuInfoStats, err = cpu.InfoWithContext(ctx); err != nil {
|
||||||
|
initErr = errors.Join(initErr, fmt.Errorf("Unable to obtain CPU information: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, infoStat := range cpuInfoStats {
|
||||||
|
cpuModelName = infoStat.ModelName
|
||||||
|
cpuPowerCoreMHz = uint64(infoStat.Mhz)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// compute ticks using only power core, until we add support for
|
||||||
|
// detecting little cores on non-apple platforms
|
||||||
|
cpuTotalTicks = uint64(cpuPowerCoreCount) * cpuPowerCoreMHz
|
||||||
|
|
||||||
|
initErr = err
|
||||||
}
|
}
|
||||||
|
|
||||||
var cpuInfo []cpu.InfoStat
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), cpuInfoTimeout)
|
|
||||||
defer cancel()
|
|
||||||
if cpuInfo, err = cpu.InfoWithContext(ctx); err != nil {
|
|
||||||
merrs = multierror.Append(merrs, fmt.Errorf("Unable to obtain CPU information: %v", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, cpu := range cpuInfo {
|
|
||||||
cpuModelName = cpu.ModelName
|
|
||||||
cpuMhzPerCore = cpu.Mhz
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
// Floor all of the values such that small difference don't cause the
|
|
||||||
// node to fall into a unique computed node class
|
|
||||||
cpuMhzPerCore = math.Floor(cpuMhzPerCore)
|
|
||||||
cpuTotalTicks = math.Floor(float64(cpuNumCores) * cpuMhzPerCore)
|
|
||||||
|
|
||||||
// Set any errors that occurred
|
|
||||||
initErr = merrs.ErrorOrNil()
|
|
||||||
})
|
})
|
||||||
return initErr
|
return initErr
|
||||||
}
|
}
|
||||||
|
|
||||||
// CPUNumCores returns the number of CPU cores available
|
// CPUNumCores returns the number of CPU cores available.
|
||||||
func CPUNumCores() int {
|
//
|
||||||
return cpuNumCores
|
// This is represented with two values - (Power (P), Efficiency (E)) so we can
|
||||||
|
// correctly compute total compute for processors with asymetric cores such as
|
||||||
|
// Apple Silicon.
|
||||||
|
//
|
||||||
|
// For platforms with symetric cores (or where we do not correcly detect asymetric
|
||||||
|
// cores), all cores are presented as P cores.
|
||||||
|
func CPUNumCores() (int, int) {
|
||||||
|
return cpuPowerCoreCount, cpuEfficiencyCoreCount
|
||||||
}
|
}
|
||||||
|
|
||||||
// CPUMHzPerCore returns the MHz per CPU core
|
// CPUMHzPerCore returns the MHz per CPU (P, E) core type.
|
||||||
func CPUMHzPerCore() float64 {
|
//
|
||||||
return cpuMhzPerCore
|
// As with CPUNumCores, asymetric core detection currently only works with
|
||||||
|
// Apple Silicon CPUs.
|
||||||
|
func CPUMHzPerCore() (uint64, uint64) {
|
||||||
|
return cpuPowerCoreMHz, cpuEfficiencyCoreMHz
|
||||||
}
|
}
|
||||||
|
|
||||||
// CPUModelName returns the model name of the CPU
|
// CPUModelName returns the model name of the CPU.
|
||||||
func CPUModelName() string {
|
func CPUModelName() string {
|
||||||
return cpuModelName
|
return cpuModelName
|
||||||
}
|
}
|
||||||
|
|
||||||
// TotalTicksAvailable calculates the total Mhz available across all cores
|
// TotalTicksAvailable calculates the total MHz available across all cores.
|
||||||
func TotalTicksAvailable() float64 {
|
//
|
||||||
|
// Where asymetric cores are correctly detected, the total ticks is the sum of
|
||||||
|
// the performance across both core types.
|
||||||
|
//
|
||||||
|
// Where asymetric cores are not correctly detected (such as Intel 13th gen),
|
||||||
|
// the total ticks available is over-estimated, as we assume all cores are P
|
||||||
|
// cores.
|
||||||
|
func TotalTicksAvailable() uint64 {
|
||||||
return cpuTotalTicks
|
return cpuTotalTicks
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
//go:build darwin && arm64 && cgo
|
||||||
|
|
||||||
|
package stats
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/shoenig/test/must"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCPU_Init(t *testing.T) {
|
||||||
|
must.NoError(t, Init())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPU_CPUNumCores(t *testing.T) {
|
||||||
|
big, little := CPUNumCores()
|
||||||
|
must.Between(t, 4, big, 32)
|
||||||
|
must.Between(t, 2, little, 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPU_CPUMHzPerCore(t *testing.T) {
|
||||||
|
big, little := CPUMHzPerCore()
|
||||||
|
must.Between(t, 3_000, big, 6_000)
|
||||||
|
must.Between(t, 2_000, little, 4_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPU_CPUModelName(t *testing.T) {
|
||||||
|
name := CPUModelName()
|
||||||
|
must.NotEq(t, "", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPU_CPUTotalTicksAvailable(t *testing.T) {
|
||||||
|
ticks := TotalTicksAvailable()
|
||||||
|
must.Positive(t, ticks)
|
||||||
|
}
|
Loading…
Reference in New Issue