From 0870aa31dcd547d81266d677e826a59a764ab3f7 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 7 Apr 2022 08:52:07 -0500 Subject: [PATCH 1/2] client: set environment variable indicating set of reserved cpu cores This PR injects the 'NOMAD_CPU_CORES' environment variable into tasks that have been allocated reserved cpu cores. The value uses normal cpuset notation, as found in cpuset.cpu cgroup interface files. Note this value is not necessiarly the same as the content of the actual cpuset.cpus interface file, which will also include shared cpu cores when using cgroups v2. This variable is a workaround for users who used to be able to read the reserved cgroup cpuset file, but lose the information about distinct reserved cores when using cgroups v2. Side discussion in: https://github.com/hashicorp/nomad/issues/12374 --- client/taskenv/env.go | 11 ++++++++++- client/taskenv/env_test.go | 8 +++++++- website/content/partials/envvars.mdx | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/client/taskenv/env.go b/client/taskenv/env.go index 1b6d4ce78..eaed60567 100644 --- a/client/taskenv/env.go +++ b/client/taskenv/env.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/nomad/helper" hargs "github.com/hashicorp/nomad/helper/args" + "github.com/hashicorp/nomad/lib/cpuset" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" "github.com/zclconf/go-cty/cty" @@ -40,6 +41,9 @@ const ( // CpuLimit is the environment variable with the tasks CPU limit in MHz. CpuLimit = "NOMAD_CPU_LIMIT" + // CpuCores is the environment variable for passing the task's reserved cpu cores + CpuCores = "NOMAD_CPU_CORES" + // AllocID is the environment variable for passing the allocation ID. AllocID = "NOMAD_ALLOC_ID" @@ -397,6 +401,7 @@ type Builder struct { // clientTaskSecretsDir is the secrets dir from the client's perspective; eg /secrets clientTaskSecretsDir string + cpuCores string cpuLimit int64 memLimit int64 memMaxLimit int64 @@ -493,6 +498,9 @@ func (b *Builder) buildEnv(allocDir, localDir, secretsDir string, if b.cpuLimit != 0 { envMap[CpuLimit] = strconv.FormatInt(b.cpuLimit, 10) } + if b.cpuCores != "" { + envMap[CpuCores] = b.cpuCores + } // Add the task metadata if b.allocId != "" { @@ -742,6 +750,7 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder { // Populate task resources if tr, ok := alloc.AllocatedResources.Tasks[b.taskName]; ok { b.cpuLimit = tr.Cpu.CpuShares + b.cpuCores = cpuset.New(tr.Cpu.ReservedCores...).String() b.memLimit = tr.Memory.MemoryMB b.memMaxLimit = tr.Memory.MemoryMaxMB @@ -788,7 +797,7 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder { } } - upstreams := []structs.ConsulUpstream{} + var upstreams []structs.ConsulUpstream for _, svc := range tg.Services { if svc.Connect.HasSidecar() && svc.Connect.SidecarService.HasUpstreams() { upstreams = append(upstreams, svc.Connect.SidecarService.Proxy.Upstreams...) diff --git a/client/taskenv/env_test.go b/client/taskenv/env_test.go index 32e623816..5d9bf75e0 100644 --- a/client/taskenv/env_test.go +++ b/client/taskenv/env_test.go @@ -159,7 +159,10 @@ func TestEnvironment_AsList(t *testing.T) { a := mock.Alloc() a.Job.ParentID = fmt.Sprintf("mock-parent-service-%s", uuid.Generate()) a.AllocatedResources.Tasks["web"] = &structs.AllocatedTaskResources{ - Cpu: structs.AllocatedCpuResources{CpuShares: 500}, + Cpu: structs.AllocatedCpuResources{ + CpuShares: 500, + ReservedCores: []uint16{0, 5, 6, 7}, + }, Memory: structs.AllocatedMemoryResources{ MemoryMB: 256, MemoryMaxMB: 512, @@ -215,6 +218,7 @@ func TestEnvironment_AsList(t *testing.T) { "NOMAD_PORT_ssh_other=1234", "NOMAD_PORT_ssh_ssh=22", "NOMAD_CPU_LIMIT=500", + "NOMAD_CPU_CORES=0,5-7", "NOMAD_DC=dc1", "NOMAD_NAMESPACE=not-default", "NOMAD_REGION=global", @@ -260,6 +264,7 @@ func TestEnvironment_AllValues(t *testing.T) { MBits: 50, DynamicPorts: []structs.Port{{Label: "http", Value: 80}}, } + a.AllocatedResources.Tasks["web"].Cpu.ReservedCores = []uint16{0, 5, 6, 7} a.AllocatedResources.Tasks["ssh"] = &structs.AllocatedTaskResources{ Networks: []*structs.NetworkResource{ { @@ -378,6 +383,7 @@ func TestEnvironment_AllValues(t *testing.T) { "NOMAD_PORT_ssh_other": "1234", "NOMAD_PORT_ssh_ssh": "22", "NOMAD_CPU_LIMIT": "500", + "NOMAD_CPU_CORES": "0,5-7", "NOMAD_DC": "dc1", "NOMAD_PARENT_CGROUP": "abc.slice", "NOMAD_NAMESPACE": "default", diff --git a/website/content/partials/envvars.mdx b/website/content/partials/envvars.mdx index 019603d74..9edca49a9 100644 --- a/website/content/partials/envvars.mdx +++ b/website/content/partials/envvars.mdx @@ -54,6 +54,15 @@ CPU limit in MHz for the task + + + NOMAD_CPU_CORES + + + The specific CPU cores reserved for the task in cpuset list notation. + Omitted if the the task does not request cpu cores. E.g. 0-2,7,12-14 + + NOMAD_ALLOC_ID From 9236fe390453e661cda74b8954c74cb1738f5f0e Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 7 Apr 2022 10:02:00 -0500 Subject: [PATCH 2/2] docs: update cl --- .changelog/12496.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/12496.txt diff --git a/.changelog/12496.txt b/.changelog/12496.txt new file mode 100644 index 000000000..ed5ca473c --- /dev/null +++ b/.changelog/12496.txt @@ -0,0 +1,3 @@ +```release-note:improvement +client: set NOMAD_CPU_CORES environment variable when reserving cpu cores +```