From 04c14b3cb2f467ed5c5397dbf7f1ed5ec72de40d Mon Sep 17 00:00:00 2001 From: Jainin Shah Date: Thu, 1 Feb 2018 10:09:12 -0800 Subject: [PATCH 1/7] add a flag for cpu_hard_limit --- client/driver/docker.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/client/driver/docker.go b/client/driver/docker.go index 5f2dedc29..32e179073 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -120,6 +120,11 @@ const ( // https://docs.docker.com/engine/reference/run/#block-io-bandwidth-blkio-constraint dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," + "SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE" + + // This is cpu.cfs_period_us: the length of a period. The default values is 100 microsecnds represented in nano Seconds, below is the documnentation + // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt + // https://docs.docker.com/engine/admin/resource_constraints/#cpu + defaultCFSPeriod = 100000 ) type DockerDriver struct { @@ -217,6 +222,7 @@ type DockerDriverConfig struct { CapAdd []string `mapstructure:"cap_add"` // Flags to pass directly to cap-add CapDrop []string `mapstructure:"cap_drop"` // Flags to pass directly to cap-drop ReadonlyRootfs bool `mapstructure:"readonly_rootfs"` // Mount the container’s root filesystem as read only + CPUHardLimit bool `mapstructure:"cpu_hard_limit"` // Flag to pass whether to enforce CPU hard limit. } func sliceMergeUlimit(ulimitsRaw map[string]string) ([]docker.ULimit, error) { @@ -674,6 +680,9 @@ func (d *DockerDriver) Validate(config map[string]interface{}) error { "readonly_rootfs": { Type: fields.TypeBool, }, + "cpu_hard_limit": { + Type: fields.TypeBool, + }, }, } @@ -1119,6 +1128,12 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas VolumeDriver: driverConfig.VolumeDriver, } + // Calculate CPU Quota + if driverConfig.CPUHardLimit { + percentTicks := float64(task.Resources.CPU) / shelpers.TotalTicksAvailable() + hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod) + } + // Windows does not support MemorySwap/MemorySwappiness #2193 if runtime.GOOS == "windows" { hostConfig.MemorySwap = 0 @@ -1137,6 +1152,9 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name) + if driverConfig.CPUHardLimit { + d.logger.Printf("[DEBUG] driver.docker: using %d cpu quota (cpu-period: %d) for %s", hostConfig.CPUQuota, defaultCFSPeriod , task.Name) + } d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name) // set privileged mode From 0d99f256de266bd33cccc40becd70981ad73da34 Mon Sep 17 00:00:00 2001 From: Jainin Shah Date: Thu, 1 Feb 2018 12:07:05 -0800 Subject: [PATCH 2/7] changes after running go fmt --- client/driver/docker.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 32e179073..d15cb073a 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -121,7 +121,7 @@ const ( dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," + "SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE" - // This is cpu.cfs_period_us: the length of a period. The default values is 100 microsecnds represented in nano Seconds, below is the documnentation + // This is cpu.cfs_period_us: the length of a period. The default values is 100 microsecnds represented in nano Seconds, below is the documnentation // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt // https://docs.docker.com/engine/admin/resource_constraints/#cpu defaultCFSPeriod = 100000 @@ -222,7 +222,7 @@ type DockerDriverConfig struct { CapAdd []string `mapstructure:"cap_add"` // Flags to pass directly to cap-add CapDrop []string `mapstructure:"cap_drop"` // Flags to pass directly to cap-drop ReadonlyRootfs bool `mapstructure:"readonly_rootfs"` // Mount the container’s root filesystem as read only - CPUHardLimit bool `mapstructure:"cpu_hard_limit"` // Flag to pass whether to enforce CPU hard limit. + CPUHardLimit bool `mapstructure:"cpu_hard_limit"` // Enforce CPU hard limit. } func sliceMergeUlimit(ulimitsRaw map[string]string) ([]docker.ULimit, error) { @@ -1128,11 +1128,11 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas VolumeDriver: driverConfig.VolumeDriver, } - // Calculate CPU Quota - if driverConfig.CPUHardLimit { - percentTicks := float64(task.Resources.CPU) / shelpers.TotalTicksAvailable() - hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod) - } + // Calculate CPU Quota + if driverConfig.CPUHardLimit { + percentTicks := float64(task.Resources.CPU) / shelpers.TotalTicksAvailable() + hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod) + } // Windows does not support MemorySwap/MemorySwappiness #2193 if runtime.GOOS == "windows" { @@ -1153,7 +1153,7 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name) if driverConfig.CPUHardLimit { - d.logger.Printf("[DEBUG] driver.docker: using %d cpu quota (cpu-period: %d) for %s", hostConfig.CPUQuota, defaultCFSPeriod , task.Name) + d.logger.Printf("[DEBUG] driver.docker: using %d cpu quota (cpu-period: %d) for %s", hostConfig.CPUQuota, defaultCFSPeriod, task.Name) } d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name) From 94d0ce6006a2597e883c38facc18cda54147b0fa Mon Sep 17 00:00:00 2001 From: Jainin Shah Date: Thu, 1 Feb 2018 14:16:38 -0800 Subject: [PATCH 3/7] wrapping the line to less than 80 characters --- client/driver/docker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index d15cb073a..cf61ffc5c 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -121,7 +121,9 @@ const ( dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," + "SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE" - // This is cpu.cfs_period_us: the length of a period. The default values is 100 microsecnds represented in nano Seconds, below is the documnentation + // This is cpu.cfs_period_us: the length of a period. + // The default values is 100 microsecnds represented in nano Seconds. + // Below is the documnentation: // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt // https://docs.docker.com/engine/admin/resource_constraints/#cpu defaultCFSPeriod = 100000 From d3087d60698b2d78fb8fb9b17f949368fdffe51e Mon Sep 17 00:00:00 2001 From: Jainin Shah Date: Tue, 6 Feb 2018 14:52:15 -0800 Subject: [PATCH 4/7] using d.node.Resources.CPU as suggested --- client/driver/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index cf61ffc5c..8a4978d55 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -1132,7 +1132,7 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas // Calculate CPU Quota if driverConfig.CPUHardLimit { - percentTicks := float64(task.Resources.CPU) / shelpers.TotalTicksAvailable() + percentTicks := float64(task.Resources.CPU) / float64(d.node.Resources.CPU) hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod) } From 8149587abe830590dfddbf6fdca12288b67b4dd3 Mon Sep 17 00:00:00 2001 From: Jainin Shah Date: Tue, 6 Feb 2018 19:11:39 -0800 Subject: [PATCH 5/7] clearing the confusion between microsecond,nanosecond and millisecond --- client/driver/docker.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 8a4978d55..13b16aff7 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -122,11 +122,11 @@ const ( "SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE" // This is cpu.cfs_period_us: the length of a period. - // The default values is 100 microsecnds represented in nano Seconds. + // The default values is 100 milliseconds (ms) represented in microseconds (us). // Below is the documnentation: // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt - // https://docs.docker.com/engine/admin/resource_constraints/#cpu - defaultCFSPeriod = 100000 + // https://docs.docker.com/engine/api/v1.35/# + defaultCFSPeriod_us = 100000 ) type DockerDriver struct { @@ -1133,7 +1133,7 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas // Calculate CPU Quota if driverConfig.CPUHardLimit { percentTicks := float64(task.Resources.CPU) / float64(d.node.Resources.CPU) - hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod) + hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod_us) } // Windows does not support MemorySwap/MemorySwappiness #2193 @@ -1155,7 +1155,7 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name) if driverConfig.CPUHardLimit { - d.logger.Printf("[DEBUG] driver.docker: using %d cpu quota (cpu-period: %d) for %s", hostConfig.CPUQuota, defaultCFSPeriod, task.Name) + d.logger.Printf("[DEBUG] driver.docker: using %dms cpu quota and %dms cpu period for %s", hostConfig.CPUQuota, defaultCFSPeriod_us, task.Name) } d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name) From a4516aa71a3c71c5b83d49a40632caee66c5ba50 Mon Sep 17 00:00:00 2001 From: Jainin Shah Date: Wed, 7 Feb 2018 16:28:43 -0800 Subject: [PATCH 6/7] removing underscore in variable name --- client/driver/docker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 13b16aff7..8a5451cb2 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -126,7 +126,7 @@ const ( // Below is the documnentation: // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt // https://docs.docker.com/engine/api/v1.35/# - defaultCFSPeriod_us = 100000 + defaultCFSPeriodUS = 100000 ) type DockerDriver struct { @@ -1133,7 +1133,7 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas // Calculate CPU Quota if driverConfig.CPUHardLimit { percentTicks := float64(task.Resources.CPU) / float64(d.node.Resources.CPU) - hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriod_us) + hostConfig.CPUQuota = int64(percentTicks * defaultCFSPeriodUS) } // Windows does not support MemorySwap/MemorySwappiness #2193 @@ -1155,7 +1155,7 @@ func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Tas d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name) if driverConfig.CPUHardLimit { - d.logger.Printf("[DEBUG] driver.docker: using %dms cpu quota and %dms cpu period for %s", hostConfig.CPUQuota, defaultCFSPeriod_us, task.Name) + d.logger.Printf("[DEBUG] driver.docker: using %dms cpu quota and %dms cpu period for %s", hostConfig.CPUQuota, defaultCFSPeriodUS, task.Name) } d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name) From c081fdc449e6585d1ad503daf0077176a0a50254 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 8 Feb 2018 20:20:26 -0800 Subject: [PATCH 7/7] docker: add cpu_hard_limit docs --- CHANGELOG.md | 1 + website/source/docs/drivers/docker.html.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3077c6977..de628047a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ IMPROVEMENTS: [[GH-3781](https://github.com/hashicorp/nomad/issues/3781)] * discovery: Allow `check_restart` to be specified in the `service` stanza. [[GH-3718](https://github.com/hashicorp/nomad/issues/3718)] + * driver/docker: Support hard CPU limits [[GH-3825](https://github.com/hashicorp/nomad/issues/3825)] * driver/docker: Support advertising IPv6 addresses [[GH-3790](https://github.com/hashicorp/nomad/issues/3790)] * driver/docker; Support overriding image entrypoint [[GH-3788](https://github.com/hashicorp/nomad/issues/3788)] * driver/docker: Support adding or dropping capabilities [[GH-3754](https://github.com/hashicorp/nomad/issues/3754)] diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 99ef6de58..e20667bf2 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -355,6 +355,12 @@ The `docker` driver supports the following configuration in the job spec. Only ] } ``` + +* `cpu_hard_limit` - (Optional) `true` or `false` (default). Use hard CPU + limiting instead of soft limiting. By default this is `false` which means + soft limiting is used and containers are able to burst above their CPU limit + when there is idle capacity. + * `advertise_ipv6_address` - (Optional) `true` or `false` (default). Use the container's IPv6 address (GlobalIPv6Address in Docker) when registering services and checks. See [IPv6 Docker containers](/docs/job-specification/service.html#IPv6 Docker containers) for details.