From abdcf0ac17283a6e089f4e80d21f6ba0a1b7b825 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Mon, 17 May 2021 13:11:04 -0600 Subject: [PATCH] Merge pull request #10600 from hashicorp/f-exec-allow_caps drivers/exec+java: reduce default set of linux capabilities --- CHANGELOG.md | 1 + drivers/docker/config.go | 43 +-- drivers/docker/driver.go | 122 +------- drivers/docker/driver_default.go | 21 -- drivers/docker/driver_test.go | 191 +----------- drivers/exec/driver.go | 53 +++- drivers/exec/driver_test.go | 125 +++++--- drivers/exec/driver_unix_test.go | 129 +++++++- drivers/java/driver.go | 85 +++++- drivers/java/driver_test.go | 116 +++++-- drivers/shared/capabilities/defaults.go | 209 +++++++++++++ drivers/shared/capabilities/defaults_test.go | 282 ++++++++++++++++++ drivers/shared/capabilities/set.go | 134 +++++++++ drivers/shared/capabilities/set_test.go | 214 +++++++++++++ drivers/shared/executor/client.go | 1 + drivers/shared/executor/executor.go | 7 + drivers/shared/executor/executor_linux.go | 35 ++- .../shared/executor/executor_linux_test.go | 11 +- drivers/shared/executor/proto/executor.pb.go | 149 +++++---- drivers/shared/executor/proto/executor.proto | 2 + drivers/shared/executor/server.go | 1 + .../docker/docker/oci/caps/defaults.go | 21 -- .../docker/docker/oci/caps/utils.go | 169 ----------- vendor/modules.txt | 1 - website/content/docs/drivers/docker.mdx | 61 ++-- website/content/docs/drivers/exec.mdx | 43 +++ website/content/docs/drivers/java.mdx | 43 +++ .../content/docs/upgrade/upgrade-specific.mdx | 41 ++- 28 files changed, 1575 insertions(+), 735 deletions(-) create mode 100644 drivers/shared/capabilities/defaults.go create mode 100644 drivers/shared/capabilities/defaults_test.go create mode 100644 drivers/shared/capabilities/set.go create mode 100644 drivers/shared/capabilities/set_test.go delete mode 100644 vendor/github.com/docker/docker/oci/caps/defaults.go delete mode 100644 vendor/github.com/docker/docker/oci/caps/utils.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c944cd749..a2dfd29e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ FEATURES: __BACKWARDS INCOMPATIBILITIES:__ * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)] + * drivers/exec+java: Reduce set of linux capabilities enabled by default [[GH-10600](https://github.com/hashicorp/nomad/pull/10600)] * licensing: Enterprise licenses are no longer stored in raft or synced between servers. Loading the Enterprise license from disk or environment is required. The `nomad license put` command has been removed. [[GH-10458](https://github.com/hashicorp/nomad/issues/10458)] SECURITY: diff --git a/drivers/docker/config.go b/drivers/docker/config.go index ec919360d..aff5c5d69 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -10,6 +10,7 @@ import ( docker "github.com/fsouza/go-dockerclient" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" @@ -41,36 +42,6 @@ const ( dockerAuthHelperPrefix = "docker-credential-" ) -// nomadDefaultCaps is the subset of dockerDefaultCaps that Nomad enables by -// default and is used to compute the set of capabilities to add/drop given -// docker driver configuration. -func nomadDefaultCaps() []string { - return []string{ - "AUDIT_WRITE", - "CHOWN", - "DAC_OVERRIDE", - "FOWNER", - "FSETID", - "KILL", - "MKNOD", - "NET_BIND_SERVICE", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_CHROOT", - } -} - -// dockerDefaultCaps is a list of Linux capabilities enabled by docker by default -// and is used to compute the set of capabilities to add/drop given docker driver -// configuration, as well as Nomad built-in limitations. -// -// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -func dockerDefaultCaps() []string { - return append(nomadDefaultCaps(), "NET_RAW") -} - func PluginLoader(opts map[string]string) (map[string]interface{}, error) { conf := map[string]interface{}{} if v, ok := opts["docker.endpoint"]; ok { @@ -287,7 +258,7 @@ var ( "allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false), "allow_caps": hclspec.NewDefault( hclspec.NewAttr("allow_caps", "list(string)", false), - hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), ), "nvidia_runtime": hclspec.NewDefault( hclspec.NewAttr("nvidia_runtime", "string", false), @@ -427,9 +398,9 @@ var ( "work_dir": hclspec.NewAttr("work_dir", "string", false), }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ + // driverCapabilities represents the RPC response for what features are + // implemented by the docker task driver + driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, FSIsolation: drivers.FSIsolationImage, @@ -788,8 +759,10 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } +// Capabilities is returned by the Capabilities RPC and indicates what optional +// features this driver supports. func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } var _ drivers.InternalCapabilitiesDriver = (*Driver)(nil) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index c8a2981ec..7234d3467 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "runtime" - "sort" "strconv" "strings" "sync" @@ -23,10 +22,9 @@ import ( plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/drivers/docker/docklog" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/drivers/shared/eventer" - "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/drivers/shared/resolvconf" - "github.com/hashicorp/nomad/helper" nstructs "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" @@ -913,8 +911,9 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.Privileged = driverConfig.Privileged // set add/drop capabilities - hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig) - if err != nil { + if hostConfig.CapAdd, hostConfig.CapDrop, err = capabilities.Delta( + capabilities.DockerDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ); err != nil { return c, err } @@ -1184,119 +1183,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T }, nil } -// getCaps computes the capabilities to supply to the --add-cap and --drop-cap -// options to the docker driver, which override the default capabilities enabled -// by docker itself. -func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) { - - // capabilities allowable by client docker plugin configuration - allowCaps := expandAllowCaps(d.config.AllowCaps) - - // capabilities the task docker config is asking for based on the default - // capabilities allowable by nomad - desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop) - if err != nil { - return nil, nil, err - } - - // capabilities the task is requesting that are NOT allowed by the docker plugin - if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 { - return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing) - } - - // capabilities that should be dropped relative to the docker default capabilities - dropCaps := capDrops(taskConfig.CapDrop, allowCaps) - - return taskConfig.CapAdd, dropCaps, nil -} - -// capDrops will compute the total dropped capabilities set -// -// {task cap_drop} U ({docker defaults} \ {driver allow caps}) -func capDrops(dropCaps []string, allowCaps []string) []string { - dropSet := make(map[string]struct{}) - - for _, c := range normalizeCaps(dropCaps) { - dropSet[c] = struct{}{} - } - - // if dropCaps includes ALL, no need to iterate every capability - if _, exists := dropSet["ALL"]; exists { - return []string{"ALL"} - } - - dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps())) - allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps)) - - // find the docker default caps not in allowed caps - for dCap := range dockerDefaults { - if _, exists := allowedCaps[dCap]; !exists { - dropSet[dCap] = struct{}{} - } - } - - drops := make([]string, 0, len(dropSet)) - for c := range dropSet { - drops = append(drops, c) - } - sort.Strings(drops) - return drops -} - -// expandAllowCaps returns the normalized set of allowable capabilities set -// for the docker plugin configuration. -func expandAllowCaps(allowCaps []string) []string { - if len(allowCaps) == 0 { - return nil - } - - set := make(map[string]struct{}, len(allowCaps)) - - for _, rawCap := range allowCaps { - capability := strings.ToUpper(rawCap) - if capability == "ALL" { - for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) { - set[defCap] = struct{}{} - } - } else { - set[capability] = struct{}{} - } - } - - result := make([]string, 0, len(set)) - for capability := range set { - result = append(result, capability) - } - sort.Strings(result) - return result -} - -// missingCaps returns the set of elements in desired that are not present in -// allowed. The elements in desired are first upper-cased before comparison. -// The elements in allowed are assumed to be upper-cased. -func missingCaps(allowed, desired []string) []string { - _, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired)) - sort.Strings(missing) - return missing -} - -// normalizeCaps returns a copy of caps with duplicate elements removed and all -// elements upper-cased. -func normalizeCaps(caps []string) []string { - set := make(map[string]struct{}, len(caps)) - for _, c := range caps { - normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_") - set[strings.ToUpper(normal)] = struct{}{} - } - - result := make([]string, 0, len(set)) - for c := range set { - result = append(result, c) - } - sort.Strings(result) - return result -} - func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) { hm, err := m.toDockerHostMount() if err != nil { diff --git a/drivers/docker/driver_default.go b/drivers/docker/driver_default.go index 45086f7bf..b180ae8f9 100644 --- a/drivers/docker/driver_default.go +++ b/drivers/docker/driver_default.go @@ -3,30 +3,9 @@ package docker import ( - "github.com/docker/docker/oci/caps" docker "github.com/fsouza/go-dockerclient" ) func getPortBinding(ip string, port string) docker.PortBinding { return docker.PortBinding{HostIP: ip, HostPort: port} } - -func tweakCapabilities(basics, adds, drops []string) ([]string, error) { - // Moby mixes 2 different capabilities formats: prefixed with "CAP_" - // and not. We do the conversion here to have a consistent, - // non-prefixed format on the Nomad side. - for i, cap := range basics { - basics[i] = "CAP_" + cap - } - - effectiveCaps, err := caps.TweakCapabilities(basics, adds, drops, nil, false) - if err != nil { - return effectiveCaps, err - } - - for i, cap := range effectiveCaps { - effectiveCaps[i] = cap[len("CAP_"):] - } - - return effectiveCaps, nil -} diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 67c22e88e..f51b8b3f0 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -19,7 +19,6 @@ import ( hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" - "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/helper/freeport" "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" @@ -1387,44 +1386,44 @@ func TestDockerDriver_Capabilities(t *testing.T) { { Name: "default-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"ALL"}, + CapDrop: []string{"all"}, }, { Name: "default-allowlist-add-forbidden", CapAdd: []string{"net_admin"}, - StartError: "NET_ADMIN", + StartError: "net_admin", }, { Name: "default-allowlist-drop-existing", - CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, + CapDrop: []string{"fowner", "mknod", "net_raw"}, }, { Name: "restrictive-allowlist-drop-all", - CapDrop: []string{"ALL"}, - Allowlist: "FOWNER,MKNOD", + CapDrop: []string{"all"}, + Allowlist: "fowner,mknod", }, { Name: "restrictive-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"ALL"}, - Allowlist: "fowner,mknod", + CapDrop: []string{"all"}, + Allowlist: "mknod,fowner", }, { Name: "restrictive-allowlist-add-forbidden", CapAdd: []string{"net_admin", "mknod"}, - CapDrop: []string{"ALL"}, + CapDrop: []string{"all"}, Allowlist: "fowner,mknod", - StartError: "NET_ADMIN", + StartError: "net_admin", }, { Name: "permissive-allowlist", - CapAdd: []string{"net_admin", "mknod"}, - Allowlist: "ALL", + CapAdd: []string{"mknod", "net_admin"}, + Allowlist: "all", }, { Name: "permissive-allowlist-add-all", CapAdd: []string{"all"}, - Allowlist: "ALL", + Allowlist: "all", }, } @@ -3064,169 +3063,3 @@ func TestDockerDriver_StopSignal(t *testing.T) { }) } } - -func TestDockerCaps_normalizeCaps(t *testing.T) { - t.Run("empty", func(t *testing.T) { - result := normalizeCaps(nil) - require.Len(t, result, 0) - }) - - t.Run("mixed", func(t *testing.T) { - result := normalizeCaps([]string{ - "DAC_OVERRIDE", "sys_chroot", "kill", "KILL", - }) - require.Equal(t, []string{ - "DAC_OVERRIDE", "KILL", "SYS_CHROOT", - }, result) - }) -} - -func TestDockerCaps_missingCaps(t *testing.T) { - allowed := []string{ - "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", - } - - t.Run("none missing", func(t *testing.T) { - result := missingCaps(allowed, []string{ - "SYS_CHROOT", "chown", "KILL", - }) - require.Equal(t, []string(nil), result) - }) - - t.Run("some missing", func(t *testing.T) { - result := missingCaps(allowed, []string{ - "chown", "audit_write", "SETPCAP", "dac_override", - }) - require.Equal(t, []string{"AUDIT_WRITE", "SETPCAP"}, result) - }) -} - -func TestDockerCaps_expandAllowCaps(t *testing.T) { - t.Run("empty", func(t *testing.T) { - result := expandAllowCaps(nil) - require.Empty(t, result) - }) - - t.Run("manual", func(t *testing.T) { - result := expandAllowCaps([]string{ - "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", - }) - require.Equal(t, []string{ - "CHOWN", "DAC_OVERRIDE", "KILL", "SYS_CHROOT", - }, result) - }) - - t.Run("all", func(t *testing.T) { - result := expandAllowCaps([]string{"all"}) - exp := normalizeCaps(executor.SupportedCaps(true)) - sort.Strings(exp) - require.Equal(t, exp, result) - }) -} - -func TestDockerCaps_capDrops(t *testing.T) { - // docker default caps is always the same, task configured drop_caps and - // plugin config allow_caps may be altered - - // This is the 90% use case, where NET_RAW is dropped, as Nomad's default - // capability allow-list is a subset of the docker default cap list. - t.Run("defaults", func(t *testing.T) { - result := capDrops(nil, nomadDefaultCaps()) - require.Equal(t, []string{"NET_RAW"}, result) - }) - - // Users want to use ICMP (ping). - t.Run("enable net_raw", func(t *testing.T) { - result := capDrops(nil, append(nomadDefaultCaps(), "net_raw")) - require.Empty(t, result) - }) - - // The plugin is reduced in ability. - t.Run("enable minimal", func(t *testing.T) { - allow := []string{"setgid", "setuid", "chown", "kill"} - exp := []string{"AUDIT_WRITE", "DAC_OVERRIDE", "FOWNER", "FSETID", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETPCAP", "SYS_CHROOT"} - result := capDrops(nil, allow) - require.Equal(t, exp, result) - }) - - // The task drops abilities. - t.Run("task drops", func(t *testing.T) { - drops := []string{"audit_write", "fowner", "kill", "chown"} - exp := []string{"AUDIT_WRITE", "CHOWN", "FOWNER", "KILL", "NET_RAW"} - result := capDrops(drops, nomadDefaultCaps()) - require.Equal(t, exp, result) - }) - - // Drop all mixed with others. - t.Run("task drops mix", func(t *testing.T) { - drops := []string{"audit_write", "all", "chown"} - exp := []string{"ALL"} // minimized - result := capDrops(drops, nomadDefaultCaps()) - require.Equal(t, exp, result) - }) -} - -func TestDockerCaps_getCaps(t *testing.T) { - testutil.ExecCompatible(t) // tests require linux - - t.Run("defaults", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, CapDrop: nil, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Equal(t, []string{"NET_RAW"}, drop) - }) - - t.Run("enable net_raw", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: append(nomadDefaultCaps(), "net_raw"), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, CapDrop: nil, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Empty(t, drop) - }) - - t.Run("block sys_time", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - _, _, err := d.getCaps(&TaskConfig{ - CapAdd: []string{"SYS_TIME"}, - CapDrop: nil, - }) - require.EqualError(t, err, `Docker driver does not have the following caps allow-listed on this Nomad agent: [SYS_TIME]`) - }) - - t.Run("enable sys_time", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: append(nomadDefaultCaps(), "sys_time"), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: []string{"SYS_TIME"}, - CapDrop: nil, - }) - require.NoError(t, err) - require.Equal(t, []string{"SYS_TIME"}, add) - require.Equal(t, []string{"NET_RAW"}, drop) - }) - - t.Run("task drops chown", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, - CapDrop: []string{"chown"}, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Equal(t, []string{"CHOWN", "NET_RAW"}, drop) - }) -} diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index b7cfb0058..422b8b42a 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -74,6 +75,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -83,11 +88,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ + // driverCapabilities represents the RPC response for what features are + // implemented by the exec task driver + driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, FSIsolation: drivers.FSIsolationChroot, @@ -141,6 +148,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -156,6 +167,11 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } @@ -174,6 +190,12 @@ type TaskConfig struct { // ModeIPC indicates whether IPC namespace isolation is enabled for the task. // Must be "private" or "host" if set. ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -189,6 +211,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -266,8 +298,10 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } +// Capabilities is returned by the Capabilities RPC and indicates what +// optional features this driver supports func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -439,6 +473,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := capabilities.Calculate( + capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ) + if err != nil { + return nil, nil, err + } + d.logger.Debug("task capabilities", "capabilities", caps) + execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, Args: driverConfig.Args, @@ -455,6 +497,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -482,7 +525,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err := handle.SetDriverState(&driverState); err != nil { d.logger.Error("failed to start task, error setting driver state", "error", err) - exec.Shutdown("", 0) + _ = exec.Shutdown("", 0) pluginClient.Kill() return nil, nil, fmt.Errorf("failed to set driver state: %v", err) } diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index be5dda5cf..8651eda20 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -764,42 +764,99 @@ func TestExecDriver_NoPivotRoot(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) } func TestDriver_TaskConfig_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "host", ipcMode: "", exp: nil}, - {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "", ipcMode: "host", exp: nil}, - {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&TaskConfig{ - ModePID: tc.pidMode, - ModeIPC: tc.ipcMode, - }).validate()) - } + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/exec/driver_unix_test.go b/drivers/exec/driver_unix_test.go index f8d77a2d7..3b2e60374 100644 --- a/drivers/exec/driver_unix_test.go +++ b/drivers/exec/driver_unix_test.go @@ -5,9 +5,13 @@ package exec import ( "context" "fmt" + "strings" "testing" "time" + "github.com/hashicorp/nomad/drivers/shared/capabilities" + "github.com/hashicorp/nomad/drivers/shared/executor" + basePlug "github.com/hashicorp/nomad/plugins/base" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -173,5 +177,128 @@ func TestExec_dnsConfig(t *testing.T) { dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) } - +} + +func TestExecDriver_Capabilities(t *testing.T) { + ctestutils.ExecCompatible(t) + + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "sleep", + } + + for _, tc := range []struct { + Name string + CapAdd []string + CapDrop []string + AllowList string + StartError string + }{ + { + Name: "default-allowlist-add-allowed", + CapAdd: []string{"fowner", "mknod"}, + CapDrop: []string{"ALL"}, + }, + { + Name: "default-allowlist-add-forbidden", + CapAdd: []string{"net_admin"}, + StartError: "net_admin", + }, + { + Name: "default-allowlist-drop-existing", + CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, + }, + { + Name: "restrictive-allowlist-drop-all", + CapDrop: []string{"ALL"}, + AllowList: "FOWNER,MKNOD", + }, + { + Name: "restrictive-allowlist-add-allowed", + CapAdd: []string{"fowner", "mknod"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + }, + { + Name: "restrictive-allowlist-add-forbidden", + CapAdd: []string{"net_admin", "mknod"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + StartError: "net_admin", + }, + { + Name: "restrictive-allowlist-add-multiple-forbidden", + CapAdd: []string{"net_admin", "mknod", "CAP_SYS_TIME"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + StartError: "net_admin, sys_time", + }, + { + Name: "permissive-allowlist", + CapAdd: []string{"net_admin", "mknod"}, + AllowList: "ALL", + }, + { + Name: "permissive-allowlist-add-all", + CapAdd: []string{"all"}, + AllowList: "ALL", + }, + } { + t.Run(tc.Name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + d := NewExecDriver(ctx, testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + config := &Config{ + NoPivotRoot: true, + DefaultModePID: executor.IsolationModePrivate, + DefaultModeIPC: executor.IsolationModePrivate, + } + + if tc.AllowList != "" { + config.AllowCaps = strings.Split(tc.AllowList, ",") + } else { + // inherit HCL defaults if not set + config.AllowCaps = capabilities.NomadDefaults().Slice(true) + } + + var data []byte + require.NoError(t, basePlug.MsgPackEncode(&data, config)) + baseConfig := &basePlug.Config{PluginConfig: data} + require.NoError(t, harness.SetConfig(baseConfig)) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + tCfg := &TaskConfig{ + Command: "/bin/sleep", + Args: []string{"9000"}, + } + if len(tc.CapAdd) > 0 { + tCfg.CapAdd = tc.CapAdd + } + if len(tc.CapDrop) > 0 { + tCfg.CapDrop = tc.CapDrop + } + require.NoError(t, task.EncodeConcreteDriverConfig(&tCfg)) + + // check the start error against expectations + _, _, err := harness.StartTask(task) + if err == nil && tc.StartError != "" { + t.Fatalf("Expected error in start: %v", tc.StartError) + } else if err != nil { + if tc.StartError == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), tc.StartError) + } + return + } + + _ = d.DestroyTask(task.ID, true) + }) + } } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index c34888f6e..f5951ad2b 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -73,6 +74,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -88,11 +93,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what + // driverCapabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports - capabilities = &drivers.Capabilities{ + driverCapabilities = &drivers.Capabilities{ SendSignals: false, Exec: false, FSIsolation: drivers.FSIsolationNone, @@ -108,8 +115,8 @@ var ( func init() { if runtime.GOOS == "linux" { - capabilities.FSIsolation = drivers.FSIsolationChroot - capabilities.MountConfigs = drivers.MountConfigSupportAll + driverCapabilities.FSIsolation = drivers.FSIsolationChroot + driverCapabilities.MountConfigs = drivers.MountConfigSupportAll } } @@ -122,6 +129,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -137,18 +148,44 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } // TaskConfig is the driver configuration of a taskConfig within a job type TaskConfig struct { - Class string `codec:"class"` - ClassPath string `codec:"class_path"` - JarPath string `codec:"jar_path"` - JvmOpts []string `codec:"jvm_options"` - Args []string `codec:"args"` // extra arguments to java executable - ModePID string `codec:"pid_mode"` - ModeIPC string `codec:"ipc_mode"` + // Class indicates which class contains the java entry point. + Class string `codec:"class"` + + // ClassPath indicates where class files are found. + ClassPath string `codec:"class_path"` + + // JarPath indicates where a jar file is found. + JarPath string `codec:"jar_path"` + + // JvmOpts are arguments to pass to the JVM + JvmOpts []string `codec:"jvm_options"` + + // Args are extra arguments to java executable + Args []string `codec:"args"` + + // ModePID indicates whether PID namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModePID string `codec:"pid_mode"` + + // ModeIPC indicates whether IPC namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { } func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", - FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot, + FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot, } exec, pluginClient, err := executor.CreateExecutor( @@ -438,6 +485,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := capabilities.Calculate( + capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ) + if err != nil { + return nil, nil, err + } + d.logger.Debug("task capabilities", "capabilities", caps) + execCmd := &executor.ExecCommand{ Cmd: absPath, Args: args, @@ -453,6 +508,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -491,7 +547,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive } func javaCmdArgs(driverConfig TaskConfig) []string { - args := []string{} + var args []string + // Look for jvm options if len(driverConfig.JvmOpts) != 0 { args = append(args, driverConfig.JvmOpts...) diff --git a/drivers/java/driver_test.go b/drivers/java/driver_test.go index 5c5624037..e407e3c19 100644 --- a/drivers/java/driver_test.go +++ b/drivers/java/driver_test.go @@ -1,6 +1,7 @@ package java import ( + "context" "errors" "fmt" "io" @@ -8,12 +9,10 @@ import ( "os" "path/filepath" "testing" + "time" dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" - "context" - "time" - ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/testlog" @@ -416,20 +415,99 @@ func Test_dnsConfig(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) +} + +func TestDriver_TaskConfig_validate(t *testing.T) { + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go new file mode 100644 index 000000000..827f41019 --- /dev/null +++ b/drivers/shared/capabilities/defaults.go @@ -0,0 +1,209 @@ +package capabilities + +import ( + "fmt" + "regexp" + + "github.com/syndtr/gocapability/capability" +) + +const ( + // HCLSpecLiteral is an equivalent list to NomadDefaults, expressed as a literal + // HCL string for use in HCL config parsing. + HCLSpecLiteral = `["AUDIT_WRITE","CHOWN","DAC_OVERRIDE","FOWNER","FSETID","KILL","MKNOD","NET_BIND_SERVICE","SETFCAP","SETGID","SETPCAP","SETUID","SYS_CHROOT"]` +) + +var ( + extractLiteral = regexp.MustCompile(`([\w]+)`) +) + +// NomadDefaults is the set of Linux capabilities that Nomad enables by +// default. This list originates from what Docker enabled by default, but then +// excludes NET_RAW for security reasons. +// +// This set is use in the as HCL configuration default, described by HCLSpecLiteral. +func NomadDefaults() *Set { + return New(extractLiteral.FindAllString(HCLSpecLiteral, -1)) +} + +// DockerDefaults is a list of Linux capabilities enabled by Docker by default +// and is used to compute the set of capabilities to add/drop given docker driver +// configuration. +// +// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +func DockerDefaults() *Set { + defaults := NomadDefaults() + defaults.Add("NET_RAW") + return defaults +} + +// Supported returns the set of capabilities supported by the operating system. +// +// This set will expand over time as new capabilities are introduced to the kernel +// and the capability library is updated (which tends to happen to keep up with +// run-container libraries). +// +// Defers to a library generated from +// https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h +func Supported() *Set { + s := New(nil) + + last := capability.CAP_LAST_CAP + + // workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap + if last == capability.Cap(63) { + last = capability.CAP_BLOCK_SUSPEND + } + + // accumulate every capability supported by this system + for _, c := range capability.List() { + if c > last { + continue + } + s.Add(c.String()) + } + + return s +} + +// LegacySupported returns the historical set of capabilities used when a task is +// configured to run as root using the exec task driver. Older versions of Nomad +// always allowed the root user to make use of any capability. Now that the exec +// task driver supports configuring the allowed capabilities, operators are +// encouraged to explicitly opt-in to capabilities beyond this legacy set. We +// maintain the legacy list here, because previous versions of Nomad deferred to +// the capability.List library function, which adds new capabilities over time. +// +// https://github.com/hashicorp/nomad/blob/v1.0.4/vendor/github.com/syndtr/gocapability/capability/enum_gen.go#L88 +func LegacySupported() *Set { + return New([]string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_MAC_OVERRIDE", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND", + "CAP_AUDIT_READ", + }) +} + +// Calculate the resulting set of linux capabilities to enable for a task, taking +// into account: +// - default capability basis +// - driver allowable capabilities +// - task capability drops +// - task capability adds +// +// Nomad establishes a standard set of enabled capabilities allowed by the task +// driver if allow_caps is not set. This is the same set that the task will be +// enabled with by default if allow_caps does not further reduce permissions, +// in which case the task capabilities will also be reduced accordingly. +// +// The task will drop any capabilities specified in cap_drop, and add back +// capabilities specified in cap_add. The task will not be allowed to add capabilities +// not set in the the allow_caps setting (which by default is the same as the basis). +// +// cap_add takes precedence over cap_drop, enabling the common pattern of dropping +// all capabilities, then adding back the desired smaller set. e.g. +// cap_drop = ["all"] +// cap_add = ["chown", "kill"] +// +// Note that the resulting capability names are upper-cased and prefixed with +// "CAP_", which is the expected input for the exec/java driver implementation. +func Calculate(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, error) { + allow := New(allowCaps) + adds := New(capAdd) + + // determine caps the task wants that are not allowed + missing := allow.Difference(adds) + if !missing.Empty() { + return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // the realized enabled capabilities starts with what is allowed both by driver + // config AND is a member of the basis (i.e. nomad defaults) + result := basis.Intersect(allow) + + // then remove capabilities the task explicitly drops + result.Remove(capDrop) + + // then add back capabilities the task explicitly adds + return result.Union(adds).Slice(true), nil +} + +// Delta calculates the set of capabilities that must be added and dropped relative +// to a basis to achieve a desired result. The use case is that the docker driver +// assumes a default set (DockerDefault), and we must calculate what to pass into +// --cap-add and --cap-drop on container creation given the inputs of the docker +// plugin config for allow_caps, and the docker task configuration for cap_add and +// cap_drop. Note that the user provided cap_add and cap_drop settings are always +// included, even if they are redundant with the basis (maintaining existing +// behavior, working with existing tests). +// +// Note that the resulting capability names are lower-cased and not prefixed with +// "CAP_", which is the existing style used with the docker driver implementation. +func Delta(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, []string, error) { + all := func(caps []string) bool { + for _, c := range caps { + if normalize(c) == "all" { + return true + } + } + return false + } + + // set of caps allowed by driver + allow := New(allowCaps) + + // determine caps the task wants that are not allowed + missing := allow.Difference(New(capAdd)) + if !missing.Empty() { + return nil, nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // add what the task is asking for + add := New(capAdd).Slice(false) + if all(capAdd) { + add = []string{"all"} + } + + // drop what the task removes plus whatever is in the basis that is not + // in the driver allow configuration + drop := New(allowCaps).Difference(basis).Union(New(capDrop)).Slice(false) + if all(capDrop) { + drop = []string{"all"} + } + + return add, drop, nil +} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go new file mode 100644 index 000000000..7fd03513e --- /dev/null +++ b/drivers/shared/capabilities/defaults_test.go @@ -0,0 +1,282 @@ +package capabilities + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSet_NomadDefaults(t *testing.T) { + result := NomadDefaults() + require.Len(t, result.Slice(false), 13) + defaults := strings.ToLower(HCLSpecLiteral) + for _, c := range result.Slice(false) { + require.Contains(t, defaults, c) + } +} + +func TestSet_DockerDefaults(t *testing.T) { + result := DockerDefaults() + require.Len(t, result.Slice(false), 14) + require.Contains(t, result.String(), "net_raw") +} + +func TestCaps_Calculate(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + exp []string + err error + skip bool // error message is linux version dependent + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow all no mods", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow selection no mods", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + exp: []string{"CAP_CHOWN"}, + err: nil, + }, + { + name: "allow selection and add them", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: []string{"net_raw", "sys_time"}, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"}, + err: nil, + }, + { + name: "allow defaults and add redundant", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + skip: true, + name: "allow defaults and add all", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"all"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"), + }, + { + name: "allow defaults and drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + { + name: "allow defaults and drop all and add back some", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "fowner"}, + capDrop: []string{"all"}, + exp: []string{"CAP_CHOWN", "CAP_FOWNER"}, + err: nil, + }, + { + name: "add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "drop some", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + exp: []string{"CAP_AUDIT_WRITE", "CAP_FSETID", "CAP_NET_BIND_SERVICE", "CAP_SETFCAP", "CAP_SETGID", "CAP_SETPCAP", "CAP_SETUID"}, + err: nil, + }, + { + name: "drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + { + name: "drop all and add back", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "mknod"}, + capDrop: []string{"all"}, + exp: []string{"CAP_CHOWN", "CAP_MKNOD"}, + err: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + caps, err := Calculate(NomadDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) + if !tc.skip { + require.Equal(t, tc.err, err) + require.Equal(t, tc.exp, caps) + } else { + require.Error(t, err) + require.Equal(t, tc.exp, caps) + } + }) + } +} + +func TestCaps_Delta(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + expAdd []string + expDrop []string + err error + skip bool // error message is linux version dependent + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{"net_raw"}, + err: nil, + }, + { + name: "allow all no mods", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{}, + err: nil, + }, + { + name: "allow non-default no mods", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{ + "audit_write", "dac_override", "fowner", "fsetid", + "kill", "mknod", "net_bind_service", "setfcap", + "setgid", "setpcap", "setuid", "sys_chroot"}, + err: nil, + }, + { + name: "allow default add from default", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + expAdd: []string{"chown", "kill"}, + expDrop: []string{"net_raw"}, + err: nil, + }, + { + name: "allow default add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + expAdd: nil, + expDrop: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "allow default drop from default", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + expAdd: []string{}, + expDrop: []string{"chown", "dac_override", "fowner", "kill", "mknod", "net_raw", "sys_chroot"}, + err: nil, + }, + { + name: "allow default drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + expAdd: []string{}, + expDrop: []string{"all"}, + err: nil, + }, + { + name: "task drop all and add back", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "fowner"}, + capDrop: []string{"all"}, + expAdd: []string{"chown", "fowner"}, + expDrop: []string{"all"}, + err: nil, + }, + { + name: "add atop allow all", + allowCaps: []string{"all"}, + capAdd: []string{"chown", "fowner"}, + capDrop: nil, + expAdd: []string{"chown", "fowner"}, + expDrop: []string{}, + err: nil, + }, + { + name: "add all atop all", + allowCaps: []string{"all"}, + capAdd: []string{"all"}, + capDrop: nil, + expAdd: []string{"all"}, + expDrop: []string{}, + err: nil, + }, + { + skip: true, + name: "add all atop defaults", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"all"}, + capDrop: nil, + expAdd: nil, + expDrop: nil, + err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + add, drop, err := Delta(DockerDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) + if !tc.skip { + require.Equal(t, tc.err, err) + require.Equal(t, tc.expAdd, add) + require.Equal(t, tc.expDrop, drop) + } else { + require.Error(t, err) + require.Equal(t, tc.expDrop, drop) + } + }) + } +} diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go new file mode 100644 index 000000000..d6fe527d9 --- /dev/null +++ b/drivers/shared/capabilities/set.go @@ -0,0 +1,134 @@ +// Package capabilities is used for managing sets of linux capabilities. +package capabilities + +import ( + "sort" + "strings" +) + +type nothing struct{} + +var null = nothing{} + +// Set represents a group linux capabilities, implementing some useful set +// operations, taking care of name normalization, and sentinel value expansions. +// +// Linux capabilities can be expressed in multiple ways when working with docker +// and/or executor, along with Nomad configuration. +// +// Capability names may be upper or lower case, and may or may not be prefixed +// with "CAP_" or "cap_". On top of that, Nomad interprets the special name "all" +// and "ALL" to mean "all capabilities supported by the operating system". +type Set struct { + data map[string]nothing +} + +// New creates a new Set setting caps as the initial elements. +func New(caps []string) *Set { + m := make(map[string]nothing, len(caps)) + for _, c := range caps { + insert(m, c) + } + return &Set{data: m} +} + +// Add cap into s. +func (s *Set) Add(cap string) { + insert(s.data, cap) +} + +func insert(data map[string]nothing, cap string) { + switch name := normalize(cap); name { + case "": + case "all": + for k, v := range Supported().data { + data[k] = v + } + return + default: + data[name] = null + } +} + +// Remove caps from s. +func (s *Set) Remove(caps []string) { + for _, c := range caps { + name := normalize(c) + if name == "all" { + s.data = make(map[string]nothing) + return + } + delete(s.data, name) + } +} + +// Union returns of Set of elements of both s and b. +func (s *Set) Union(b *Set) *Set { + data := make(map[string]nothing) + for c := range s.data { + data[c] = null + } + for c := range b.data { + data[c] = null + } + return &Set{data: data} +} + +// Difference returns the Set of elements of b not in s. +func (s *Set) Difference(b *Set) *Set { + data := make(map[string]nothing) + for c := range b.data { + if _, exists := s.data[c]; !exists { + data[c] = null + } + } + return &Set{data: data} +} + +// Intersect returns the Set of elements in both s and b. +func (s *Set) Intersect(b *Set) *Set { + data := make(map[string]nothing) + for c := range s.data { + if _, exists := b.data[c]; exists { + data[c] = null + } + } + return &Set{data: data} +} + +// Empty return true if no capabilities exist in s. +func (s *Set) Empty() bool { + return len(s.data) == 0 +} + +// String returns the normalized and sorted string representation of s. +func (s *Set) String() string { + return strings.Join(s.Slice(false), ", ") +} + +// Slice returns a sorted slice of capabilities in s. +// +// upper - indicates whether to uppercase and prefix capabilities with CAP_ +func (s *Set) Slice(upper bool) []string { + caps := make([]string, 0, len(s.data)) + for c := range s.data { + if upper { + c = "CAP_" + strings.ToUpper(c) + } + caps = append(caps, c) + } + sort.Strings(caps) + return caps +} + +// linux capabilities are often named in 4 possible ways - upper or lower case, +// and with or without a CAP_ prefix +// +// since we must do comparisons on cap names, always normalize the names before +// letting them into the Set data-structure +func normalize(name string) string { + spaces := strings.TrimSpace(name) + lower := strings.ToLower(spaces) + trim := strings.TrimPrefix(lower, "cap_") + return trim +} diff --git a/drivers/shared/capabilities/set_test.go b/drivers/shared/capabilities/set_test.go new file mode 100644 index 000000000..2134719f2 --- /dev/null +++ b/drivers/shared/capabilities/set_test.go @@ -0,0 +1,214 @@ +package capabilities + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSet_Empty(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + result := New(nil).Empty() + require.True(t, result) + }) + + t.Run("empty", func(t *testing.T) { + result := New([]string{}).Empty() + require.True(t, result) + }) + + t.Run("full", func(t *testing.T) { + result := New([]string{"chown", "sys_time"}).Empty() + require.False(t, result) + }) +} + +func TestSet_New(t *testing.T) { + t.Parallel() + + t.Run("duplicates", func(t *testing.T) { + result := New([]string{"chown", "sys_time", "chown"}) + require.Equal(t, "chown, sys_time", result.String()) + }) + + t.Run("empty string", func(t *testing.T) { + result := New([]string{""}) + require.True(t, result.Empty()) + }) + + t.Run("all", func(t *testing.T) { + result := New([]string{"all"}) + exp := len(Supported().Slice(false)) + require.Len(t, result.Slice(false), exp) + }) +} + +func TestSet_Slice(t *testing.T) { + t.Parallel() + + exp := []string{"chown", "net_raw", "sys_time"} + + t.Run("lower case", func(t *testing.T) { + s := New([]string{"net_raw", "chown", "sys_time"}) + require.Equal(t, exp, s.Slice(false)) + }) + + t.Run("upper case", func(t *testing.T) { + s := New([]string{"NET_RAW", "CHOWN", "SYS_TIME"}) + require.Equal(t, exp, s.Slice(false)) + }) + + t.Run("prefix", func(t *testing.T) { + s := New([]string{"CAP_net_raw", "sys_TIME", "cap_chown"}) + require.Equal(t, exp, s.Slice(false)) + }) +} + +func TestSet_String(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + result := New(nil).String() + require.Equal(t, "", result) + }) + + t.Run("full", func(t *testing.T) { + exp := "chown, net_raw, sys_time" + in := []string{"net_raw", "CAP_CHOWN", "cap_sys_time"} + result := New(in).String() + require.Equal(t, exp, result) + }) +} + +func TestSet_Add(t *testing.T) { + t.Parallel() + + t.Run("add one", func(t *testing.T) { + s := New([]string{"chown", "net_raw"}) + require.Equal(t, "chown, net_raw", s.String()) + + s.Add("CAP_SYS_TIME") + require.Equal(t, "chown, net_raw, sys_time", s.String()) + + s.Add("AF_NET") + require.Equal(t, "af_net, chown, net_raw, sys_time", s.String()) + }) + + t.Run("add empty string", func(t *testing.T) { + s := New([]string{"chown"}) + s.Add("") + require.Equal(t, "chown", s.String()) + }) + + t.Run("add all", func(t *testing.T) { + s := New([]string{"chown", "net_raw"}) + require.Equal(t, "chown, net_raw", s.String()) + + exp := len(Supported().Slice(false)) + s.Add("all") + require.Len(t, s.Slice(false), exp) + }) + +} + +func TestSet_Remove(t *testing.T) { + t.Parallel() + + t.Run("remove one", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"CAP_NET_RAW"}) + require.Equal(t, "af_net, chown, seteuid, sys_time", s.String()) + }) + + t.Run("remove couple", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"CAP_NET_RAW", "af_net"}) + require.Equal(t, "chown, seteuid, sys_time", s.String()) + }) + + t.Run("remove all", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"all"}) + require.True(t, s.Empty()) + require.Equal(t, "", s.String()) + }) +} + +func TestSet_Difference(t *testing.T) { + t.Parallel() + + t.Run("a is empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"chown", "af_net"}) + result := a.Difference(b) + require.Equal(t, "af_net, chown", result.String()) + }) + + t.Run("b is empty", func(t *testing.T) { + a := New([]string{"chown", "af_net"}) + b := New(nil) + result := a.Difference(b) + require.True(t, result.Empty()) + }) + + t.Run("a diff b", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f"}) + b := New([]string{"B", "x", "Y", "a"}) + result := a.Difference(b) + require.Equal(t, "x, y", result.String()) + }) +} + +func TestSet_Intersect(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"a", "b"}) + + result := a.Intersect(b) + require.True(t, result.Empty()) + + result2 := b.Intersect(a) + require.True(t, result2.Empty()) + }) + + t.Run("intersect", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f", "G"}) + b := New([]string{"Z", "B", "E", "f", "y"}) + + result := a.Intersect(b) + require.Equal(t, "b, e, f", result.String()) + + result2 := b.Intersect(a) + require.Equal(t, "b, e, f", result2.String()) + }) +} + +func TestSet_Union(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"a", "b"}) + + result := a.Union(b) + require.Equal(t, "a, b", result.String()) + + result2 := b.Union(a) + require.Equal(t, "a, b", result2.String()) + }) + + t.Run("union", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f", "G"}) + b := New([]string{"Z", "B", "E", "f", "y"}) + + result := a.Union(b) + require.Equal(t, "a, b, c, d, e, f, g, y, z", result.String()) + + result2 := b.Union(a) + require.Equal(t, "a, b, c, d, e, f, g, y, z", result2.String()) + }) +} diff --git a/drivers/shared/executor/client.go b/drivers/shared/executor/client.go index 67724cc83..1779d919a 100644 --- a/drivers/shared/executor/client.go +++ b/drivers/shared/executor/client.go @@ -47,6 +47,7 @@ func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) { NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation), DefaultPidMode: cmd.ModePID, DefaultIpcMode: cmd.ModeIPC, + Capabilities: cmd.Capabilities, } resp, err := c.client.Launch(ctx, req) if err != nil { diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index a87674d2a..10a70f0b8 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -92,6 +92,10 @@ type Executor interface { // ExecCommand holds the user command, args, and other isolation related // settings. +// +// Important (!): when adding fields, make sure to update the RPC methods in +// grpcExecutorClient.Launch and grpcExecutorServer.Launch. Number of hours +// spent tracking this down: too many. type ExecCommand struct { // Cmd is the command that the user wants to run. Cmd string @@ -147,6 +151,9 @@ type ExecCommand struct { // ModeIPC is the IPC isolation mode (private or host). ModeIPC string + + // Capabilities are the linux capabilities to be enabled by the task driver. + Capabilities []string } // SetWriters sets the writer for the process stdout and stderr. This should diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 16eb27b4c..5c4919e83 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/opencontainers/runtime-spec/specs-go" "github.com/armon/circbuf" @@ -532,28 +533,25 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc ch <- &waitResult{ps, err} } -func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error { - // TODO(shoenig): allow better control of these - // use capabilities list as prior to adopting libcontainer in 0.9 - - // match capabilities used in Nomad 0.8 - if command.User == "root" { - allCaps := SupportedCaps(true) +func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) { + switch command.User { + case "root": + // when running as root, use the legacy set of system capabilities, so + // that we do not break existing nomad clusters using this "feature" + legacyCaps := capabilities.LegacySupported().Slice(true) cfg.Capabilities = &lconfigs.Capabilities{ - Bounding: allCaps, - Permitted: allCaps, - Effective: allCaps, + Bounding: legacyCaps, + Permitted: legacyCaps, + Effective: legacyCaps, Ambient: nil, Inheritable: nil, } - } else { - allCaps := SupportedCaps(false) + default: + // otherwise apply the plugin + task capability configuration cfg.Capabilities = &lconfigs.Capabilities{ - Bounding: allCaps, + Bounding: command.Capabilities, } } - - return nil } func configureNamespaces(pidMode, ipcMode string) lconfigs.Namespaces { @@ -759,16 +757,17 @@ func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) { }, Version: "1.0.0", } + for _, device := range specconv.AllowedDevices { cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule) } - if err := configureCapabilities(cfg, command); err != nil { - return nil, err - } + configureCapabilities(cfg, command) + if err := configureIsolation(cfg, command); err != nil { return nil, err } + if err := configureCgroups(cfg, command); err != nil { return nil, err } diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index a8bfba0a4..b3576797a 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/plugins/drivers" @@ -478,7 +479,7 @@ func TestExecutor_Capabilities(t *testing.T) { CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 -CapBnd: 0000003fffffdfff +CapBnd: 00000000a80405fb CapAmb: 0000000000000000`, }, { @@ -494,7 +495,6 @@ CapAmb: 0000000000000000`, for _, c := range cases { t.Run(c.user, func(t *testing.T) { - require := require.New(t) testExecCmd := testExecutorCommandWithChroot(t) execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir @@ -504,12 +504,13 @@ CapAmb: 0000000000000000`, execCmd.ResourceLimits = true execCmd.Cmd = "/bin/bash" execCmd.Args = []string{"-c", "cat /proc/$$/status"} + execCmd.Capabilities = capabilities.NomadDefaults().Slice(true) executor := NewExecutorWithIsolation(testlog.HCLogger(t)) defer executor.Shutdown("SIGKILL", 0) _, err := executor.Launch(execCmd) - require.NoError(err) + require.NoError(t, err) ch := make(chan interface{}) go func() { @@ -521,7 +522,7 @@ CapAmb: 0000000000000000`, case <-ch: // all good case <-time.After(5 * time.Second): - require.Fail("timeout waiting for exec to shutdown") + require.Fail(t, "timeout waiting for exec to shutdown") } canonical := func(s string) string { @@ -538,7 +539,7 @@ CapAmb: 0000000000000000`, return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output) } return true, nil - }, func(err error) { require.NoError(err) }) + }, func(err error) { require.NoError(t, err) }) }) } diff --git a/drivers/shared/executor/proto/executor.pb.go b/drivers/shared/executor/proto/executor.pb.go index f6defb54a..dcf1a38c7 100644 --- a/drivers/shared/executor/proto/executor.pb.go +++ b/drivers/shared/executor/proto/executor.pb.go @@ -44,6 +44,8 @@ type LaunchRequest struct { DefaultPidMode string `protobuf:"bytes,15,opt,name=default_pid_mode,json=defaultPidMode,proto3" json:"default_pid_mode,omitempty"` DefaultIpcMode string `protobuf:"bytes,16,opt,name=default_ipc_mode,json=defaultIpcMode,proto3" json:"default_ipc_mode,omitempty"` CpusetCgroup string `protobuf:"bytes,17,opt,name=cpuset_cgroup,json=cpusetCgroup,proto3" json:"cpuset_cgroup,omitempty"` + AllowCaps []string `protobuf:"bytes,18,rep,name=allow_caps,json=allowCaps,proto3" json:"allow_caps,omitempty"` + Capabilities []string `protobuf:"bytes,19,rep,name=capabilities,proto3" json:"capabilities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -193,6 +195,20 @@ func (m *LaunchRequest) GetCpusetCgroup() string { return "" } +func (m *LaunchRequest) GetAllowCaps() []string { + if m != nil { + return m.AllowCaps + } + return nil +} + +func (m *LaunchRequest) GetCapabilities() []string { + if m != nil { + return m.Capabilities + } + return nil +} + type LaunchResponse struct { Process *ProcessState `protobuf:"bytes,1,opt,name=process,proto3" json:"process,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -858,71 +874,74 @@ func init() { } var fileDescriptor_66b85426380683f3 = []byte{ - // 1020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x5b, 0x8f, 0xdb, 0x44, - 0x14, 0xae, 0x37, 0x9b, 0xdb, 0x49, 0xb2, 0x49, 0x47, 0xa8, 0xb8, 0xe1, 0xa1, 0xc1, 0x48, 0x34, - 0x82, 0xe2, 0xac, 0xb6, 0x37, 0x24, 0x24, 0x8a, 0xd8, 0x2d, 0xa8, 0xd2, 0x76, 0x15, 0x39, 0x85, - 0x4a, 0x3c, 0x60, 0x5c, 0xcf, 0x34, 0x19, 0x6d, 0xe2, 0x31, 0x33, 0xe3, 0x74, 0x91, 0x90, 0x78, - 0xe2, 0x1f, 0x80, 0xc4, 0x5f, 0xe5, 0x0d, 0xcd, 0xcd, 0x9b, 0x6c, 0x4b, 0xe5, 0x14, 0xf1, 0x14, - 0xcf, 0xc9, 0xf7, 0x9d, 0xcb, 0x9c, 0x73, 0xbe, 0x81, 0x3b, 0x98, 0xd3, 0x35, 0xe1, 0x62, 0x22, - 0x16, 0x09, 0x27, 0x78, 0x42, 0x2e, 0x48, 0x5a, 0x48, 0xc6, 0x27, 0x39, 0x67, 0x92, 0x95, 0xc7, - 0x50, 0x1f, 0xd1, 0xc7, 0x8b, 0x44, 0x2c, 0x68, 0xca, 0x78, 0x1e, 0x66, 0x6c, 0x95, 0xe0, 0x30, - 0x5f, 0x16, 0x73, 0x9a, 0x89, 0x70, 0x1b, 0x37, 0xbc, 0x35, 0x67, 0x6c, 0xbe, 0x24, 0xc6, 0xc9, - 0x8b, 0xe2, 0xe5, 0x44, 0xd2, 0x15, 0x11, 0x32, 0x59, 0xe5, 0x16, 0x10, 0x58, 0xe2, 0xc4, 0x85, - 0x37, 0xe1, 0xcc, 0xc9, 0x60, 0x82, 0xbf, 0xeb, 0xd0, 0x3b, 0x4d, 0x8a, 0x2c, 0x5d, 0x44, 0xe4, - 0xe7, 0x82, 0x08, 0x89, 0x06, 0x50, 0x4b, 0x57, 0xd8, 0xf7, 0x46, 0xde, 0xb8, 0x1d, 0xa9, 0x4f, - 0x84, 0x60, 0x3f, 0xe1, 0x73, 0xe1, 0xef, 0x8d, 0x6a, 0xe3, 0x76, 0xa4, 0xbf, 0xd1, 0x19, 0xb4, - 0x39, 0x11, 0xac, 0xe0, 0x29, 0x11, 0x7e, 0x6d, 0xe4, 0x8d, 0x3b, 0x47, 0x87, 0xe1, 0xbf, 0x25, - 0x6e, 0xe3, 0x9b, 0x90, 0x61, 0xe4, 0x78, 0xd1, 0xa5, 0x0b, 0x74, 0x0b, 0x3a, 0x42, 0x62, 0x56, - 0xc8, 0x38, 0x4f, 0xe4, 0xc2, 0xdf, 0xd7, 0xd1, 0xc1, 0x98, 0xa6, 0x89, 0x5c, 0x58, 0x00, 0xe1, - 0xdc, 0x00, 0xea, 0x25, 0x80, 0x70, 0xae, 0x01, 0x03, 0xa8, 0x91, 0x6c, 0xed, 0x37, 0x74, 0x92, - 0xea, 0x53, 0xe5, 0x5d, 0x08, 0xc2, 0xfd, 0xa6, 0xc6, 0xea, 0x6f, 0x74, 0x13, 0x5a, 0x32, 0x11, - 0xe7, 0x31, 0xa6, 0xdc, 0x6f, 0x69, 0x7b, 0x53, 0x9d, 0x4f, 0x28, 0x47, 0xb7, 0xa1, 0xef, 0xf2, - 0x89, 0x97, 0x74, 0x45, 0xa5, 0xf0, 0xdb, 0x23, 0x6f, 0xdc, 0x8a, 0x0e, 0x9c, 0xf9, 0x54, 0x5b, - 0xd1, 0x21, 0xbc, 0xf7, 0x22, 0x11, 0x34, 0x8d, 0x73, 0xce, 0x52, 0x22, 0x44, 0x9c, 0xce, 0x39, - 0x2b, 0x72, 0x1f, 0x34, 0x1a, 0xe9, 0xff, 0xa6, 0xe6, 0xaf, 0x63, 0xfd, 0x0f, 0x3a, 0x81, 0xc6, - 0x8a, 0x15, 0x99, 0x14, 0x7e, 0x67, 0x54, 0x1b, 0x77, 0x8e, 0xee, 0x54, 0xbc, 0xaa, 0xa7, 0x8a, - 0x14, 0x59, 0x2e, 0xfa, 0x16, 0x9a, 0x98, 0xac, 0xa9, 0xba, 0xf1, 0xae, 0x76, 0xf3, 0x59, 0x45, - 0x37, 0x27, 0x9a, 0x15, 0x39, 0x36, 0x5a, 0xc0, 0xf5, 0x8c, 0xc8, 0x57, 0x8c, 0x9f, 0xc7, 0x54, - 0xb0, 0x65, 0x22, 0x29, 0xcb, 0xfc, 0x9e, 0x6e, 0xe2, 0x17, 0x15, 0x5d, 0x9e, 0x19, 0xfe, 0x13, - 0x47, 0x9f, 0xe5, 0x24, 0x8d, 0x06, 0xd9, 0x15, 0x2b, 0x0a, 0xa0, 0x97, 0xb1, 0x38, 0xa7, 0x6b, - 0x26, 0x63, 0xce, 0x98, 0xf4, 0x0f, 0xf4, 0x1d, 0x75, 0x32, 0x36, 0x55, 0xb6, 0x88, 0x31, 0x89, - 0xc6, 0x30, 0xc0, 0xe4, 0x65, 0x52, 0x2c, 0x65, 0x9c, 0x53, 0x1c, 0xaf, 0x18, 0x26, 0x7e, 0x5f, - 0xb7, 0xe6, 0xc0, 0xda, 0xa7, 0x14, 0x3f, 0x65, 0x98, 0x6c, 0x22, 0x69, 0x9e, 0x1a, 0xe4, 0x60, - 0x0b, 0xf9, 0x24, 0x4f, 0x35, 0xf2, 0x23, 0xe8, 0xa5, 0x79, 0x21, 0x88, 0x74, 0xbd, 0xb9, 0xae, - 0x61, 0x5d, 0x63, 0x34, 0x5d, 0x09, 0x7e, 0x82, 0x03, 0x37, 0xfa, 0x22, 0x67, 0x99, 0x20, 0xe8, - 0x0c, 0x9a, 0xb6, 0xa7, 0x7a, 0xfe, 0x3b, 0x47, 0xf7, 0xc2, 0x6a, 0xcb, 0x18, 0xda, 0x7e, 0xcf, - 0x64, 0x22, 0x49, 0xe4, 0x9c, 0x04, 0x3d, 0xe8, 0x3c, 0x4f, 0xa8, 0xb4, 0xab, 0x15, 0xfc, 0x08, - 0x5d, 0x73, 0xfc, 0x9f, 0xc2, 0x9d, 0x42, 0x7f, 0xb6, 0x28, 0x24, 0x66, 0xaf, 0x32, 0xb7, 0xcd, - 0x37, 0xa0, 0x21, 0xe8, 0x3c, 0x4b, 0x96, 0x76, 0xa1, 0xed, 0x09, 0x7d, 0x08, 0xdd, 0x39, 0x4f, - 0x52, 0x12, 0xe7, 0x84, 0x53, 0x86, 0xfd, 0xbd, 0x91, 0x37, 0xae, 0x45, 0x1d, 0x6d, 0x9b, 0x6a, - 0x53, 0x80, 0x60, 0x70, 0xe9, 0xcd, 0x64, 0x1c, 0x2c, 0xe0, 0xc6, 0x77, 0x39, 0x56, 0x41, 0xcb, - 0x25, 0xb6, 0x81, 0xb6, 0x04, 0xc1, 0xfb, 0xcf, 0x82, 0x10, 0xdc, 0x84, 0xf7, 0x5f, 0x8b, 0x64, - 0x93, 0x18, 0xc0, 0xc1, 0xf7, 0x84, 0x0b, 0xca, 0x5c, 0x95, 0xc1, 0xa7, 0xd0, 0x2f, 0x2d, 0xf6, - 0x6e, 0x7d, 0x68, 0xae, 0x8d, 0xc9, 0x56, 0xee, 0x8e, 0xc1, 0x27, 0xd0, 0x55, 0xf7, 0x56, 0x66, - 0x3e, 0x84, 0x16, 0xcd, 0x24, 0xe1, 0x6b, 0x7b, 0x49, 0xb5, 0xa8, 0x3c, 0x07, 0xcf, 0xa1, 0x67, - 0xb1, 0xd6, 0xed, 0x37, 0x50, 0x17, 0xca, 0xb0, 0x63, 0x89, 0xcf, 0x12, 0x71, 0x6e, 0x1c, 0x19, - 0x7a, 0x70, 0x1b, 0x7a, 0x33, 0xdd, 0x89, 0x37, 0x37, 0xaa, 0xee, 0x1a, 0xa5, 0x8a, 0x75, 0x40, - 0x5b, 0xfe, 0x39, 0x74, 0x1e, 0x5f, 0x90, 0xd4, 0x11, 0x1f, 0x40, 0x0b, 0x93, 0x04, 0x2f, 0x69, - 0x46, 0x6c, 0x52, 0xc3, 0xd0, 0xbc, 0x0c, 0xa1, 0x7b, 0x19, 0xc2, 0x67, 0xee, 0x65, 0x88, 0x4a, - 0xac, 0xd3, 0xf9, 0xbd, 0xd7, 0x75, 0xbe, 0x76, 0xa9, 0xf3, 0xc1, 0x31, 0x74, 0x4d, 0x30, 0x5b, - 0xff, 0x0d, 0x68, 0xb0, 0x42, 0xe6, 0x85, 0xd4, 0xb1, 0xba, 0x91, 0x3d, 0xa1, 0x0f, 0xa0, 0x4d, - 0x2e, 0xa8, 0x8c, 0x53, 0xb5, 0x93, 0x7b, 0xba, 0x82, 0x96, 0x32, 0x1c, 0x33, 0x4c, 0x82, 0xdf, - 0x3d, 0xe8, 0x6e, 0x4e, 0xac, 0x8a, 0x9d, 0x53, 0x6c, 0x2b, 0x55, 0x9f, 0x6f, 0xe5, 0x6f, 0xdc, - 0x4d, 0x6d, 0xf3, 0x6e, 0x50, 0x08, 0xfb, 0xea, 0xcd, 0xd3, 0xaf, 0xc5, 0xdb, 0xcb, 0xd6, 0xb8, - 0xa3, 0x3f, 0xdb, 0xd0, 0x7a, 0x6c, 0x17, 0x09, 0xfd, 0x02, 0x0d, 0xb3, 0xfd, 0xe8, 0x7e, 0xd5, - 0xad, 0xdb, 0x7a, 0x28, 0x87, 0x0f, 0x76, 0xa5, 0xd9, 0xfe, 0x5d, 0x43, 0x02, 0xf6, 0x95, 0x0e, - 0xa0, 0xbb, 0x55, 0x3d, 0x6c, 0x88, 0xc8, 0xf0, 0xde, 0x6e, 0xa4, 0x32, 0xe8, 0x6f, 0xd0, 0x72, - 0xeb, 0x8c, 0x1e, 0x56, 0xf5, 0x71, 0x45, 0x4e, 0x86, 0x9f, 0xef, 0x4e, 0x2c, 0x13, 0xf8, 0xc3, - 0x83, 0xfe, 0x95, 0x95, 0x46, 0x5f, 0x56, 0xf5, 0xf7, 0x66, 0xd5, 0x19, 0x3e, 0x7a, 0x67, 0x7e, - 0x99, 0xd6, 0xaf, 0xd0, 0xb4, 0xda, 0x81, 0x2a, 0x77, 0x74, 0x5b, 0x7e, 0x86, 0x0f, 0x77, 0xe6, - 0x95, 0xd1, 0x2f, 0xa0, 0xae, 0x75, 0x01, 0x55, 0x6e, 0xeb, 0xa6, 0x76, 0x0d, 0xef, 0xef, 0xc8, - 0x72, 0x71, 0x0f, 0x3d, 0x35, 0xff, 0x46, 0x58, 0xaa, 0xcf, 0xff, 0x96, 0x62, 0x55, 0x9f, 0xff, - 0x2b, 0xfa, 0xa5, 0xe7, 0x5f, 0xad, 0x61, 0xf5, 0xf9, 0xdf, 0xd0, 0xbb, 0xea, 0xf3, 0xbf, 0xa9, - 0x5b, 0xc1, 0x35, 0xf4, 0x97, 0x07, 0x3d, 0x65, 0x9a, 0x49, 0x4e, 0x92, 0x15, 0xcd, 0xe6, 0xe8, - 0x51, 0x45, 0xf1, 0x56, 0x2c, 0x23, 0xe0, 0x96, 0xe9, 0x52, 0xf9, 0xea, 0xdd, 0x1d, 0xb8, 0xb4, - 0xc6, 0xde, 0xa1, 0xf7, 0x75, 0xf3, 0x87, 0xba, 0xd1, 0xac, 0x86, 0xfe, 0xb9, 0xfb, 0x4f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x32, 0xbc, 0x95, 0x4b, 0x31, 0x0c, 0x00, 0x00, + // 1058 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x6d, 0x6f, 0x1b, 0x45, + 0x10, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0x2e, 0xa8, 0x5c, 0x8d, 0x50, 0xcd, 0x21, 0x51, + 0x0b, 0xca, 0x25, 0x4a, 0xdf, 0x90, 0x90, 0x28, 0x22, 0x2d, 0xa8, 0x52, 0x1a, 0x45, 0x97, 0x42, + 0x25, 0x3e, 0x70, 0x6c, 0xee, 0xb6, 0xf6, 0x2a, 0xf6, 0xed, 0xb2, 0xbb, 0xe7, 0x04, 0x09, 0x89, + 0x4f, 0xfc, 0x03, 0x90, 0xf8, 0x31, 0xfc, 0x38, 0xb4, 0x6f, 0x17, 0x3b, 0x2d, 0xd5, 0xb9, 0x88, + 0x4f, 0xbe, 0x1d, 0x3f, 0xcf, 0xcc, 0xec, 0xce, 0xcc, 0x33, 0x70, 0x27, 0x17, 0x74, 0x41, 0x84, + 0xdc, 0x95, 0x53, 0x2c, 0x48, 0xbe, 0x4b, 0x2e, 0x48, 0x56, 0x2a, 0x26, 0x76, 0xb9, 0x60, 0x8a, + 0x55, 0xc7, 0xd8, 0x1c, 0xd1, 0xc7, 0x53, 0x2c, 0xa7, 0x34, 0x63, 0x82, 0xc7, 0x05, 0x9b, 0xe3, + 0x3c, 0xe6, 0xb3, 0x72, 0x42, 0x0b, 0x19, 0xaf, 0xe2, 0x86, 0xb7, 0x26, 0x8c, 0x4d, 0x66, 0xc4, + 0x3a, 0x39, 0x2d, 0x5f, 0xee, 0x2a, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0x77, + 0x7d, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0xfa, 0xbb, 0x09, 0xfd, 0x43, 0x5c, 0x16, 0xd9, 0x34, + 0x21, 0x3f, 0x97, 0x44, 0x2a, 0x34, 0x80, 0x46, 0x36, 0xcf, 0xc3, 0x60, 0x14, 0x8c, 0x3b, 0x89, + 0xfe, 0x44, 0x08, 0x36, 0xb1, 0x98, 0xc8, 0x70, 0x63, 0xd4, 0x18, 0x77, 0x12, 0xf3, 0x8d, 0x8e, + 0xa0, 0x23, 0x88, 0x64, 0xa5, 0xc8, 0x88, 0x0c, 0x1b, 0xa3, 0x60, 0xdc, 0xdd, 0xdf, 0x8b, 0xff, + 0x2d, 0x71, 0x17, 0xdf, 0x86, 0x8c, 0x13, 0xcf, 0x4b, 0x2e, 0x5d, 0xa0, 0x5b, 0xd0, 0x95, 0x2a, + 0x67, 0xa5, 0x4a, 0x39, 0x56, 0xd3, 0x70, 0xd3, 0x44, 0x07, 0x6b, 0x3a, 0xc6, 0x6a, 0xea, 0x00, + 0x44, 0x08, 0x0b, 0xd8, 0xaa, 0x00, 0x44, 0x08, 0x03, 0x18, 0x40, 0x83, 0x14, 0x8b, 0xb0, 0x69, + 0x92, 0xd4, 0x9f, 0x3a, 0xef, 0x52, 0x12, 0x11, 0xb6, 0x0c, 0xd6, 0x7c, 0xa3, 0x9b, 0xd0, 0x56, + 0x58, 0x9e, 0xa5, 0x39, 0x15, 0x61, 0xdb, 0xd8, 0x5b, 0xfa, 0xfc, 0x98, 0x0a, 0x74, 0x1b, 0x76, + 0x7c, 0x3e, 0xe9, 0x8c, 0xce, 0xa9, 0x92, 0x61, 0x67, 0x14, 0x8c, 0xdb, 0xc9, 0xb6, 0x37, 0x1f, + 0x1a, 0x2b, 0xda, 0x83, 0x77, 0x4f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, + 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x1d, 0xdb, 0xbf, 0x0e, 0xcc, 0x3f, 0xe8, 0x31, + 0x34, 0xe7, 0xac, 0x2c, 0x94, 0x0c, 0xbb, 0xa3, 0xc6, 0xb8, 0xbb, 0x7f, 0xa7, 0xe6, 0x53, 0x3d, + 0xd3, 0xa4, 0xc4, 0x71, 0xd1, 0xb7, 0xd0, 0xca, 0xc9, 0x82, 0xea, 0x17, 0xef, 0x19, 0x37, 0x9f, + 0xd5, 0x74, 0xf3, 0xd8, 0xb0, 0x12, 0xcf, 0x46, 0x53, 0xb8, 0x5e, 0x10, 0x75, 0xce, 0xc4, 0x59, + 0x4a, 0x25, 0x9b, 0x61, 0x45, 0x59, 0x11, 0xf6, 0x4d, 0x11, 0xbf, 0xa8, 0xe9, 0xf2, 0xc8, 0xf2, + 0x9f, 0x7a, 0xfa, 0x09, 0x27, 0x59, 0x32, 0x28, 0xae, 0x58, 0x51, 0x04, 0xfd, 0x82, 0xa5, 0x9c, + 0x2e, 0x98, 0x4a, 0x05, 0x63, 0x2a, 0xdc, 0x36, 0x6f, 0xd4, 0x2d, 0xd8, 0xb1, 0xb6, 0x25, 0x8c, + 0x29, 0x34, 0x86, 0x41, 0x4e, 0x5e, 0xe2, 0x72, 0xa6, 0x52, 0x4e, 0xf3, 0x74, 0xce, 0x72, 0x12, + 0xee, 0x98, 0xd2, 0x6c, 0x3b, 0xfb, 0x31, 0xcd, 0x9f, 0xb1, 0x9c, 0x2c, 0x23, 0x29, 0xcf, 0x2c, + 0x72, 0xb0, 0x82, 0x7c, 0xca, 0x33, 0x83, 0xfc, 0x08, 0xfa, 0x19, 0x2f, 0x25, 0x51, 0xbe, 0x36, + 0xd7, 0x0d, 0xac, 0x67, 0x8d, 0xae, 0x2a, 0x1f, 0x00, 0xe0, 0xd9, 0x8c, 0x9d, 0xa7, 0x19, 0xe6, + 0x32, 0x44, 0xa6, 0x71, 0x3a, 0xc6, 0x72, 0x80, 0xb9, 0x44, 0x11, 0xf4, 0x32, 0xcc, 0xf1, 0x29, + 0x9d, 0x51, 0x45, 0x89, 0x0c, 0xdf, 0x31, 0x80, 0x15, 0x5b, 0xf4, 0x13, 0x6c, 0xfb, 0xe9, 0x91, + 0x9c, 0x15, 0x92, 0xa0, 0x23, 0x68, 0xb9, 0xb6, 0x30, 0x23, 0xd4, 0xdd, 0xbf, 0x17, 0xd7, 0x9b, + 0xe7, 0xd8, 0xb5, 0xcc, 0x89, 0xc2, 0x8a, 0x24, 0xde, 0x49, 0xd4, 0x87, 0xee, 0x0b, 0x4c, 0x95, + 0x9b, 0xce, 0xe8, 0x47, 0xe8, 0xd9, 0xe3, 0xff, 0x14, 0xee, 0x10, 0x76, 0x4e, 0xa6, 0xa5, 0xca, + 0xd9, 0x79, 0xe1, 0x05, 0xe1, 0x06, 0x34, 0x25, 0x9d, 0x14, 0x78, 0xe6, 0x34, 0xc1, 0x9d, 0xd0, + 0x87, 0xd0, 0x9b, 0x08, 0x9c, 0x91, 0x94, 0x13, 0x41, 0x59, 0x1e, 0x6e, 0x8c, 0x82, 0x71, 0x23, + 0xe9, 0x1a, 0xdb, 0xb1, 0x31, 0x45, 0x08, 0x06, 0x97, 0xde, 0x6c, 0xc6, 0xd1, 0x14, 0x6e, 0x7c, + 0xc7, 0x73, 0x1d, 0xb4, 0xd2, 0x01, 0x17, 0x68, 0x45, 0x53, 0x82, 0xff, 0xac, 0x29, 0xd1, 0x4d, + 0x78, 0xef, 0x95, 0x48, 0x2e, 0x89, 0x01, 0x6c, 0x7f, 0x4f, 0x84, 0xa4, 0xcc, 0xdf, 0x32, 0xfa, + 0x14, 0x76, 0x2a, 0x8b, 0x7b, 0xdb, 0x10, 0x5a, 0x0b, 0x6b, 0x72, 0x37, 0xf7, 0xc7, 0xe8, 0x13, + 0xe8, 0xe9, 0x77, 0xab, 0x32, 0x1f, 0x42, 0x9b, 0x16, 0x8a, 0x88, 0x85, 0x7b, 0xa4, 0x46, 0x52, + 0x9d, 0xa3, 0x17, 0xd0, 0x77, 0x58, 0xe7, 0xf6, 0x1b, 0xd8, 0x92, 0xda, 0xb0, 0xe6, 0x15, 0x9f, + 0x63, 0x79, 0x66, 0x1d, 0x59, 0x7a, 0x74, 0x1b, 0xfa, 0x27, 0xa6, 0x12, 0xaf, 0x2f, 0xd4, 0x96, + 0x2f, 0x94, 0xbe, 0xac, 0x07, 0xba, 0xeb, 0x9f, 0x41, 0xf7, 0xc9, 0x05, 0xc9, 0x3c, 0xf1, 0x01, + 0xb4, 0x73, 0x82, 0xf3, 0x19, 0x2d, 0x88, 0x4b, 0x6a, 0x18, 0xdb, 0xe5, 0x12, 0xfb, 0xe5, 0x12, + 0x3f, 0xf7, 0xcb, 0x25, 0xa9, 0xb0, 0x7e, 0x55, 0x6c, 0xbc, 0xba, 0x2a, 0x1a, 0x97, 0xab, 0x22, + 0x3a, 0x80, 0x9e, 0x0d, 0xe6, 0xee, 0x7f, 0x03, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, + 0x71, 0x27, 0xf4, 0x3e, 0x74, 0xc8, 0x05, 0x55, 0x69, 0xa6, 0xc7, 0x7a, 0xc3, 0xdc, 0xa0, 0xad, + 0x0d, 0x07, 0x2c, 0x27, 0xd1, 0xef, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, + 0xd5, 0x9f, 0x6f, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x6d, 0x9a, + 0x85, 0xf3, 0xe6, 0x6b, 0x1b, 0xdc, 0xfe, 0x9f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x81, + 0xa6, 0x9d, 0x7e, 0x74, 0xbf, 0xee, 0xd4, 0xad, 0xec, 0xda, 0xe1, 0x83, 0x75, 0x69, 0xae, 0x7e, + 0xd7, 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x6e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xb7, 0x1e, + 0xa9, 0x0a, 0xfa, 0x1b, 0xb4, 0xfd, 0x38, 0xa3, 0x87, 0x75, 0x7d, 0x5c, 0x91, 0x93, 0xe1, 0xe7, + 0xeb, 0x13, 0xab, 0x04, 0xfe, 0x08, 0x60, 0xe7, 0xca, 0x48, 0xa3, 0x2f, 0xeb, 0xfa, 0x7b, 0xbd, + 0xea, 0x0c, 0x1f, 0xbd, 0x35, 0xbf, 0x4a, 0xeb, 0x57, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, + 0x2a, 0x3f, 0xc3, 0x87, 0x6b, 0xf3, 0xaa, 0xe8, 0x17, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, + 0xb2, 0x76, 0x0d, 0xef, 0xaf, 0xc9, 0xf2, 0x71, 0xf7, 0x02, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, + 0xff, 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x2b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, + 0xd2, 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x35, 0xf4, 0x57, 0x00, 0x7d, 0x6d, 0x3a, 0x51, + 0x82, 0xe0, 0x39, 0x2d, 0x26, 0xe8, 0x51, 0x4d, 0xf1, 0xd6, 0x2c, 0x2b, 0xe0, 0x8e, 0xe9, 0x53, + 0xf9, 0xea, 0xed, 0x1d, 0xf8, 0xb4, 0xc6, 0xc1, 0x5e, 0xf0, 0x75, 0xeb, 0x87, 0x2d, 0xab, 0x59, + 0x4d, 0xf3, 0x73, 0xf7, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2e, 0x63, 0xc4, 0xd3, 0x74, 0x0c, + 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/drivers/shared/executor/proto/executor.proto b/drivers/shared/executor/proto/executor.proto index db0a7b3cb..aa349d265 100644 --- a/drivers/shared/executor/proto/executor.proto +++ b/drivers/shared/executor/proto/executor.proto @@ -45,6 +45,8 @@ message LaunchRequest { string default_pid_mode = 15; string default_ipc_mode = 16; string cpuset_cgroup = 17; + repeated string allow_caps = 18; + repeated string capabilities = 19; } message LaunchResponse { diff --git a/drivers/shared/executor/server.go b/drivers/shared/executor/server.go index 7ef91f4d3..9583bbb7b 100644 --- a/drivers/shared/executor/server.go +++ b/drivers/shared/executor/server.go @@ -37,6 +37,7 @@ func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchReques NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation), ModePID: req.DefaultPidMode, ModeIPC: req.DefaultIpcMode, + Capabilities: req.Capabilities, }) if err != nil { diff --git a/vendor/github.com/docker/docker/oci/caps/defaults.go b/vendor/github.com/docker/docker/oci/caps/defaults.go deleted file mode 100644 index 242ee5811..000000000 --- a/vendor/github.com/docker/docker/oci/caps/defaults.go +++ /dev/null @@ -1,21 +0,0 @@ -package caps // import "github.com/docker/docker/oci/caps" - -// DefaultCapabilities returns a Linux kernel default capabilities -func DefaultCapabilities() []string { - return []string{ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE", - } -} diff --git a/vendor/github.com/docker/docker/oci/caps/utils.go b/vendor/github.com/docker/docker/oci/caps/utils.go deleted file mode 100644 index ffd3f6f50..000000000 --- a/vendor/github.com/docker/docker/oci/caps/utils.go +++ /dev/null @@ -1,169 +0,0 @@ -package caps // import "github.com/docker/docker/oci/caps" - -import ( - "fmt" - "strings" - - "github.com/docker/docker/errdefs" - "github.com/syndtr/gocapability/capability" -) - -var capabilityList Capabilities - -func init() { - last := capability.CAP_LAST_CAP - // hack for RHEL6 which has no /proc/sys/kernel/cap_last_cap - if last == capability.Cap(63) { - last = capability.CAP_BLOCK_SUSPEND - } - for _, cap := range capability.List() { - if cap > last { - continue - } - capabilityList = append(capabilityList, - &CapabilityMapping{ - Key: "CAP_" + strings.ToUpper(cap.String()), - Value: cap, - }, - ) - } -} - -type ( - // CapabilityMapping maps linux capability name to its value of capability.Cap type - // Capabilities is one of the security systems in Linux Security Module (LSM) - // framework provided by the kernel. - // For more details on capabilities, see http://man7.org/linux/man-pages/man7/capabilities.7.html - CapabilityMapping struct { - Key string `json:"key,omitempty"` - Value capability.Cap `json:"value,omitempty"` - } - // Capabilities contains all CapabilityMapping - Capabilities []*CapabilityMapping -) - -// String returns of CapabilityMapping -func (c *CapabilityMapping) String() string { - return c.Key -} - -// GetCapability returns CapabilityMapping which contains specific key -func GetCapability(key string) *CapabilityMapping { - for _, capp := range capabilityList { - if capp.Key == key { - cpy := *capp - return &cpy - } - } - return nil -} - -// GetAllCapabilities returns all of the capabilities -func GetAllCapabilities() []string { - output := make([]string, len(capabilityList)) - for i, capability := range capabilityList { - output[i] = capability.String() - } - return output -} - -// inSlice tests whether a string is contained in a slice of strings or not. -func inSlice(slice []string, s string) bool { - for _, ss := range slice { - if s == ss { - return true - } - } - return false -} - -const allCapabilities = "ALL" - -// NormalizeLegacyCapabilities normalizes, and validates CapAdd/CapDrop capabilities -// by upper-casing them, and adding a CAP_ prefix (if not yet present). -// -// This function also accepts the "ALL" magic-value, that's used by CapAdd/CapDrop. -func NormalizeLegacyCapabilities(caps []string) ([]string, error) { - var normalized []string - - valids := GetAllCapabilities() - for _, c := range caps { - c = strings.ToUpper(c) - if c == allCapabilities { - normalized = append(normalized, c) - continue - } - if !strings.HasPrefix(c, "CAP_") { - c = "CAP_" + c - } - if !inSlice(valids, c) { - return nil, errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c)) - } - normalized = append(normalized, c) - } - return normalized, nil -} - -// ValidateCapabilities validates if caps only contains valid capabilities -func ValidateCapabilities(caps []string) error { - valids := GetAllCapabilities() - for _, c := range caps { - if !inSlice(valids, c) { - return errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c)) - } - } - return nil -} - -// TweakCapabilities tweaks capabilities by adding, dropping, or overriding -// capabilities in the basics capabilities list. -func TweakCapabilities(basics, adds, drops, capabilities []string, privileged bool) ([]string, error) { - switch { - case privileged: - // Privileged containers get all capabilities - return GetAllCapabilities(), nil - case capabilities != nil: - // Use custom set of capabilities - if err := ValidateCapabilities(capabilities); err != nil { - return nil, err - } - return capabilities, nil - case len(adds) == 0 && len(drops) == 0: - // Nothing to tweak; we're done - return basics, nil - } - - capDrop, err := NormalizeLegacyCapabilities(drops) - if err != nil { - return nil, err - } - capAdd, err := NormalizeLegacyCapabilities(adds) - if err != nil { - return nil, err - } - - var caps []string - - switch { - case inSlice(capAdd, allCapabilities): - // Add all capabilities except ones on capDrop - for _, c := range GetAllCapabilities() { - if !inSlice(capDrop, c) { - caps = append(caps, c) - } - } - case inSlice(capDrop, allCapabilities): - // "Drop" all capabilities; use what's in capAdd instead - caps = capAdd - default: - // First drop some capabilities - for _, c := range basics { - if !inSlice(capDrop, c) { - caps = append(caps, c) - } - } - // Then add the list of capabilities from capAdd - caps = append(caps, capAdd...) - } - return caps, nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 37bcd8f6c..063bfceb2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -241,7 +241,6 @@ github.com/docker/docker/api/types/swarm github.com/docker/docker/api/types/swarm/runtime github.com/docker/docker/api/types/versions github.com/docker/docker/errdefs -github.com/docker/docker/oci/caps github.com/docker/docker/pkg/archive github.com/docker/docker/pkg/fileutils github.com/docker/docker/pkg/homedir diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index 68e2bf4f4..16e248be7 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -452,30 +452,26 @@ config { - `cap_add` - (Optional) A list of Linux capabilities as strings to pass directly to [`--cap-add`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist. - The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration. + The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration. For example: - ```hcl - config { - cap_add = [ - "SYS_TIME", - ] - } - ``` +```hcl +config { + cap_add = ["net_raw", sys_time"] +} +``` - `cap_drop` - (Optional) A list of Linux capabilities as strings to pass directly to [`--cap-drop`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist. - The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration. + The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration. For example: - ```hcl - config { - cap_drop = [ - "MKNOD", - ] - } - ``` +```hcl +config { + cap_drop = ["mknod"] +} +``` - `cpu_hard_limit` - (Optional) `true` or `false` (default). Use hard CPU limiting instead of soft limiting. By default this is `false` which means @@ -797,10 +793,7 @@ plugin "docker" { } allow_privileged = false - allow_caps = ["CHOWN", "NET_RAW"] - - # allow_caps can also be set to "ALL" - # allow_caps = ["ALL"] + allow_caps = ["chown", "net_raw"] } } ``` @@ -823,13 +816,22 @@ plugin "docker" { from the Docker engine during an image pull within this timeframe, Nomad will timeout the request that initiated the pull command. (Minimum of `1m`) -- `allow_caps` - A list of allowed Linux capabilities. - Defaults to - `CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP, NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE` which is the list of - capabilities allowed by docker by default, as defined here. Allows the - operator to control which capabilities can be obtained by tasks using cap_add - and cap_drop options. Supports the value "ALL" as a shortcut for allowlisting - all capabilities. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is the same list of capabilities allowed by [docker by default][docker_caps] + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained + by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports + the value `"all"` as a shortcut for allow-listing all capabilities supported by + the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. - `allow_runtimes` - defaults to `["runc", "nvidia"]` - A list of the allowed docker runtimes a task may use. @@ -1136,3 +1138,8 @@ Windows is relatively new and rapidly evolving you may want to consult the [plugin-stanza]: /docs/configuration/plugin [allocation working directory]: /docs/runtime/environment#task-directories 'Task Directories' [`auth_soft_fail=true`]: #auth_soft_fail +[cap_add]: /docs/drivers/docker#cap_add +[cap_drop]: /docs/drivers/docker#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +[allow_caps]: /docs/drivers/docker#allow_caps diff --git a/website/content/docs/drivers/exec.mdx b/website/content/docs/drivers/exec.mdx index 71e491c99..065693233 100644 --- a/website/content/docs/drivers/exec.mdx +++ b/website/content/docs/drivers/exec.mdx @@ -54,6 +54,27 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `cap_add` - (Optional) A list of Linux capabilities to enable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_add = ["net_raw", "sys_time"] +} +``` + +- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_drop = ["all"] + cap_add = ["chown", "sys_chroot", "mknod"] +} +``` + ## Examples To run a binary present on the Node: @@ -138,6 +159,23 @@ able to make use of IPC features, like sending unexpected POSIX signals. for file system isolation without `pivot_root`. This is useful for systems where the root is on a ramdisk. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is modeled after the capabilities allowed by [docker by default][docker_caps] + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. + Supports the value `"all"` as a shortcut for allow-listing all capabilities supported + by the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. + ## Client Attributes The `exec` driver will set the following client attributes: @@ -200,3 +238,8 @@ This list is configurable through the agent client [default_pid_mode]: /docs/drivers/exec#default_pid_mode [default_ipc_mode]: /docs/drivers/exec#default_ipc_mode +[cap_add]: /docs/drivers/exec#cap_add +[cap_drop]: /docs/drivers/exec#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps]: /docs/drivers/exec#allow_caps +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities diff --git a/website/content/docs/drivers/java.mdx b/website/content/docs/drivers/java.mdx index 8124d491c..ad0957dbf 100644 --- a/website/content/docs/drivers/java.mdx +++ b/website/content/docs/drivers/java.mdx @@ -61,6 +61,27 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `cap_add` - (Optional) A list of Linux capabilities to enable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_add = ["net_raw", "sys_time"] +} +``` + +- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_drop = ["all"] + cap_add = ["chown", "sys_chroot", "mknod"] +} +``` + ## Examples A simple config block to run a Java Jar: @@ -138,6 +159,23 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is modeled after the capabilities allowed by [docker by default][docker_caps] + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. + Supports the value `"all"` as a shortcut for allow-listing all capabilities supported + by the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. + ## Client Requirements The `java` driver requires Java to be installed and in your system's `$PATH`. On @@ -208,3 +246,8 @@ This list is configurable through the agent client [default_pid_mode]: /docs/drivers/java#default_pid_mode [default_ipc_mode]: /docs/drivers/java#default_ipc_mode +[cap_add]: /docs/drivers/java#cap_add +[cap_drop]: /docs/drivers/java#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps]: /docs/drivers/java#allow_caps +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 2ff1ade19..0a5c1ac61 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -54,7 +54,37 @@ these fields. Connect native tasks running in host networking mode will now have `CONSUL_HTTP_ADDR` set automatically. Before this was only the case for bridge networking. If an operator -already explicitly set `CONSUL_HTTP_ADDR` then it will not get overriden. +already explicitly set `CONSUL_HTTP_ADDR` then it will not get overridden. + +#### Linux capabilities in exec/java + +Following the security [remediation][no_net_raw] in Nomad versions 0.12.12, 1.0.5, +and 1.1.0-rc1, the `exec` and `java` task drivers will additionally no longer enable +the following linux capabilities by default. + +``` +AUDIT_CONTROL AUDIT_READ BLOCK_SUSPEND DAC_READ_SEARCH IPC_LOCK IPC_OWNER LEASE +LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST NET_RAW SYS_ADMIN +SYS_BOOT SYSLOG SYS_MODULE SYS_NICE SYS_PACCT SYS_PTRACE SYS_RAWIO SYS_RESOURCE +SYS_TIME SYS_TTY_CONFIG WAKE_ALARM +``` + +The capabilities now enabled by default are modeled after Docker default +[`linux capabilities`] (excluding `NET_RAW`). + +``` +AUDIT_WRITE CHOWN DAC_OVERRIDE FOWNER FSETID KILL MKNOD NET_BIND_SERVICE +SETFCAP SETGID SETPCAP SETUID SYS_CHROOT +``` + +A new `allow_caps` plugin configuration parameter for [`exec`][allow_caps_exec] +and [`java`][allow_caps_java] task drivers can be used to restrict the set of +capabilities allowed for use by tasks. + +Tasks using the `exec` or `java` task drivers can add or remove desired linux +capabilities using the [`cap_add`][cap_add_exec] and [`cap_drop`][cap_drop_exec] +task configuration options. + #### iptables @@ -63,9 +93,9 @@ inserting them as the first rule. This allows better control for user-defined iptables rules but users who append rules currently should verify that their rules are being appended in the correct order. -## Nomad 1.1.0, 1.0.5, 0.12.12 +## Nomad 1.1.0-rc1, 1.0.5, 0.12.12 -Nomad versions 1.1.0, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, +Nomad versions 1.1.0-rc1, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, and `java` task drivers so that the [`CAP_NET_RAW`] linux capability is disabled by default. This is one of the [`linux capabilities`] that Docker itself enables by default, as this capability enables the generation of ICMP packets - used by @@ -1111,3 +1141,8 @@ deleted and then Nomad 0.3.0 can be launched. [`CAP_NET_RAW`]: https://security.stackexchange.com/a/128988 [`linux capabilities`]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities [`allow_caps`]: /docs/drivers/docker#allow_caps +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps_exec]: /docs/drivers/exec#allow_caps +[allow_caps_java]: /docs/drivers/java#allow_caps +[cap_add_exec]: /docs/drivers/exec#cap_add +[cap_drop_exec]: /docs/drivers/exec#cap_drop