From 5b8a32f23d09a42e93e27d4bcbe072c25fc88bfc Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 13 May 2021 14:21:52 -0600 Subject: [PATCH 1/8] drivers/exec: enable setting allow_caps on exec driver This PR enables setting allow_caps on the exec driver plugin configuration, as well as cap_add and cap_drop in exec task configuration. These options replicate the functionality already present in the docker task driver. Important: this change also reduces the default set of capabilities enabled by the exec driver to match the default set enabled by the docker driver. Until v1.0.5 the exec task driver would enable all capabilities supported by the operating system. v1.0.5 removed NET_RAW from that list of default capabilities, but left may others which could potentially also be leveraged by compromised tasks. Important: the "root" user is still special cased when used with the exec driver. Older versions of Nomad enabled enabled all capabilities supported by the operating system for tasks set with the root user. To maintain compatibility with existing clusters we continue supporting this "feature", however we maintain support for the legacy set of capabilities rather than enabling all capabilities now supported on modern operating systems. --- drivers/docker/config.go | 13 +- drivers/exec/driver.go | 73 +++++++- drivers/exec/driver_test.go | 125 ++++++++++---- drivers/exec/driver_unix_test.go | 129 +++++++++++++- drivers/shared/capabilities/defaults.go | 114 ++++++++++++ drivers/shared/capabilities/defaults_test.go | 23 +++ drivers/shared/capabilities/set.go | 110 ++++++++++++ drivers/shared/capabilities/set_test.go | 162 ++++++++++++++++++ drivers/shared/executor/executor.go | 3 + drivers/shared/executor/executor_linux.go | 34 ++-- .../shared/executor/executor_linux_test.go | 11 +- drivers/shared/executor/proto/executor.pb.go | 140 ++++++++------- drivers/shared/executor/proto/executor.proto | 1 + 13 files changed, 806 insertions(+), 132 deletions(-) create mode 100644 drivers/shared/capabilities/defaults.go create mode 100644 drivers/shared/capabilities/defaults_test.go create mode 100644 drivers/shared/capabilities/set.go create mode 100644 drivers/shared/capabilities/set_test.go diff --git a/drivers/docker/config.go b/drivers/docker/config.go index ec919360d..898d9172c 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -10,6 +10,7 @@ import ( docker "github.com/fsouza/go-dockerclient" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" @@ -287,7 +288,7 @@ var ( "allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false), "allow_caps": hclspec.NewDefault( hclspec.NewAttr("allow_caps", "list(string)", false), - hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), ), "nvidia_runtime": hclspec.NewDefault( hclspec.NewAttr("nvidia_runtime", "string", false), @@ -427,9 +428,9 @@ var ( "work_dir": hclspec.NewAttr("work_dir", "string", false), }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ + // driverCapabilities represents the RPC response for what features are + // implemented by the docker task driver + driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, FSIsolation: drivers.FSIsolationImage, @@ -788,8 +789,10 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } +// Capabilities is returned by the Capabilities RPC and indicates what optional +// features this driver supports. func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } var _ drivers.InternalCapabilitiesDriver = (*Driver)(nil) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index b7cfb0058..435796508 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -74,6 +75,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -83,11 +88,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ + // driverCapabilities represents the RPC response for what features are + // implemented by the exec task driver + driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, FSIsolation: drivers.FSIsolationChroot, @@ -141,6 +148,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -156,6 +167,11 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } @@ -174,6 +190,12 @@ type TaskConfig struct { // ModeIPC indicates whether IPC namespace isolation is enabled for the task. // Must be "private" or "host" if set. ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -189,6 +211,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -266,8 +298,33 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } +// getCaps computes the complete set of linux capabilities to enable for driver, +// which gets passed along to libcontainer. +func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) { + driverAllowed := capabilities.New(d.config.AllowCaps) + + // determine caps the task wants that are not allowed + taskCaps := capabilities.New(tc.CapAdd) + missing := driverAllowed.Difference(taskCaps) + if !missing.Empty() { + return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // if task did not specify allowed caps, use nomad defaults minus task drops + if len(tc.CapAdd) == 0 { + driverAllowed.Remove(tc.CapDrop) + return driverAllowed.Slice(true), nil + } + + // otherwise task did specify allowed caps, enable exactly those + taskAdd := capabilities.New(tc.CapAdd) + return taskAdd.Slice(true), nil +} + +// Capabilities is returned by the Capabilities RPC and indicates what +// optional features this driver supports func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -439,6 +496,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := d.getCaps(&driverConfig) + if err != nil { + return nil, nil, err + } + execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, Args: driverConfig.Args, @@ -455,6 +517,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -482,7 +545,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err := handle.SetDriverState(&driverState); err != nil { d.logger.Error("failed to start task, error setting driver state", "error", err) - exec.Shutdown("", 0) + _ = exec.Shutdown("", 0) pluginClient.Kill() return nil, nil, fmt.Errorf("failed to set driver state: %v", err) } diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index be5dda5cf..8651eda20 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -764,42 +764,99 @@ func TestExecDriver_NoPivotRoot(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) } func TestDriver_TaskConfig_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "host", ipcMode: "", exp: nil}, - {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "", ipcMode: "host", exp: nil}, - {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&TaskConfig{ - ModePID: tc.pidMode, - ModeIPC: tc.ipcMode, - }).validate()) - } + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/exec/driver_unix_test.go b/drivers/exec/driver_unix_test.go index f8d77a2d7..3b2e60374 100644 --- a/drivers/exec/driver_unix_test.go +++ b/drivers/exec/driver_unix_test.go @@ -5,9 +5,13 @@ package exec import ( "context" "fmt" + "strings" "testing" "time" + "github.com/hashicorp/nomad/drivers/shared/capabilities" + "github.com/hashicorp/nomad/drivers/shared/executor" + basePlug "github.com/hashicorp/nomad/plugins/base" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -173,5 +177,128 @@ func TestExec_dnsConfig(t *testing.T) { dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) } - +} + +func TestExecDriver_Capabilities(t *testing.T) { + ctestutils.ExecCompatible(t) + + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "sleep", + } + + for _, tc := range []struct { + Name string + CapAdd []string + CapDrop []string + AllowList string + StartError string + }{ + { + Name: "default-allowlist-add-allowed", + CapAdd: []string{"fowner", "mknod"}, + CapDrop: []string{"ALL"}, + }, + { + Name: "default-allowlist-add-forbidden", + CapAdd: []string{"net_admin"}, + StartError: "net_admin", + }, + { + Name: "default-allowlist-drop-existing", + CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, + }, + { + Name: "restrictive-allowlist-drop-all", + CapDrop: []string{"ALL"}, + AllowList: "FOWNER,MKNOD", + }, + { + Name: "restrictive-allowlist-add-allowed", + CapAdd: []string{"fowner", "mknod"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + }, + { + Name: "restrictive-allowlist-add-forbidden", + CapAdd: []string{"net_admin", "mknod"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + StartError: "net_admin", + }, + { + Name: "restrictive-allowlist-add-multiple-forbidden", + CapAdd: []string{"net_admin", "mknod", "CAP_SYS_TIME"}, + CapDrop: []string{"ALL"}, + AllowList: "fowner,mknod", + StartError: "net_admin, sys_time", + }, + { + Name: "permissive-allowlist", + CapAdd: []string{"net_admin", "mknod"}, + AllowList: "ALL", + }, + { + Name: "permissive-allowlist-add-all", + CapAdd: []string{"all"}, + AllowList: "ALL", + }, + } { + t.Run(tc.Name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + d := NewExecDriver(ctx, testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + config := &Config{ + NoPivotRoot: true, + DefaultModePID: executor.IsolationModePrivate, + DefaultModeIPC: executor.IsolationModePrivate, + } + + if tc.AllowList != "" { + config.AllowCaps = strings.Split(tc.AllowList, ",") + } else { + // inherit HCL defaults if not set + config.AllowCaps = capabilities.NomadDefaults().Slice(true) + } + + var data []byte + require.NoError(t, basePlug.MsgPackEncode(&data, config)) + baseConfig := &basePlug.Config{PluginConfig: data} + require.NoError(t, harness.SetConfig(baseConfig)) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + tCfg := &TaskConfig{ + Command: "/bin/sleep", + Args: []string{"9000"}, + } + if len(tc.CapAdd) > 0 { + tCfg.CapAdd = tc.CapAdd + } + if len(tc.CapDrop) > 0 { + tCfg.CapDrop = tc.CapDrop + } + require.NoError(t, task.EncodeConcreteDriverConfig(&tCfg)) + + // check the start error against expectations + _, _, err := harness.StartTask(task) + if err == nil && tc.StartError != "" { + t.Fatalf("Expected error in start: %v", tc.StartError) + } else if err != nil { + if tc.StartError == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), tc.StartError) + } + return + } + + _ = d.DestroyTask(task.ID, true) + }) + } } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go new file mode 100644 index 000000000..a1a6e3531 --- /dev/null +++ b/drivers/shared/capabilities/defaults.go @@ -0,0 +1,114 @@ +package capabilities + +import ( + "regexp" + + "github.com/syndtr/gocapability/capability" +) + +const ( + // HCLSpecLiteral is an equivalent list to NomadDefaults, expressed as a literal + // HCL string for use in HCL config parsing. + HCLSpecLiteral = `["AUDIT_WRITE","CHOWN","DAC_OVERRIDE","FOWNER","FSETID","KILL","MKNOD","NET_BIND_SERVICE","SETFCAP","SETGID","SETPCAP","SETUID","SYS_CHROOT"]` +) + +var ( + extractLiteral = regexp.MustCompile(`("[\w]+)`) +) + +// NomadDefaults is the set of Linux capabilities that Nomad enables by +// default. This list originates from what Docker enabled by default, but then +// excludes NET_RAW for security reasons. +// +// This set is use in the as HCL configuration default, described by HCLSpecLiteral. +func NomadDefaults() *Set { + return New(extractLiteral.FindAllString(HCLSpecLiteral, -1)) +} + +// DockerDefaults is a list of Linux capabilities enabled by Docker by default +// and is used to compute the set of capabilities to add/drop given docker driver +// configuration. +// +// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +func DockerDefaults() *Set { + defaults := NomadDefaults() + defaults.Add("NET_RAW") + return defaults +} + +// Supported returns the set of capabilities supported by the operating system. +// +// Defers to a library generated from +// https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h +func Supported() *Set { + s := New(nil) + + last := capability.CAP_LAST_CAP + + // workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap + if last == capability.Cap(63) { + last = capability.CAP_BLOCK_SUSPEND + } + + // accumulate every capability supported by this system + for _, c := range capability.List() { + if c > last { + continue + } + s.Add(c.String()) + } + + return s +} + +// LegacySupported returns the historical set of capabilities used when a task is +// configured to run as root using the exec task driver. Older versions of Nomad +// always allowed the root user to make use of any capability. Now that the exec +// task driver supports configuring the allowed capabilities, operators are +// encouraged to explicitly opt-in to capabilities beyond this legacy set. We +// maintain the legacy list here, because previous versions of Nomad deferred to +// the capability.List library function, which adds new capabilities over time. +// +// https://github.com/hashicorp/nomad/blob/v1.0.4/vendor/github.com/syndtr/gocapability/capability/enum_gen.go#L88 +func LegacySupported() *Set { + return New([]string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_MAC_OVERRIDE", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND", + "CAP_AUDIT_READ", + }) +} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go new file mode 100644 index 000000000..2a07afa0e --- /dev/null +++ b/drivers/shared/capabilities/defaults_test.go @@ -0,0 +1,23 @@ +package capabilities + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSet_NomadDefaults(t *testing.T) { + result := NomadDefaults() + require.Len(t, result.Slice(false), 13) + defaults := strings.ToLower(HCLSpecLiteral) + for _, c := range result.Slice(false) { + require.Contains(t, defaults, c) + } +} + +func TestSet_DockerDefaults(t *testing.T) { + result := DockerDefaults() + require.Len(t, result.Slice(false), 14) + require.Contains(t, result.String(), "net_raw") +} diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go new file mode 100644 index 000000000..046573e2a --- /dev/null +++ b/drivers/shared/capabilities/set.go @@ -0,0 +1,110 @@ +// Package capabilities is used for managing sets of linux capabilities. +package capabilities + +import ( + "sort" + "strings" +) + +type nothing struct{} + +var null = nothing{} + +// Set represents a group linux capabilities, implementing some useful set +// operations, taking care of name normalization, and sentinel value expansions. +// +// Linux capabilities can be expressed in multiple ways when working with docker +// and/or libcontainer, along with Nomad. +// +// Capability names may be upper or lower case, and may or may not be prefixed +// with "CAP_" or "cap_". On top of that, Nomad interprets the special name "all" +// and "ALL" to mean "all capabilities supported by the operating system". +type Set struct { + data map[string]nothing +} + +func New(caps []string) *Set { + m := make(map[string]nothing, len(caps)) + for _, c := range caps { + insert(m, c) + } + return &Set{data: m} +} + +// Add cap into s. +func (s *Set) Add(cap string) { + insert(s.data, cap) +} + +func insert(data map[string]nothing, cap string) { + switch name := normalize(cap); name { + case "": + case "all": + for k, v := range Supported().data { + data[k] = v + } + return + default: + data[name] = null + } +} + +// Remove caps from s. +func (s *Set) Remove(caps []string) { + for _, c := range caps { + name := normalize(c) + if name == "all" { + s.data = make(map[string]nothing) + return + } + delete(s.data, name) + } +} + +// Difference returns the Set of elements of b not in s. +func (s *Set) Difference(b *Set) *Set { + data := make(map[string]nothing) + for c := range b.data { + if _, exists := s.data[c]; !exists { + data[c] = null + } + } + return &Set{data: data} +} + +// Empty return true if no capabilities exist in s. +func (s *Set) Empty() bool { + return len(s.data) == 0 +} + +// String returns the normalized and sorted string representation of s. +func (s *Set) String() string { + return strings.Join(s.Slice(false), ", ") +} + +// Slice returns a sorted slice of capabilities in s. +// +// big - indicates whether to uppercase and prefix capabilities with CAP_ +func (s *Set) Slice(upper bool) []string { + caps := make([]string, 0, len(s.data)) + for c := range s.data { + if upper { + c = "CAP_" + strings.ToUpper(c) + } + caps = append(caps, c) + } + sort.Strings(caps) + return caps +} + +// linux capabilities are often named in 4 possible ways - upper or lower case, +// and with or without a CAP_ prefix +// +// since we must do comparisons on cap names, always normalize the names before +// letting them into the Set data-structure +func normalize(name string) string { + spaces := strings.TrimSpace(name) + lower := strings.ToLower(spaces) + trim := strings.TrimPrefix(lower, "cap_") + return trim +} diff --git a/drivers/shared/capabilities/set_test.go b/drivers/shared/capabilities/set_test.go new file mode 100644 index 000000000..8e072e8b0 --- /dev/null +++ b/drivers/shared/capabilities/set_test.go @@ -0,0 +1,162 @@ +package capabilities + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSet_Empty(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + result := New(nil).Empty() + require.True(t, result) + }) + + t.Run("empty", func(t *testing.T) { + result := New([]string{}).Empty() + require.True(t, result) + }) + + t.Run("full", func(t *testing.T) { + result := New([]string{"chown", "sys_time"}).Empty() + require.False(t, result) + }) +} + +func TestSet_New(t *testing.T) { + t.Parallel() + + t.Run("duplicates", func(t *testing.T) { + result := New([]string{"chown", "sys_time", "chown"}) + require.Equal(t, "chown, sys_time", result.String()) + }) + + t.Run("empty string", func(t *testing.T) { + result := New([]string{""}) + require.True(t, result.Empty()) + }) + + t.Run("all", func(t *testing.T) { + result := New([]string{"all"}) + exp := len(Supported().Slice(false)) + require.Len(t, result.Slice(false), exp) + }) +} + +func TestSet_Slice(t *testing.T) { + t.Parallel() + + exp := []string{"chown", "net_raw", "sys_time"} + + t.Run("lower case", func(t *testing.T) { + s := New([]string{"net_raw", "chown", "sys_time"}) + require.Equal(t, exp, s.Slice(false)) + }) + + t.Run("upper case", func(t *testing.T) { + s := New([]string{"NET_RAW", "CHOWN", "SYS_TIME"}) + require.Equal(t, exp, s.Slice(false)) + }) + + t.Run("prefix", func(t *testing.T) { + s := New([]string{"CAP_net_raw", "sys_TIME", "cap_chown"}) + require.Equal(t, exp, s.Slice(false)) + }) +} + +func TestSet_String(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + result := New(nil).String() + require.Equal(t, "", result) + }) + + t.Run("full", func(t *testing.T) { + exp := "chown, net_raw, sys_time" + in := []string{"net_raw", "CAP_CHOWN", "cap_sys_time"} + result := New(in).String() + require.Equal(t, exp, result) + }) +} + +func TestSet_Add(t *testing.T) { + t.Parallel() + + t.Run("add one", func(t *testing.T) { + s := New([]string{"chown", "net_raw"}) + require.Equal(t, "chown, net_raw", s.String()) + + s.Add("CAP_SYS_TIME") + require.Equal(t, "chown, net_raw, sys_time", s.String()) + + s.Add("AF_NET") + require.Equal(t, "af_net, chown, net_raw, sys_time", s.String()) + }) + + t.Run("add empty string", func(t *testing.T) { + s := New([]string{"chown"}) + s.Add("") + require.Equal(t, "chown", s.String()) + }) + + t.Run("add all", func(t *testing.T) { + s := New([]string{"chown", "net_raw"}) + require.Equal(t, "chown, net_raw", s.String()) + + exp := len(Supported().Slice(false)) + s.Add("all") + require.Len(t, s.Slice(false), exp) + }) + +} + +func TestSet_Remove(t *testing.T) { + t.Parallel() + + t.Run("remove one", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"CAP_NET_RAW"}) + require.Equal(t, "af_net, chown, seteuid, sys_time", s.String()) + }) + + t.Run("remove couple", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"CAP_NET_RAW", "af_net"}) + require.Equal(t, "chown, seteuid, sys_time", s.String()) + }) + + t.Run("remove all", func(t *testing.T) { + s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"}) + s.Remove([]string{"all"}) + require.True(t, s.Empty()) + require.Equal(t, "", s.String()) + }) +} + +func TestSet_Difference(t *testing.T) { + t.Parallel() + + t.Run("a is empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"chown", "af_net"}) + result := a.Difference(b) + require.Equal(t, "af_net, chown", result.String()) + }) + + t.Run("b is empty", func(t *testing.T) { + a := New([]string{"chown", "af_net"}) + b := New(nil) + result := a.Difference(b) + require.True(t, result.Empty()) + }) + + t.Run("a diff b", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f"}) + b := New([]string{"B", "x", "Y", "a"}) + result := a.Difference(b) + require.Equal(t, "x, y", result.String()) + }) +} diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index a87674d2a..7d738e45f 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -147,6 +147,9 @@ type ExecCommand struct { // ModeIPC is the IPC isolation mode (private or host). ModeIPC string + + // Capabilities are the linux capabilities to be enabled by the task driver. + Capabilities []string } // SetWriters sets the writer for the process stdout and stderr. This should diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 16eb27b4c..1bbb2839c 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/opencontainers/runtime-spec/specs-go" "github.com/armon/circbuf" @@ -532,28 +533,26 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc ch <- &waitResult{ps, err} } -func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error { - // TODO(shoenig): allow better control of these - // use capabilities list as prior to adopting libcontainer in 0.9 +func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) { - // match capabilities used in Nomad 0.8 - if command.User == "root" { - allCaps := SupportedCaps(true) + switch command.User { + case "root": + // when running as root, use the legacy set of system capabilities, so + // that we do not break existing nomad clusters using this "feature" + legacyCaps := capabilities.LegacySupported().Slice(true) cfg.Capabilities = &lconfigs.Capabilities{ - Bounding: allCaps, - Permitted: allCaps, - Effective: allCaps, + Bounding: legacyCaps, + Permitted: legacyCaps, + Effective: legacyCaps, Ambient: nil, Inheritable: nil, } - } else { - allCaps := SupportedCaps(false) + default: + // otherwise apply the plugin + task capability configuration cfg.Capabilities = &lconfigs.Capabilities{ - Bounding: allCaps, + Bounding: command.Capabilities, } } - - return nil } func configureNamespaces(pidMode, ipcMode string) lconfigs.Namespaces { @@ -759,16 +758,17 @@ func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) { }, Version: "1.0.0", } + for _, device := range specconv.AllowedDevices { cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule) } - if err := configureCapabilities(cfg, command); err != nil { - return nil, err - } + configureCapabilities(cfg, command) + if err := configureIsolation(cfg, command); err != nil { return nil, err } + if err := configureCgroups(cfg, command); err != nil { return nil, err } diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index a8bfba0a4..b3576797a 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/plugins/drivers" @@ -478,7 +479,7 @@ func TestExecutor_Capabilities(t *testing.T) { CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 -CapBnd: 0000003fffffdfff +CapBnd: 00000000a80405fb CapAmb: 0000000000000000`, }, { @@ -494,7 +495,6 @@ CapAmb: 0000000000000000`, for _, c := range cases { t.Run(c.user, func(t *testing.T) { - require := require.New(t) testExecCmd := testExecutorCommandWithChroot(t) execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir @@ -504,12 +504,13 @@ CapAmb: 0000000000000000`, execCmd.ResourceLimits = true execCmd.Cmd = "/bin/bash" execCmd.Args = []string{"-c", "cat /proc/$$/status"} + execCmd.Capabilities = capabilities.NomadDefaults().Slice(true) executor := NewExecutorWithIsolation(testlog.HCLogger(t)) defer executor.Shutdown("SIGKILL", 0) _, err := executor.Launch(execCmd) - require.NoError(err) + require.NoError(t, err) ch := make(chan interface{}) go func() { @@ -521,7 +522,7 @@ CapAmb: 0000000000000000`, case <-ch: // all good case <-time.After(5 * time.Second): - require.Fail("timeout waiting for exec to shutdown") + require.Fail(t, "timeout waiting for exec to shutdown") } canonical := func(s string) string { @@ -538,7 +539,7 @@ CapAmb: 0000000000000000`, return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output) } return true, nil - }, func(err error) { require.NoError(err) }) + }, func(err error) { require.NoError(t, err) }) }) } diff --git a/drivers/shared/executor/proto/executor.pb.go b/drivers/shared/executor/proto/executor.pb.go index f6defb54a..bf6c980c2 100644 --- a/drivers/shared/executor/proto/executor.pb.go +++ b/drivers/shared/executor/proto/executor.pb.go @@ -44,6 +44,7 @@ type LaunchRequest struct { DefaultPidMode string `protobuf:"bytes,15,opt,name=default_pid_mode,json=defaultPidMode,proto3" json:"default_pid_mode,omitempty"` DefaultIpcMode string `protobuf:"bytes,16,opt,name=default_ipc_mode,json=defaultIpcMode,proto3" json:"default_ipc_mode,omitempty"` CpusetCgroup string `protobuf:"bytes,17,opt,name=cpuset_cgroup,json=cpusetCgroup,proto3" json:"cpuset_cgroup,omitempty"` + AllowCaps []string `protobuf:"bytes,18,rep,name=allow_caps,json=allowCaps,proto3" json:"allow_caps,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -193,6 +194,13 @@ func (m *LaunchRequest) GetCpusetCgroup() string { return "" } +func (m *LaunchRequest) GetAllowCaps() []string { + if m != nil { + return m.AllowCaps + } + return nil +} + type LaunchResponse struct { Process *ProcessState `protobuf:"bytes,1,opt,name=process,proto3" json:"process,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -858,71 +866,73 @@ func init() { } var fileDescriptor_66b85426380683f3 = []byte{ - // 1020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x5b, 0x8f, 0xdb, 0x44, - 0x14, 0xae, 0x37, 0x9b, 0xdb, 0x49, 0xb2, 0x49, 0x47, 0xa8, 0xb8, 0xe1, 0xa1, 0xc1, 0x48, 0x34, - 0x82, 0xe2, 0xac, 0xb6, 0x37, 0x24, 0x24, 0x8a, 0xd8, 0x2d, 0xa8, 0xd2, 0x76, 0x15, 0x39, 0x85, - 0x4a, 0x3c, 0x60, 0x5c, 0xcf, 0x34, 0x19, 0x6d, 0xe2, 0x31, 0x33, 0xe3, 0x74, 0x91, 0x90, 0x78, - 0xe2, 0x1f, 0x80, 0xc4, 0x5f, 0xe5, 0x0d, 0xcd, 0xcd, 0x9b, 0x6c, 0x4b, 0xe5, 0x14, 0xf1, 0x14, - 0xcf, 0xc9, 0xf7, 0x9d, 0xcb, 0x9c, 0x73, 0xbe, 0x81, 0x3b, 0x98, 0xd3, 0x35, 0xe1, 0x62, 0x22, - 0x16, 0x09, 0x27, 0x78, 0x42, 0x2e, 0x48, 0x5a, 0x48, 0xc6, 0x27, 0x39, 0x67, 0x92, 0x95, 0xc7, - 0x50, 0x1f, 0xd1, 0xc7, 0x8b, 0x44, 0x2c, 0x68, 0xca, 0x78, 0x1e, 0x66, 0x6c, 0x95, 0xe0, 0x30, - 0x5f, 0x16, 0x73, 0x9a, 0x89, 0x70, 0x1b, 0x37, 0xbc, 0x35, 0x67, 0x6c, 0xbe, 0x24, 0xc6, 0xc9, - 0x8b, 0xe2, 0xe5, 0x44, 0xd2, 0x15, 0x11, 0x32, 0x59, 0xe5, 0x16, 0x10, 0x58, 0xe2, 0xc4, 0x85, - 0x37, 0xe1, 0xcc, 0xc9, 0x60, 0x82, 0xbf, 0xeb, 0xd0, 0x3b, 0x4d, 0x8a, 0x2c, 0x5d, 0x44, 0xe4, - 0xe7, 0x82, 0x08, 0x89, 0x06, 0x50, 0x4b, 0x57, 0xd8, 0xf7, 0x46, 0xde, 0xb8, 0x1d, 0xa9, 0x4f, - 0x84, 0x60, 0x3f, 0xe1, 0x73, 0xe1, 0xef, 0x8d, 0x6a, 0xe3, 0x76, 0xa4, 0xbf, 0xd1, 0x19, 0xb4, - 0x39, 0x11, 0xac, 0xe0, 0x29, 0x11, 0x7e, 0x6d, 0xe4, 0x8d, 0x3b, 0x47, 0x87, 0xe1, 0xbf, 0x25, - 0x6e, 0xe3, 0x9b, 0x90, 0x61, 0xe4, 0x78, 0xd1, 0xa5, 0x0b, 0x74, 0x0b, 0x3a, 0x42, 0x62, 0x56, - 0xc8, 0x38, 0x4f, 0xe4, 0xc2, 0xdf, 0xd7, 0xd1, 0xc1, 0x98, 0xa6, 0x89, 0x5c, 0x58, 0x00, 0xe1, - 0xdc, 0x00, 0xea, 0x25, 0x80, 0x70, 0xae, 0x01, 0x03, 0xa8, 0x91, 0x6c, 0xed, 0x37, 0x74, 0x92, - 0xea, 0x53, 0xe5, 0x5d, 0x08, 0xc2, 0xfd, 0xa6, 0xc6, 0xea, 0x6f, 0x74, 0x13, 0x5a, 0x32, 0x11, - 0xe7, 0x31, 0xa6, 0xdc, 0x6f, 0x69, 0x7b, 0x53, 0x9d, 0x4f, 0x28, 0x47, 0xb7, 0xa1, 0xef, 0xf2, - 0x89, 0x97, 0x74, 0x45, 0xa5, 0xf0, 0xdb, 0x23, 0x6f, 0xdc, 0x8a, 0x0e, 0x9c, 0xf9, 0x54, 0x5b, - 0xd1, 0x21, 0xbc, 0xf7, 0x22, 0x11, 0x34, 0x8d, 0x73, 0xce, 0x52, 0x22, 0x44, 0x9c, 0xce, 0x39, - 0x2b, 0x72, 0x1f, 0x34, 0x1a, 0xe9, 0xff, 0xa6, 0xe6, 0xaf, 0x63, 0xfd, 0x0f, 0x3a, 0x81, 0xc6, - 0x8a, 0x15, 0x99, 0x14, 0x7e, 0x67, 0x54, 0x1b, 0x77, 0x8e, 0xee, 0x54, 0xbc, 0xaa, 0xa7, 0x8a, - 0x14, 0x59, 0x2e, 0xfa, 0x16, 0x9a, 0x98, 0xac, 0xa9, 0xba, 0xf1, 0xae, 0x76, 0xf3, 0x59, 0x45, - 0x37, 0x27, 0x9a, 0x15, 0x39, 0x36, 0x5a, 0xc0, 0xf5, 0x8c, 0xc8, 0x57, 0x8c, 0x9f, 0xc7, 0x54, - 0xb0, 0x65, 0x22, 0x29, 0xcb, 0xfc, 0x9e, 0x6e, 0xe2, 0x17, 0x15, 0x5d, 0x9e, 0x19, 0xfe, 0x13, - 0x47, 0x9f, 0xe5, 0x24, 0x8d, 0x06, 0xd9, 0x15, 0x2b, 0x0a, 0xa0, 0x97, 0xb1, 0x38, 0xa7, 0x6b, - 0x26, 0x63, 0xce, 0x98, 0xf4, 0x0f, 0xf4, 0x1d, 0x75, 0x32, 0x36, 0x55, 0xb6, 0x88, 0x31, 0x89, - 0xc6, 0x30, 0xc0, 0xe4, 0x65, 0x52, 0x2c, 0x65, 0x9c, 0x53, 0x1c, 0xaf, 0x18, 0x26, 0x7e, 0x5f, - 0xb7, 0xe6, 0xc0, 0xda, 0xa7, 0x14, 0x3f, 0x65, 0x98, 0x6c, 0x22, 0x69, 0x9e, 0x1a, 0xe4, 0x60, - 0x0b, 0xf9, 0x24, 0x4f, 0x35, 0xf2, 0x23, 0xe8, 0xa5, 0x79, 0x21, 0x88, 0x74, 0xbd, 0xb9, 0xae, - 0x61, 0x5d, 0x63, 0x34, 0x5d, 0x09, 0x7e, 0x82, 0x03, 0x37, 0xfa, 0x22, 0x67, 0x99, 0x20, 0xe8, - 0x0c, 0x9a, 0xb6, 0xa7, 0x7a, 0xfe, 0x3b, 0x47, 0xf7, 0xc2, 0x6a, 0xcb, 0x18, 0xda, 0x7e, 0xcf, - 0x64, 0x22, 0x49, 0xe4, 0x9c, 0x04, 0x3d, 0xe8, 0x3c, 0x4f, 0xa8, 0xb4, 0xab, 0x15, 0xfc, 0x08, - 0x5d, 0x73, 0xfc, 0x9f, 0xc2, 0x9d, 0x42, 0x7f, 0xb6, 0x28, 0x24, 0x66, 0xaf, 0x32, 0xb7, 0xcd, - 0x37, 0xa0, 0x21, 0xe8, 0x3c, 0x4b, 0x96, 0x76, 0xa1, 0xed, 0x09, 0x7d, 0x08, 0xdd, 0x39, 0x4f, - 0x52, 0x12, 0xe7, 0x84, 0x53, 0x86, 0xfd, 0xbd, 0x91, 0x37, 0xae, 0x45, 0x1d, 0x6d, 0x9b, 0x6a, - 0x53, 0x80, 0x60, 0x70, 0xe9, 0xcd, 0x64, 0x1c, 0x2c, 0xe0, 0xc6, 0x77, 0x39, 0x56, 0x41, 0xcb, - 0x25, 0xb6, 0x81, 0xb6, 0x04, 0xc1, 0xfb, 0xcf, 0x82, 0x10, 0xdc, 0x84, 0xf7, 0x5f, 0x8b, 0x64, - 0x93, 0x18, 0xc0, 0xc1, 0xf7, 0x84, 0x0b, 0xca, 0x5c, 0x95, 0xc1, 0xa7, 0xd0, 0x2f, 0x2d, 0xf6, - 0x6e, 0x7d, 0x68, 0xae, 0x8d, 0xc9, 0x56, 0xee, 0x8e, 0xc1, 0x27, 0xd0, 0x55, 0xf7, 0x56, 0x66, - 0x3e, 0x84, 0x16, 0xcd, 0x24, 0xe1, 0x6b, 0x7b, 0x49, 0xb5, 0xa8, 0x3c, 0x07, 0xcf, 0xa1, 0x67, - 0xb1, 0xd6, 0xed, 0x37, 0x50, 0x17, 0xca, 0xb0, 0x63, 0x89, 0xcf, 0x12, 0x71, 0x6e, 0x1c, 0x19, - 0x7a, 0x70, 0x1b, 0x7a, 0x33, 0xdd, 0x89, 0x37, 0x37, 0xaa, 0xee, 0x1a, 0xa5, 0x8a, 0x75, 0x40, - 0x5b, 0xfe, 0x39, 0x74, 0x1e, 0x5f, 0x90, 0xd4, 0x11, 0x1f, 0x40, 0x0b, 0x93, 0x04, 0x2f, 0x69, - 0x46, 0x6c, 0x52, 0xc3, 0xd0, 0xbc, 0x0c, 0xa1, 0x7b, 0x19, 0xc2, 0x67, 0xee, 0x65, 0x88, 0x4a, - 0xac, 0xd3, 0xf9, 0xbd, 0xd7, 0x75, 0xbe, 0x76, 0xa9, 0xf3, 0xc1, 0x31, 0x74, 0x4d, 0x30, 0x5b, - 0xff, 0x0d, 0x68, 0xb0, 0x42, 0xe6, 0x85, 0xd4, 0xb1, 0xba, 0x91, 0x3d, 0xa1, 0x0f, 0xa0, 0x4d, - 0x2e, 0xa8, 0x8c, 0x53, 0xb5, 0x93, 0x7b, 0xba, 0x82, 0x96, 0x32, 0x1c, 0x33, 0x4c, 0x82, 0xdf, - 0x3d, 0xe8, 0x6e, 0x4e, 0xac, 0x8a, 0x9d, 0x53, 0x6c, 0x2b, 0x55, 0x9f, 0x6f, 0xe5, 0x6f, 0xdc, - 0x4d, 0x6d, 0xf3, 0x6e, 0x50, 0x08, 0xfb, 0xea, 0xcd, 0xd3, 0xaf, 0xc5, 0xdb, 0xcb, 0xd6, 0xb8, - 0xa3, 0x3f, 0xdb, 0xd0, 0x7a, 0x6c, 0x17, 0x09, 0xfd, 0x02, 0x0d, 0xb3, 0xfd, 0xe8, 0x7e, 0xd5, - 0xad, 0xdb, 0x7a, 0x28, 0x87, 0x0f, 0x76, 0xa5, 0xd9, 0xfe, 0x5d, 0x43, 0x02, 0xf6, 0x95, 0x0e, - 0xa0, 0xbb, 0x55, 0x3d, 0x6c, 0x88, 0xc8, 0xf0, 0xde, 0x6e, 0xa4, 0x32, 0xe8, 0x6f, 0xd0, 0x72, - 0xeb, 0x8c, 0x1e, 0x56, 0xf5, 0x71, 0x45, 0x4e, 0x86, 0x9f, 0xef, 0x4e, 0x2c, 0x13, 0xf8, 0xc3, - 0x83, 0xfe, 0x95, 0x95, 0x46, 0x5f, 0x56, 0xf5, 0xf7, 0x66, 0xd5, 0x19, 0x3e, 0x7a, 0x67, 0x7e, - 0x99, 0xd6, 0xaf, 0xd0, 0xb4, 0xda, 0x81, 0x2a, 0x77, 0x74, 0x5b, 0x7e, 0x86, 0x0f, 0x77, 0xe6, - 0x95, 0xd1, 0x2f, 0xa0, 0xae, 0x75, 0x01, 0x55, 0x6e, 0xeb, 0xa6, 0x76, 0x0d, 0xef, 0xef, 0xc8, - 0x72, 0x71, 0x0f, 0x3d, 0x35, 0xff, 0x46, 0x58, 0xaa, 0xcf, 0xff, 0x96, 0x62, 0x55, 0x9f, 0xff, - 0x2b, 0xfa, 0xa5, 0xe7, 0x5f, 0xad, 0x61, 0xf5, 0xf9, 0xdf, 0xd0, 0xbb, 0xea, 0xf3, 0xbf, 0xa9, - 0x5b, 0xc1, 0x35, 0xf4, 0x97, 0x07, 0x3d, 0x65, 0x9a, 0x49, 0x4e, 0x92, 0x15, 0xcd, 0xe6, 0xe8, - 0x51, 0x45, 0xf1, 0x56, 0x2c, 0x23, 0xe0, 0x96, 0xe9, 0x52, 0xf9, 0xea, 0xdd, 0x1d, 0xb8, 0xb4, - 0xc6, 0xde, 0xa1, 0xf7, 0x75, 0xf3, 0x87, 0xba, 0xd1, 0xac, 0x86, 0xfe, 0xb9, 0xfb, 0x4f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x32, 0xbc, 0x95, 0x4b, 0x31, 0x0c, 0x00, 0x00, + // 1041 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xed, 0x6f, 0x1b, 0xc5, + 0x13, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0xae, 0x7e, 0xca, 0xef, 0x6a, 0x84, 0x6a, 0x0e, + 0x89, 0x5a, 0x50, 0x2e, 0x51, 0xfa, 0x86, 0x84, 0x44, 0x11, 0x49, 0x41, 0x95, 0xd2, 0x28, 0xba, + 0x14, 0x2a, 0xf1, 0x81, 0x63, 0x7b, 0xb7, 0xb5, 0x57, 0xb1, 0x6f, 0x97, 0xdd, 0x3d, 0x27, 0x48, + 0x48, 0x7c, 0xe2, 0x3f, 0x00, 0xa9, 0x7f, 0x2e, 0xda, 0xb7, 0x8b, 0x9d, 0x96, 0xea, 0x5c, 0xc4, + 0x27, 0xdf, 0x8e, 0x9f, 0x67, 0x66, 0x76, 0x67, 0xe6, 0x19, 0xb8, 0x9b, 0x0b, 0xba, 0x20, 0x42, + 0xee, 0xc9, 0x29, 0x16, 0x24, 0xdf, 0x23, 0x97, 0x24, 0x2b, 0x15, 0x13, 0x7b, 0x5c, 0x30, 0xc5, + 0xaa, 0x63, 0x6c, 0x8e, 0xe8, 0x93, 0x29, 0x96, 0x53, 0x9a, 0x31, 0xc1, 0xe3, 0x82, 0xcd, 0x71, + 0x1e, 0xf3, 0x59, 0x39, 0xa1, 0x85, 0x8c, 0x57, 0x71, 0xc3, 0xdb, 0x13, 0xc6, 0x26, 0x33, 0x62, + 0x9d, 0xbc, 0x2c, 0x5f, 0xed, 0x29, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0xf7, + 0x7c, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0x7a, 0xdd, 0x84, 0xfe, 0x31, 0x2e, 0x8b, 0x6c, 0x9a, + 0x90, 0x5f, 0x4a, 0x22, 0x15, 0x1a, 0x40, 0x23, 0x9b, 0xe7, 0x61, 0x30, 0x0a, 0xc6, 0x9d, 0x44, + 0x7f, 0x22, 0x04, 0x9b, 0x58, 0x4c, 0x64, 0xb8, 0x31, 0x6a, 0x8c, 0x3b, 0x89, 0xf9, 0x46, 0x27, + 0xd0, 0x11, 0x44, 0xb2, 0x52, 0x64, 0x44, 0x86, 0x8d, 0x51, 0x30, 0xee, 0x1e, 0xec, 0xc7, 0xff, + 0x94, 0xb8, 0x8b, 0x6f, 0x43, 0xc6, 0x89, 0xe7, 0x25, 0x57, 0x2e, 0xd0, 0x6d, 0xe8, 0x4a, 0x95, + 0xb3, 0x52, 0xa5, 0x1c, 0xab, 0x69, 0xb8, 0x69, 0xa2, 0x83, 0x35, 0x9d, 0x62, 0x35, 0x75, 0x00, + 0x22, 0x84, 0x05, 0x6c, 0x55, 0x00, 0x22, 0x84, 0x01, 0x0c, 0xa0, 0x41, 0x8a, 0x45, 0xd8, 0x34, + 0x49, 0xea, 0x4f, 0x9d, 0x77, 0x29, 0x89, 0x08, 0x5b, 0x06, 0x6b, 0xbe, 0xd1, 0x2d, 0x68, 0x2b, + 0x2c, 0xcf, 0xd3, 0x9c, 0x8a, 0xb0, 0x6d, 0xec, 0x2d, 0x7d, 0x3e, 0xa2, 0x02, 0xdd, 0x81, 0x1d, + 0x9f, 0x4f, 0x3a, 0xa3, 0x73, 0xaa, 0x64, 0xd8, 0x19, 0x05, 0xe3, 0x76, 0xb2, 0xed, 0xcd, 0xc7, + 0xc6, 0x8a, 0xf6, 0xe1, 0x7f, 0x2f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, + 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x9d, 0xda, 0xbf, 0x0e, 0xcd, 0x3f, 0xe8, 0x08, + 0x9a, 0x73, 0x56, 0x16, 0x4a, 0x86, 0xdd, 0x51, 0x63, 0xdc, 0x3d, 0xb8, 0x5b, 0xf3, 0xa9, 0x9e, + 0x69, 0x52, 0xe2, 0xb8, 0xe8, 0x3b, 0x68, 0xe5, 0x64, 0x41, 0xf5, 0x8b, 0xf7, 0x8c, 0x9b, 0xcf, + 0x6b, 0xba, 0x39, 0x32, 0xac, 0xc4, 0xb3, 0xd1, 0x14, 0x6e, 0x16, 0x44, 0x5d, 0x30, 0x71, 0x9e, + 0x52, 0xc9, 0x66, 0x58, 0x51, 0x56, 0x84, 0x7d, 0x53, 0xc4, 0x2f, 0x6b, 0xba, 0x3c, 0xb1, 0xfc, + 0xa7, 0x9e, 0x7e, 0xc6, 0x49, 0x96, 0x0c, 0x8a, 0x6b, 0x56, 0x14, 0x41, 0xbf, 0x60, 0x29, 0xa7, + 0x0b, 0xa6, 0x52, 0xc1, 0x98, 0x0a, 0xb7, 0xcd, 0x1b, 0x75, 0x0b, 0x76, 0xaa, 0x6d, 0x09, 0x63, + 0x0a, 0x8d, 0x61, 0x90, 0x93, 0x57, 0xb8, 0x9c, 0xa9, 0x94, 0xd3, 0x3c, 0x9d, 0xb3, 0x9c, 0x84, + 0x3b, 0xa6, 0x34, 0xdb, 0xce, 0x7e, 0x4a, 0xf3, 0x67, 0x2c, 0x27, 0xcb, 0x48, 0xca, 0x33, 0x8b, + 0x1c, 0xac, 0x20, 0x9f, 0xf2, 0xcc, 0x20, 0x3f, 0x86, 0x7e, 0xc6, 0x4b, 0x49, 0x94, 0xaf, 0xcd, + 0x4d, 0x03, 0xeb, 0x59, 0xa3, 0xab, 0xca, 0x87, 0x00, 0x78, 0x36, 0x63, 0x17, 0x69, 0x86, 0xb9, + 0x0c, 0x91, 0x69, 0x9c, 0x8e, 0xb1, 0x1c, 0x62, 0x2e, 0xa3, 0x9f, 0x61, 0xdb, 0x4f, 0x86, 0xe4, + 0xac, 0x90, 0x04, 0x9d, 0x40, 0xcb, 0x95, 0xdc, 0x8c, 0x47, 0xf7, 0xe0, 0x7e, 0x5c, 0x6f, 0x56, + 0x63, 0xd7, 0x0e, 0x67, 0x0a, 0x2b, 0x92, 0x78, 0x27, 0x51, 0x1f, 0xba, 0x2f, 0x30, 0x55, 0x6e, + 0xf2, 0xa2, 0x9f, 0xa0, 0x67, 0x8f, 0xff, 0x51, 0xb8, 0x63, 0xd8, 0x39, 0x9b, 0x96, 0x2a, 0x67, + 0x17, 0x85, 0x1f, 0xf6, 0x5d, 0x68, 0x4a, 0x3a, 0x29, 0xf0, 0xcc, 0xcd, 0xbb, 0x3b, 0xa1, 0x8f, + 0xa0, 0x37, 0x11, 0x38, 0x23, 0x29, 0x27, 0x82, 0xb2, 0x3c, 0xdc, 0x18, 0x05, 0xe3, 0x46, 0xd2, + 0x35, 0xb6, 0x53, 0x63, 0x8a, 0x10, 0x0c, 0xae, 0xbc, 0xd9, 0x8c, 0xa3, 0x29, 0xec, 0x7e, 0xcf, + 0x73, 0x1d, 0xb4, 0x9a, 0x71, 0x17, 0x68, 0x45, 0x2f, 0x82, 0x7f, 0xad, 0x17, 0xd1, 0x2d, 0xf8, + 0xff, 0x1b, 0x91, 0x5c, 0x12, 0x03, 0xd8, 0xfe, 0x81, 0x08, 0x49, 0x99, 0xbf, 0x65, 0xf4, 0x19, + 0xec, 0x54, 0x16, 0xf7, 0xb6, 0x21, 0xb4, 0x16, 0xd6, 0xe4, 0x6e, 0xee, 0x8f, 0xd1, 0xa7, 0xd0, + 0xd3, 0xef, 0x56, 0x65, 0x3e, 0x84, 0x36, 0x2d, 0x14, 0x11, 0x0b, 0xf7, 0x48, 0x8d, 0xa4, 0x3a, + 0x47, 0x2f, 0xa0, 0xef, 0xb0, 0xce, 0xed, 0xb7, 0xb0, 0x25, 0xb5, 0x61, 0xcd, 0x2b, 0x3e, 0xc7, + 0xf2, 0xdc, 0x3a, 0xb2, 0xf4, 0xe8, 0x0e, 0xf4, 0xcf, 0x4c, 0x25, 0xde, 0x5e, 0xa8, 0x2d, 0x5f, + 0x28, 0x7d, 0x59, 0x0f, 0x74, 0xd7, 0x3f, 0x87, 0xee, 0x93, 0x4b, 0x92, 0x79, 0xe2, 0x43, 0x68, + 0xe7, 0x04, 0xe7, 0x33, 0x5a, 0x10, 0x97, 0xd4, 0x30, 0xb6, 0x8b, 0x23, 0xf6, 0x8b, 0x23, 0x7e, + 0xee, 0x17, 0x47, 0x52, 0x61, 0xfd, 0x1a, 0xd8, 0x78, 0x73, 0x0d, 0x34, 0xae, 0xd6, 0x40, 0x74, + 0x08, 0x3d, 0x1b, 0xcc, 0xdd, 0x7f, 0x17, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, 0x71, + 0x27, 0xf4, 0x01, 0x74, 0xc8, 0x25, 0x55, 0x69, 0xa6, 0x47, 0x76, 0xc3, 0xdc, 0xa0, 0xad, 0x0d, + 0x87, 0x2c, 0x27, 0xd1, 0x1f, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, 0xd5, + 0x9f, 0xef, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x25, 0x9a, 0x65, + 0xf2, 0xee, 0x6b, 0x1b, 0xdc, 0xc1, 0x5f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x85, 0xa6, + 0x9d, 0x7e, 0xf4, 0xa0, 0xee, 0xd4, 0xad, 0xec, 0xd1, 0xe1, 0xc3, 0x75, 0x69, 0xae, 0x7e, 0x37, + 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x5e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xbf, 0x1e, 0xa9, + 0x0a, 0xfa, 0x3b, 0xb4, 0xfd, 0x38, 0xa3, 0x47, 0x75, 0x7d, 0x5c, 0x93, 0x93, 0xe1, 0x17, 0xeb, + 0x13, 0xab, 0x04, 0xfe, 0x0c, 0x60, 0xe7, 0xda, 0x48, 0xa3, 0xaf, 0xea, 0xfa, 0x7b, 0xbb, 0xea, + 0x0c, 0x1f, 0xbf, 0x37, 0xbf, 0x4a, 0xeb, 0x37, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, 0x2a, + 0x3f, 0xc3, 0x47, 0x6b, 0xf3, 0xaa, 0xe8, 0x97, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, 0xb2, + 0x76, 0x0d, 0x1f, 0xac, 0xc9, 0xf2, 0x71, 0xf7, 0x03, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, 0xff, + 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x6b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, 0xd2, + 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x0d, 0xf4, 0x3a, 0x80, 0xbe, 0x36, 0x9d, 0x29, 0x41, + 0xf0, 0x9c, 0x16, 0x13, 0xf4, 0xb8, 0xa6, 0x78, 0x6b, 0x96, 0x15, 0x70, 0xc7, 0xf4, 0xa9, 0x7c, + 0xfd, 0xfe, 0x0e, 0x7c, 0x5a, 0xe3, 0x60, 0x3f, 0xf8, 0xa6, 0xf5, 0xe3, 0x96, 0xd5, 0xac, 0xa6, + 0xf9, 0xb9, 0xf7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x41, 0x13, 0xe3, 0x8e, 0x50, 0x0c, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/drivers/shared/executor/proto/executor.proto b/drivers/shared/executor/proto/executor.proto index db0a7b3cb..9ac241209 100644 --- a/drivers/shared/executor/proto/executor.proto +++ b/drivers/shared/executor/proto/executor.proto @@ -45,6 +45,7 @@ message LaunchRequest { string default_pid_mode = 15; string default_ipc_mode = 16; string cpuset_cgroup = 17; + repeated string allow_caps = 18; } message LaunchResponse { From 2361a91938f70320ea3d6f6c95c7aceb7b1bfe9d Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 10:55:44 -0600 Subject: [PATCH 2/8] drivers/java: enable setting allow_caps on java driver Enable setting allow_caps on the java task driver plugin, along with the associated cap_add and cap_drop options in java task configuration. --- drivers/exec/driver.go | 25 +--- drivers/java/driver.go | 82 ++++++++++--- drivers/java/driver_test.go | 116 ++++++++++++++++--- drivers/shared/capabilities/defaults.go | 33 +++++- drivers/shared/capabilities/defaults_test.go | 79 +++++++++++++ 5 files changed, 277 insertions(+), 58 deletions(-) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 435796508..0bd6864c6 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -298,29 +298,6 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { return taskConfigSpec, nil } -// getCaps computes the complete set of linux capabilities to enable for driver, -// which gets passed along to libcontainer. -func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) { - driverAllowed := capabilities.New(d.config.AllowCaps) - - // determine caps the task wants that are not allowed - taskCaps := capabilities.New(tc.CapAdd) - missing := driverAllowed.Difference(taskCaps) - if !missing.Empty() { - return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) - } - - // if task did not specify allowed caps, use nomad defaults minus task drops - if len(tc.CapAdd) == 0 { - driverAllowed.Remove(tc.CapDrop) - return driverAllowed.Slice(true), nil - } - - // otherwise task did specify allowed caps, enable exactly those - taskAdd := capabilities.New(tc.CapAdd) - return taskAdd.Slice(true), nil -} - // Capabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports func (d *Driver) Capabilities() (*drivers.Capabilities, error) { @@ -496,7 +473,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := d.getCaps(&driverConfig) + caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) if err != nil { return nil, nil, err } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index c34888f6e..a34a56af4 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/nomad/client/lib/cgutil" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" @@ -73,6 +74,10 @@ var ( hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewLiteral(`"private"`), ), + "allow_caps": hclspec.NewDefault( + hclspec.NewAttr("allow_caps", "list(string)", false), + hclspec.NewLiteral(capabilities.HCLSpecLiteral), + ), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -88,11 +93,13 @@ var ( "args": hclspec.NewAttr("args", "list(string)", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), }) - // capabilities is returned by the Capabilities RPC and indicates what + // driverCapabilities is returned by the Capabilities RPC and indicates what // optional features this driver supports - capabilities = &drivers.Capabilities{ + driverCapabilities = &drivers.Capabilities{ SendSignals: false, Exec: false, FSIsolation: drivers.FSIsolationNone, @@ -108,8 +115,8 @@ var ( func init() { if runtime.GOOS == "linux" { - capabilities.FSIsolation = drivers.FSIsolationChroot - capabilities.MountConfigs = drivers.MountConfigSupportAll + driverCapabilities.FSIsolation = drivers.FSIsolationChroot + driverCapabilities.MountConfigs = drivers.MountConfigSupportAll } } @@ -122,6 +129,10 @@ type Config struct { // DefaultModeIPC is the default IPC isolation set for all tasks using // exec-based task drivers. DefaultModeIPC string `codec:"default_ipc_mode"` + + // AllowCaps configures which Linux Capabilities are enabled for tasks + // running on this node. + AllowCaps []string `codec:"allow_caps"` } func (c *Config) validate() error { @@ -137,18 +148,44 @@ func (c *Config) validate() error { return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC) } + badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps)) + if !badCaps.Empty() { + return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps) + } + return nil } // TaskConfig is the driver configuration of a taskConfig within a job type TaskConfig struct { - Class string `codec:"class"` - ClassPath string `codec:"class_path"` - JarPath string `codec:"jar_path"` - JvmOpts []string `codec:"jvm_options"` - Args []string `codec:"args"` // extra arguments to java executable - ModePID string `codec:"pid_mode"` - ModeIPC string `codec:"ipc_mode"` + // Class indicates which class contains the java entry point. + Class string `codec:"class"` + + // ClassPath indicates where class files are found. + ClassPath string `codec:"class_path"` + + // JarPath indicates where a jar file is found. + JarPath string `codec:"jar_path"` + + // JvmOpts are arguments to pass to the JVM + JvmOpts []string `codec:"jvm_options"` + + // Args are extra arguments to java executable + Args []string `codec:"args"` + + // ModePID indicates whether PID namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModePID string `codec:"pid_mode"` + + // ModeIPC indicates whether IPC namespace isolation is enabled for the task. + // Must be "private" or "host" if set. + ModeIPC string `codec:"ipc_mode"` + + // CapAdd is a set of linux capabilities to enable. + CapAdd []string `codec:"cap_add"` + + // CapDrop is a set of linux capabilities to disable. + CapDrop []string `codec:"cap_drop"` } func (tc *TaskConfig) validate() error { @@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error { return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC) } + supported := capabilities.Supported() + badAdds := supported.Difference(capabilities.New(tc.CapAdd)) + if !badAdds.Empty() { + return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds) + } + badDrops := supported.Difference(capabilities.New(tc.CapDrop)) + if !badDrops.Empty() { + return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops) + } + return nil } @@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { } func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return driverCapabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", - FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot, + FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot, } exec, pluginClient, err := executor.CreateExecutor( @@ -438,6 +485,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } + caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + if err != nil { + return nil, nil, err + } + execCmd := &executor.ExecCommand{ Cmd: absPath, Args: args, @@ -453,6 +505,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive NetworkIsolation: cfg.NetworkIsolation, ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID), ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC), + Capabilities: caps, } ps, err := exec.Launch(execCmd) @@ -491,7 +544,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive } func javaCmdArgs(driverConfig TaskConfig) []string { - args := []string{} + var args []string + // Look for jvm options if len(driverConfig.JvmOpts) != 0 { args = append(args, driverConfig.JvmOpts...) diff --git a/drivers/java/driver_test.go b/drivers/java/driver_test.go index 5c5624037..e407e3c19 100644 --- a/drivers/java/driver_test.go +++ b/drivers/java/driver_test.go @@ -1,6 +1,7 @@ package java import ( + "context" "errors" "fmt" "io" @@ -8,12 +9,10 @@ import ( "os" "path/filepath" "testing" + "time" dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" - "context" - "time" - ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/testlog" @@ -416,20 +415,99 @@ func Test_dnsConfig(t *testing.T) { } func TestDriver_Config_validate(t *testing.T) { - for _, tc := range []struct { - pidMode, ipcMode string - exp error - }{ - {pidMode: "host", ipcMode: "host", exp: nil}, - {pidMode: "private", ipcMode: "host", exp: nil}, - {pidMode: "host", ipcMode: "private", exp: nil}, - {pidMode: "private", ipcMode: "private", exp: nil}, - {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, - {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, - } { - require.Equal(t, tc.exp, (&Config{ - DefaultModePID: tc.pidMode, - DefaultModeIPC: tc.ipcMode, - }).validate()) - } + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "private", ipcMode: "private", exp: nil}, + {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, + {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: tc.pidMode, + DefaultModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("allow_caps", func(t *testing.T) { + for _, tc := range []struct { + ac []string + exp error + }{ + {ac: []string{}, exp: nil}, + {ac: []string{"all"}, exp: nil}, + {ac: []string{"chown", "sys_time"}, exp: nil}, + {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, + {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&Config{ + DefaultModePID: "private", + DefaultModeIPC: "private", + AllowCaps: tc.ac, + }).validate()) + } + }) +} + +func TestDriver_TaskConfig_validate(t *testing.T) { + t.Run("pid/ipc", func(t *testing.T) { + for _, tc := range []struct { + pidMode, ipcMode string + exp error + }{ + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "host", ipcMode: "private", exp: nil}, + {pidMode: "host", ipcMode: "", exp: nil}, + {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, + + {pidMode: "host", ipcMode: "host", exp: nil}, + {pidMode: "private", ipcMode: "host", exp: nil}, + {pidMode: "", ipcMode: "host", exp: nil}, + {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + ModePID: tc.pidMode, + ModeIPC: tc.ipcMode, + }).validate()) + } + }) + + t.Run("cap_add", func(t *testing.T) { + for _, tc := range []struct { + adds []string + exp error + }{ + {adds: nil, exp: nil}, + {adds: []string{"chown"}, exp: nil}, + {adds: []string{"CAP_CHOWN"}, exp: nil}, + {adds: []string{"chown", "sys_time"}, exp: nil}, + {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapAdd: tc.adds, + }).validate()) + } + }) + + t.Run("cap_drop", func(t *testing.T) { + for _, tc := range []struct { + drops []string + exp error + }{ + {drops: nil, exp: nil}, + {drops: []string{"chown"}, exp: nil}, + {drops: []string{"CAP_CHOWN"}, exp: nil}, + {drops: []string{"chown", "sys_time"}, exp: nil}, + {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, + } { + require.Equal(t, tc.exp, (&TaskConfig{ + CapDrop: tc.drops, + }).validate()) + } + }) } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go index a1a6e3531..30a69b65b 100644 --- a/drivers/shared/capabilities/defaults.go +++ b/drivers/shared/capabilities/defaults.go @@ -1,6 +1,7 @@ package capabilities import ( + "fmt" "regexp" "github.com/syndtr/gocapability/capability" @@ -13,7 +14,7 @@ const ( ) var ( - extractLiteral = regexp.MustCompile(`("[\w]+)`) + extractLiteral = regexp.MustCompile(`([\w]+)`) ) // NomadDefaults is the set of Linux capabilities that Nomad enables by @@ -112,3 +113,33 @@ func LegacySupported() *Set { "CAP_AUDIT_READ", }) } + +// Calculate the reduced set of linux capabilities to enable for driver, taking +// into account the capabilities allowed by the driver and the capabilities +// explicitly requested / removed by the task configuration. +// +// capAdd if set indicates the minimal set of capabilities that should be enabled. +// capDrop if set indicates capabilities that should be dropped from the driver defaults +// +// If the task requests a capability not allowed by the driver, an error is +// returned. +func Calculate(allowCaps, capAdd, capDrop []string) ([]string, error) { + driverAllowed := New(allowCaps) + + // determine caps the task wants that are not allowed + taskCaps := New(capAdd) + missing := driverAllowed.Difference(taskCaps) + if !missing.Empty() { + return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // if task did not specify allowed caps, use nomad defaults minus task drops + if len(capAdd) == 0 { + driverAllowed.Remove(capDrop) + return driverAllowed.Slice(true), nil + } + + // otherwise task did specify allowed caps, enable exactly those + taskAdd := New(capAdd) + return taskAdd.Slice(true), nil +} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 2a07afa0e..2fd9f70d4 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -1,6 +1,7 @@ package capabilities import ( + "errors" "strings" "testing" @@ -21,3 +22,81 @@ func TestSet_DockerDefaults(t *testing.T) { require.Len(t, result.Slice(false), 14) require.Contains(t, result.String(), "net_raw") } + +func TestCaps_Calculate(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + exp []string + err error + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow all", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + exp: Supported().Slice(true), + err: nil, + }, + { + name: "allow selection", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"}, + err: nil, + }, + { + name: "add allowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + exp: []string{"CAP_CHOWN", "CAP_KILL"}, + err: nil, + }, + { + name: "add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "drop some", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + exp: []string{"CAP_AUDIT_WRITE", "CAP_FSETID", "CAP_NET_BIND_SERVICE", "CAP_SETFCAP", "CAP_SETGID", "CAP_SETPCAP", "CAP_SETUID"}, + err: nil, + }, + { + name: "drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + caps, err := Calculate(tc.allowCaps, tc.capAdd, tc.capDrop) + require.Equal(t, tc.err, err) + require.Equal(t, tc.exp, caps) + }) + } +} From 87c96eed1109f7a76b42abe92b5ef98e0292a2e3 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 14:48:01 -0600 Subject: [PATCH 3/8] drivers/docker: reuse capabilities plumbing in docker driver This changeset does not introduce any functional change for the docker driver, but rather cleans up the implementation around computing configured capabilities by re-using code written for the exec/java task drivers. --- drivers/docker/driver.go | 122 +----------- drivers/docker/driver_test.go | 191 ++----------------- drivers/exec/driver.go | 4 +- drivers/java/driver.go | 4 +- drivers/shared/capabilities/defaults.go | 100 ++++++++-- drivers/shared/capabilities/defaults_test.go | 178 ++++++++++++++++- drivers/shared/capabilities/set.go | 27 ++- drivers/shared/capabilities/set_test.go | 52 +++++ 8 files changed, 353 insertions(+), 325 deletions(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index c8a2981ec..7234d3467 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "runtime" - "sort" "strconv" "strings" "sync" @@ -23,10 +22,9 @@ import ( plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/drivers/docker/docklog" + "github.com/hashicorp/nomad/drivers/shared/capabilities" "github.com/hashicorp/nomad/drivers/shared/eventer" - "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/drivers/shared/resolvconf" - "github.com/hashicorp/nomad/helper" nstructs "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" @@ -913,8 +911,9 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.Privileged = driverConfig.Privileged // set add/drop capabilities - hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig) - if err != nil { + if hostConfig.CapAdd, hostConfig.CapDrop, err = capabilities.Delta( + capabilities.DockerDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ); err != nil { return c, err } @@ -1184,119 +1183,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T }, nil } -// getCaps computes the capabilities to supply to the --add-cap and --drop-cap -// options to the docker driver, which override the default capabilities enabled -// by docker itself. -func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) { - - // capabilities allowable by client docker plugin configuration - allowCaps := expandAllowCaps(d.config.AllowCaps) - - // capabilities the task docker config is asking for based on the default - // capabilities allowable by nomad - desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop) - if err != nil { - return nil, nil, err - } - - // capabilities the task is requesting that are NOT allowed by the docker plugin - if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 { - return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing) - } - - // capabilities that should be dropped relative to the docker default capabilities - dropCaps := capDrops(taskConfig.CapDrop, allowCaps) - - return taskConfig.CapAdd, dropCaps, nil -} - -// capDrops will compute the total dropped capabilities set -// -// {task cap_drop} U ({docker defaults} \ {driver allow caps}) -func capDrops(dropCaps []string, allowCaps []string) []string { - dropSet := make(map[string]struct{}) - - for _, c := range normalizeCaps(dropCaps) { - dropSet[c] = struct{}{} - } - - // if dropCaps includes ALL, no need to iterate every capability - if _, exists := dropSet["ALL"]; exists { - return []string{"ALL"} - } - - dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps())) - allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps)) - - // find the docker default caps not in allowed caps - for dCap := range dockerDefaults { - if _, exists := allowedCaps[dCap]; !exists { - dropSet[dCap] = struct{}{} - } - } - - drops := make([]string, 0, len(dropSet)) - for c := range dropSet { - drops = append(drops, c) - } - sort.Strings(drops) - return drops -} - -// expandAllowCaps returns the normalized set of allowable capabilities set -// for the docker plugin configuration. -func expandAllowCaps(allowCaps []string) []string { - if len(allowCaps) == 0 { - return nil - } - - set := make(map[string]struct{}, len(allowCaps)) - - for _, rawCap := range allowCaps { - capability := strings.ToUpper(rawCap) - if capability == "ALL" { - for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) { - set[defCap] = struct{}{} - } - } else { - set[capability] = struct{}{} - } - } - - result := make([]string, 0, len(set)) - for capability := range set { - result = append(result, capability) - } - sort.Strings(result) - return result -} - -// missingCaps returns the set of elements in desired that are not present in -// allowed. The elements in desired are first upper-cased before comparison. -// The elements in allowed are assumed to be upper-cased. -func missingCaps(allowed, desired []string) []string { - _, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired)) - sort.Strings(missing) - return missing -} - -// normalizeCaps returns a copy of caps with duplicate elements removed and all -// elements upper-cased. -func normalizeCaps(caps []string) []string { - set := make(map[string]struct{}, len(caps)) - for _, c := range caps { - normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_") - set[strings.ToUpper(normal)] = struct{}{} - } - - result := make([]string, 0, len(set)) - for c := range set { - result = append(result, c) - } - sort.Strings(result) - return result -} - func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) { hm, err := m.toDockerHostMount() if err != nil { diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 67c22e88e..f51b8b3f0 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -19,7 +19,6 @@ import ( hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" - "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/helper/freeport" "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" @@ -1387,44 +1386,44 @@ func TestDockerDriver_Capabilities(t *testing.T) { { Name: "default-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"ALL"}, + CapDrop: []string{"all"}, }, { Name: "default-allowlist-add-forbidden", CapAdd: []string{"net_admin"}, - StartError: "NET_ADMIN", + StartError: "net_admin", }, { Name: "default-allowlist-drop-existing", - CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, + CapDrop: []string{"fowner", "mknod", "net_raw"}, }, { Name: "restrictive-allowlist-drop-all", - CapDrop: []string{"ALL"}, - Allowlist: "FOWNER,MKNOD", + CapDrop: []string{"all"}, + Allowlist: "fowner,mknod", }, { Name: "restrictive-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"ALL"}, - Allowlist: "fowner,mknod", + CapDrop: []string{"all"}, + Allowlist: "mknod,fowner", }, { Name: "restrictive-allowlist-add-forbidden", CapAdd: []string{"net_admin", "mknod"}, - CapDrop: []string{"ALL"}, + CapDrop: []string{"all"}, Allowlist: "fowner,mknod", - StartError: "NET_ADMIN", + StartError: "net_admin", }, { Name: "permissive-allowlist", - CapAdd: []string{"net_admin", "mknod"}, - Allowlist: "ALL", + CapAdd: []string{"mknod", "net_admin"}, + Allowlist: "all", }, { Name: "permissive-allowlist-add-all", CapAdd: []string{"all"}, - Allowlist: "ALL", + Allowlist: "all", }, } @@ -3064,169 +3063,3 @@ func TestDockerDriver_StopSignal(t *testing.T) { }) } } - -func TestDockerCaps_normalizeCaps(t *testing.T) { - t.Run("empty", func(t *testing.T) { - result := normalizeCaps(nil) - require.Len(t, result, 0) - }) - - t.Run("mixed", func(t *testing.T) { - result := normalizeCaps([]string{ - "DAC_OVERRIDE", "sys_chroot", "kill", "KILL", - }) - require.Equal(t, []string{ - "DAC_OVERRIDE", "KILL", "SYS_CHROOT", - }, result) - }) -} - -func TestDockerCaps_missingCaps(t *testing.T) { - allowed := []string{ - "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", - } - - t.Run("none missing", func(t *testing.T) { - result := missingCaps(allowed, []string{ - "SYS_CHROOT", "chown", "KILL", - }) - require.Equal(t, []string(nil), result) - }) - - t.Run("some missing", func(t *testing.T) { - result := missingCaps(allowed, []string{ - "chown", "audit_write", "SETPCAP", "dac_override", - }) - require.Equal(t, []string{"AUDIT_WRITE", "SETPCAP"}, result) - }) -} - -func TestDockerCaps_expandAllowCaps(t *testing.T) { - t.Run("empty", func(t *testing.T) { - result := expandAllowCaps(nil) - require.Empty(t, result) - }) - - t.Run("manual", func(t *testing.T) { - result := expandAllowCaps([]string{ - "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", - }) - require.Equal(t, []string{ - "CHOWN", "DAC_OVERRIDE", "KILL", "SYS_CHROOT", - }, result) - }) - - t.Run("all", func(t *testing.T) { - result := expandAllowCaps([]string{"all"}) - exp := normalizeCaps(executor.SupportedCaps(true)) - sort.Strings(exp) - require.Equal(t, exp, result) - }) -} - -func TestDockerCaps_capDrops(t *testing.T) { - // docker default caps is always the same, task configured drop_caps and - // plugin config allow_caps may be altered - - // This is the 90% use case, where NET_RAW is dropped, as Nomad's default - // capability allow-list is a subset of the docker default cap list. - t.Run("defaults", func(t *testing.T) { - result := capDrops(nil, nomadDefaultCaps()) - require.Equal(t, []string{"NET_RAW"}, result) - }) - - // Users want to use ICMP (ping). - t.Run("enable net_raw", func(t *testing.T) { - result := capDrops(nil, append(nomadDefaultCaps(), "net_raw")) - require.Empty(t, result) - }) - - // The plugin is reduced in ability. - t.Run("enable minimal", func(t *testing.T) { - allow := []string{"setgid", "setuid", "chown", "kill"} - exp := []string{"AUDIT_WRITE", "DAC_OVERRIDE", "FOWNER", "FSETID", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETPCAP", "SYS_CHROOT"} - result := capDrops(nil, allow) - require.Equal(t, exp, result) - }) - - // The task drops abilities. - t.Run("task drops", func(t *testing.T) { - drops := []string{"audit_write", "fowner", "kill", "chown"} - exp := []string{"AUDIT_WRITE", "CHOWN", "FOWNER", "KILL", "NET_RAW"} - result := capDrops(drops, nomadDefaultCaps()) - require.Equal(t, exp, result) - }) - - // Drop all mixed with others. - t.Run("task drops mix", func(t *testing.T) { - drops := []string{"audit_write", "all", "chown"} - exp := []string{"ALL"} // minimized - result := capDrops(drops, nomadDefaultCaps()) - require.Equal(t, exp, result) - }) -} - -func TestDockerCaps_getCaps(t *testing.T) { - testutil.ExecCompatible(t) // tests require linux - - t.Run("defaults", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, CapDrop: nil, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Equal(t, []string{"NET_RAW"}, drop) - }) - - t.Run("enable net_raw", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: append(nomadDefaultCaps(), "net_raw"), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, CapDrop: nil, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Empty(t, drop) - }) - - t.Run("block sys_time", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - _, _, err := d.getCaps(&TaskConfig{ - CapAdd: []string{"SYS_TIME"}, - CapDrop: nil, - }) - require.EqualError(t, err, `Docker driver does not have the following caps allow-listed on this Nomad agent: [SYS_TIME]`) - }) - - t.Run("enable sys_time", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: append(nomadDefaultCaps(), "sys_time"), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: []string{"SYS_TIME"}, - CapDrop: nil, - }) - require.NoError(t, err) - require.Equal(t, []string{"SYS_TIME"}, add) - require.Equal(t, []string{"NET_RAW"}, drop) - }) - - t.Run("task drops chown", func(t *testing.T) { - d := Driver{config: &DriverConfig{ - AllowCaps: nomadDefaultCaps(), - }} - add, drop, err := d.getCaps(&TaskConfig{ - CapAdd: nil, - CapDrop: []string{"chown"}, - }) - require.NoError(t, err) - require.Empty(t, add) - require.Equal(t, []string{"CHOWN", "NET_RAW"}, drop) - }) -} diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 0bd6864c6..ef95a374c 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -473,7 +473,9 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + caps, err := capabilities.Calculate( + capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ) if err != nil { return nil, nil, err } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index a34a56af4..d3272cef9 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -485,7 +485,9 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive cfg.Mounts = append(cfg.Mounts, dnsMount) } - caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop) + caps, err := capabilities.Calculate( + capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop, + ) if err != nil { return nil, nil, err } diff --git a/drivers/shared/capabilities/defaults.go b/drivers/shared/capabilities/defaults.go index 30a69b65b..827f41019 100644 --- a/drivers/shared/capabilities/defaults.go +++ b/drivers/shared/capabilities/defaults.go @@ -39,6 +39,10 @@ func DockerDefaults() *Set { // Supported returns the set of capabilities supported by the operating system. // +// This set will expand over time as new capabilities are introduced to the kernel +// and the capability library is updated (which tends to happen to keep up with +// run-container libraries). +// // Defers to a library generated from // https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h func Supported() *Set { @@ -114,32 +118,92 @@ func LegacySupported() *Set { }) } -// Calculate the reduced set of linux capabilities to enable for driver, taking -// into account the capabilities allowed by the driver and the capabilities -// explicitly requested / removed by the task configuration. +// Calculate the resulting set of linux capabilities to enable for a task, taking +// into account: +// - default capability basis +// - driver allowable capabilities +// - task capability drops +// - task capability adds // -// capAdd if set indicates the minimal set of capabilities that should be enabled. -// capDrop if set indicates capabilities that should be dropped from the driver defaults +// Nomad establishes a standard set of enabled capabilities allowed by the task +// driver if allow_caps is not set. This is the same set that the task will be +// enabled with by default if allow_caps does not further reduce permissions, +// in which case the task capabilities will also be reduced accordingly. // -// If the task requests a capability not allowed by the driver, an error is -// returned. -func Calculate(allowCaps, capAdd, capDrop []string) ([]string, error) { - driverAllowed := New(allowCaps) +// The task will drop any capabilities specified in cap_drop, and add back +// capabilities specified in cap_add. The task will not be allowed to add capabilities +// not set in the the allow_caps setting (which by default is the same as the basis). +// +// cap_add takes precedence over cap_drop, enabling the common pattern of dropping +// all capabilities, then adding back the desired smaller set. e.g. +// cap_drop = ["all"] +// cap_add = ["chown", "kill"] +// +// Note that the resulting capability names are upper-cased and prefixed with +// "CAP_", which is the expected input for the exec/java driver implementation. +func Calculate(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, error) { + allow := New(allowCaps) + adds := New(capAdd) // determine caps the task wants that are not allowed - taskCaps := New(capAdd) - missing := driverAllowed.Difference(taskCaps) + missing := allow.Difference(adds) if !missing.Empty() { return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) } - // if task did not specify allowed caps, use nomad defaults minus task drops - if len(capAdd) == 0 { - driverAllowed.Remove(capDrop) - return driverAllowed.Slice(true), nil + // the realized enabled capabilities starts with what is allowed both by driver + // config AND is a member of the basis (i.e. nomad defaults) + result := basis.Intersect(allow) + + // then remove capabilities the task explicitly drops + result.Remove(capDrop) + + // then add back capabilities the task explicitly adds + return result.Union(adds).Slice(true), nil +} + +// Delta calculates the set of capabilities that must be added and dropped relative +// to a basis to achieve a desired result. The use case is that the docker driver +// assumes a default set (DockerDefault), and we must calculate what to pass into +// --cap-add and --cap-drop on container creation given the inputs of the docker +// plugin config for allow_caps, and the docker task configuration for cap_add and +// cap_drop. Note that the user provided cap_add and cap_drop settings are always +// included, even if they are redundant with the basis (maintaining existing +// behavior, working with existing tests). +// +// Note that the resulting capability names are lower-cased and not prefixed with +// "CAP_", which is the existing style used with the docker driver implementation. +func Delta(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, []string, error) { + all := func(caps []string) bool { + for _, c := range caps { + if normalize(c) == "all" { + return true + } + } + return false } - // otherwise task did specify allowed caps, enable exactly those - taskAdd := New(capAdd) - return taskAdd.Slice(true), nil + // set of caps allowed by driver + allow := New(allowCaps) + + // determine caps the task wants that are not allowed + missing := allow.Difference(New(capAdd)) + if !missing.Empty() { + return nil, nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing) + } + + // add what the task is asking for + add := New(capAdd).Slice(false) + if all(capAdd) { + add = []string{"all"} + } + + // drop what the task removes plus whatever is in the basis that is not + // in the driver allow configuration + drop := New(allowCaps).Difference(basis).Union(New(capDrop)).Slice(false) + if all(capDrop) { + drop = []string{"all"} + } + + return add, drop, nil } diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 2fd9f70d4..408f954ea 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -45,27 +45,59 @@ func TestCaps_Calculate(t *testing.T) { err: nil, }, { - name: "allow all", + name: "allow all no mods", allowCaps: []string{"all"}, capAdd: nil, capDrop: nil, - exp: Supported().Slice(true), + exp: NomadDefaults().Slice(true), err: nil, }, { - name: "allow selection", + name: "allow selection no mods", allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, capAdd: nil, capDrop: nil, + exp: []string{"CAP_CHOWN"}, + err: nil, + }, + { + name: "allow selection and add them", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: []string{"net_raw", "sys_time"}, + capDrop: nil, exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"}, err: nil, }, { - name: "add allowed", + name: "allow defaults and add redundant", allowCaps: NomadDefaults().Slice(false), capAdd: []string{"chown", "KILL"}, capDrop: nil, - exp: []string{"CAP_CHOWN", "CAP_KILL"}, + exp: NomadDefaults().Slice(true), + err: nil, + }, + { + name: "allow defaults and add all", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"all"}, + capDrop: nil, + exp: nil, + err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"), + }, + { + name: "allow defaults and drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + exp: []string{}, + err: nil, + }, + { + name: "allow defaults and drop all and add back some", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "fowner"}, + capDrop: []string{"all"}, + exp: []string{"CAP_CHOWN", "CAP_FOWNER"}, err: nil, }, { @@ -92,11 +124,145 @@ func TestCaps_Calculate(t *testing.T) { exp: []string{}, err: nil, }, + { + name: "drop all and add back", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "mknod"}, + capDrop: []string{"all"}, + exp: []string{"CAP_CHOWN", "CAP_MKNOD"}, + err: nil, + }, } { t.Run(tc.name, func(t *testing.T) { - caps, err := Calculate(tc.allowCaps, tc.capAdd, tc.capDrop) + caps, err := Calculate(NomadDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) require.Equal(t, tc.err, err) require.Equal(t, tc.exp, caps) }) } } + +func TestCaps_Delta(t *testing.T) { + for _, tc := range []struct { + name string + + // input + allowCaps []string // driver config + capAdd []string // task config + capDrop []string // task config + + // output + expAdd []string + expDrop []string + err error + }{ + { + name: "the default setting", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{"net_raw"}, + err: nil, + }, + { + name: "allow all no mods", + allowCaps: []string{"all"}, + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{}, + err: nil, + }, + { + name: "allow non-default no mods", + allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"}, + capAdd: nil, + capDrop: nil, + expAdd: []string{}, + expDrop: []string{ + "audit_write", "dac_override", "fowner", "fsetid", + "kill", "mknod", "net_bind_service", "setfcap", + "setgid", "setpcap", "setuid", "sys_chroot"}, + err: nil, + }, + { + name: "allow default add from default", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "KILL"}, + capDrop: nil, + expAdd: []string{"chown", "kill"}, + expDrop: []string{"net_raw"}, + err: nil, + }, + { + name: "allow default add disallowed", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "net_raw"}, + capDrop: nil, + expAdd: nil, + expDrop: nil, + err: errors.New("driver does not allow the following capabilities: net_raw"), + }, + { + name: "allow default drop from default", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"}, + expAdd: []string{}, + expDrop: []string{"chown", "dac_override", "fowner", "kill", "mknod", "net_raw", "sys_chroot"}, + err: nil, + }, + { + name: "allow default drop all", + allowCaps: NomadDefaults().Slice(false), + capAdd: nil, + capDrop: []string{"all"}, + expAdd: []string{}, + expDrop: []string{"all"}, + err: nil, + }, + { + name: "task drop all and add back", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"chown", "fowner"}, + capDrop: []string{"all"}, + expAdd: []string{"chown", "fowner"}, + expDrop: []string{"all"}, + err: nil, + }, + { + name: "add atop allow all", + allowCaps: []string{"all"}, + capAdd: []string{"chown", "fowner"}, + capDrop: nil, + expAdd: []string{"chown", "fowner"}, + expDrop: []string{}, + err: nil, + }, + { + name: "add all atop all", + allowCaps: []string{"all"}, + capAdd: []string{"all"}, + capDrop: nil, + expAdd: []string{"all"}, + expDrop: []string{}, + err: nil, + }, + { + name: "add all atop defaults", + allowCaps: NomadDefaults().Slice(false), + capAdd: []string{"all"}, + capDrop: nil, + expAdd: nil, + expDrop: nil, + err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + add, drop, err := Delta(DockerDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) + require.Equal(t, tc.err, err) + require.Equal(t, tc.expAdd, add) + require.Equal(t, tc.expDrop, drop) + }) + } +} diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go index 046573e2a..916e3d4d3 100644 --- a/drivers/shared/capabilities/set.go +++ b/drivers/shared/capabilities/set.go @@ -14,7 +14,7 @@ var null = nothing{} // operations, taking care of name normalization, and sentinel value expansions. // // Linux capabilities can be expressed in multiple ways when working with docker -// and/or libcontainer, along with Nomad. +// and/or executor, along with Nomad configuration. // // Capability names may be upper or lower case, and may or may not be prefixed // with "CAP_" or "cap_". On top of that, Nomad interprets the special name "all" @@ -61,6 +61,18 @@ func (s *Set) Remove(caps []string) { } } +// Union returns of Set of elements of both s and b. +func (s *Set) Union(b *Set) *Set { + data := make(map[string]nothing) + for c := range s.data { + data[c] = null + } + for c := range b.data { + data[c] = null + } + return &Set{data: data} +} + // Difference returns the Set of elements of b not in s. func (s *Set) Difference(b *Set) *Set { data := make(map[string]nothing) @@ -72,6 +84,17 @@ func (s *Set) Difference(b *Set) *Set { return &Set{data: data} } +// Intersect returns the Set of elements in both s and b. +func (s *Set) Intersect(b *Set) *Set { + data := make(map[string]nothing) + for c := range s.data { + if _, exists := b.data[c]; exists { + data[c] = null + } + } + return &Set{data: data} +} + // Empty return true if no capabilities exist in s. func (s *Set) Empty() bool { return len(s.data) == 0 @@ -84,7 +107,7 @@ func (s *Set) String() string { // Slice returns a sorted slice of capabilities in s. // -// big - indicates whether to uppercase and prefix capabilities with CAP_ +// upper - indicates whether to uppercase and prefix capabilities with CAP_ func (s *Set) Slice(upper bool) []string { caps := make([]string, 0, len(s.data)) for c := range s.data { diff --git a/drivers/shared/capabilities/set_test.go b/drivers/shared/capabilities/set_test.go index 8e072e8b0..2134719f2 100644 --- a/drivers/shared/capabilities/set_test.go +++ b/drivers/shared/capabilities/set_test.go @@ -160,3 +160,55 @@ func TestSet_Difference(t *testing.T) { require.Equal(t, "x, y", result.String()) }) } + +func TestSet_Intersect(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"a", "b"}) + + result := a.Intersect(b) + require.True(t, result.Empty()) + + result2 := b.Intersect(a) + require.True(t, result2.Empty()) + }) + + t.Run("intersect", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f", "G"}) + b := New([]string{"Z", "B", "E", "f", "y"}) + + result := a.Intersect(b) + require.Equal(t, "b, e, f", result.String()) + + result2 := b.Intersect(a) + require.Equal(t, "b, e, f", result2.String()) + }) +} + +func TestSet_Union(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + a := New(nil) + b := New([]string{"a", "b"}) + + result := a.Union(b) + require.Equal(t, "a, b", result.String()) + + result2 := b.Union(a) + require.Equal(t, "a, b", result2.String()) + }) + + t.Run("union", func(t *testing.T) { + a := New([]string{"A", "b", "C", "d", "e", "f", "G"}) + b := New([]string{"Z", "B", "E", "f", "y"}) + + result := a.Union(b) + require.Equal(t, "a, b, c, d, e, f, g, y, z", result.String()) + + result2 := b.Union(a) + require.Equal(t, "a, b, c, d, e, f, g, y, z", result2.String()) + }) +} From f64baec276c3d6678ba72e2a213c629dfc47a933 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 17:19:23 -0600 Subject: [PATCH 4/8] docs: update docs for linux capabilities in exec/java/docker drivers Update docs for allow_caps, cap_add, cap_drop in exec/java/docker driver pages. Also update upgrade guide with guidance on new default linux capabilities for exec and java drivers. --- drivers/shared/capabilities/set.go | 1 + website/content/docs/drivers/docker.mdx | 61 +++++++++++-------- website/content/docs/drivers/exec.mdx | 43 +++++++++++++ website/content/docs/drivers/java.mdx | 43 +++++++++++++ .../content/docs/upgrade/upgrade-specific.mdx | 40 +++++++++++- 5 files changed, 158 insertions(+), 30 deletions(-) diff --git a/drivers/shared/capabilities/set.go b/drivers/shared/capabilities/set.go index 916e3d4d3..d6fe527d9 100644 --- a/drivers/shared/capabilities/set.go +++ b/drivers/shared/capabilities/set.go @@ -23,6 +23,7 @@ type Set struct { data map[string]nothing } +// New creates a new Set setting caps as the initial elements. func New(caps []string) *Set { m := make(map[string]nothing, len(caps)) for _, c := range caps { diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index 68e2bf4f4..3c202e107 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -452,30 +452,26 @@ config { - `cap_add` - (Optional) A list of Linux capabilities as strings to pass directly to [`--cap-add`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist. - The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration. + The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration. For example: - ```hcl - config { - cap_add = [ - "SYS_TIME", - ] - } - ``` +```hcl +config { + cap_add = ["net_raw", sys_time"] +} +``` - `cap_drop` - (Optional) A list of Linux capabilities as strings to pass directly to [`--cap-drop`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist. - The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration. + The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration. For example: - ```hcl - config { - cap_drop = [ - "MKNOD", - ] - } - ``` +```hcl +config { + cap_drop = ["mknod"] +} +``` - `cpu_hard_limit` - (Optional) `true` or `false` (default). Use hard CPU limiting instead of soft limiting. By default this is `false` which means @@ -797,10 +793,7 @@ plugin "docker" { } allow_privileged = false - allow_caps = ["CHOWN", "NET_RAW"] - - # allow_caps can also be set to "ALL" - # allow_caps = ["ALL"] + allow_caps = ["chown", "net_raw"] } } ``` @@ -823,13 +816,22 @@ plugin "docker" { from the Docker engine during an image pull within this timeframe, Nomad will timeout the request that initiated the pull command. (Minimum of `1m`) -- `allow_caps` - A list of allowed Linux capabilities. - Defaults to - `CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP, NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE` which is the list of - capabilities allowed by docker by default, as defined here. Allows the - operator to control which capabilities can be obtained by tasks using cap_add - and cap_drop options. Supports the value "ALL" as a shortcut for allowlisting - all capabilities. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is the same list of capabilities allowed by [docker by default][docker_caps] + (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained + by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports + the value `"all"` as a shortcut for allow-listing all capabilities supported by + the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. - `allow_runtimes` - defaults to `["runc", "nvidia"]` - A list of the allowed docker runtimes a task may use. @@ -1136,3 +1138,8 @@ Windows is relatively new and rapidly evolving you may want to consult the [plugin-stanza]: /docs/configuration/plugin [allocation working directory]: /docs/runtime/environment#task-directories 'Task Directories' [`auth_soft_fail=true`]: #auth_soft_fail +[cap_add]: /docs/drivers/docker#cap_add +[cap_drop]: /docs/drivers/docker#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +[allow_caps]: /docs/drivers/docker#allow_caps diff --git a/website/content/docs/drivers/exec.mdx b/website/content/docs/drivers/exec.mdx index 71e491c99..df557b7c7 100644 --- a/website/content/docs/drivers/exec.mdx +++ b/website/content/docs/drivers/exec.mdx @@ -54,6 +54,27 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `cap_add` - (Optional) A list of Linux capabilities to enable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_add = ["net_raw", "sys_time"] +} +``` + +- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_drop = ["all"] + cap_add = ["chown", "sys_chroot", "mknod"] +} +``` + ## Examples To run a binary present on the Node: @@ -138,6 +159,23 @@ able to make use of IPC features, like sending unexpected POSIX signals. for file system isolation without `pivot_root`. This is useful for systems where the root is on a ramdisk. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is modeled after the capabilities allowed by [docker by default][docker_caps] + (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. + Supports the value `"all"` as a shortcut for allow-listing all capabilities supported + by the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. + ## Client Attributes The `exec` driver will set the following client attributes: @@ -200,3 +238,8 @@ This list is configurable through the agent client [default_pid_mode]: /docs/drivers/exec#default_pid_mode [default_ipc_mode]: /docs/drivers/exec#default_ipc_mode +[cap_add]: /docs/drivers/exec#cap_add +[cap_drop]: /docs/drivers/exec#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps]: /docs/drivers/exec#allow_caps +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities diff --git a/website/content/docs/drivers/java.mdx b/website/content/docs/drivers/java.mdx index 8124d491c..f7e180a52 100644 --- a/website/content/docs/drivers/java.mdx +++ b/website/content/docs/drivers/java.mdx @@ -61,6 +61,27 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `cap_add` - (Optional) A list of Linux capabilities to enable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_add = ["net_raw", "sys_time"] +} +``` + +- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task. + Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset + of the allowed capabilities configured with [`allow_caps`][allow_caps]. + +```hcl +config { + cap_drop = ["all"] + cap_add = ["chown", "sys_chroot", "mknod"] +} +``` + ## Examples A simple config block to run a Java Jar: @@ -138,6 +159,23 @@ be able to access sensitive process information like environment variables. !> **Warning:** If set to `"host"`, other processes running as the same user will be able to make use of IPC features, like sending unexpected POSIX signals. +- `allow_caps` - A list of allowed Linux capabilities. Defaults to + +```hcl +["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", + "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"] +``` + + which is modeled after the capabilities allowed by [docker by default][docker_caps] + (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. + Supports the value `"all"` as a shortcut for allow-listing all capabilities supported + by the operating system. + +!> **Warning:** Allowing more capabilities beyond the default may lead to +undesirable consequences, including untrusted tasks being able to compromise the +host system. + ## Client Requirements The `java` driver requires Java to be installed and in your system's `$PATH`. On @@ -208,3 +246,8 @@ This list is configurable through the agent client [default_pid_mode]: /docs/drivers/java#default_pid_mode [default_ipc_mode]: /docs/drivers/java#default_ipc_mode +[cap_add]: /docs/drivers/java#cap_add +[cap_drop]: /docs/drivers/java#cap_drop +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps]: /docs/drivers/java#allow_caps +[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 2ff1ade19..e74d16f3c 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -54,7 +54,36 @@ these fields. Connect native tasks running in host networking mode will now have `CONSUL_HTTP_ADDR` set automatically. Before this was only the case for bridge networking. If an operator -already explicitly set `CONSUL_HTTP_ADDR` then it will not get overriden. +already explicitly set `CONSUL_HTTP_ADDR` then it will not get overridden. + +#### Linux capabilities in exec/java + +Following the security [remediation][no_net_raw] in Nomad versions 0.12.12, 1.0.5, +and 1.1.0-rc1, the `exec` and `java` task drivers will additionally no longer enable +the following linux capabilities by default: + +``` +AUDIT_CONTROL AUDIT_READ BLOCK_SUSPEND DAC_READ_SEARCH IPC_LOCK IPC_OWNER LEASE +LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST SYS_ADMIN +SYS_BOOT SYSLOG SYS_MODULE SYS_NICE SYS_PACCT SYS_PTRACE SYS_RAWIO SYS_RESOURCE +SYS_TIME SYS_TTY_CONFIG WAKE_ALARM +``` + +The capabilities now enabled by default are modeled after Docker default [`linux capabilities`]: + +``` +AUDIT_WRITE CHOWN DAC_OVERRIDE FOWNER FSETID KILL MKNOD NET_BIND_SERVICE +NET_RAW SETFCAP SETGID SETPCAP SETUID SYS_CHROOT +``` + +A new `allow_caps` plugin configuration parameter for [`exec`][allow_caps_exec] +and [`java`][allow_caps_java] task drivers can be used to restrict the set of +capabilities allowed for use by tasks. + +Tasks using the `exec` or `java` task drivers can add or remove desired linux +capabilities using the [`cap_add`][cap_add_exec] and [`cap_drop`][cap_drop_exec] +task configuration options. + #### iptables @@ -63,9 +92,9 @@ inserting them as the first rule. This allows better control for user-defined iptables rules but users who append rules currently should verify that their rules are being appended in the correct order. -## Nomad 1.1.0, 1.0.5, 0.12.12 +## Nomad 1.1.0-rc1, 1.0.5, 0.12.12 -Nomad versions 1.1.0, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, +Nomad versions 1.1.0-rc1, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, and `java` task drivers so that the [`CAP_NET_RAW`] linux capability is disabled by default. This is one of the [`linux capabilities`] that Docker itself enables by default, as this capability enables the generation of ICMP packets - used by @@ -1111,3 +1140,8 @@ deleted and then Nomad 0.3.0 can be launched. [`CAP_NET_RAW`]: https://security.stackexchange.com/a/128988 [`linux capabilities`]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities [`allow_caps`]: /docs/drivers/docker#allow_caps +[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12 +[allow_caps_exec]: /docs/drivers/exec#allow_caps +[allow_caps_java]: /docs/drivers/java#allow_caps +[cap_add_exec]: /docs/drivers/exec#cap_add +[cap_drop_exec]: /docs/drivers/exec#cap_drop From e365652e8103b8c102bc6383380329355662258c Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 17:45:39 -0600 Subject: [PATCH 5/8] drivers: fixup linux version dependent test cases The error output being checked depends on the linux caps supported by the particular operating system. Fix these test cases to just check that an error did occur. --- drivers/docker/config.go | 30 -------------------- drivers/docker/driver_default.go | 21 -------------- drivers/shared/capabilities/defaults_test.go | 28 +++++++++++++----- 3 files changed, 21 insertions(+), 58 deletions(-) diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 898d9172c..aff5c5d69 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -42,36 +42,6 @@ const ( dockerAuthHelperPrefix = "docker-credential-" ) -// nomadDefaultCaps is the subset of dockerDefaultCaps that Nomad enables by -// default and is used to compute the set of capabilities to add/drop given -// docker driver configuration. -func nomadDefaultCaps() []string { - return []string{ - "AUDIT_WRITE", - "CHOWN", - "DAC_OVERRIDE", - "FOWNER", - "FSETID", - "KILL", - "MKNOD", - "NET_BIND_SERVICE", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_CHROOT", - } -} - -// dockerDefaultCaps is a list of Linux capabilities enabled by docker by default -// and is used to compute the set of capabilities to add/drop given docker driver -// configuration, as well as Nomad built-in limitations. -// -// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -func dockerDefaultCaps() []string { - return append(nomadDefaultCaps(), "NET_RAW") -} - func PluginLoader(opts map[string]string) (map[string]interface{}, error) { conf := map[string]interface{}{} if v, ok := opts["docker.endpoint"]; ok { diff --git a/drivers/docker/driver_default.go b/drivers/docker/driver_default.go index 45086f7bf..b180ae8f9 100644 --- a/drivers/docker/driver_default.go +++ b/drivers/docker/driver_default.go @@ -3,30 +3,9 @@ package docker import ( - "github.com/docker/docker/oci/caps" docker "github.com/fsouza/go-dockerclient" ) func getPortBinding(ip string, port string) docker.PortBinding { return docker.PortBinding{HostIP: ip, HostPort: port} } - -func tweakCapabilities(basics, adds, drops []string) ([]string, error) { - // Moby mixes 2 different capabilities formats: prefixed with "CAP_" - // and not. We do the conversion here to have a consistent, - // non-prefixed format on the Nomad side. - for i, cap := range basics { - basics[i] = "CAP_" + cap - } - - effectiveCaps, err := caps.TweakCapabilities(basics, adds, drops, nil, false) - if err != nil { - return effectiveCaps, err - } - - for i, cap := range effectiveCaps { - effectiveCaps[i] = cap[len("CAP_"):] - } - - return effectiveCaps, nil -} diff --git a/drivers/shared/capabilities/defaults_test.go b/drivers/shared/capabilities/defaults_test.go index 408f954ea..7fd03513e 100644 --- a/drivers/shared/capabilities/defaults_test.go +++ b/drivers/shared/capabilities/defaults_test.go @@ -33,8 +33,9 @@ func TestCaps_Calculate(t *testing.T) { capDrop []string // task config // output - exp []string - err error + exp []string + err error + skip bool // error message is linux version dependent }{ { name: "the default setting", @@ -77,6 +78,7 @@ func TestCaps_Calculate(t *testing.T) { err: nil, }, { + skip: true, name: "allow defaults and add all", allowCaps: NomadDefaults().Slice(false), capAdd: []string{"all"}, @@ -135,8 +137,13 @@ func TestCaps_Calculate(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { caps, err := Calculate(NomadDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) - require.Equal(t, tc.err, err) - require.Equal(t, tc.exp, caps) + if !tc.skip { + require.Equal(t, tc.err, err) + require.Equal(t, tc.exp, caps) + } else { + require.Error(t, err) + require.Equal(t, tc.exp, caps) + } }) } } @@ -154,6 +161,7 @@ func TestCaps_Delta(t *testing.T) { expAdd []string expDrop []string err error + skip bool // error message is linux version dependent }{ { name: "the default setting", @@ -249,6 +257,7 @@ func TestCaps_Delta(t *testing.T) { err: nil, }, { + skip: true, name: "add all atop defaults", allowCaps: NomadDefaults().Slice(false), capAdd: []string{"all"}, @@ -260,9 +269,14 @@ func TestCaps_Delta(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { add, drop, err := Delta(DockerDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop) - require.Equal(t, tc.err, err) - require.Equal(t, tc.expAdd, add) - require.Equal(t, tc.expDrop, drop) + if !tc.skip { + require.Equal(t, tc.err, err) + require.Equal(t, tc.expAdd, add) + require.Equal(t, tc.expDrop, drop) + } else { + require.Error(t, err) + require.Equal(t, tc.expDrop, drop) + } }) } } From 2e041e0813702d75615cc85fbdbf41385bb3ac20 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sat, 15 May 2021 18:22:06 -0600 Subject: [PATCH 6/8] deps: update go mod tidy Looks like we no longer need a package. --- .../docker/docker/oci/caps/defaults.go | 21 --- .../docker/docker/oci/caps/utils.go | 169 ------------------ vendor/modules.txt | 1 - 3 files changed, 191 deletions(-) delete mode 100644 vendor/github.com/docker/docker/oci/caps/defaults.go delete mode 100644 vendor/github.com/docker/docker/oci/caps/utils.go diff --git a/vendor/github.com/docker/docker/oci/caps/defaults.go b/vendor/github.com/docker/docker/oci/caps/defaults.go deleted file mode 100644 index 242ee5811..000000000 --- a/vendor/github.com/docker/docker/oci/caps/defaults.go +++ /dev/null @@ -1,21 +0,0 @@ -package caps // import "github.com/docker/docker/oci/caps" - -// DefaultCapabilities returns a Linux kernel default capabilities -func DefaultCapabilities() []string { - return []string{ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE", - } -} diff --git a/vendor/github.com/docker/docker/oci/caps/utils.go b/vendor/github.com/docker/docker/oci/caps/utils.go deleted file mode 100644 index ffd3f6f50..000000000 --- a/vendor/github.com/docker/docker/oci/caps/utils.go +++ /dev/null @@ -1,169 +0,0 @@ -package caps // import "github.com/docker/docker/oci/caps" - -import ( - "fmt" - "strings" - - "github.com/docker/docker/errdefs" - "github.com/syndtr/gocapability/capability" -) - -var capabilityList Capabilities - -func init() { - last := capability.CAP_LAST_CAP - // hack for RHEL6 which has no /proc/sys/kernel/cap_last_cap - if last == capability.Cap(63) { - last = capability.CAP_BLOCK_SUSPEND - } - for _, cap := range capability.List() { - if cap > last { - continue - } - capabilityList = append(capabilityList, - &CapabilityMapping{ - Key: "CAP_" + strings.ToUpper(cap.String()), - Value: cap, - }, - ) - } -} - -type ( - // CapabilityMapping maps linux capability name to its value of capability.Cap type - // Capabilities is one of the security systems in Linux Security Module (LSM) - // framework provided by the kernel. - // For more details on capabilities, see http://man7.org/linux/man-pages/man7/capabilities.7.html - CapabilityMapping struct { - Key string `json:"key,omitempty"` - Value capability.Cap `json:"value,omitempty"` - } - // Capabilities contains all CapabilityMapping - Capabilities []*CapabilityMapping -) - -// String returns of CapabilityMapping -func (c *CapabilityMapping) String() string { - return c.Key -} - -// GetCapability returns CapabilityMapping which contains specific key -func GetCapability(key string) *CapabilityMapping { - for _, capp := range capabilityList { - if capp.Key == key { - cpy := *capp - return &cpy - } - } - return nil -} - -// GetAllCapabilities returns all of the capabilities -func GetAllCapabilities() []string { - output := make([]string, len(capabilityList)) - for i, capability := range capabilityList { - output[i] = capability.String() - } - return output -} - -// inSlice tests whether a string is contained in a slice of strings or not. -func inSlice(slice []string, s string) bool { - for _, ss := range slice { - if s == ss { - return true - } - } - return false -} - -const allCapabilities = "ALL" - -// NormalizeLegacyCapabilities normalizes, and validates CapAdd/CapDrop capabilities -// by upper-casing them, and adding a CAP_ prefix (if not yet present). -// -// This function also accepts the "ALL" magic-value, that's used by CapAdd/CapDrop. -func NormalizeLegacyCapabilities(caps []string) ([]string, error) { - var normalized []string - - valids := GetAllCapabilities() - for _, c := range caps { - c = strings.ToUpper(c) - if c == allCapabilities { - normalized = append(normalized, c) - continue - } - if !strings.HasPrefix(c, "CAP_") { - c = "CAP_" + c - } - if !inSlice(valids, c) { - return nil, errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c)) - } - normalized = append(normalized, c) - } - return normalized, nil -} - -// ValidateCapabilities validates if caps only contains valid capabilities -func ValidateCapabilities(caps []string) error { - valids := GetAllCapabilities() - for _, c := range caps { - if !inSlice(valids, c) { - return errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c)) - } - } - return nil -} - -// TweakCapabilities tweaks capabilities by adding, dropping, or overriding -// capabilities in the basics capabilities list. -func TweakCapabilities(basics, adds, drops, capabilities []string, privileged bool) ([]string, error) { - switch { - case privileged: - // Privileged containers get all capabilities - return GetAllCapabilities(), nil - case capabilities != nil: - // Use custom set of capabilities - if err := ValidateCapabilities(capabilities); err != nil { - return nil, err - } - return capabilities, nil - case len(adds) == 0 && len(drops) == 0: - // Nothing to tweak; we're done - return basics, nil - } - - capDrop, err := NormalizeLegacyCapabilities(drops) - if err != nil { - return nil, err - } - capAdd, err := NormalizeLegacyCapabilities(adds) - if err != nil { - return nil, err - } - - var caps []string - - switch { - case inSlice(capAdd, allCapabilities): - // Add all capabilities except ones on capDrop - for _, c := range GetAllCapabilities() { - if !inSlice(capDrop, c) { - caps = append(caps, c) - } - } - case inSlice(capDrop, allCapabilities): - // "Drop" all capabilities; use what's in capAdd instead - caps = capAdd - default: - // First drop some capabilities - for _, c := range basics { - if !inSlice(capDrop, c) { - caps = append(caps, c) - } - } - // Then add the list of capabilities from capAdd - caps = append(caps, capAdd...) - } - return caps, nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 37bcd8f6c..063bfceb2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -241,7 +241,6 @@ github.com/docker/docker/api/types/swarm github.com/docker/docker/api/types/swarm/runtime github.com/docker/docker/api/types/versions github.com/docker/docker/errdefs -github.com/docker/docker/oci/caps github.com/docker/docker/pkg/archive github.com/docker/docker/pkg/fileutils github.com/docker/docker/pkg/homedir From fe9258b7541bd708b93527b8073152a609e77b79 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Sun, 16 May 2021 10:06:03 -0600 Subject: [PATCH 7/8] drivers/exec: pass capabilities through executor RPC Add capabilities to the LaunchRequest proto so that the capabilities set actually gets plumbed all the way through to task launch. --- drivers/exec/driver.go | 1 + drivers/java/driver.go | 1 + drivers/shared/executor/client.go | 1 + drivers/shared/executor/executor.go | 4 + drivers/shared/executor/executor_linux.go | 1 - drivers/shared/executor/proto/executor.pb.go | 143 ++++++++++--------- drivers/shared/executor/proto/executor.proto | 1 + drivers/shared/executor/server.go | 1 + 8 files changed, 85 insertions(+), 68 deletions(-) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index ef95a374c..422b8b42a 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -479,6 +479,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err != nil { return nil, nil, err } + d.logger.Debug("task capabilities", "capabilities", caps) execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, diff --git a/drivers/java/driver.go b/drivers/java/driver.go index d3272cef9..f5951ad2b 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -491,6 +491,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive if err != nil { return nil, nil, err } + d.logger.Debug("task capabilities", "capabilities", caps) execCmd := &executor.ExecCommand{ Cmd: absPath, diff --git a/drivers/shared/executor/client.go b/drivers/shared/executor/client.go index 67724cc83..1779d919a 100644 --- a/drivers/shared/executor/client.go +++ b/drivers/shared/executor/client.go @@ -47,6 +47,7 @@ func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) { NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation), DefaultPidMode: cmd.ModePID, DefaultIpcMode: cmd.ModeIPC, + Capabilities: cmd.Capabilities, } resp, err := c.client.Launch(ctx, req) if err != nil { diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index 7d738e45f..10a70f0b8 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -92,6 +92,10 @@ type Executor interface { // ExecCommand holds the user command, args, and other isolation related // settings. +// +// Important (!): when adding fields, make sure to update the RPC methods in +// grpcExecutorClient.Launch and grpcExecutorServer.Launch. Number of hours +// spent tracking this down: too many. type ExecCommand struct { // Cmd is the command that the user wants to run. Cmd string diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 1bbb2839c..5c4919e83 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -534,7 +534,6 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc } func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) { - switch command.User { case "root": // when running as root, use the legacy set of system capabilities, so diff --git a/drivers/shared/executor/proto/executor.pb.go b/drivers/shared/executor/proto/executor.pb.go index bf6c980c2..dcf1a38c7 100644 --- a/drivers/shared/executor/proto/executor.pb.go +++ b/drivers/shared/executor/proto/executor.pb.go @@ -45,6 +45,7 @@ type LaunchRequest struct { DefaultIpcMode string `protobuf:"bytes,16,opt,name=default_ipc_mode,json=defaultIpcMode,proto3" json:"default_ipc_mode,omitempty"` CpusetCgroup string `protobuf:"bytes,17,opt,name=cpuset_cgroup,json=cpusetCgroup,proto3" json:"cpuset_cgroup,omitempty"` AllowCaps []string `protobuf:"bytes,18,rep,name=allow_caps,json=allowCaps,proto3" json:"allow_caps,omitempty"` + Capabilities []string `protobuf:"bytes,19,rep,name=capabilities,proto3" json:"capabilities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -201,6 +202,13 @@ func (m *LaunchRequest) GetAllowCaps() []string { return nil } +func (m *LaunchRequest) GetCapabilities() []string { + if m != nil { + return m.Capabilities + } + return nil +} + type LaunchResponse struct { Process *ProcessState `protobuf:"bytes,1,opt,name=process,proto3" json:"process,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -866,73 +874,74 @@ func init() { } var fileDescriptor_66b85426380683f3 = []byte{ - // 1041 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xed, 0x6f, 0x1b, 0xc5, - 0x13, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0xae, 0x7e, 0xca, 0xef, 0x6a, 0x84, 0x6a, 0x0e, - 0x89, 0x5a, 0x50, 0x2e, 0x51, 0xfa, 0x86, 0x84, 0x44, 0x11, 0x49, 0x41, 0x95, 0xd2, 0x28, 0xba, - 0x14, 0x2a, 0xf1, 0x81, 0x63, 0x7b, 0xb7, 0xb5, 0x57, 0xb1, 0x6f, 0x97, 0xdd, 0x3d, 0x27, 0x48, - 0x48, 0x7c, 0xe2, 0x3f, 0x00, 0xa9, 0x7f, 0x2e, 0xda, 0xb7, 0x8b, 0x9d, 0x96, 0xea, 0x5c, 0xc4, - 0x27, 0xdf, 0x8e, 0x9f, 0x67, 0x66, 0x76, 0x67, 0xe6, 0x19, 0xb8, 0x9b, 0x0b, 0xba, 0x20, 0x42, - 0xee, 0xc9, 0x29, 0x16, 0x24, 0xdf, 0x23, 0x97, 0x24, 0x2b, 0x15, 0x13, 0x7b, 0x5c, 0x30, 0xc5, - 0xaa, 0x63, 0x6c, 0x8e, 0xe8, 0x93, 0x29, 0x96, 0x53, 0x9a, 0x31, 0xc1, 0xe3, 0x82, 0xcd, 0x71, - 0x1e, 0xf3, 0x59, 0x39, 0xa1, 0x85, 0x8c, 0x57, 0x71, 0xc3, 0xdb, 0x13, 0xc6, 0x26, 0x33, 0x62, - 0x9d, 0xbc, 0x2c, 0x5f, 0xed, 0x29, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0xf7, - 0x7c, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0x7a, 0xdd, 0x84, 0xfe, 0x31, 0x2e, 0x8b, 0x6c, 0x9a, - 0x90, 0x5f, 0x4a, 0x22, 0x15, 0x1a, 0x40, 0x23, 0x9b, 0xe7, 0x61, 0x30, 0x0a, 0xc6, 0x9d, 0x44, - 0x7f, 0x22, 0x04, 0x9b, 0x58, 0x4c, 0x64, 0xb8, 0x31, 0x6a, 0x8c, 0x3b, 0x89, 0xf9, 0x46, 0x27, - 0xd0, 0x11, 0x44, 0xb2, 0x52, 0x64, 0x44, 0x86, 0x8d, 0x51, 0x30, 0xee, 0x1e, 0xec, 0xc7, 0xff, - 0x94, 0xb8, 0x8b, 0x6f, 0x43, 0xc6, 0x89, 0xe7, 0x25, 0x57, 0x2e, 0xd0, 0x6d, 0xe8, 0x4a, 0x95, - 0xb3, 0x52, 0xa5, 0x1c, 0xab, 0x69, 0xb8, 0x69, 0xa2, 0x83, 0x35, 0x9d, 0x62, 0x35, 0x75, 0x00, - 0x22, 0x84, 0x05, 0x6c, 0x55, 0x00, 0x22, 0x84, 0x01, 0x0c, 0xa0, 0x41, 0x8a, 0x45, 0xd8, 0x34, - 0x49, 0xea, 0x4f, 0x9d, 0x77, 0x29, 0x89, 0x08, 0x5b, 0x06, 0x6b, 0xbe, 0xd1, 0x2d, 0x68, 0x2b, - 0x2c, 0xcf, 0xd3, 0x9c, 0x8a, 0xb0, 0x6d, 0xec, 0x2d, 0x7d, 0x3e, 0xa2, 0x02, 0xdd, 0x81, 0x1d, - 0x9f, 0x4f, 0x3a, 0xa3, 0x73, 0xaa, 0x64, 0xd8, 0x19, 0x05, 0xe3, 0x76, 0xb2, 0xed, 0xcd, 0xc7, - 0xc6, 0x8a, 0xf6, 0xe1, 0x7f, 0x2f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, - 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x9d, 0xda, 0xbf, 0x0e, 0xcd, 0x3f, 0xe8, 0x08, - 0x9a, 0x73, 0x56, 0x16, 0x4a, 0x86, 0xdd, 0x51, 0x63, 0xdc, 0x3d, 0xb8, 0x5b, 0xf3, 0xa9, 0x9e, - 0x69, 0x52, 0xe2, 0xb8, 0xe8, 0x3b, 0x68, 0xe5, 0x64, 0x41, 0xf5, 0x8b, 0xf7, 0x8c, 0x9b, 0xcf, - 0x6b, 0xba, 0x39, 0x32, 0xac, 0xc4, 0xb3, 0xd1, 0x14, 0x6e, 0x16, 0x44, 0x5d, 0x30, 0x71, 0x9e, - 0x52, 0xc9, 0x66, 0x58, 0x51, 0x56, 0x84, 0x7d, 0x53, 0xc4, 0x2f, 0x6b, 0xba, 0x3c, 0xb1, 0xfc, - 0xa7, 0x9e, 0x7e, 0xc6, 0x49, 0x96, 0x0c, 0x8a, 0x6b, 0x56, 0x14, 0x41, 0xbf, 0x60, 0x29, 0xa7, - 0x0b, 0xa6, 0x52, 0xc1, 0x98, 0x0a, 0xb7, 0xcd, 0x1b, 0x75, 0x0b, 0x76, 0xaa, 0x6d, 0x09, 0x63, - 0x0a, 0x8d, 0x61, 0x90, 0x93, 0x57, 0xb8, 0x9c, 0xa9, 0x94, 0xd3, 0x3c, 0x9d, 0xb3, 0x9c, 0x84, - 0x3b, 0xa6, 0x34, 0xdb, 0xce, 0x7e, 0x4a, 0xf3, 0x67, 0x2c, 0x27, 0xcb, 0x48, 0xca, 0x33, 0x8b, - 0x1c, 0xac, 0x20, 0x9f, 0xf2, 0xcc, 0x20, 0x3f, 0x86, 0x7e, 0xc6, 0x4b, 0x49, 0x94, 0xaf, 0xcd, - 0x4d, 0x03, 0xeb, 0x59, 0xa3, 0xab, 0xca, 0x87, 0x00, 0x78, 0x36, 0x63, 0x17, 0x69, 0x86, 0xb9, - 0x0c, 0x91, 0x69, 0x9c, 0x8e, 0xb1, 0x1c, 0x62, 0x2e, 0xa3, 0x9f, 0x61, 0xdb, 0x4f, 0x86, 0xe4, - 0xac, 0x90, 0x04, 0x9d, 0x40, 0xcb, 0x95, 0xdc, 0x8c, 0x47, 0xf7, 0xe0, 0x7e, 0x5c, 0x6f, 0x56, - 0x63, 0xd7, 0x0e, 0x67, 0x0a, 0x2b, 0x92, 0x78, 0x27, 0x51, 0x1f, 0xba, 0x2f, 0x30, 0x55, 0x6e, - 0xf2, 0xa2, 0x9f, 0xa0, 0x67, 0x8f, 0xff, 0x51, 0xb8, 0x63, 0xd8, 0x39, 0x9b, 0x96, 0x2a, 0x67, - 0x17, 0x85, 0x1f, 0xf6, 0x5d, 0x68, 0x4a, 0x3a, 0x29, 0xf0, 0xcc, 0xcd, 0xbb, 0x3b, 0xa1, 0x8f, - 0xa0, 0x37, 0x11, 0x38, 0x23, 0x29, 0x27, 0x82, 0xb2, 0x3c, 0xdc, 0x18, 0x05, 0xe3, 0x46, 0xd2, - 0x35, 0xb6, 0x53, 0x63, 0x8a, 0x10, 0x0c, 0xae, 0xbc, 0xd9, 0x8c, 0xa3, 0x29, 0xec, 0x7e, 0xcf, - 0x73, 0x1d, 0xb4, 0x9a, 0x71, 0x17, 0x68, 0x45, 0x2f, 0x82, 0x7f, 0xad, 0x17, 0xd1, 0x2d, 0xf8, - 0xff, 0x1b, 0x91, 0x5c, 0x12, 0x03, 0xd8, 0xfe, 0x81, 0x08, 0x49, 0x99, 0xbf, 0x65, 0xf4, 0x19, - 0xec, 0x54, 0x16, 0xf7, 0xb6, 0x21, 0xb4, 0x16, 0xd6, 0xe4, 0x6e, 0xee, 0x8f, 0xd1, 0xa7, 0xd0, - 0xd3, 0xef, 0x56, 0x65, 0x3e, 0x84, 0x36, 0x2d, 0x14, 0x11, 0x0b, 0xf7, 0x48, 0x8d, 0xa4, 0x3a, - 0x47, 0x2f, 0xa0, 0xef, 0xb0, 0xce, 0xed, 0xb7, 0xb0, 0x25, 0xb5, 0x61, 0xcd, 0x2b, 0x3e, 0xc7, - 0xf2, 0xdc, 0x3a, 0xb2, 0xf4, 0xe8, 0x0e, 0xf4, 0xcf, 0x4c, 0x25, 0xde, 0x5e, 0xa8, 0x2d, 0x5f, - 0x28, 0x7d, 0x59, 0x0f, 0x74, 0xd7, 0x3f, 0x87, 0xee, 0x93, 0x4b, 0x92, 0x79, 0xe2, 0x43, 0x68, - 0xe7, 0x04, 0xe7, 0x33, 0x5a, 0x10, 0x97, 0xd4, 0x30, 0xb6, 0x8b, 0x23, 0xf6, 0x8b, 0x23, 0x7e, - 0xee, 0x17, 0x47, 0x52, 0x61, 0xfd, 0x1a, 0xd8, 0x78, 0x73, 0x0d, 0x34, 0xae, 0xd6, 0x40, 0x74, - 0x08, 0x3d, 0x1b, 0xcc, 0xdd, 0x7f, 0x17, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, 0x71, - 0x27, 0xf4, 0x01, 0x74, 0xc8, 0x25, 0x55, 0x69, 0xa6, 0x47, 0x76, 0xc3, 0xdc, 0xa0, 0xad, 0x0d, - 0x87, 0x2c, 0x27, 0xd1, 0x1f, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, 0xd5, - 0x9f, 0xef, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x25, 0x9a, 0x65, - 0xf2, 0xee, 0x6b, 0x1b, 0xdc, 0xc1, 0x5f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x85, 0xa6, - 0x9d, 0x7e, 0xf4, 0xa0, 0xee, 0xd4, 0xad, 0xec, 0xd1, 0xe1, 0xc3, 0x75, 0x69, 0xae, 0x7e, 0x37, - 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x5e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xbf, 0x1e, 0xa9, - 0x0a, 0xfa, 0x3b, 0xb4, 0xfd, 0x38, 0xa3, 0x47, 0x75, 0x7d, 0x5c, 0x93, 0x93, 0xe1, 0x17, 0xeb, - 0x13, 0xab, 0x04, 0xfe, 0x0c, 0x60, 0xe7, 0xda, 0x48, 0xa3, 0xaf, 0xea, 0xfa, 0x7b, 0xbb, 0xea, - 0x0c, 0x1f, 0xbf, 0x37, 0xbf, 0x4a, 0xeb, 0x37, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, 0x2a, - 0x3f, 0xc3, 0x47, 0x6b, 0xf3, 0xaa, 0xe8, 0x97, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, 0xb2, - 0x76, 0x0d, 0x1f, 0xac, 0xc9, 0xf2, 0x71, 0xf7, 0x03, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, 0xff, - 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x6b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, 0xd2, - 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x0d, 0xf4, 0x3a, 0x80, 0xbe, 0x36, 0x9d, 0x29, 0x41, - 0xf0, 0x9c, 0x16, 0x13, 0xf4, 0xb8, 0xa6, 0x78, 0x6b, 0x96, 0x15, 0x70, 0xc7, 0xf4, 0xa9, 0x7c, - 0xfd, 0xfe, 0x0e, 0x7c, 0x5a, 0xe3, 0x60, 0x3f, 0xf8, 0xa6, 0xf5, 0xe3, 0x96, 0xd5, 0xac, 0xa6, - 0xf9, 0xb9, 0xf7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x41, 0x13, 0xe3, 0x8e, 0x50, 0x0c, 0x00, - 0x00, + // 1058 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x6d, 0x6f, 0x1b, 0x45, + 0x10, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0x2e, 0xa8, 0x5c, 0x8d, 0x50, 0xcd, 0x21, 0x51, + 0x0b, 0xca, 0x25, 0x4a, 0xdf, 0x90, 0x90, 0x28, 0x22, 0x2d, 0xa8, 0x52, 0x1a, 0x45, 0x97, 0x42, + 0x25, 0x3e, 0x70, 0x6c, 0xee, 0xb6, 0xf6, 0x2a, 0xf6, 0xed, 0xb2, 0xbb, 0xe7, 0x04, 0x09, 0x89, + 0x4f, 0xfc, 0x03, 0x90, 0xf8, 0x31, 0xfc, 0x38, 0xb4, 0x6f, 0x17, 0x3b, 0x2d, 0xd5, 0xb9, 0x88, + 0x4f, 0xbe, 0x1d, 0x3f, 0xcf, 0xcc, 0xec, 0xce, 0xcc, 0x33, 0x70, 0x27, 0x17, 0x74, 0x41, 0x84, + 0xdc, 0x95, 0x53, 0x2c, 0x48, 0xbe, 0x4b, 0x2e, 0x48, 0x56, 0x2a, 0x26, 0x76, 0xb9, 0x60, 0x8a, + 0x55, 0xc7, 0xd8, 0x1c, 0xd1, 0xc7, 0x53, 0x2c, 0xa7, 0x34, 0x63, 0x82, 0xc7, 0x05, 0x9b, 0xe3, + 0x3c, 0xe6, 0xb3, 0x72, 0x42, 0x0b, 0x19, 0xaf, 0xe2, 0x86, 0xb7, 0x26, 0x8c, 0x4d, 0x66, 0xc4, + 0x3a, 0x39, 0x2d, 0x5f, 0xee, 0x2a, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0x77, + 0x7d, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0xfa, 0xbb, 0x09, 0xfd, 0x43, 0x5c, 0x16, 0xd9, 0x34, + 0x21, 0x3f, 0x97, 0x44, 0x2a, 0x34, 0x80, 0x46, 0x36, 0xcf, 0xc3, 0x60, 0x14, 0x8c, 0x3b, 0x89, + 0xfe, 0x44, 0x08, 0x36, 0xb1, 0x98, 0xc8, 0x70, 0x63, 0xd4, 0x18, 0x77, 0x12, 0xf3, 0x8d, 0x8e, + 0xa0, 0x23, 0x88, 0x64, 0xa5, 0xc8, 0x88, 0x0c, 0x1b, 0xa3, 0x60, 0xdc, 0xdd, 0xdf, 0x8b, 0xff, + 0x2d, 0x71, 0x17, 0xdf, 0x86, 0x8c, 0x13, 0xcf, 0x4b, 0x2e, 0x5d, 0xa0, 0x5b, 0xd0, 0x95, 0x2a, + 0x67, 0xa5, 0x4a, 0x39, 0x56, 0xd3, 0x70, 0xd3, 0x44, 0x07, 0x6b, 0x3a, 0xc6, 0x6a, 0xea, 0x00, + 0x44, 0x08, 0x0b, 0xd8, 0xaa, 0x00, 0x44, 0x08, 0x03, 0x18, 0x40, 0x83, 0x14, 0x8b, 0xb0, 0x69, + 0x92, 0xd4, 0x9f, 0x3a, 0xef, 0x52, 0x12, 0x11, 0xb6, 0x0c, 0xd6, 0x7c, 0xa3, 0x9b, 0xd0, 0x56, + 0x58, 0x9e, 0xa5, 0x39, 0x15, 0x61, 0xdb, 0xd8, 0x5b, 0xfa, 0xfc, 0x98, 0x0a, 0x74, 0x1b, 0x76, + 0x7c, 0x3e, 0xe9, 0x8c, 0xce, 0xa9, 0x92, 0x61, 0x67, 0x14, 0x8c, 0xdb, 0xc9, 0xb6, 0x37, 0x1f, + 0x1a, 0x2b, 0xda, 0x83, 0x77, 0x4f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36, + 0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x1d, 0xdb, 0xbf, 0x0e, 0xcc, 0x3f, 0xe8, 0x31, + 0x34, 0xe7, 0xac, 0x2c, 0x94, 0x0c, 0xbb, 0xa3, 0xc6, 0xb8, 0xbb, 0x7f, 0xa7, 0xe6, 0x53, 0x3d, + 0xd3, 0xa4, 0xc4, 0x71, 0xd1, 0xb7, 0xd0, 0xca, 0xc9, 0x82, 0xea, 0x17, 0xef, 0x19, 0x37, 0x9f, + 0xd5, 0x74, 0xf3, 0xd8, 0xb0, 0x12, 0xcf, 0x46, 0x53, 0xb8, 0x5e, 0x10, 0x75, 0xce, 0xc4, 0x59, + 0x4a, 0x25, 0x9b, 0x61, 0x45, 0x59, 0x11, 0xf6, 0x4d, 0x11, 0xbf, 0xa8, 0xe9, 0xf2, 0xc8, 0xf2, + 0x9f, 0x7a, 0xfa, 0x09, 0x27, 0x59, 0x32, 0x28, 0xae, 0x58, 0x51, 0x04, 0xfd, 0x82, 0xa5, 0x9c, + 0x2e, 0x98, 0x4a, 0x05, 0x63, 0x2a, 0xdc, 0x36, 0x6f, 0xd4, 0x2d, 0xd8, 0xb1, 0xb6, 0x25, 0x8c, + 0x29, 0x34, 0x86, 0x41, 0x4e, 0x5e, 0xe2, 0x72, 0xa6, 0x52, 0x4e, 0xf3, 0x74, 0xce, 0x72, 0x12, + 0xee, 0x98, 0xd2, 0x6c, 0x3b, 0xfb, 0x31, 0xcd, 0x9f, 0xb1, 0x9c, 0x2c, 0x23, 0x29, 0xcf, 0x2c, + 0x72, 0xb0, 0x82, 0x7c, 0xca, 0x33, 0x83, 0xfc, 0x08, 0xfa, 0x19, 0x2f, 0x25, 0x51, 0xbe, 0x36, + 0xd7, 0x0d, 0xac, 0x67, 0x8d, 0xae, 0x2a, 0x1f, 0x00, 0xe0, 0xd9, 0x8c, 0x9d, 0xa7, 0x19, 0xe6, + 0x32, 0x44, 0xa6, 0x71, 0x3a, 0xc6, 0x72, 0x80, 0xb9, 0x44, 0x11, 0xf4, 0x32, 0xcc, 0xf1, 0x29, + 0x9d, 0x51, 0x45, 0x89, 0x0c, 0xdf, 0x31, 0x80, 0x15, 0x5b, 0xf4, 0x13, 0x6c, 0xfb, 0xe9, 0x91, + 0x9c, 0x15, 0x92, 0xa0, 0x23, 0x68, 0xb9, 0xb6, 0x30, 0x23, 0xd4, 0xdd, 0xbf, 0x17, 0xd7, 0x9b, + 0xe7, 0xd8, 0xb5, 0xcc, 0x89, 0xc2, 0x8a, 0x24, 0xde, 0x49, 0xd4, 0x87, 0xee, 0x0b, 0x4c, 0x95, + 0x9b, 0xce, 0xe8, 0x47, 0xe8, 0xd9, 0xe3, 0xff, 0x14, 0xee, 0x10, 0x76, 0x4e, 0xa6, 0xa5, 0xca, + 0xd9, 0x79, 0xe1, 0x05, 0xe1, 0x06, 0x34, 0x25, 0x9d, 0x14, 0x78, 0xe6, 0x34, 0xc1, 0x9d, 0xd0, + 0x87, 0xd0, 0x9b, 0x08, 0x9c, 0x91, 0x94, 0x13, 0x41, 0x59, 0x1e, 0x6e, 0x8c, 0x82, 0x71, 0x23, + 0xe9, 0x1a, 0xdb, 0xb1, 0x31, 0x45, 0x08, 0x06, 0x97, 0xde, 0x6c, 0xc6, 0xd1, 0x14, 0x6e, 0x7c, + 0xc7, 0x73, 0x1d, 0xb4, 0xd2, 0x01, 0x17, 0x68, 0x45, 0x53, 0x82, 0xff, 0xac, 0x29, 0xd1, 0x4d, + 0x78, 0xef, 0x95, 0x48, 0x2e, 0x89, 0x01, 0x6c, 0x7f, 0x4f, 0x84, 0xa4, 0xcc, 0xdf, 0x32, 0xfa, + 0x14, 0x76, 0x2a, 0x8b, 0x7b, 0xdb, 0x10, 0x5a, 0x0b, 0x6b, 0x72, 0x37, 0xf7, 0xc7, 0xe8, 0x13, + 0xe8, 0xe9, 0x77, 0xab, 0x32, 0x1f, 0x42, 0x9b, 0x16, 0x8a, 0x88, 0x85, 0x7b, 0xa4, 0x46, 0x52, + 0x9d, 0xa3, 0x17, 0xd0, 0x77, 0x58, 0xe7, 0xf6, 0x1b, 0xd8, 0x92, 0xda, 0xb0, 0xe6, 0x15, 0x9f, + 0x63, 0x79, 0x66, 0x1d, 0x59, 0x7a, 0x74, 0x1b, 0xfa, 0x27, 0xa6, 0x12, 0xaf, 0x2f, 0xd4, 0x96, + 0x2f, 0x94, 0xbe, 0xac, 0x07, 0xba, 0xeb, 0x9f, 0x41, 0xf7, 0xc9, 0x05, 0xc9, 0x3c, 0xf1, 0x01, + 0xb4, 0x73, 0x82, 0xf3, 0x19, 0x2d, 0x88, 0x4b, 0x6a, 0x18, 0xdb, 0xe5, 0x12, 0xfb, 0xe5, 0x12, + 0x3f, 0xf7, 0xcb, 0x25, 0xa9, 0xb0, 0x7e, 0x55, 0x6c, 0xbc, 0xba, 0x2a, 0x1a, 0x97, 0xab, 0x22, + 0x3a, 0x80, 0x9e, 0x0d, 0xe6, 0xee, 0x7f, 0x03, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f, + 0x71, 0x27, 0xf4, 0x3e, 0x74, 0xc8, 0x05, 0x55, 0x69, 0xa6, 0xc7, 0x7a, 0xc3, 0xdc, 0xa0, 0xad, + 0x0d, 0x07, 0x2c, 0x27, 0xd1, 0xef, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37, + 0xd5, 0x9f, 0x6f, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x6d, 0x9a, + 0x85, 0xf3, 0xe6, 0x6b, 0x1b, 0xdc, 0xfe, 0x9f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x81, + 0xa6, 0x9d, 0x7e, 0x74, 0xbf, 0xee, 0xd4, 0xad, 0xec, 0xda, 0xe1, 0x83, 0x75, 0x69, 0xae, 0x7e, + 0xd7, 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x6e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xb7, 0x1e, + 0xa9, 0x0a, 0xfa, 0x1b, 0xb4, 0xfd, 0x38, 0xa3, 0x87, 0x75, 0x7d, 0x5c, 0x91, 0x93, 0xe1, 0xe7, + 0xeb, 0x13, 0xab, 0x04, 0xfe, 0x08, 0x60, 0xe7, 0xca, 0x48, 0xa3, 0x2f, 0xeb, 0xfa, 0x7b, 0xbd, + 0xea, 0x0c, 0x1f, 0xbd, 0x35, 0xbf, 0x4a, 0xeb, 0x57, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba, + 0x2a, 0x3f, 0xc3, 0x87, 0x6b, 0xf3, 0xaa, 0xe8, 0x17, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb, + 0xb2, 0x76, 0x0d, 0xef, 0xaf, 0xc9, 0xf2, 0x71, 0xf7, 0x02, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7, + 0xff, 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x2b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f, + 0xd2, 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x35, 0xf4, 0x57, 0x00, 0x7d, 0x6d, 0x3a, 0x51, + 0x82, 0xe0, 0x39, 0x2d, 0x26, 0xe8, 0x51, 0x4d, 0xf1, 0xd6, 0x2c, 0x2b, 0xe0, 0x8e, 0xe9, 0x53, + 0xf9, 0xea, 0xed, 0x1d, 0xf8, 0xb4, 0xc6, 0xc1, 0x5e, 0xf0, 0x75, 0xeb, 0x87, 0x2d, 0xab, 0x59, + 0x4d, 0xf3, 0x73, 0xf7, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2e, 0x63, 0xc4, 0xd3, 0x74, 0x0c, + 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/drivers/shared/executor/proto/executor.proto b/drivers/shared/executor/proto/executor.proto index 9ac241209..aa349d265 100644 --- a/drivers/shared/executor/proto/executor.proto +++ b/drivers/shared/executor/proto/executor.proto @@ -46,6 +46,7 @@ message LaunchRequest { string default_ipc_mode = 16; string cpuset_cgroup = 17; repeated string allow_caps = 18; + repeated string capabilities = 19; } message LaunchResponse { diff --git a/drivers/shared/executor/server.go b/drivers/shared/executor/server.go index 7ef91f4d3..9583bbb7b 100644 --- a/drivers/shared/executor/server.go +++ b/drivers/shared/executor/server.go @@ -37,6 +37,7 @@ func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchReques NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation), ModePID: req.DefaultPidMode, ModeIPC: req.DefaultIpcMode, + Capabilities: req.Capabilities, }) if err != nil { From 591f0f02199313ef7481201f5ad7303623369664 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Mon, 17 May 2021 12:52:52 -0600 Subject: [PATCH 8/8] docs: minor wording tweaks + cl --- CHANGELOG.md | 1 + website/content/docs/drivers/docker.mdx | 2 +- website/content/docs/drivers/exec.mdx | 2 +- website/content/docs/drivers/java.mdx | 2 +- website/content/docs/upgrade/upgrade-specific.mdx | 9 +++++---- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c944cd749..a2dfd29e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ FEATURES: __BACKWARDS INCOMPATIBILITIES:__ * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)] + * drivers/exec+java: Reduce set of linux capabilities enabled by default [[GH-10600](https://github.com/hashicorp/nomad/pull/10600)] * licensing: Enterprise licenses are no longer stored in raft or synced between servers. Loading the Enterprise license from disk or environment is required. The `nomad license put` command has been removed. [[GH-10458](https://github.com/hashicorp/nomad/issues/10458)] SECURITY: diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index 3c202e107..16e248be7 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -824,7 +824,7 @@ plugin "docker" { ``` which is the same list of capabilities allowed by [docker by default][docker_caps] - (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports the value `"all"` as a shortcut for allow-listing all capabilities supported by the operating system. diff --git a/website/content/docs/drivers/exec.mdx b/website/content/docs/drivers/exec.mdx index df557b7c7..065693233 100644 --- a/website/content/docs/drivers/exec.mdx +++ b/website/content/docs/drivers/exec.mdx @@ -167,7 +167,7 @@ able to make use of IPC features, like sending unexpected POSIX signals. ``` which is modeled after the capabilities allowed by [docker by default][docker_caps] - (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports the value `"all"` as a shortcut for allow-listing all capabilities supported by the operating system. diff --git a/website/content/docs/drivers/java.mdx b/website/content/docs/drivers/java.mdx index f7e180a52..ad0957dbf 100644 --- a/website/content/docs/drivers/java.mdx +++ b/website/content/docs/drivers/java.mdx @@ -167,7 +167,7 @@ able to make use of IPC features, like sending unexpected POSIX signals. ``` which is modeled after the capabilities allowed by [docker by default][docker_caps] - (sans [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities + (without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports the value `"all"` as a shortcut for allow-listing all capabilities supported by the operating system. diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index e74d16f3c..0a5c1ac61 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -60,20 +60,21 @@ already explicitly set `CONSUL_HTTP_ADDR` then it will not get overridden. Following the security [remediation][no_net_raw] in Nomad versions 0.12.12, 1.0.5, and 1.1.0-rc1, the `exec` and `java` task drivers will additionally no longer enable -the following linux capabilities by default: +the following linux capabilities by default. ``` AUDIT_CONTROL AUDIT_READ BLOCK_SUSPEND DAC_READ_SEARCH IPC_LOCK IPC_OWNER LEASE -LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST SYS_ADMIN +LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST NET_RAW SYS_ADMIN SYS_BOOT SYSLOG SYS_MODULE SYS_NICE SYS_PACCT SYS_PTRACE SYS_RAWIO SYS_RESOURCE SYS_TIME SYS_TTY_CONFIG WAKE_ALARM ``` -The capabilities now enabled by default are modeled after Docker default [`linux capabilities`]: +The capabilities now enabled by default are modeled after Docker default +[`linux capabilities`] (excluding `NET_RAW`). ``` AUDIT_WRITE CHOWN DAC_OVERRIDE FOWNER FSETID KILL MKNOD NET_BIND_SERVICE -NET_RAW SETFCAP SETGID SETPCAP SETUID SYS_CHROOT +SETFCAP SETGID SETPCAP SETUID SYS_CHROOT ``` A new `allow_caps` plugin configuration parameter for [`exec`][allow_caps_exec]