luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge pull request #10600 from hashicorp/f-exec-allow_caps 

drivers/exec+java: reduce default set of linux capabilities
This commit is contained in:
Seth Hoenig 2021-05-17 13:11:04 -06:00 committed by Tim Gross
parent a99decde7d
commit abdcf0ac17
28 changed files with 1575 additions and 735 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -13,6 +13,7 @@ FEATURES:
__BACKWARDS INCOMPATIBILITIES:__
* csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)]
* drivers/exec+java: Reduce set of linux capabilities enabled by default [[GH-10600](https://github.com/hashicorp/nomad/pull/10600)]
* licensing: Enterprise licenses are no longer stored in raft or synced between servers. Loading the Enterprise license from disk or environment is required. The `nomad license put` command has been removed. [[GH-10458](https://github.com/hashicorp/nomad/issues/10458)]
SECURITY:

43
drivers/docker/config.go
View File

@ -10,6 +10,7 @@ import (
docker "github.com/fsouza/go-dockerclient"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
"github.com/hashicorp/nomad/helper/pluginutils/loader"
"github.com/hashicorp/nomad/plugins/base"
@ -41,36 +42,6 @@ const (
dockerAuthHelperPrefix = "docker-credential-"
)
// nomadDefaultCaps is the subset of dockerDefaultCaps that Nomad enables by
// default and is used to compute the set of capabilities to add/drop given
// docker driver configuration.
func nomadDefaultCaps() []string {
return []string{
"AUDIT_WRITE",
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"FSETID",
"KILL",
"MKNOD",
"NET_BIND_SERVICE",
"SETFCAP",
"SETGID",
"SETPCAP",
"SETUID",
"SYS_CHROOT",
}
}
// dockerDefaultCaps is a list of Linux capabilities enabled by docker by default
// and is used to compute the set of capabilities to add/drop given docker driver
// configuration, as well as Nomad built-in limitations.
//
// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
func dockerDefaultCaps() []string {
return append(nomadDefaultCaps(), "NET_RAW")
}
func PluginLoader(opts map[string]string) (map[string]interface{}, error) {
conf := map[string]interface{}{}
if v, ok := opts["docker.endpoint"]; ok {
@ -287,7 +258,7 @@ var (
"allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false),
"allow_caps": hclspec.NewDefault(
hclspec.NewAttr("allow_caps", "list(string)", false),
hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`),
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
),
"nvidia_runtime": hclspec.NewDefault(
hclspec.NewAttr("nvidia_runtime", "string", false),
@ -427,9 +398,9 @@ var (
"work_dir": hclspec.NewAttr("work_dir", "string", false),
})
// capabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
capabilities = &drivers.Capabilities{
// driverCapabilities represents the RPC response for what features are
// implemented by the docker task driver
driverCapabilities = &drivers.Capabilities{
SendSignals: true,
Exec: true,
FSIsolation: drivers.FSIsolationImage,
@ -788,8 +759,10 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
return taskConfigSpec, nil
}
// Capabilities is returned by the Capabilities RPC and indicates what optional
// features this driver supports.
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
return capabilities, nil
return driverCapabilities, nil
}
var _ drivers.InternalCapabilitiesDriver = (*Driver)(nil)

122
drivers/docker/driver.go
View File

@ -10,7 +10,6 @@ import (
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
@ -23,10 +22,9 @@ import (
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/drivers/docker/docklog"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/drivers/shared/eventer"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/drivers/shared/resolvconf"
"github.com/hashicorp/nomad/helper"
nstructs "github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/plugins/drivers"
@ -913,8 +911,9 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
hostConfig.Privileged = driverConfig.Privileged
// set add/drop capabilities
hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig)
if err != nil {
if hostConfig.CapAdd, hostConfig.CapDrop, err = capabilities.Delta(
capabilities.DockerDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
); err != nil {
return c, err
}
@ -1184,119 +1183,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
}, nil
}
// getCaps computes the capabilities to supply to the --add-cap and --drop-cap
// options to the docker driver, which override the default capabilities enabled
// by docker itself.
func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) {
// capabilities allowable by client docker plugin configuration
allowCaps := expandAllowCaps(d.config.AllowCaps)
// capabilities the task docker config is asking for based on the default
// capabilities allowable by nomad
desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop)
if err != nil {
return nil, nil, err
}
// capabilities the task is requesting that are NOT allowed by the docker plugin
if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 {
return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing)
}
// capabilities that should be dropped relative to the docker default capabilities
dropCaps := capDrops(taskConfig.CapDrop, allowCaps)
return taskConfig.CapAdd, dropCaps, nil
}
// capDrops will compute the total dropped capabilities set
//
// {task cap_drop} U ({docker defaults} \ {driver allow caps})
func capDrops(dropCaps []string, allowCaps []string) []string {
dropSet := make(map[string]struct{})
for _, c := range normalizeCaps(dropCaps) {
dropSet[c] = struct{}{}
}
// if dropCaps includes ALL, no need to iterate every capability
if _, exists := dropSet["ALL"]; exists {
return []string{"ALL"}
}
dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps()))
allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps))
// find the docker default caps not in allowed caps
for dCap := range dockerDefaults {
if _, exists := allowedCaps[dCap]; !exists {
dropSet[dCap] = struct{}{}
}
}
drops := make([]string, 0, len(dropSet))
for c := range dropSet {
drops = append(drops, c)
}
sort.Strings(drops)
return drops
}
// expandAllowCaps returns the normalized set of allowable capabilities set
// for the docker plugin configuration.
func expandAllowCaps(allowCaps []string) []string {
if len(allowCaps) == 0 {
return nil
}
set := make(map[string]struct{}, len(allowCaps))
for _, rawCap := range allowCaps {
capability := strings.ToUpper(rawCap)
if capability == "ALL" {
for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) {
set[defCap] = struct{}{}
}
} else {
set[capability] = struct{}{}
}
}
result := make([]string, 0, len(set))
for capability := range set {
result = append(result, capability)
}
sort.Strings(result)
return result
}
// missingCaps returns the set of elements in desired that are not present in
// allowed. The elements in desired are first upper-cased before comparison.
// The elements in allowed are assumed to be upper-cased.
func missingCaps(allowed, desired []string) []string {
_, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired))
sort.Strings(missing)
return missing
}
// normalizeCaps returns a copy of caps with duplicate elements removed and all
// elements upper-cased.
func normalizeCaps(caps []string) []string {
set := make(map[string]struct{}, len(caps))
for _, c := range caps {
normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_")
set[strings.ToUpper(normal)] = struct{}{}
}
result := make([]string, 0, len(set))
for c := range set {
result = append(result, c)
}
sort.Strings(result)
return result
}
func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) {
hm, err := m.toDockerHostMount()
if err != nil {

21
drivers/docker/driver_default.go
View File

@ -3,30 +3,9 @@
package docker
import (
"github.com/docker/docker/oci/caps"
docker "github.com/fsouza/go-dockerclient"
)
func getPortBinding(ip string, port string) docker.PortBinding {
return docker.PortBinding{HostIP: ip, HostPort: port}
}
func tweakCapabilities(basics, adds, drops []string) ([]string, error) {
// Moby mixes 2 different capabilities formats: prefixed with "CAP_"
// and not. We do the conversion here to have a consistent,
// non-prefixed format on the Nomad side.
for i, cap := range basics {
basics[i] = "CAP_" + cap
}
effectiveCaps, err := caps.TweakCapabilities(basics, adds, drops, nil, false)
if err != nil {
return effectiveCaps, err
}
for i, cap := range effectiveCaps {
effectiveCaps[i] = cap[len("CAP_"):]
}
return effectiveCaps, nil
}

191
drivers/docker/driver_test.go
View File

@ -19,7 +19,6 @@ import (
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper/freeport"
"github.com/hashicorp/nomad/helper/pluginutils/hclspecutils"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
@ -1387,44 +1386,44 @@ func TestDockerDriver_Capabilities(t *testing.T) {
{
Name: "default-allowlist-add-allowed",
CapAdd: []string{"fowner", "mknod"},
CapDrop: []string{"ALL"},
CapDrop: []string{"all"},
},
{
Name: "default-allowlist-add-forbidden",
CapAdd: []string{"net_admin"},
StartError: "NET_ADMIN",
StartError: "net_admin",
},
{
Name: "default-allowlist-drop-existing",
CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"},
CapDrop: []string{"fowner", "mknod", "net_raw"},
},
{
Name: "restrictive-allowlist-drop-all",
CapDrop: []string{"ALL"},
Allowlist: "FOWNER,MKNOD",
CapDrop: []string{"all"},
Allowlist: "fowner,mknod",
},
{
Name: "restrictive-allowlist-add-allowed",
CapAdd: []string{"fowner", "mknod"},
CapDrop: []string{"ALL"},
Allowlist: "fowner,mknod",
CapDrop: []string{"all"},
Allowlist: "mknod,fowner",
},
{
Name: "restrictive-allowlist-add-forbidden",
CapAdd: []string{"net_admin", "mknod"},
CapDrop: []string{"ALL"},
CapDrop: []string{"all"},
Allowlist: "fowner,mknod",
StartError: "NET_ADMIN",
StartError: "net_admin",
},
{
Name: "permissive-allowlist",
CapAdd: []string{"net_admin", "mknod"},
Allowlist: "ALL",
CapAdd: []string{"mknod", "net_admin"},
Allowlist: "all",
},
{
Name: "permissive-allowlist-add-all",
CapAdd: []string{"all"},
Allowlist: "ALL",
Allowlist: "all",
},
}
@ -3064,169 +3063,3 @@ func TestDockerDriver_StopSignal(t *testing.T) {
})
}
}
func TestDockerCaps_normalizeCaps(t *testing.T) {
t.Run("empty", func(t *testing.T) {
result := normalizeCaps(nil)
require.Len(t, result, 0)
})
t.Run("mixed", func(t *testing.T) {
result := normalizeCaps([]string{
"DAC_OVERRIDE", "sys_chroot", "kill", "KILL",
})
require.Equal(t, []string{
"DAC_OVERRIDE", "KILL", "SYS_CHROOT",
}, result)
})
}
func TestDockerCaps_missingCaps(t *testing.T) {
allowed := []string{
"DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN",
}
t.Run("none missing", func(t *testing.T) {
result := missingCaps(allowed, []string{
"SYS_CHROOT", "chown", "KILL",
})
require.Equal(t, []string(nil), result)
})
t.Run("some missing", func(t *testing.T) {
result := missingCaps(allowed, []string{
"chown", "audit_write", "SETPCAP", "dac_override",
})
require.Equal(t, []string{"AUDIT_WRITE", "SETPCAP"}, result)
})
}
func TestDockerCaps_expandAllowCaps(t *testing.T) {
t.Run("empty", func(t *testing.T) {
result := expandAllowCaps(nil)
require.Empty(t, result)
})
t.Run("manual", func(t *testing.T) {
result := expandAllowCaps([]string{
"DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN",
})
require.Equal(t, []string{
"CHOWN", "DAC_OVERRIDE", "KILL", "SYS_CHROOT",
}, result)
})
t.Run("all", func(t *testing.T) {
result := expandAllowCaps([]string{"all"})
exp := normalizeCaps(executor.SupportedCaps(true))
sort.Strings(exp)
require.Equal(t, exp, result)
})
}
func TestDockerCaps_capDrops(t *testing.T) {
// docker default caps is always the same, task configured drop_caps and
// plugin config allow_caps may be altered
// This is the 90% use case, where NET_RAW is dropped, as Nomad's default
// capability allow-list is a subset of the docker default cap list.
t.Run("defaults", func(t *testing.T) {
result := capDrops(nil, nomadDefaultCaps())
require.Equal(t, []string{"NET_RAW"}, result)
})
// Users want to use ICMP (ping).
t.Run("enable net_raw", func(t *testing.T) {
result := capDrops(nil, append(nomadDefaultCaps(), "net_raw"))
require.Empty(t, result)
})
// The plugin is reduced in ability.
t.Run("enable minimal", func(t *testing.T) {
allow := []string{"setgid", "setuid", "chown", "kill"}
exp := []string{"AUDIT_WRITE", "DAC_OVERRIDE", "FOWNER", "FSETID", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETPCAP", "SYS_CHROOT"}
result := capDrops(nil, allow)
require.Equal(t, exp, result)
})
// The task drops abilities.
t.Run("task drops", func(t *testing.T) {
drops := []string{"audit_write", "fowner", "kill", "chown"}
exp := []string{"AUDIT_WRITE", "CHOWN", "FOWNER", "KILL", "NET_RAW"}
result := capDrops(drops, nomadDefaultCaps())
require.Equal(t, exp, result)
})
// Drop all mixed with others.
t.Run("task drops mix", func(t *testing.T) {
drops := []string{"audit_write", "all", "chown"}
exp := []string{"ALL"} // minimized
result := capDrops(drops, nomadDefaultCaps())
require.Equal(t, exp, result)
})
}
func TestDockerCaps_getCaps(t *testing.T) {
testutil.ExecCompatible(t) // tests require linux
t.Run("defaults", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: nomadDefaultCaps(),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: nil, CapDrop: nil,
})
require.NoError(t, err)
require.Empty(t, add)
require.Equal(t, []string{"NET_RAW"}, drop)
})
t.Run("enable net_raw", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: append(nomadDefaultCaps(), "net_raw"),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: nil, CapDrop: nil,
})
require.NoError(t, err)
require.Empty(t, add)
require.Empty(t, drop)
})
t.Run("block sys_time", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: nomadDefaultCaps(),
}}
_, _, err := d.getCaps(&TaskConfig{
CapAdd: []string{"SYS_TIME"},
CapDrop: nil,
})
require.EqualError(t, err, `Docker driver does not have the following caps allow-listed on this Nomad agent: [SYS_TIME]`)
})
t.Run("enable sys_time", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: append(nomadDefaultCaps(), "sys_time"),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: []string{"SYS_TIME"},
CapDrop: nil,
})
require.NoError(t, err)
require.Equal(t, []string{"SYS_TIME"}, add)
require.Equal(t, []string{"NET_RAW"}, drop)
})
t.Run("task drops chown", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: nomadDefaultCaps(),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: nil,
CapDrop: []string{"chown"},
})
require.NoError(t, err)
require.Empty(t, add)
require.Equal(t, []string{"CHOWN", "NET_RAW"}, drop)
})
}

53
drivers/exec/driver.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/hashicorp/nomad/client/lib/cgutil"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/consul-template/signals"
hclog "github.com/hashicorp/go-hclog"
@ -74,6 +75,10 @@ var (
hclspec.NewAttr("default_ipc_mode", "string", false),
hclspec.NewLiteral(`"private"`),
),
"allow_caps": hclspec.NewDefault(
hclspec.NewAttr("allow_caps", "list(string)", false),
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
),
})
// taskConfigSpec is the hcl specification for the driver config section of
@ -83,11 +88,13 @@ var (
"args": hclspec.NewAttr("args", "list(string)", false),
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
})
// capabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
capabilities = &drivers.Capabilities{
// driverCapabilities represents the RPC response for what features are
// implemented by the exec task driver
driverCapabilities = &drivers.Capabilities{
SendSignals: true,
Exec: true,
FSIsolation: drivers.FSIsolationChroot,
@ -141,6 +148,10 @@ type Config struct {
// DefaultModeIPC is the default IPC isolation set for all tasks using
// exec-based task drivers.
DefaultModeIPC string `codec:"default_ipc_mode"`
// AllowCaps configures which Linux Capabilities are enabled for tasks
// running on this node.
AllowCaps []string `codec:"allow_caps"`
}
func (c *Config) validate() error {
@ -156,6 +167,11 @@ func (c *Config) validate() error {
return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
}
badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps))
if !badCaps.Empty() {
return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps)
}
return nil
}
@ -174,6 +190,12 @@ type TaskConfig struct {
// ModeIPC indicates whether IPC namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModeIPC string `codec:"ipc_mode"`
// CapAdd is a set of linux capabilities to enable.
CapAdd []string `codec:"cap_add"`
// CapDrop is a set of linux capabilities to disable.
CapDrop []string `codec:"cap_drop"`
}
func (tc *TaskConfig) validate() error {
@ -189,6 +211,16 @@ func (tc *TaskConfig) validate() error {
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
}
supported := capabilities.Supported()
badAdds := supported.Difference(capabilities.New(tc.CapAdd))
if !badAdds.Empty() {
return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds)
}
badDrops := supported.Difference(capabilities.New(tc.CapDrop))
if !badDrops.Empty() {
return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
}
return nil
}
@ -266,8 +298,10 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
return taskConfigSpec, nil
}
// Capabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
return capabilities, nil
return driverCapabilities, nil
}
func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
@ -439,6 +473,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}
caps, err := capabilities.Calculate(
capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
)
if err != nil {
return nil, nil, err
}
d.logger.Debug("task capabilities", "capabilities", caps)
execCmd := &executor.ExecCommand{
Cmd: driverConfig.Command,
Args: driverConfig.Args,
@ -455,6 +497,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
NetworkIsolation: cfg.NetworkIsolation,
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
Capabilities: caps,
}
ps, err := exec.Launch(execCmd)
@ -482,7 +525,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
if err := handle.SetDriverState(&driverState); err != nil {
d.logger.Error("failed to start task, error setting driver state", "error", err)
exec.Shutdown("", 0)
_ = exec.Shutdown("", 0)
pluginClient.Kill()
return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
}

57
drivers/exec/driver_test.go
View File

@ -764,6 +764,7 @@ func TestExecDriver_NoPivotRoot(t *testing.T) {
}
func TestDriver_Config_validate(t *testing.T) {
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
@ -780,9 +781,30 @@ func TestDriver_Config_validate(t *testing.T) {
DefaultModeIPC: tc.ipcMode,
}).validate())
}
})
t.Run("allow_caps", func(t *testing.T) {
for _, tc := range []struct {
ac []string
exp error
}{
{ac: []string{}, exp: nil},
{ac: []string{"all"}, exp: nil},
{ac: []string{"chown", "sys_time"}, exp: nil},
{ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil},
{ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: "private",
DefaultModeIPC: "private",
AllowCaps: tc.ac,
}).validate())
}
})
}
func TestDriver_TaskConfig_validate(t *testing.T) {
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
@ -802,4 +824,39 @@ func TestDriver_TaskConfig_validate(t *testing.T) {
ModeIPC: tc.ipcMode,
}).validate())
}
})
t.Run("cap_add", func(t *testing.T) {
for _, tc := range []struct {
adds []string
exp error
}{
{adds: nil, exp: nil},
{adds: []string{"chown"}, exp: nil},
{adds: []string{"CAP_CHOWN"}, exp: nil},
{adds: []string{"chown", "sys_time"}, exp: nil},
{adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapAdd: tc.adds,
}).validate())
}
})
t.Run("cap_drop", func(t *testing.T) {
for _, tc := range []struct {
drops []string
exp error
}{
{drops: nil, exp: nil},
{drops: []string{"chown"}, exp: nil},
{drops: []string{"CAP_CHOWN"}, exp: nil},
{drops: []string{"chown", "sys_time"}, exp: nil},
{drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapDrop: tc.drops,
}).validate())
}
})
}

129
drivers/exec/driver_unix_test.go
View File

@ -5,9 +5,13 @@ package exec
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/drivers/shared/executor"
basePlug "github.com/hashicorp/nomad/plugins/base"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
@ -173,5 +177,128 @@ func TestExec_dnsConfig(t *testing.T) {
dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg)
}
}
func TestExecDriver_Capabilities(t *testing.T) {
ctestutils.ExecCompatible(t)
task := &drivers.TaskConfig{
ID: uuid.Generate(),
Name: "sleep",
}
for _, tc := range []struct {
Name string
CapAdd []string
CapDrop []string
AllowList string
StartError string
}{
{
Name: "default-allowlist-add-allowed",
CapAdd: []string{"fowner", "mknod"},
CapDrop: []string{"ALL"},
},
{
Name: "default-allowlist-add-forbidden",
CapAdd: []string{"net_admin"},
StartError: "net_admin",
},
{
Name: "default-allowlist-drop-existing",
CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"},
},
{
Name: "restrictive-allowlist-drop-all",
CapDrop: []string{"ALL"},
AllowList: "FOWNER,MKNOD",
},
{
Name: "restrictive-allowlist-add-allowed",
CapAdd: []string{"fowner", "mknod"},
CapDrop: []string{"ALL"},
AllowList: "fowner,mknod",
},
{
Name: "restrictive-allowlist-add-forbidden",
CapAdd: []string{"net_admin", "mknod"},
CapDrop: []string{"ALL"},
AllowList: "fowner,mknod",
StartError: "net_admin",
},
{
Name: "restrictive-allowlist-add-multiple-forbidden",
CapAdd: []string{"net_admin", "mknod", "CAP_SYS_TIME"},
CapDrop: []string{"ALL"},
AllowList: "fowner,mknod",
StartError: "net_admin, sys_time",
},
{
Name: "permissive-allowlist",
CapAdd: []string{"net_admin", "mknod"},
AllowList: "ALL",
},
{
Name: "permissive-allowlist-add-all",
CapAdd: []string{"all"},
AllowList: "ALL",
},
} {
t.Run(tc.Name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
d := NewExecDriver(ctx, testlog.HCLogger(t))
harness := dtestutil.NewDriverHarness(t, d)
defer harness.Kill()
config := &Config{
NoPivotRoot: true,
DefaultModePID: executor.IsolationModePrivate,
DefaultModeIPC: executor.IsolationModePrivate,
}
if tc.AllowList != "" {
config.AllowCaps = strings.Split(tc.AllowList, ",")
} else {
// inherit HCL defaults if not set
config.AllowCaps = capabilities.NomadDefaults().Slice(true)
}
var data []byte
require.NoError(t, basePlug.MsgPackEncode(&data, config))
baseConfig := &basePlug.Config{PluginConfig: data}
require.NoError(t, harness.SetConfig(baseConfig))
cleanup := harness.MkAllocDir(task, false)
defer cleanup()
tCfg := &TaskConfig{
Command: "/bin/sleep",
Args: []string{"9000"},
}
if len(tc.CapAdd) > 0 {
tCfg.CapAdd = tc.CapAdd
}
if len(tc.CapDrop) > 0 {
tCfg.CapDrop = tc.CapDrop
}
require.NoError(t, task.EncodeConcreteDriverConfig(&tCfg))
// check the start error against expectations
_, _, err := harness.StartTask(task)
if err == nil && tc.StartError != "" {
t.Fatalf("Expected error in start: %v", tc.StartError)
} else if err != nil {
if tc.StartError == "" {
require.NoError(t, err)
} else {
require.Contains(t, err.Error(), tc.StartError)
}
return
}
_ = d.DestroyTask(task.ID, true)
})
}
}

73
drivers/java/driver.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/hashicorp/nomad/client/lib/cgutil"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/consul-template/signals"
hclog "github.com/hashicorp/go-hclog"
@ -73,6 +74,10 @@ var (
hclspec.NewAttr("default_ipc_mode", "string", false),
hclspec.NewLiteral(`"private"`),
),
"allow_caps": hclspec.NewDefault(
hclspec.NewAttr("allow_caps", "list(string)", false),
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
),
})
// taskConfigSpec is the hcl specification for the driver config section of
@ -88,11 +93,13 @@ var (
"args": hclspec.NewAttr("args", "list(string)", false),
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
})
// capabilities is returned by the Capabilities RPC and indicates what
// driverCapabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
capabilities = &drivers.Capabilities{
driverCapabilities = &drivers.Capabilities{
SendSignals: false,
Exec: false,
FSIsolation: drivers.FSIsolationNone,
@ -108,8 +115,8 @@ var (
func init() {
if runtime.GOOS == "linux" {
capabilities.FSIsolation = drivers.FSIsolationChroot
capabilities.MountConfigs = drivers.MountConfigSupportAll
driverCapabilities.FSIsolation = drivers.FSIsolationChroot
driverCapabilities.MountConfigs = drivers.MountConfigSupportAll
}
}
@ -122,6 +129,10 @@ type Config struct {
// DefaultModeIPC is the default IPC isolation set for all tasks using
// exec-based task drivers.
DefaultModeIPC string `codec:"default_ipc_mode"`
// AllowCaps configures which Linux Capabilities are enabled for tasks
// running on this node.
AllowCaps []string `codec:"allow_caps"`
}
func (c *Config) validate() error {
@ -137,18 +148,44 @@ func (c *Config) validate() error {
return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
}
badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps))
if !badCaps.Empty() {
return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps)
}
return nil
}
// TaskConfig is the driver configuration of a taskConfig within a job
type TaskConfig struct {
// Class indicates which class contains the java entry point.
Class string `codec:"class"`
// ClassPath indicates where class files are found.
ClassPath string `codec:"class_path"`
// JarPath indicates where a jar file is found.
JarPath string `codec:"jar_path"`
// JvmOpts are arguments to pass to the JVM
JvmOpts []string `codec:"jvm_options"`
Args []string `codec:"args"` // extra arguments to java executable
// Args are extra arguments to java executable
Args []string `codec:"args"`
// ModePID indicates whether PID namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModePID string `codec:"pid_mode"`
// ModeIPC indicates whether IPC namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModeIPC string `codec:"ipc_mode"`
// CapAdd is a set of linux capabilities to enable.
CapAdd []string `codec:"cap_add"`
// CapDrop is a set of linux capabilities to disable.
CapDrop []string `codec:"cap_drop"`
}
func (tc *TaskConfig) validate() error {
@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error {
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
}
supported := capabilities.Supported()
badAdds := supported.Difference(capabilities.New(tc.CapAdd))
if !badAdds.Empty() {
return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds)
}
badDrops := supported.Difference(capabilities.New(tc.CapDrop))
if !badDrops.Empty() {
return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
}
return nil
}
@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
}
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
return capabilities, nil
return driverCapabilities, nil
}
func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
executorConfig := &executor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: "debug",
FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot,
FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot,
}
exec, pluginClient, err := executor.CreateExecutor(
@ -438,6 +485,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}
caps, err := capabilities.Calculate(
capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
)
if err != nil {
return nil, nil, err
}
d.logger.Debug("task capabilities", "capabilities", caps)
execCmd := &executor.ExecCommand{
Cmd: absPath,
Args: args,
@ -453,6 +508,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
NetworkIsolation: cfg.NetworkIsolation,
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
Capabilities: caps,
}
ps, err := exec.Launch(execCmd)
@ -491,7 +547,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
}
func javaCmdArgs(driverConfig TaskConfig) []string {
args := []string{}
var args []string
// Look for jvm options
if len(driverConfig.JvmOpts) != 0 {
args = append(args, driverConfig.JvmOpts...)

84
drivers/java/driver_test.go
View File

@ -1,6 +1,7 @@
package java
import (
"context"
"errors"
"fmt"
"io"
@ -8,12 +9,10 @@ import (
"os"
"path/filepath"
"testing"
"time"
dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
"context"
"time"
ctestutil "github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
"github.com/hashicorp/nomad/helper/testlog"
@ -416,6 +415,7 @@ func Test_dnsConfig(t *testing.T) {
}
func TestDriver_Config_validate(t *testing.T) {
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
@ -432,4 +432,82 @@ func TestDriver_Config_validate(t *testing.T) {
DefaultModeIPC: tc.ipcMode,
}).validate())
}
})
t.Run("allow_caps", func(t *testing.T) {
for _, tc := range []struct {
ac []string
exp error
}{
{ac: []string{}, exp: nil},
{ac: []string{"all"}, exp: nil},
{ac: []string{"chown", "sys_time"}, exp: nil},
{ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil},
{ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: "private",
DefaultModeIPC: "private",
AllowCaps: tc.ac,
}).validate())
}
})
}
func TestDriver_TaskConfig_validate(t *testing.T) {
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "host", ipcMode: "", exp: nil},
{pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)},
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "", ipcMode: "host", exp: nil},
{pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&TaskConfig{
ModePID: tc.pidMode,
ModeIPC: tc.ipcMode,
}).validate())
}
})
t.Run("cap_add", func(t *testing.T) {
for _, tc := range []struct {
adds []string
exp error
}{
{adds: nil, exp: nil},
{adds: []string{"chown"}, exp: nil},
{adds: []string{"CAP_CHOWN"}, exp: nil},
{adds: []string{"chown", "sys_time"}, exp: nil},
{adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapAdd: tc.adds,
}).validate())
}
})
t.Run("cap_drop", func(t *testing.T) {
for _, tc := range []struct {
drops []string
exp error
}{
{drops: nil, exp: nil},
{drops: []string{"chown"}, exp: nil},
{drops: []string{"CAP_CHOWN"}, exp: nil},
{drops: []string{"chown", "sys_time"}, exp: nil},
{drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapDrop: tc.drops,
}).validate())
}
})
}

209
drivers/shared/capabilities/defaults.go Normal file
View File

@ -0,0 +1,209 @@
package capabilities
import (
"fmt"
"regexp"
"github.com/syndtr/gocapability/capability"
)
const (
// HCLSpecLiteral is an equivalent list to NomadDefaults, expressed as a literal
// HCL string for use in HCL config parsing.
HCLSpecLiteral = `["AUDIT_WRITE","CHOWN","DAC_OVERRIDE","FOWNER","FSETID","KILL","MKNOD","NET_BIND_SERVICE","SETFCAP","SETGID","SETPCAP","SETUID","SYS_CHROOT"]`
)
var (
extractLiteral = regexp.MustCompile(`([\w]+)`)
)
// NomadDefaults is the set of Linux capabilities that Nomad enables by
// default. This list originates from what Docker enabled by default, but then
// excludes NET_RAW for security reasons.
//
// This set is use in the as HCL configuration default, described by HCLSpecLiteral.
func NomadDefaults() *Set {
return New(extractLiteral.FindAllString(HCLSpecLiteral, -1))
}
// DockerDefaults is a list of Linux capabilities enabled by Docker by default
// and is used to compute the set of capabilities to add/drop given docker driver
// configuration.
//
// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
func DockerDefaults() *Set {
defaults := NomadDefaults()
defaults.Add("NET_RAW")
return defaults
}
// Supported returns the set of capabilities supported by the operating system.
//
// This set will expand over time as new capabilities are introduced to the kernel
// and the capability library is updated (which tends to happen to keep up with
// run-container libraries).
//
// Defers to a library generated from
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
func Supported() *Set {
s := New(nil)
last := capability.CAP_LAST_CAP
// workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap
if last == capability.Cap(63) {
last = capability.CAP_BLOCK_SUSPEND
}
// accumulate every capability supported by this system
for _, c := range capability.List() {
if c > last {
continue
}
s.Add(c.String())
}
return s
}
// LegacySupported returns the historical set of capabilities used when a task is
// configured to run as root using the exec task driver. Older versions of Nomad
// always allowed the root user to make use of any capability. Now that the exec
// task driver supports configuring the allowed capabilities, operators are
// encouraged to explicitly opt-in to capabilities beyond this legacy set. We
// maintain the legacy list here, because previous versions of Nomad deferred to
// the capability.List library function, which adds new capabilities over time.
//
// https://github.com/hashicorp/nomad/blob/v1.0.4/vendor/github.com/syndtr/gocapability/capability/enum_gen.go#L88
func LegacySupported() *Set {
return New([]string{
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_DAC_READ_SEARCH",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETPCAP",
"CAP_LINUX_IMMUTABLE",
"CAP_NET_BIND_SERVICE",
"CAP_NET_BROADCAST",
"CAP_NET_ADMIN",
"CAP_NET_RAW",
"CAP_IPC_LOCK",
"CAP_IPC_OWNER",
"CAP_SYS_MODULE",
"CAP_SYS_RAWIO",
"CAP_SYS_CHROOT",
"CAP_SYS_PTRACE",
"CAP_SYS_PACCT",
"CAP_SYS_ADMIN",
"CAP_SYS_BOOT",
"CAP_SYS_NICE",
"CAP_SYS_RESOURCE",
"CAP_SYS_TIME",
"CAP_SYS_TTY_CONFIG",
"CAP_MKNOD",
"CAP_LEASE",
"CAP_AUDIT_WRITE",
"CAP_AUDIT_CONTROL",
"CAP_SETFCAP",
"CAP_MAC_OVERRIDE",
"CAP_MAC_ADMIN",
"CAP_SYSLOG",
"CAP_WAKE_ALARM",
"CAP_BLOCK_SUSPEND",
"CAP_AUDIT_READ",
})
}
// Calculate the resulting set of linux capabilities to enable for a task, taking
// into account:
// - default capability basis
// - driver allowable capabilities
// - task capability drops
// - task capability adds
//
// Nomad establishes a standard set of enabled capabilities allowed by the task
// driver if allow_caps is not set. This is the same set that the task will be
// enabled with by default if allow_caps does not further reduce permissions,
// in which case the task capabilities will also be reduced accordingly.
//
// The task will drop any capabilities specified in cap_drop, and add back
// capabilities specified in cap_add. The task will not be allowed to add capabilities
// not set in the the allow_caps setting (which by default is the same as the basis).
//
// cap_add takes precedence over cap_drop, enabling the common pattern of dropping
// all capabilities, then adding back the desired smaller set. e.g.
// cap_drop = ["all"]
// cap_add = ["chown", "kill"]
//
// Note that the resulting capability names are upper-cased and prefixed with
// "CAP_", which is the expected input for the exec/java driver implementation.
func Calculate(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, error) {
allow := New(allowCaps)
adds := New(capAdd)
// determine caps the task wants that are not allowed
missing := allow.Difference(adds)
if !missing.Empty() {
return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing)
}
// the realized enabled capabilities starts with what is allowed both by driver
// config AND is a member of the basis (i.e. nomad defaults)
result := basis.Intersect(allow)
// then remove capabilities the task explicitly drops
result.Remove(capDrop)
// then add back capabilities the task explicitly adds
return result.Union(adds).Slice(true), nil
}
// Delta calculates the set of capabilities that must be added and dropped relative
// to a basis to achieve a desired result. The use case is that the docker driver
// assumes a default set (DockerDefault), and we must calculate what to pass into
// --cap-add and --cap-drop on container creation given the inputs of the docker
// plugin config for allow_caps, and the docker task configuration for cap_add and
// cap_drop. Note that the user provided cap_add and cap_drop settings are always
// included, even if they are redundant with the basis (maintaining existing
// behavior, working with existing tests).
//
// Note that the resulting capability names are lower-cased and not prefixed with
// "CAP_", which is the existing style used with the docker driver implementation.
func Delta(basis *Set, allowCaps, capAdd, capDrop []string) ([]string, []string, error) {
all := func(caps []string) bool {
for _, c := range caps {
if normalize(c) == "all" {
return true
}
}
return false
}
// set of caps allowed by driver
allow := New(allowCaps)
// determine caps the task wants that are not allowed
missing := allow.Difference(New(capAdd))
if !missing.Empty() {
return nil, nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing)
}
// add what the task is asking for
add := New(capAdd).Slice(false)
if all(capAdd) {
add = []string{"all"}
}
// drop what the task removes plus whatever is in the basis that is not
// in the driver allow configuration
drop := New(allowCaps).Difference(basis).Union(New(capDrop)).Slice(false)
if all(capDrop) {
drop = []string{"all"}
}
return add, drop, nil
}

282
drivers/shared/capabilities/defaults_test.go Normal file
View File

@ -0,0 +1,282 @@
package capabilities
import (
"errors"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestSet_NomadDefaults(t *testing.T) {
result := NomadDefaults()
require.Len(t, result.Slice(false), 13)
defaults := strings.ToLower(HCLSpecLiteral)
for _, c := range result.Slice(false) {
require.Contains(t, defaults, c)
}
}
func TestSet_DockerDefaults(t *testing.T) {
result := DockerDefaults()
require.Len(t, result.Slice(false), 14)
require.Contains(t, result.String(), "net_raw")
}
func TestCaps_Calculate(t *testing.T) {
for _, tc := range []struct {
name string
// input
allowCaps []string // driver config
capAdd []string // task config
capDrop []string // task config
// output
exp []string
err error
skip bool // error message is linux version dependent
}{
{
name: "the default setting",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: nil,
exp: NomadDefaults().Slice(true),
err: nil,
},
{
name: "allow all no mods",
allowCaps: []string{"all"},
capAdd: nil,
capDrop: nil,
exp: NomadDefaults().Slice(true),
err: nil,
},
{
name: "allow selection no mods",
allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"},
capAdd: nil,
capDrop: nil,
exp: []string{"CAP_CHOWN"},
err: nil,
},
{
name: "allow selection and add them",
allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"},
capAdd: []string{"net_raw", "sys_time"},
capDrop: nil,
exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"},
err: nil,
},
{
name: "allow defaults and add redundant",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "KILL"},
capDrop: nil,
exp: NomadDefaults().Slice(true),
err: nil,
},
{
skip: true,
name: "allow defaults and add all",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"all"},
capDrop: nil,
exp: nil,
err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"),
},
{
name: "allow defaults and drop all",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"all"},
exp: []string{},
err: nil,
},
{
name: "allow defaults and drop all and add back some",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "fowner"},
capDrop: []string{"all"},
exp: []string{"CAP_CHOWN", "CAP_FOWNER"},
err: nil,
},
{
name: "add disallowed",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "net_raw"},
capDrop: nil,
exp: nil,
err: errors.New("driver does not allow the following capabilities: net_raw"),
},
{
name: "drop some",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"},
exp: []string{"CAP_AUDIT_WRITE", "CAP_FSETID", "CAP_NET_BIND_SERVICE", "CAP_SETFCAP", "CAP_SETGID", "CAP_SETPCAP", "CAP_SETUID"},
err: nil,
},
{
name: "drop all",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"all"},
exp: []string{},
err: nil,
},
{
name: "drop all and add back",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "mknod"},
capDrop: []string{"all"},
exp: []string{"CAP_CHOWN", "CAP_MKNOD"},
err: nil,
},
} {
t.Run(tc.name, func(t *testing.T) {
caps, err := Calculate(NomadDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop)
if !tc.skip {
require.Equal(t, tc.err, err)
require.Equal(t, tc.exp, caps)
} else {
require.Error(t, err)
require.Equal(t, tc.exp, caps)
}
})
}
}
func TestCaps_Delta(t *testing.T) {
for _, tc := range []struct {
name string
// input
allowCaps []string // driver config
capAdd []string // task config
capDrop []string // task config
// output
expAdd []string
expDrop []string
err error
skip bool // error message is linux version dependent
}{
{
name: "the default setting",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: nil,
expAdd: []string{},
expDrop: []string{"net_raw"},
err: nil,
},
{
name: "allow all no mods",
allowCaps: []string{"all"},
capAdd: nil,
capDrop: nil,
expAdd: []string{},
expDrop: []string{},
err: nil,
},
{
name: "allow non-default no mods",
allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"},
capAdd: nil,
capDrop: nil,
expAdd: []string{},
expDrop: []string{
"audit_write", "dac_override", "fowner", "fsetid",
"kill", "mknod", "net_bind_service", "setfcap",
"setgid", "setpcap", "setuid", "sys_chroot"},
err: nil,
},
{
name: "allow default add from default",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "KILL"},
capDrop: nil,
expAdd: []string{"chown", "kill"},
expDrop: []string{"net_raw"},
err: nil,
},
{
name: "allow default add disallowed",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "net_raw"},
capDrop: nil,
expAdd: nil,
expDrop: nil,
err: errors.New("driver does not allow the following capabilities: net_raw"),
},
{
name: "allow default drop from default",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"},
expAdd: []string{},
expDrop: []string{"chown", "dac_override", "fowner", "kill", "mknod", "net_raw", "sys_chroot"},
err: nil,
},
{
name: "allow default drop all",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"all"},
expAdd: []string{},
expDrop: []string{"all"},
err: nil,
},
{
name: "task drop all and add back",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "fowner"},
capDrop: []string{"all"},
expAdd: []string{"chown", "fowner"},
expDrop: []string{"all"},
err: nil,
},
{
name: "add atop allow all",
allowCaps: []string{"all"},
capAdd: []string{"chown", "fowner"},
capDrop: nil,
expAdd: []string{"chown", "fowner"},
expDrop: []string{},
err: nil,
},
{
name: "add all atop all",
allowCaps: []string{"all"},
capAdd: []string{"all"},
capDrop: nil,
expAdd: []string{"all"},
expDrop: []string{},
err: nil,
},
{
skip: true,
name: "add all atop defaults",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"all"},
capDrop: nil,
expAdd: nil,
expDrop: nil,
err: errors.New("driver does not allow the following capabilities: audit_control, audit_read, block_suspend, bpf, dac_read_search, ipc_lock, ipc_owner, lease, linux_immutable, mac_admin, mac_override, net_admin, net_broadcast, net_raw, perfmon, sys_admin, sys_boot, sys_module, sys_nice, sys_pacct, sys_ptrace, sys_rawio, sys_resource, sys_time, sys_tty_config, syslog, wake_alarm"),
},
} {
t.Run(tc.name, func(t *testing.T) {
add, drop, err := Delta(DockerDefaults(), tc.allowCaps, tc.capAdd, tc.capDrop)
if !tc.skip {
require.Equal(t, tc.err, err)
require.Equal(t, tc.expAdd, add)
require.Equal(t, tc.expDrop, drop)
} else {
require.Error(t, err)
require.Equal(t, tc.expDrop, drop)
}
})
}
}

134
drivers/shared/capabilities/set.go Normal file
View File

@ -0,0 +1,134 @@
// Package capabilities is used for managing sets of linux capabilities.
package capabilities
import (
"sort"
"strings"
)
type nothing struct{}
var null = nothing{}
// Set represents a group linux capabilities, implementing some useful set
// operations, taking care of name normalization, and sentinel value expansions.
//
// Linux capabilities can be expressed in multiple ways when working with docker
// and/or executor, along with Nomad configuration.
//
// Capability names may be upper or lower case, and may or may not be prefixed
// with "CAP_" or "cap_". On top of that, Nomad interprets the special name "all"
// and "ALL" to mean "all capabilities supported by the operating system".
type Set struct {
data map[string]nothing
}
// New creates a new Set setting caps as the initial elements.
func New(caps []string) *Set {
m := make(map[string]nothing, len(caps))
for _, c := range caps {
insert(m, c)
}
return &Set{data: m}
}
// Add cap into s.
func (s *Set) Add(cap string) {
insert(s.data, cap)
}
func insert(data map[string]nothing, cap string) {
switch name := normalize(cap); name {
case "":
case "all":
for k, v := range Supported().data {
data[k] = v
}
return
default:
data[name] = null
}
}
// Remove caps from s.
func (s *Set) Remove(caps []string) {
for _, c := range caps {
name := normalize(c)
if name == "all" {
s.data = make(map[string]nothing)
return
}
delete(s.data, name)
}
}
// Union returns of Set of elements of both s and b.
func (s *Set) Union(b *Set) *Set {
data := make(map[string]nothing)
for c := range s.data {
data[c] = null
}
for c := range b.data {
data[c] = null
}
return &Set{data: data}
}
// Difference returns the Set of elements of b not in s.
func (s *Set) Difference(b *Set) *Set {
data := make(map[string]nothing)
for c := range b.data {
if _, exists := s.data[c]; !exists {
data[c] = null
}
}
return &Set{data: data}
}
// Intersect returns the Set of elements in both s and b.
func (s *Set) Intersect(b *Set) *Set {
data := make(map[string]nothing)
for c := range s.data {
if _, exists := b.data[c]; exists {
data[c] = null
}
}
return &Set{data: data}
}
// Empty return true if no capabilities exist in s.
func (s *Set) Empty() bool {
return len(s.data) == 0
}
// String returns the normalized and sorted string representation of s.
func (s *Set) String() string {
return strings.Join(s.Slice(false), ", ")
}
// Slice returns a sorted slice of capabilities in s.
//
// upper - indicates whether to uppercase and prefix capabilities with CAP_
func (s *Set) Slice(upper bool) []string {
caps := make([]string, 0, len(s.data))
for c := range s.data {
if upper {
c = "CAP_" + strings.ToUpper(c)
}
caps = append(caps, c)
}
sort.Strings(caps)
return caps
}
// linux capabilities are often named in 4 possible ways - upper or lower case,
// and with or without a CAP_ prefix
//
// since we must do comparisons on cap names, always normalize the names before
// letting them into the Set data-structure
func normalize(name string) string {
spaces := strings.TrimSpace(name)
lower := strings.ToLower(spaces)
trim := strings.TrimPrefix(lower, "cap_")
return trim
}

214
drivers/shared/capabilities/set_test.go Normal file
View File

@ -0,0 +1,214 @@
package capabilities
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestSet_Empty(t *testing.T) {
t.Parallel()
t.Run("nil", func(t *testing.T) {
result := New(nil).Empty()
require.True(t, result)
})
t.Run("empty", func(t *testing.T) {
result := New([]string{}).Empty()
require.True(t, result)
})
t.Run("full", func(t *testing.T) {
result := New([]string{"chown", "sys_time"}).Empty()
require.False(t, result)
})
}
func TestSet_New(t *testing.T) {
t.Parallel()
t.Run("duplicates", func(t *testing.T) {
result := New([]string{"chown", "sys_time", "chown"})
require.Equal(t, "chown, sys_time", result.String())
})
t.Run("empty string", func(t *testing.T) {
result := New([]string{""})
require.True(t, result.Empty())
})
t.Run("all", func(t *testing.T) {
result := New([]string{"all"})
exp := len(Supported().Slice(false))
require.Len(t, result.Slice(false), exp)
})
}
func TestSet_Slice(t *testing.T) {
t.Parallel()
exp := []string{"chown", "net_raw", "sys_time"}
t.Run("lower case", func(t *testing.T) {
s := New([]string{"net_raw", "chown", "sys_time"})
require.Equal(t, exp, s.Slice(false))
})
t.Run("upper case", func(t *testing.T) {
s := New([]string{"NET_RAW", "CHOWN", "SYS_TIME"})
require.Equal(t, exp, s.Slice(false))
})
t.Run("prefix", func(t *testing.T) {
s := New([]string{"CAP_net_raw", "sys_TIME", "cap_chown"})
require.Equal(t, exp, s.Slice(false))
})
}
func TestSet_String(t *testing.T) {
t.Parallel()
t.Run("empty", func(t *testing.T) {
result := New(nil).String()
require.Equal(t, "", result)
})
t.Run("full", func(t *testing.T) {
exp := "chown, net_raw, sys_time"
in := []string{"net_raw", "CAP_CHOWN", "cap_sys_time"}
result := New(in).String()
require.Equal(t, exp, result)
})
}
func TestSet_Add(t *testing.T) {
t.Parallel()
t.Run("add one", func(t *testing.T) {
s := New([]string{"chown", "net_raw"})
require.Equal(t, "chown, net_raw", s.String())
s.Add("CAP_SYS_TIME")
require.Equal(t, "chown, net_raw, sys_time", s.String())
s.Add("AF_NET")
require.Equal(t, "af_net, chown, net_raw, sys_time", s.String())
})
t.Run("add empty string", func(t *testing.T) {
s := New([]string{"chown"})
s.Add("")
require.Equal(t, "chown", s.String())
})
t.Run("add all", func(t *testing.T) {
s := New([]string{"chown", "net_raw"})
require.Equal(t, "chown, net_raw", s.String())
exp := len(Supported().Slice(false))
s.Add("all")
require.Len(t, s.Slice(false), exp)
})
}
func TestSet_Remove(t *testing.T) {
t.Parallel()
t.Run("remove one", func(t *testing.T) {
s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"})
s.Remove([]string{"CAP_NET_RAW"})
require.Equal(t, "af_net, chown, seteuid, sys_time", s.String())
})
t.Run("remove couple", func(t *testing.T) {
s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"})
s.Remove([]string{"CAP_NET_RAW", "af_net"})
require.Equal(t, "chown, seteuid, sys_time", s.String())
})
t.Run("remove all", func(t *testing.T) {
s := New([]string{"af_net", "chown", "net_raw", "seteuid", "sys_time"})
s.Remove([]string{"all"})
require.True(t, s.Empty())
require.Equal(t, "", s.String())
})
}
func TestSet_Difference(t *testing.T) {
t.Parallel()
t.Run("a is empty", func(t *testing.T) {
a := New(nil)
b := New([]string{"chown", "af_net"})
result := a.Difference(b)
require.Equal(t, "af_net, chown", result.String())
})
t.Run("b is empty", func(t *testing.T) {
a := New([]string{"chown", "af_net"})
b := New(nil)
result := a.Difference(b)
require.True(t, result.Empty())
})
t.Run("a diff b", func(t *testing.T) {
a := New([]string{"A", "b", "C", "d", "e", "f"})
b := New([]string{"B", "x", "Y", "a"})
result := a.Difference(b)
require.Equal(t, "x, y", result.String())
})
}
func TestSet_Intersect(t *testing.T) {
t.Parallel()
t.Run("empty", func(t *testing.T) {
a := New(nil)
b := New([]string{"a", "b"})
result := a.Intersect(b)
require.True(t, result.Empty())
result2 := b.Intersect(a)
require.True(t, result2.Empty())
})
t.Run("intersect", func(t *testing.T) {
a := New([]string{"A", "b", "C", "d", "e", "f", "G"})
b := New([]string{"Z", "B", "E", "f", "y"})
result := a.Intersect(b)
require.Equal(t, "b, e, f", result.String())
result2 := b.Intersect(a)
require.Equal(t, "b, e, f", result2.String())
})
}
func TestSet_Union(t *testing.T) {
t.Parallel()
t.Run("empty", func(t *testing.T) {
a := New(nil)
b := New([]string{"a", "b"})
result := a.Union(b)
require.Equal(t, "a, b", result.String())
result2 := b.Union(a)
require.Equal(t, "a, b", result2.String())
})
t.Run("union", func(t *testing.T) {
a := New([]string{"A", "b", "C", "d", "e", "f", "G"})
b := New([]string{"Z", "B", "E", "f", "y"})
result := a.Union(b)
require.Equal(t, "a, b, c, d, e, f, g, y, z", result.String())
result2 := b.Union(a)
require.Equal(t, "a, b, c, d, e, f, g, y, z", result2.String())
})
}

1
drivers/shared/executor/client.go
View File

@ -47,6 +47,7 @@ func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) {
NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation),
DefaultPidMode: cmd.ModePID,
DefaultIpcMode: cmd.ModeIPC,
Capabilities: cmd.Capabilities,
}
resp, err := c.client.Launch(ctx, req)
if err != nil {

7
drivers/shared/executor/executor.go
View File

@ -92,6 +92,10 @@ type Executor interface {
// ExecCommand holds the user command, args, and other isolation related
// settings.
//
// Important (!): when adding fields, make sure to update the RPC methods in
// grpcExecutorClient.Launch and grpcExecutorServer.Launch. Number of hours
// spent tracking this down: too many.
type ExecCommand struct {
// Cmd is the command that the user wants to run.
Cmd string
@ -147,6 +151,9 @@ type ExecCommand struct {
// ModeIPC is the IPC isolation mode (private or host).
ModeIPC string
// Capabilities are the linux capabilities to be enabled by the task driver.
Capabilities []string
}
// SetWriters sets the writer for the process stdout and stderr. This should

35
drivers/shared/executor/executor_linux.go
View File

@ -14,6 +14,7 @@ import (
"syscall"
"time"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/armon/circbuf"
@ -532,28 +533,25 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc
ch <- &waitResult{ps, err}
}
func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error {
// TODO(shoenig): allow better control of these
// use capabilities list as prior to adopting libcontainer in 0.9
// match capabilities used in Nomad 0.8
if command.User == "root" {
allCaps := SupportedCaps(true)
func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) {
switch command.User {
case "root":
// when running as root, use the legacy set of system capabilities, so
// that we do not break existing nomad clusters using this "feature"
legacyCaps := capabilities.LegacySupported().Slice(true)
cfg.Capabilities = &lconfigs.Capabilities{
Bounding: allCaps,
Permitted: allCaps,
Effective: allCaps,
Bounding: legacyCaps,
Permitted: legacyCaps,
Effective: legacyCaps,
Ambient: nil,
Inheritable: nil,
}
} else {
allCaps := SupportedCaps(false)
default:
// otherwise apply the plugin + task capability configuration
cfg.Capabilities = &lconfigs.Capabilities{
Bounding: allCaps,
Bounding: command.Capabilities,
}
}
return nil
}
func configureNamespaces(pidMode, ipcMode string) lconfigs.Namespaces {
@ -759,16 +757,17 @@ func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) {
},
Version: "1.0.0",
}
for _, device := range specconv.AllowedDevices {
cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule)
}
if err := configureCapabilities(cfg, command); err != nil {
return nil, err
}
configureCapabilities(cfg, command)
if err := configureIsolation(cfg, command); err != nil {
return nil, err
}
if err := configureCgroups(cfg, command); err != nil {
return nil, err
}

11
drivers/shared/executor/executor_linux_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/plugins/drivers"
@ -478,7 +479,7 @@ func TestExecutor_Capabilities(t *testing.T) {
CapInh: 0000000000000000
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 0000003fffffdfff
CapBnd: 00000000a80405fb
CapAmb: 0000000000000000`,
},
{
@ -494,7 +495,6 @@ CapAmb: 0000000000000000`,
for _, c := range cases {
t.Run(c.user, func(t *testing.T) {
require := require.New(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
@ -504,12 +504,13 @@ CapAmb: 0000000000000000`,
execCmd.ResourceLimits = true
execCmd.Cmd = "/bin/bash"
execCmd.Args = []string{"-c", "cat /proc/$$/status"}
execCmd.Capabilities = capabilities.NomadDefaults().Slice(true)
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
_, err := executor.Launch(execCmd)
require.NoError(err)
require.NoError(t, err)
ch := make(chan interface{})
go func() {
@ -521,7 +522,7 @@ CapAmb: 0000000000000000`,
case <-ch:
// all good
case <-time.After(5 * time.Second):
require.Fail("timeout waiting for exec to shutdown")
require.Fail(t, "timeout waiting for exec to shutdown")
}
canonical := func(s string) string {
@ -538,7 +539,7 @@ CapAmb: 0000000000000000`,
return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output)
}
return true, nil
}, func(err error) { require.NoError(err) })
}, func(err error) { require.NoError(t, err) })
})
}

149
drivers/shared/executor/proto/executor.pb.go
View File

@ -44,6 +44,8 @@ type LaunchRequest struct {
DefaultPidMode string `protobuf:"bytes,15,opt,name=default_pid_mode,json=defaultPidMode,proto3" json:"default_pid_mode,omitempty"`
DefaultIpcMode string `protobuf:"bytes,16,opt,name=default_ipc_mode,json=defaultIpcMode,proto3" json:"default_ipc_mode,omitempty"`
CpusetCgroup string `protobuf:"bytes,17,opt,name=cpuset_cgroup,json=cpusetCgroup,proto3" json:"cpuset_cgroup,omitempty"`
AllowCaps []string `protobuf:"bytes,18,rep,name=allow_caps,json=allowCaps,proto3" json:"allow_caps,omitempty"`
Capabilities []string `protobuf:"bytes,19,rep,name=capabilities,proto3" json:"capabilities,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -193,6 +195,20 @@ func (m *LaunchRequest) GetCpusetCgroup() string {
return ""
}
func (m *LaunchRequest) GetAllowCaps() []string {
if m != nil {
return m.AllowCaps
}
return nil
}
func (m *LaunchRequest) GetCapabilities() []string {
if m != nil {
return m.Capabilities
}
return nil
}
type LaunchResponse struct {
Process *ProcessState `protobuf:"bytes,1,opt,name=process,proto3" json:"process,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -858,71 +874,74 @@ func init() {
}
var fileDescriptor_66b85426380683f3 = []byte{
// 1020 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x5b, 0x8f, 0xdb, 0x44,
0x14, 0xae, 0x37, 0x9b, 0xdb, 0x49, 0xb2, 0x49, 0x47, 0xa8, 0xb8, 0xe1, 0xa1, 0xc1, 0x48, 0x34,
0x82, 0xe2, 0xac, 0xb6, 0x37, 0x24, 0x24, 0x8a, 0xd8, 0x2d, 0xa8, 0xd2, 0x76, 0x15, 0x39, 0x85,
0x4a, 0x3c, 0x60, 0x5c, 0xcf, 0x34, 0x19, 0x6d, 0xe2, 0x31, 0x33, 0xe3, 0x74, 0x91, 0x90, 0x78,
0xe2, 0x1f, 0x80, 0xc4, 0x5f, 0xe5, 0x0d, 0xcd, 0xcd, 0x9b, 0x6c, 0x4b, 0xe5, 0x14, 0xf1, 0x14,
0xcf, 0xc9, 0xf7, 0x9d, 0xcb, 0x9c, 0x73, 0xbe, 0x81, 0x3b, 0x98, 0xd3, 0x35, 0xe1, 0x62, 0x22,
0x16, 0x09, 0x27, 0x78, 0x42, 0x2e, 0x48, 0x5a, 0x48, 0xc6, 0x27, 0x39, 0x67, 0x92, 0x95, 0xc7,
0x50, 0x1f, 0xd1, 0xc7, 0x8b, 0x44, 0x2c, 0x68, 0xca, 0x78, 0x1e, 0x66, 0x6c, 0x95, 0xe0, 0x30,
0x5f, 0x16, 0x73, 0x9a, 0x89, 0x70, 0x1b, 0x37, 0xbc, 0x35, 0x67, 0x6c, 0xbe, 0x24, 0xc6, 0xc9,
0x8b, 0xe2, 0xe5, 0x44, 0xd2, 0x15, 0x11, 0x32, 0x59, 0xe5, 0x16, 0x10, 0x58, 0xe2, 0xc4, 0x85,
0x37, 0xe1, 0xcc, 0xc9, 0x60, 0x82, 0xbf, 0xeb, 0xd0, 0x3b, 0x4d, 0x8a, 0x2c, 0x5d, 0x44, 0xe4,
0xe7, 0x82, 0x08, 0x89, 0x06, 0x50, 0x4b, 0x57, 0xd8, 0xf7, 0x46, 0xde, 0xb8, 0x1d, 0xa9, 0x4f,
0x84, 0x60, 0x3f, 0xe1, 0x73, 0xe1, 0xef, 0x8d, 0x6a, 0xe3, 0x76, 0xa4, 0xbf, 0xd1, 0x19, 0xb4,
0x39, 0x11, 0xac, 0xe0, 0x29, 0x11, 0x7e, 0x6d, 0xe4, 0x8d, 0x3b, 0x47, 0x87, 0xe1, 0xbf, 0x25,
0x6e, 0xe3, 0x9b, 0x90, 0x61, 0xe4, 0x78, 0xd1, 0xa5, 0x0b, 0x74, 0x0b, 0x3a, 0x42, 0x62, 0x56,
0xc8, 0x38, 0x4f, 0xe4, 0xc2, 0xdf, 0xd7, 0xd1, 0xc1, 0x98, 0xa6, 0x89, 0x5c, 0x58, 0x00, 0xe1,
0xdc, 0x00, 0xea, 0x25, 0x80, 0x70, 0xae, 0x01, 0x03, 0xa8, 0x91, 0x6c, 0xed, 0x37, 0x74, 0x92,
0xea, 0x53, 0xe5, 0x5d, 0x08, 0xc2, 0xfd, 0xa6, 0xc6, 0xea, 0x6f, 0x74, 0x13, 0x5a, 0x32, 0x11,
0xe7, 0x31, 0xa6, 0xdc, 0x6f, 0x69, 0x7b, 0x53, 0x9d, 0x4f, 0x28, 0x47, 0xb7, 0xa1, 0xef, 0xf2,
0x89, 0x97, 0x74, 0x45, 0xa5, 0xf0, 0xdb, 0x23, 0x6f, 0xdc, 0x8a, 0x0e, 0x9c, 0xf9, 0x54, 0x5b,
0xd1, 0x21, 0xbc, 0xf7, 0x22, 0x11, 0x34, 0x8d, 0x73, 0xce, 0x52, 0x22, 0x44, 0x9c, 0xce, 0x39,
0x2b, 0x72, 0x1f, 0x34, 0x1a, 0xe9, 0xff, 0xa6, 0xe6, 0xaf, 0x63, 0xfd, 0x0f, 0x3a, 0x81, 0xc6,
0x8a, 0x15, 0x99, 0x14, 0x7e, 0x67, 0x54, 0x1b, 0x77, 0x8e, 0xee, 0x54, 0xbc, 0xaa, 0xa7, 0x8a,
0x14, 0x59, 0x2e, 0xfa, 0x16, 0x9a, 0x98, 0xac, 0xa9, 0xba, 0xf1, 0xae, 0x76, 0xf3, 0x59, 0x45,
0x37, 0x27, 0x9a, 0x15, 0x39, 0x36, 0x5a, 0xc0, 0xf5, 0x8c, 0xc8, 0x57, 0x8c, 0x9f, 0xc7, 0x54,
0xb0, 0x65, 0x22, 0x29, 0xcb, 0xfc, 0x9e, 0x6e, 0xe2, 0x17, 0x15, 0x5d, 0x9e, 0x19, 0xfe, 0x13,
0x47, 0x9f, 0xe5, 0x24, 0x8d, 0x06, 0xd9, 0x15, 0x2b, 0x0a, 0xa0, 0x97, 0xb1, 0x38, 0xa7, 0x6b,
0x26, 0x63, 0xce, 0x98, 0xf4, 0x0f, 0xf4, 0x1d, 0x75, 0x32, 0x36, 0x55, 0xb6, 0x88, 0x31, 0x89,
0xc6, 0x30, 0xc0, 0xe4, 0x65, 0x52, 0x2c, 0x65, 0x9c, 0x53, 0x1c, 0xaf, 0x18, 0x26, 0x7e, 0x5f,
0xb7, 0xe6, 0xc0, 0xda, 0xa7, 0x14, 0x3f, 0x65, 0x98, 0x6c, 0x22, 0x69, 0x9e, 0x1a, 0xe4, 0x60,
0x0b, 0xf9, 0x24, 0x4f, 0x35, 0xf2, 0x23, 0xe8, 0xa5, 0x79, 0x21, 0x88, 0x74, 0xbd, 0xb9, 0xae,
0x61, 0x5d, 0x63, 0x34, 0x5d, 0x09, 0x7e, 0x82, 0x03, 0x37, 0xfa, 0x22, 0x67, 0x99, 0x20, 0xe8,
0x0c, 0x9a, 0xb6, 0xa7, 0x7a, 0xfe, 0x3b, 0x47, 0xf7, 0xc2, 0x6a, 0xcb, 0x18, 0xda, 0x7e, 0xcf,
0x64, 0x22, 0x49, 0xe4, 0x9c, 0x04, 0x3d, 0xe8, 0x3c, 0x4f, 0xa8, 0xb4, 0xab, 0x15, 0xfc, 0x08,
0x5d, 0x73, 0xfc, 0x9f, 0xc2, 0x9d, 0x42, 0x7f, 0xb6, 0x28, 0x24, 0x66, 0xaf, 0x32, 0xb7, 0xcd,
0x37, 0xa0, 0x21, 0xe8, 0x3c, 0x4b, 0x96, 0x76, 0xa1, 0xed, 0x09, 0x7d, 0x08, 0xdd, 0x39, 0x4f,
0x52, 0x12, 0xe7, 0x84, 0x53, 0x86, 0xfd, 0xbd, 0x91, 0x37, 0xae, 0x45, 0x1d, 0x6d, 0x9b, 0x6a,
0x53, 0x80, 0x60, 0x70, 0xe9, 0xcd, 0x64, 0x1c, 0x2c, 0xe0, 0xc6, 0x77, 0x39, 0x56, 0x41, 0xcb,
0x25, 0xb6, 0x81, 0xb6, 0x04, 0xc1, 0xfb, 0xcf, 0x82, 0x10, 0xdc, 0x84, 0xf7, 0x5f, 0x8b, 0x64,
0x93, 0x18, 0xc0, 0xc1, 0xf7, 0x84, 0x0b, 0xca, 0x5c, 0x95, 0xc1, 0xa7, 0xd0, 0x2f, 0x2d, 0xf6,
0x6e, 0x7d, 0x68, 0xae, 0x8d, 0xc9, 0x56, 0xee, 0x8e, 0xc1, 0x27, 0xd0, 0x55, 0xf7, 0x56, 0x66,
0x3e, 0x84, 0x16, 0xcd, 0x24, 0xe1, 0x6b, 0x7b, 0x49, 0xb5, 0xa8, 0x3c, 0x07, 0xcf, 0xa1, 0x67,
0xb1, 0xd6, 0xed, 0x37, 0x50, 0x17, 0xca, 0xb0, 0x63, 0x89, 0xcf, 0x12, 0x71, 0x6e, 0x1c, 0x19,
0x7a, 0x70, 0x1b, 0x7a, 0x33, 0xdd, 0x89, 0x37, 0x37, 0xaa, 0xee, 0x1a, 0xa5, 0x8a, 0x75, 0x40,
0x5b, 0xfe, 0x39, 0x74, 0x1e, 0x5f, 0x90, 0xd4, 0x11, 0x1f, 0x40, 0x0b, 0x93, 0x04, 0x2f, 0x69,
0x46, 0x6c, 0x52, 0xc3, 0xd0, 0xbc, 0x0c, 0xa1, 0x7b, 0x19, 0xc2, 0x67, 0xee, 0x65, 0x88, 0x4a,
0xac, 0xd3, 0xf9, 0xbd, 0xd7, 0x75, 0xbe, 0x76, 0xa9, 0xf3, 0xc1, 0x31, 0x74, 0x4d, 0x30, 0x5b,
0xff, 0x0d, 0x68, 0xb0, 0x42, 0xe6, 0x85, 0xd4, 0xb1, 0xba, 0x91, 0x3d, 0xa1, 0x0f, 0xa0, 0x4d,
0x2e, 0xa8, 0x8c, 0x53, 0xb5, 0x93, 0x7b, 0xba, 0x82, 0x96, 0x32, 0x1c, 0x33, 0x4c, 0x82, 0xdf,
0x3d, 0xe8, 0x6e, 0x4e, 0xac, 0x8a, 0x9d, 0x53, 0x6c, 0x2b, 0x55, 0x9f, 0x6f, 0xe5, 0x6f, 0xdc,
0x4d, 0x6d, 0xf3, 0x6e, 0x50, 0x08, 0xfb, 0xea, 0xcd, 0xd3, 0xaf, 0xc5, 0xdb, 0xcb, 0xd6, 0xb8,
0xa3, 0x3f, 0xdb, 0xd0, 0x7a, 0x6c, 0x17, 0x09, 0xfd, 0x02, 0x0d, 0xb3, 0xfd, 0xe8, 0x7e, 0xd5,
0xad, 0xdb, 0x7a, 0x28, 0x87, 0x0f, 0x76, 0xa5, 0xd9, 0xfe, 0x5d, 0x43, 0x02, 0xf6, 0x95, 0x0e,
0xa0, 0xbb, 0x55, 0x3d, 0x6c, 0x88, 0xc8, 0xf0, 0xde, 0x6e, 0xa4, 0x32, 0xe8, 0x6f, 0xd0, 0x72,
0xeb, 0x8c, 0x1e, 0x56, 0xf5, 0x71, 0x45, 0x4e, 0x86, 0x9f, 0xef, 0x4e, 0x2c, 0x13, 0xf8, 0xc3,
0x83, 0xfe, 0x95, 0x95, 0x46, 0x5f, 0x56, 0xf5, 0xf7, 0x66, 0xd5, 0x19, 0x3e, 0x7a, 0x67, 0x7e,
0x99, 0xd6, 0xaf, 0xd0, 0xb4, 0xda, 0x81, 0x2a, 0x77, 0x74, 0x5b, 0x7e, 0x86, 0x0f, 0x77, 0xe6,
0x95, 0xd1, 0x2f, 0xa0, 0xae, 0x75, 0x01, 0x55, 0x6e, 0xeb, 0xa6, 0x76, 0x0d, 0xef, 0xef, 0xc8,
0x72, 0x71, 0x0f, 0x3d, 0x35, 0xff, 0x46, 0x58, 0xaa, 0xcf, 0xff, 0x96, 0x62, 0x55, 0x9f, 0xff,
0x2b, 0xfa, 0xa5, 0xe7, 0x5f, 0xad, 0x61, 0xf5, 0xf9, 0xdf, 0xd0, 0xbb, 0xea, 0xf3, 0xbf, 0xa9,
0x5b, 0xc1, 0x35, 0xf4, 0x97, 0x07, 0x3d, 0x65, 0x9a, 0x49, 0x4e, 0x92, 0x15, 0xcd, 0xe6, 0xe8,
0x51, 0x45, 0xf1, 0x56, 0x2c, 0x23, 0xe0, 0x96, 0xe9, 0x52, 0xf9, 0xea, 0xdd, 0x1d, 0xb8, 0xb4,
0xc6, 0xde, 0xa1, 0xf7, 0x75, 0xf3, 0x87, 0xba, 0xd1, 0xac, 0x86, 0xfe, 0xb9, 0xfb, 0x4f, 0x00,
0x00, 0x00, 0xff, 0xff, 0x32, 0xbc, 0x95, 0x4b, 0x31, 0x0c, 0x00, 0x00,
// 1058 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x6d, 0x6f, 0x1b, 0x45,
0x10, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0x2e, 0xa8, 0x5c, 0x8d, 0x50, 0xcd, 0x21, 0x51,
0x0b, 0xca, 0x25, 0x4a, 0xdf, 0x90, 0x90, 0x28, 0x22, 0x2d, 0xa8, 0x52, 0x1a, 0x45, 0x97, 0x42,
0x25, 0x3e, 0x70, 0x6c, 0xee, 0xb6, 0xf6, 0x2a, 0xf6, 0xed, 0xb2, 0xbb, 0xe7, 0x04, 0x09, 0x89,
0x4f, 0xfc, 0x03, 0x90, 0xf8, 0x31, 0xfc, 0x38, 0xb4, 0x6f, 0x17, 0x3b, 0x2d, 0xd5, 0xb9, 0x88,
0x4f, 0xbe, 0x1d, 0x3f, 0xcf, 0xcc, 0xec, 0xce, 0xcc, 0x33, 0x70, 0x27, 0x17, 0x74, 0x41, 0x84,
0xdc, 0x95, 0x53, 0x2c, 0x48, 0xbe, 0x4b, 0x2e, 0x48, 0x56, 0x2a, 0x26, 0x76, 0xb9, 0x60, 0x8a,
0x55, 0xc7, 0xd8, 0x1c, 0xd1, 0xc7, 0x53, 0x2c, 0xa7, 0x34, 0x63, 0x82, 0xc7, 0x05, 0x9b, 0xe3,
0x3c, 0xe6, 0xb3, 0x72, 0x42, 0x0b, 0x19, 0xaf, 0xe2, 0x86, 0xb7, 0x26, 0x8c, 0x4d, 0x66, 0xc4,
0x3a, 0x39, 0x2d, 0x5f, 0xee, 0x2a, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0x77,
0x7d, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0xfa, 0xbb, 0x09, 0xfd, 0x43, 0x5c, 0x16, 0xd9, 0x34,
0x21, 0x3f, 0x97, 0x44, 0x2a, 0x34, 0x80, 0x46, 0x36, 0xcf, 0xc3, 0x60, 0x14, 0x8c, 0x3b, 0x89,
0xfe, 0x44, 0x08, 0x36, 0xb1, 0x98, 0xc8, 0x70, 0x63, 0xd4, 0x18, 0x77, 0x12, 0xf3, 0x8d, 0x8e,
0xa0, 0x23, 0x88, 0x64, 0xa5, 0xc8, 0x88, 0x0c, 0x1b, 0xa3, 0x60, 0xdc, 0xdd, 0xdf, 0x8b, 0xff,
0x2d, 0x71, 0x17, 0xdf, 0x86, 0x8c, 0x13, 0xcf, 0x4b, 0x2e, 0x5d, 0xa0, 0x5b, 0xd0, 0x95, 0x2a,
0x67, 0xa5, 0x4a, 0x39, 0x56, 0xd3, 0x70, 0xd3, 0x44, 0x07, 0x6b, 0x3a, 0xc6, 0x6a, 0xea, 0x00,
0x44, 0x08, 0x0b, 0xd8, 0xaa, 0x00, 0x44, 0x08, 0x03, 0x18, 0x40, 0x83, 0x14, 0x8b, 0xb0, 0x69,
0x92, 0xd4, 0x9f, 0x3a, 0xef, 0x52, 0x12, 0x11, 0xb6, 0x0c, 0xd6, 0x7c, 0xa3, 0x9b, 0xd0, 0x56,
0x58, 0x9e, 0xa5, 0x39, 0x15, 0x61, 0xdb, 0xd8, 0x5b, 0xfa, 0xfc, 0x98, 0x0a, 0x74, 0x1b, 0x76,
0x7c, 0x3e, 0xe9, 0x8c, 0xce, 0xa9, 0x92, 0x61, 0x67, 0x14, 0x8c, 0xdb, 0xc9, 0xb6, 0x37, 0x1f,
0x1a, 0x2b, 0xda, 0x83, 0x77, 0x4f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36,
0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x1d, 0xdb, 0xbf, 0x0e, 0xcc, 0x3f, 0xe8, 0x31,
0x34, 0xe7, 0xac, 0x2c, 0x94, 0x0c, 0xbb, 0xa3, 0xc6, 0xb8, 0xbb, 0x7f, 0xa7, 0xe6, 0x53, 0x3d,
0xd3, 0xa4, 0xc4, 0x71, 0xd1, 0xb7, 0xd0, 0xca, 0xc9, 0x82, 0xea, 0x17, 0xef, 0x19, 0x37, 0x9f,
0xd5, 0x74, 0xf3, 0xd8, 0xb0, 0x12, 0xcf, 0x46, 0x53, 0xb8, 0x5e, 0x10, 0x75, 0xce, 0xc4, 0x59,
0x4a, 0x25, 0x9b, 0x61, 0x45, 0x59, 0x11, 0xf6, 0x4d, 0x11, 0xbf, 0xa8, 0xe9, 0xf2, 0xc8, 0xf2,
0x9f, 0x7a, 0xfa, 0x09, 0x27, 0x59, 0x32, 0x28, 0xae, 0x58, 0x51, 0x04, 0xfd, 0x82, 0xa5, 0x9c,
0x2e, 0x98, 0x4a, 0x05, 0x63, 0x2a, 0xdc, 0x36, 0x6f, 0xd4, 0x2d, 0xd8, 0xb1, 0xb6, 0x25, 0x8c,
0x29, 0x34, 0x86, 0x41, 0x4e, 0x5e, 0xe2, 0x72, 0xa6, 0x52, 0x4e, 0xf3, 0x74, 0xce, 0x72, 0x12,
0xee, 0x98, 0xd2, 0x6c, 0x3b, 0xfb, 0x31, 0xcd, 0x9f, 0xb1, 0x9c, 0x2c, 0x23, 0x29, 0xcf, 0x2c,
0x72, 0xb0, 0x82, 0x7c, 0xca, 0x33, 0x83, 0xfc, 0x08, 0xfa, 0x19, 0x2f, 0x25, 0x51, 0xbe, 0x36,
0xd7, 0x0d, 0xac, 0x67, 0x8d, 0xae, 0x2a, 0x1f, 0x00, 0xe0, 0xd9, 0x8c, 0x9d, 0xa7, 0x19, 0xe6,
0x32, 0x44, 0xa6, 0x71, 0x3a, 0xc6, 0x72, 0x80, 0xb9, 0x44, 0x11, 0xf4, 0x32, 0xcc, 0xf1, 0x29,
0x9d, 0x51, 0x45, 0x89, 0x0c, 0xdf, 0x31, 0x80, 0x15, 0x5b, 0xf4, 0x13, 0x6c, 0xfb, 0xe9, 0x91,
0x9c, 0x15, 0x92, 0xa0, 0x23, 0x68, 0xb9, 0xb6, 0x30, 0x23, 0xd4, 0xdd, 0xbf, 0x17, 0xd7, 0x9b,
0xe7, 0xd8, 0xb5, 0xcc, 0x89, 0xc2, 0x8a, 0x24, 0xde, 0x49, 0xd4, 0x87, 0xee, 0x0b, 0x4c, 0x95,
0x9b, 0xce, 0xe8, 0x47, 0xe8, 0xd9, 0xe3, 0xff, 0x14, 0xee, 0x10, 0x76, 0x4e, 0xa6, 0xa5, 0xca,
0xd9, 0x79, 0xe1, 0x05, 0xe1, 0x06, 0x34, 0x25, 0x9d, 0x14, 0x78, 0xe6, 0x34, 0xc1, 0x9d, 0xd0,
0x87, 0xd0, 0x9b, 0x08, 0x9c, 0x91, 0x94, 0x13, 0x41, 0x59, 0x1e, 0x6e, 0x8c, 0x82, 0x71, 0x23,
0xe9, 0x1a, 0xdb, 0xb1, 0x31, 0x45, 0x08, 0x06, 0x97, 0xde, 0x6c, 0xc6, 0xd1, 0x14, 0x6e, 0x7c,
0xc7, 0x73, 0x1d, 0xb4, 0xd2, 0x01, 0x17, 0x68, 0x45, 0x53, 0x82, 0xff, 0xac, 0x29, 0xd1, 0x4d,
0x78, 0xef, 0x95, 0x48, 0x2e, 0x89, 0x01, 0x6c, 0x7f, 0x4f, 0x84, 0xa4, 0xcc, 0xdf, 0x32, 0xfa,
0x14, 0x76, 0x2a, 0x8b, 0x7b, 0xdb, 0x10, 0x5a, 0x0b, 0x6b, 0x72, 0x37, 0xf7, 0xc7, 0xe8, 0x13,
0xe8, 0xe9, 0x77, 0xab, 0x32, 0x1f, 0x42, 0x9b, 0x16, 0x8a, 0x88, 0x85, 0x7b, 0xa4, 0x46, 0x52,
0x9d, 0xa3, 0x17, 0xd0, 0x77, 0x58, 0xe7, 0xf6, 0x1b, 0xd8, 0x92, 0xda, 0xb0, 0xe6, 0x15, 0x9f,
0x63, 0x79, 0x66, 0x1d, 0x59, 0x7a, 0x74, 0x1b, 0xfa, 0x27, 0xa6, 0x12, 0xaf, 0x2f, 0xd4, 0x96,
0x2f, 0x94, 0xbe, 0xac, 0x07, 0xba, 0xeb, 0x9f, 0x41, 0xf7, 0xc9, 0x05, 0xc9, 0x3c, 0xf1, 0x01,
0xb4, 0x73, 0x82, 0xf3, 0x19, 0x2d, 0x88, 0x4b, 0x6a, 0x18, 0xdb, 0xe5, 0x12, 0xfb, 0xe5, 0x12,
0x3f, 0xf7, 0xcb, 0x25, 0xa9, 0xb0, 0x7e, 0x55, 0x6c, 0xbc, 0xba, 0x2a, 0x1a, 0x97, 0xab, 0x22,
0x3a, 0x80, 0x9e, 0x0d, 0xe6, 0xee, 0x7f, 0x03, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f,
0x71, 0x27, 0xf4, 0x3e, 0x74, 0xc8, 0x05, 0x55, 0x69, 0xa6, 0xc7, 0x7a, 0xc3, 0xdc, 0xa0, 0xad,
0x0d, 0x07, 0x2c, 0x27, 0xd1, 0xef, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37,
0xd5, 0x9f, 0x6f, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x6d, 0x9a,
0x85, 0xf3, 0xe6, 0x6b, 0x1b, 0xdc, 0xfe, 0x9f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x81,
0xa6, 0x9d, 0x7e, 0x74, 0xbf, 0xee, 0xd4, 0xad, 0xec, 0xda, 0xe1, 0x83, 0x75, 0x69, 0xae, 0x7e,
0xd7, 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x6e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xb7, 0x1e,
0xa9, 0x0a, 0xfa, 0x1b, 0xb4, 0xfd, 0x38, 0xa3, 0x87, 0x75, 0x7d, 0x5c, 0x91, 0x93, 0xe1, 0xe7,
0xeb, 0x13, 0xab, 0x04, 0xfe, 0x08, 0x60, 0xe7, 0xca, 0x48, 0xa3, 0x2f, 0xeb, 0xfa, 0x7b, 0xbd,
0xea, 0x0c, 0x1f, 0xbd, 0x35, 0xbf, 0x4a, 0xeb, 0x57, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba,
0x2a, 0x3f, 0xc3, 0x87, 0x6b, 0xf3, 0xaa, 0xe8, 0x17, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb,
0xb2, 0x76, 0x0d, 0xef, 0xaf, 0xc9, 0xf2, 0x71, 0xf7, 0x02, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7,
0xff, 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x2b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f,
0xd2, 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x35, 0xf4, 0x57, 0x00, 0x7d, 0x6d, 0x3a, 0x51,
0x82, 0xe0, 0x39, 0x2d, 0x26, 0xe8, 0x51, 0x4d, 0xf1, 0xd6, 0x2c, 0x2b, 0xe0, 0x8e, 0xe9, 0x53,
0xf9, 0xea, 0xed, 0x1d, 0xf8, 0xb4, 0xc6, 0xc1, 0x5e, 0xf0, 0x75, 0xeb, 0x87, 0x2d, 0xab, 0x59,
0x4d, 0xf3, 0x73, 0xf7, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2e, 0x63, 0xc4, 0xd3, 0x74, 0x0c,
0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

2
drivers/shared/executor/proto/executor.proto
View File

@ -45,6 +45,8 @@ message LaunchRequest {
string default_pid_mode = 15;
string default_ipc_mode = 16;
string cpuset_cgroup = 17;
repeated string allow_caps = 18;
repeated string capabilities = 19;
}
message LaunchResponse {

1
drivers/shared/executor/server.go
View File

@ -37,6 +37,7 @@ func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchReques
NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation),
ModePID: req.DefaultPidMode,
ModeIPC: req.DefaultIpcMode,
Capabilities: req.Capabilities,
})
if err != nil {

21
vendor/github.com/docker/docker/oci/caps/defaults.go generated vendored
View File

@ -1,21 +0,0 @@
package caps // import "github.com/docker/docker/oci/caps"
// DefaultCapabilities returns a Linux kernel default capabilities
func DefaultCapabilities() []string {
return []string{
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE",
}
}

169
vendor/github.com/docker/docker/oci/caps/utils.go generated vendored
View File

@ -1,169 +0,0 @@
package caps // import "github.com/docker/docker/oci/caps"
import (
"fmt"
"strings"
"github.com/docker/docker/errdefs"
"github.com/syndtr/gocapability/capability"
)
var capabilityList Capabilities
func init() {
last := capability.CAP_LAST_CAP
// hack for RHEL6 which has no /proc/sys/kernel/cap_last_cap
if last == capability.Cap(63) {
last = capability.CAP_BLOCK_SUSPEND
}
for _, cap := range capability.List() {
if cap > last {
continue
}
capabilityList = append(capabilityList,
&CapabilityMapping{
Key: "CAP_" + strings.ToUpper(cap.String()),
Value: cap,
},
)
}
}
type (
// CapabilityMapping maps linux capability name to its value of capability.Cap type
// Capabilities is one of the security systems in Linux Security Module (LSM)
// framework provided by the kernel.
// For more details on capabilities, see http://man7.org/linux/man-pages/man7/capabilities.7.html
CapabilityMapping struct {
Key string `json:"key,omitempty"`
Value capability.Cap `json:"value,omitempty"`
}
// Capabilities contains all CapabilityMapping
Capabilities []*CapabilityMapping
)
// String returns <key> of CapabilityMapping
func (c *CapabilityMapping) String() string {
return c.Key
}
// GetCapability returns CapabilityMapping which contains specific key
func GetCapability(key string) *CapabilityMapping {
for _, capp := range capabilityList {
if capp.Key == key {
cpy := *capp
return &cpy
}
}
return nil
}
// GetAllCapabilities returns all of the capabilities
func GetAllCapabilities() []string {
output := make([]string, len(capabilityList))
for i, capability := range capabilityList {
output[i] = capability.String()
}
return output
}
// inSlice tests whether a string is contained in a slice of strings or not.
func inSlice(slice []string, s string) bool {
for _, ss := range slice {
if s == ss {
return true
}
}
return false
}
const allCapabilities = "ALL"
// NormalizeLegacyCapabilities normalizes, and validates CapAdd/CapDrop capabilities
// by upper-casing them, and adding a CAP_ prefix (if not yet present).
//
// This function also accepts the "ALL" magic-value, that's used by CapAdd/CapDrop.
func NormalizeLegacyCapabilities(caps []string) ([]string, error) {
var normalized []string
valids := GetAllCapabilities()
for _, c := range caps {
c = strings.ToUpper(c)
if c == allCapabilities {
normalized = append(normalized, c)
continue
}
if !strings.HasPrefix(c, "CAP_") {
c = "CAP_" + c
}
if !inSlice(valids, c) {
return nil, errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c))
}
normalized = append(normalized, c)
}
return normalized, nil
}
// ValidateCapabilities validates if caps only contains valid capabilities
func ValidateCapabilities(caps []string) error {
valids := GetAllCapabilities()
for _, c := range caps {
if !inSlice(valids, c) {
return errdefs.InvalidParameter(fmt.Errorf("unknown capability: %q", c))
}
}
return nil
}
// TweakCapabilities tweaks capabilities by adding, dropping, or overriding
// capabilities in the basics capabilities list.
func TweakCapabilities(basics, adds, drops, capabilities []string, privileged bool) ([]string, error) {
switch {
case privileged:
// Privileged containers get all capabilities
return GetAllCapabilities(), nil
case capabilities != nil:
// Use custom set of capabilities
if err := ValidateCapabilities(capabilities); err != nil {
return nil, err
}
return capabilities, nil
case len(adds) == 0 && len(drops) == 0:
// Nothing to tweak; we're done
return basics, nil
}
capDrop, err := NormalizeLegacyCapabilities(drops)
if err != nil {
return nil, err
}
capAdd, err := NormalizeLegacyCapabilities(adds)
if err != nil {
return nil, err
}
var caps []string
switch {
case inSlice(capAdd, allCapabilities):
// Add all capabilities except ones on capDrop
for _, c := range GetAllCapabilities() {
if !inSlice(capDrop, c) {
caps = append(caps, c)
}
}
case inSlice(capDrop, allCapabilities):
// "Drop" all capabilities; use what's in capAdd instead
caps = capAdd
default:
// First drop some capabilities
for _, c := range basics {
if !inSlice(capDrop, c) {
caps = append(caps, c)
}
}
// Then add the list of capabilities from capAdd
caps = append(caps, capAdd...)
}
return caps, nil
}

1
vendor/modules.txt vendored
View File

@ -241,7 +241,6 @@ github.com/docker/docker/api/types/swarm
github.com/docker/docker/api/types/swarm/runtime
github.com/docker/docker/api/types/versions
github.com/docker/docker/errdefs
github.com/docker/docker/oci/caps
github.com/docker/docker/pkg/archive
github.com/docker/docker/pkg/fileutils
github.com/docker/docker/pkg/homedir

45
website/content/docs/drivers/docker.mdx
View File

@ -452,28 +452,24 @@ config {
- `cap_add` - (Optional) A list of Linux capabilities as strings to pass directly to
[`--cap-add`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities).
Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist.
The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration.
The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration.
For example:
```hcl
config {
cap_add = [
"SYS_TIME",
]
cap_add = ["net_raw", sys_time"]
}
```
- `cap_drop` - (Optional) A list of Linux capabilities as strings to pass directly to
[`--cap-drop`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities).
Effective capabilities (computed from `cap_add` and `cap_drop`) have to match the configured allowlist.
The allowlist can be customized using the [`allow_caps`](#plugin_caps) plugin option key in the client node's configuration.
The allowlist can be customized using the [`allow_caps`][allow_caps] plugin option key in the client node's configuration.
For example:
```hcl
config {
cap_drop = [
"MKNOD",
]
cap_drop = ["mknod"]
}
```
@ -797,10 +793,7 @@ plugin "docker" {
}
allow_privileged = false
allow_caps = ["CHOWN", "NET_RAW"]
# allow_caps can also be set to "ALL"
# allow_caps = ["ALL"]
allow_caps = ["chown", "net_raw"]
}
}
```
@ -823,13 +816,22 @@ plugin "docker" {
from the Docker engine during an image pull within this timeframe, Nomad will
timeout the request that initiated the pull command. (Minimum of `1m`)
- `allow_caps`<a id="plugin_caps"></a> - A list of allowed Linux capabilities.
Defaults to
`CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP, NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE` which is the list of
capabilities allowed by docker by default, as defined here. Allows the
operator to control which capabilities can be obtained by tasks using cap_add
and cap_drop options. Supports the value "ALL" as a shortcut for allowlisting
all capabilities.
- `allow_caps` - A list of allowed Linux capabilities. Defaults to
```hcl
["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod",
"net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"]
```
which is the same list of capabilities allowed by [docker by default][docker_caps]
(without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities can be obtained
by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options. Supports
the value `"all"` as a shortcut for allow-listing all capabilities supported by
the operating system.
!> **Warning:** Allowing more capabilities beyond the default may lead to
undesirable consequences, including untrusted tasks being able to compromise the
host system.
- `allow_runtimes` - defaults to `["runc", "nvidia"]` - A list of the allowed
docker runtimes a task may use.
@ -1136,3 +1138,8 @@ Windows is relatively new and rapidly evolving you may want to consult the
[plugin-stanza]: /docs/configuration/plugin
[allocation working directory]: /docs/runtime/environment#task-directories 'Task Directories'
[`auth_soft_fail=true`]: #auth_soft_fail
[cap_add]: /docs/drivers/docker#cap_add
[cap_drop]: /docs/drivers/docker#cap_drop
[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12
[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
[allow_caps]: /docs/drivers/docker#allow_caps

43
website/content/docs/drivers/exec.mdx
View File

@ -54,6 +54,27 @@ be able to access sensitive process information like environment variables.
!> **Warning:** If set to `"host"`, other processes running as the same user will be
able to make use of IPC features, like sending unexpected POSIX signals.
- `cap_add` - (Optional) A list of Linux capabilities to enable for the task.
Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset
of the allowed capabilities configured with [`allow_caps`][allow_caps].
```hcl
config {
cap_add = ["net_raw", "sys_time"]
}
```
- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task.
Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset
of the allowed capabilities configured with [`allow_caps`][allow_caps].
```hcl
config {
cap_drop = ["all"]
cap_add = ["chown", "sys_chroot", "mknod"]
}
```
## Examples
To run a binary present on the Node:
@ -138,6 +159,23 @@ able to make use of IPC features, like sending unexpected POSIX signals.
for file system isolation without `pivot_root`. This is useful for systems
where the root is on a ramdisk.
- `allow_caps` - A list of allowed Linux capabilities. Defaults to
```hcl
["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod",
"net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"]
```
which is modeled after the capabilities allowed by [docker by default][docker_caps]
(without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities
can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options.
Supports the value `"all"` as a shortcut for allow-listing all capabilities supported
by the operating system.
!> **Warning:** Allowing more capabilities beyond the default may lead to
undesirable consequences, including untrusted tasks being able to compromise the
host system.
## Client Attributes
The `exec` driver will set the following client attributes:
@ -200,3 +238,8 @@ This list is configurable through the agent client
[default_pid_mode]: /docs/drivers/exec#default_pid_mode
[default_ipc_mode]: /docs/drivers/exec#default_ipc_mode
[cap_add]: /docs/drivers/exec#cap_add
[cap_drop]: /docs/drivers/exec#cap_drop
[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12
[allow_caps]: /docs/drivers/exec#allow_caps
[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities

43
website/content/docs/drivers/java.mdx
View File

@ -61,6 +61,27 @@ be able to access sensitive process information like environment variables.
!> **Warning:** If set to `"host"`, other processes running as the same user will be
able to make use of IPC features, like sending unexpected POSIX signals.
- `cap_add` - (Optional) A list of Linux capabilities to enable for the task.
Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset
of the allowed capabilities configured with [`allow_caps`][allow_caps].
```hcl
config {
cap_add = ["net_raw", "sys_time"]
}
```
- `cap_drop` - (Optional) A list of Linux capabilities to disable for the task.
Effective capabilities (computed from `cap_add` and `cap_drop`) must be a subset
of the allowed capabilities configured with [`allow_caps`][allow_caps].
```hcl
config {
cap_drop = ["all"]
cap_add = ["chown", "sys_chroot", "mknod"]
}
```
## Examples
A simple config block to run a Java Jar:
@ -138,6 +159,23 @@ be able to access sensitive process information like environment variables.
!> **Warning:** If set to `"host"`, other processes running as the same user will be
able to make use of IPC features, like sending unexpected POSIX signals.
- `allow_caps` - A list of allowed Linux capabilities. Defaults to
```hcl
["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod",
"net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"]
```
which is modeled after the capabilities allowed by [docker by default][docker_caps]
(without [`NET_RAW`][no_net_raw]). Allows the operator to control which capabilities
can be obtained by tasks using [`cap_add`][cap_add] and [`cap_drop`][cap_drop] options.
Supports the value `"all"` as a shortcut for allow-listing all capabilities supported
by the operating system.
!> **Warning:** Allowing more capabilities beyond the default may lead to
undesirable consequences, including untrusted tasks being able to compromise the
host system.
## Client Requirements
The `java` driver requires Java to be installed and in your system's `$PATH`. On
@ -208,3 +246,8 @@ This list is configurable through the agent client
[default_pid_mode]: /docs/drivers/java#default_pid_mode
[default_ipc_mode]: /docs/drivers/java#default_ipc_mode
[cap_add]: /docs/drivers/java#cap_add
[cap_drop]: /docs/drivers/java#cap_drop
[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12
[allow_caps]: /docs/drivers/java#allow_caps
[docker_caps]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities

41
website/content/docs/upgrade/upgrade-specific.mdx
View File

@ -54,7 +54,37 @@ these fields.
Connect native tasks running in host networking mode will now have `CONSUL_HTTP_ADDR`
set automatically. Before this was only the case for bridge networking. If an operator
already explicitly set `CONSUL_HTTP_ADDR` then it will not get overriden.
already explicitly set `CONSUL_HTTP_ADDR` then it will not get overridden.
#### Linux capabilities in exec/java
Following the security [remediation][no_net_raw] in Nomad versions 0.12.12, 1.0.5,
and 1.1.0-rc1, the `exec` and `java` task drivers will additionally no longer enable
the following linux capabilities by default.
```
AUDIT_CONTROL AUDIT_READ BLOCK_SUSPEND DAC_READ_SEARCH IPC_LOCK IPC_OWNER LEASE
LINUX_IMMUTABLE MAC_ADMIN MAC_OVERRIDE NET_ADMIN NET_BROADCAST NET_RAW SYS_ADMIN
SYS_BOOT SYSLOG SYS_MODULE SYS_NICE SYS_PACCT SYS_PTRACE SYS_RAWIO SYS_RESOURCE
SYS_TIME SYS_TTY_CONFIG WAKE_ALARM
```
The capabilities now enabled by default are modeled after Docker default
[`linux capabilities`] (excluding `NET_RAW`).
```
AUDIT_WRITE CHOWN DAC_OVERRIDE FOWNER FSETID KILL MKNOD NET_BIND_SERVICE
SETFCAP SETGID SETPCAP SETUID SYS_CHROOT
```
A new `allow_caps` plugin configuration parameter for [`exec`][allow_caps_exec]
and [`java`][allow_caps_java] task drivers can be used to restrict the set of
capabilities allowed for use by tasks.
Tasks using the `exec` or `java` task drivers can add or remove desired linux
capabilities using the [`cap_add`][cap_add_exec] and [`cap_drop`][cap_drop_exec]
task configuration options.
#### iptables
@ -63,9 +93,9 @@ inserting them as the first rule. This allows better control for user-defined
iptables rules but users who append rules currently should verify that their
rules are being appended in the correct order.
## Nomad 1.1.0, 1.0.5, 0.12.12
## Nomad 1.1.0-rc1, 1.0.5, 0.12.12
Nomad versions 1.1.0, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`,
Nomad versions 1.1.0-rc1, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`,
and `java` task drivers so that the [`CAP_NET_RAW`] linux capability is disabled
by default. This is one of the [`linux capabilities`] that Docker itself enables
by default, as this capability enables the generation of ICMP packets - used by
@ -1111,3 +1141,8 @@ deleted and then Nomad 0.3.0 can be launched.
[`CAP_NET_RAW`]: https://security.stackexchange.com/a/128988
[`linux capabilities`]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
[`allow_caps`]: /docs/drivers/docker#allow_caps
[no_net_raw]: /docs/upgrade/upgrade-specific#nomad-1-1-0-rc1-1-0-5-0-12-12
[allow_caps_exec]: /docs/drivers/exec#allow_caps
[allow_caps_java]: /docs/drivers/java#allow_caps
[cap_add_exec]: /docs/drivers/exec#cap_add
[cap_drop_exec]: /docs/drivers/exec#cap_drop