luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

drivers/java: enable setting allow_caps on java driver

Browse source 
Enable setting allow_caps on the java task driver plugin, along
with the associated cap_add and cap_drop options in java task
configuration.
This commit is contained in:
Seth Hoenig 2021-05-15 10:55:44 -06:00
parent 5b8a32f23d
commit 2361a91938
5 changed files with 277 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
drivers/exec/driver.go
View file

@ -298,29 +298,6 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
return taskConfigSpec, nil
}
// getCaps computes the complete set of linux capabilities to enable for driver,
// which gets passed along to libcontainer.
func (d *Driver) getCaps(tc *TaskConfig) ([]string, error) {
driverAllowed := capabilities.New(d.config.AllowCaps)
// determine caps the task wants that are not allowed
taskCaps := capabilities.New(tc.CapAdd)
missing := driverAllowed.Difference(taskCaps)
if !missing.Empty() {
return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing)
}
// if task did not specify allowed caps, use nomad defaults minus task drops
if len(tc.CapAdd) == 0 {
driverAllowed.Remove(tc.CapDrop)
return driverAllowed.Slice(true), nil
}
// otherwise task did specify allowed caps, enable exactly those
taskAdd := capabilities.New(tc.CapAdd)
return taskAdd.Slice(true), nil
}
// Capabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
@ -496,7 +473,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}
caps, err := d.getCaps(&driverConfig)
caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop)
if err != nil {
return nil, nil, err
}

82
drivers/java/driver.go
View file

@ -10,6 +10,7 @@ import (
"time"
"github.com/hashicorp/nomad/client/lib/cgutil"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/consul-template/signals"
hclog "github.com/hashicorp/go-hclog"
@ -73,6 +74,10 @@ var (
hclspec.NewAttr("default_ipc_mode", "string", false),
hclspec.NewLiteral(`"private"`),
),
"allow_caps": hclspec.NewDefault(
hclspec.NewAttr("allow_caps", "list(string)", false),
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
),
})
// taskConfigSpec is the hcl specification for the driver config section of
@ -88,11 +93,13 @@ var (
"args": hclspec.NewAttr("args", "list(string)", false),
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
})
// capabilities is returned by the Capabilities RPC and indicates what
// driverCapabilities is returned by the Capabilities RPC and indicates what
// optional features this driver supports
capabilities = &drivers.Capabilities{
driverCapabilities = &drivers.Capabilities{
SendSignals: false,
Exec: false,
FSIsolation: drivers.FSIsolationNone,
@ -108,8 +115,8 @@ var (
func init() {
if runtime.GOOS == "linux" {
capabilities.FSIsolation = drivers.FSIsolationChroot
capabilities.MountConfigs = drivers.MountConfigSupportAll
driverCapabilities.FSIsolation = drivers.FSIsolationChroot
driverCapabilities.MountConfigs = drivers.MountConfigSupportAll
}
}
@ -122,6 +129,10 @@ type Config struct {
// DefaultModeIPC is the default IPC isolation set for all tasks using
// exec-based task drivers.
DefaultModeIPC string `codec:"default_ipc_mode"`
// AllowCaps configures which Linux Capabilities are enabled for tasks
// running on this node.
AllowCaps []string `codec:"allow_caps"`
}
func (c *Config) validate() error {
@ -137,18 +148,44 @@ func (c *Config) validate() error {
return fmt.Errorf("default_ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, c.DefaultModeIPC)
}
badCaps := capabilities.Supported().Difference(capabilities.New(c.AllowCaps))
if !badCaps.Empty() {
return fmt.Errorf("allow_caps configured with capabilities not supported by system: %s", badCaps)
}
return nil
}
// TaskConfig is the driver configuration of a taskConfig within a job
type TaskConfig struct {
Class string `codec:"class"`
ClassPath string `codec:"class_path"`
JarPath string `codec:"jar_path"`
JvmOpts []string `codec:"jvm_options"`
Args []string `codec:"args"` // extra arguments to java executable
ModePID string `codec:"pid_mode"`
ModeIPC string `codec:"ipc_mode"`
// Class indicates which class contains the java entry point.
Class string `codec:"class"`
// ClassPath indicates where class files are found.
ClassPath string `codec:"class_path"`
// JarPath indicates where a jar file is found.
JarPath string `codec:"jar_path"`
// JvmOpts are arguments to pass to the JVM
JvmOpts []string `codec:"jvm_options"`
// Args are extra arguments to java executable
Args []string `codec:"args"`
// ModePID indicates whether PID namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModePID string `codec:"pid_mode"`
// ModeIPC indicates whether IPC namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModeIPC string `codec:"ipc_mode"`
// CapAdd is a set of linux capabilities to enable.
CapAdd []string `codec:"cap_add"`
// CapDrop is a set of linux capabilities to disable.
CapDrop []string `codec:"cap_drop"`
}
func (tc *TaskConfig) validate() error {
@ -165,6 +202,16 @@ func (tc *TaskConfig) validate() error {
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
}
supported := capabilities.Supported()
badAdds := supported.Difference(capabilities.New(tc.CapAdd))
if !badAdds.Empty() {
return fmt.Errorf("cap_add configured with capabilities not supported by system: %s", badAdds)
}
badDrops := supported.Difference(capabilities.New(tc.CapDrop))
if !badDrops.Empty() {
return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
}
return nil
}
@ -243,7 +290,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
}
func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
return capabilities, nil
return driverCapabilities, nil
}
func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
@ -415,7 +462,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
executorConfig := &executor.ExecutorConfig{
LogFile: pluginLogFile,
LogLevel: "debug",
FSIsolation: capabilities.FSIsolation == drivers.FSIsolationChroot,
FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot,
}
exec, pluginClient, err := executor.CreateExecutor(
@ -438,6 +485,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}
caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop)
if err != nil {
return nil, nil, err
}
execCmd := &executor.ExecCommand{
Cmd: absPath,
Args: args,
@ -453,6 +505,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
NetworkIsolation: cfg.NetworkIsolation,
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
Capabilities: caps,
}
ps, err := exec.Launch(execCmd)
@ -491,7 +544,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
}
func javaCmdArgs(driverConfig TaskConfig) []string {
args := []string{}
var args []string
// Look for jvm options
if len(driverConfig.JvmOpts) != 0 {
args = append(args, driverConfig.JvmOpts...)

116
drivers/java/driver_test.go
View file

@ -1,6 +1,7 @@
package java
import (
"context"
"errors"
"fmt"
"io"
@ -8,12 +9,10 @@ import (
"os"
"path/filepath"
"testing"
"time"
dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
"context"
"time"
ctestutil "github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
"github.com/hashicorp/nomad/helper/testlog"
@ -416,20 +415,99 @@ func Test_dnsConfig(t *testing.T) {
}
func TestDriver_Config_validate(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "private", ipcMode: "private", exp: nil},
{pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)},
{pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: tc.pidMode,
DefaultModeIPC: tc.ipcMode,
}).validate())
}
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "private", ipcMode: "private", exp: nil},
{pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)},
{pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: tc.pidMode,
DefaultModeIPC: tc.ipcMode,
}).validate())
}
})
t.Run("allow_caps", func(t *testing.T) {
for _, tc := range []struct {
ac []string
exp error
}{
{ac: []string{}, exp: nil},
{ac: []string{"all"}, exp: nil},
{ac: []string{"chown", "sys_time"}, exp: nil},
{ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil},
{ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&Config{
DefaultModePID: "private",
DefaultModeIPC: "private",
AllowCaps: tc.ac,
}).validate())
}
})
}
func TestDriver_TaskConfig_validate(t *testing.T) {
t.Run("pid/ipc", func(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "host", ipcMode: "", exp: nil},
{pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)},
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "", ipcMode: "host", exp: nil},
{pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&TaskConfig{
ModePID: tc.pidMode,
ModeIPC: tc.ipcMode,
}).validate())
}
})
t.Run("cap_add", func(t *testing.T) {
for _, tc := range []struct {
adds []string
exp error
}{
{adds: nil, exp: nil},
{adds: []string{"chown"}, exp: nil},
{adds: []string{"CAP_CHOWN"}, exp: nil},
{adds: []string{"chown", "sys_time"}, exp: nil},
{adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapAdd: tc.adds,
}).validate())
}
})
t.Run("cap_drop", func(t *testing.T) {
for _, tc := range []struct {
drops []string
exp error
}{
{drops: nil, exp: nil},
{drops: []string{"chown"}, exp: nil},
{drops: []string{"CAP_CHOWN"}, exp: nil},
{drops: []string{"chown", "sys_time"}, exp: nil},
{drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")},
} {
require.Equal(t, tc.exp, (&TaskConfig{
CapDrop: tc.drops,
}).validate())
}
})
}

33
drivers/shared/capabilities/defaults.go
View file

@ -1,6 +1,7 @@
package capabilities
import (
"fmt"
"regexp"
"github.com/syndtr/gocapability/capability"
@ -13,7 +14,7 @@ const (
)
var (
extractLiteral = regexp.MustCompile(`("[\w]+)`)
extractLiteral = regexp.MustCompile(`([\w]+)`)
)
// NomadDefaults is the set of Linux capabilities that Nomad enables by
@ -112,3 +113,33 @@ func LegacySupported() *Set {
"CAP_AUDIT_READ",
})
}
// Calculate the reduced set of linux capabilities to enable for driver, taking
// into account the capabilities allowed by the driver and the capabilities
// explicitly requested / removed by the task configuration.
//
// capAdd if set indicates the minimal set of capabilities that should be enabled.
// capDrop if set indicates capabilities that should be dropped from the driver defaults
//
// If the task requests a capability not allowed by the driver, an error is
// returned.
func Calculate(allowCaps, capAdd, capDrop []string) ([]string, error) {
driverAllowed := New(allowCaps)
// determine caps the task wants that are not allowed
taskCaps := New(capAdd)
missing := driverAllowed.Difference(taskCaps)
if !missing.Empty() {
return nil, fmt.Errorf("driver does not allow the following capabilities: %s", missing)
}
// if task did not specify allowed caps, use nomad defaults minus task drops
if len(capAdd) == 0 {
driverAllowed.Remove(capDrop)
return driverAllowed.Slice(true), nil
}
// otherwise task did specify allowed caps, enable exactly those
taskAdd := New(capAdd)
return taskAdd.Slice(true), nil
}

79
drivers/shared/capabilities/defaults_test.go
View file

@ -1,6 +1,7 @@
package capabilities
import (
"errors"
"strings"
"testing"
@ -21,3 +22,81 @@ func TestSet_DockerDefaults(t *testing.T) {
require.Len(t, result.Slice(false), 14)
require.Contains(t, result.String(), "net_raw")
}
func TestCaps_Calculate(t *testing.T) {
for _, tc := range []struct {
name string
// input
allowCaps []string // driver config
capAdd []string // task config
capDrop []string // task config
// output
exp []string
err error
}{
{
name: "the default setting",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: nil,
exp: NomadDefaults().Slice(true),
err: nil,
},
{
name: "allow all",
allowCaps: []string{"all"},
capAdd: nil,
capDrop: nil,
exp: Supported().Slice(true),
err: nil,
},
{
name: "allow selection",
allowCaps: []string{"cap_net_raw", "chown", "SYS_TIME"},
capAdd: nil,
capDrop: nil,
exp: []string{"CAP_CHOWN", "CAP_NET_RAW", "CAP_SYS_TIME"},
err: nil,
},
{
name: "add allowed",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "KILL"},
capDrop: nil,
exp: []string{"CAP_CHOWN", "CAP_KILL"},
err: nil,
},
{
name: "add disallowed",
allowCaps: NomadDefaults().Slice(false),
capAdd: []string{"chown", "net_raw"},
capDrop: nil,
exp: nil,
err: errors.New("driver does not allow the following capabilities: net_raw"),
},
{
name: "drop some",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"chown", "fowner", "CAP_KILL", "SYS_CHROOT", "mknod", "dac_override"},
exp: []string{"CAP_AUDIT_WRITE", "CAP_FSETID", "CAP_NET_BIND_SERVICE", "CAP_SETFCAP", "CAP_SETGID", "CAP_SETPCAP", "CAP_SETUID"},
err: nil,
},
{
name: "drop all",
allowCaps: NomadDefaults().Slice(false),
capAdd: nil,
capDrop: []string{"all"},
exp: []string{},
err: nil,
},
} {
t.Run(tc.name, func(t *testing.T) {
caps, err := Calculate(tc.allowCaps, tc.capAdd, tc.capDrop)
require.Equal(t, tc.err, err)
require.Equal(t, tc.exp, caps)
})
}
}