From 74bd0be6ea124b82d6fd87aa6280a59e47287efa Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 9 Dec 2018 22:30:23 -0500 Subject: [PATCH] drivers/exec: support device binds and mounts --- drivers/exec/driver.go | 2 + drivers/exec/driver_test.go | 91 +++++++++++++++++++ drivers/java/driver.go | 2 + drivers/shared/executor/executor.go | 7 ++ drivers/shared/executor/executor_linux.go | 81 +++++++++++++++-- .../shared/executor/executor_linux_test.go | 66 ++++++++++++++ 6 files changed, 243 insertions(+), 6 deletions(-) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 14eca02c3..213b5dd92 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -308,6 +308,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru TaskDir: cfg.TaskDir().Dir, StdoutPath: cfg.StdoutPath, StderrPath: cfg.StderrPath, + Mounts: cfg.Mounts, + Devices: cfg.Devices, } ps, err := exec.Launch(execCmd) diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index 1299c1136..6de0a7c72 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -490,6 +490,97 @@ func TestExecDriver_HandlerExec(t *testing.T) { require.NoError(harness.DestroyTask(task.ID, true)) } +func TestExecDriver_DevicesAndMounts(t *testing.T) { + t.Parallel() + require := require.New(t) + ctestutils.ExecCompatible(t) + + tmpDir, err := ioutil.TempDir("", "exec_binds_mounts") + require.NoError(err) + defer os.RemoveAll(tmpDir) + + err = ioutil.WriteFile(filepath.Join(tmpDir, "testfile"), []byte("from-host"), 600) + require.NoError(err) + + d := NewExecDriver(testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "test", + StdoutPath: filepath.Join(tmpDir, "task-stdout"), + StderrPath: filepath.Join(tmpDir, "task-stderr"), + Devices: []*drivers.DeviceConfig{ + { + TaskPath: "/dev/inserted-random", + HostPath: "/dev/random", + Permissions: "rw", + }, + }, + Mounts: []*drivers.MountConfig{ + { + TaskPath: "/tmp/task-path-rw", + HostPath: tmpDir, + Readonly: false, + }, + { + TaskPath: "/tmp/task-path-ro", + HostPath: tmpDir, + Readonly: true, + }, + }, + } + + require.NoError(ioutil.WriteFile(task.StdoutPath, []byte{}, 660)) + require.NoError(ioutil.WriteFile(task.StderrPath, []byte{}, 660)) + + taskConfig := map[string]interface{}{ + "command": "/bin/bash", + "args": []string{"-c", ` +export LANG=en.UTF-8 +echo "mounted device /inserted-random: $(stat -c '%t:%T' /dev/inserted-random)" +echo "reading from ro path: $(cat /tmp/task-path-ro/testfile)" +echo "reading from rw path: $(cat /tmp/task-path-rw/testfile)" +touch /tmp/task-path-rw/testfile && echo 'overwriting file in rw succeeded' +touch /tmp/task-path-rw/testfile-from-rw && echo from-exec > /tmp/task-path-rw/testfile-from-rw && echo 'writing new file in rw succeeded' +touch /tmp/task-path-ro/testfile && echo 'overwriting file in ro succeeded' +touch /tmp/task-path-ro/testfile-from-ro && echo from-exec > /tmp/task-path-ro/testfile-from-ro && echo 'writing new file in ro succeeded' +exit 0 +`}, + } + encodeDriverHelper(require, task, taskConfig) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + handle, _, err := harness.StartTask(task) + require.NoError(err) + + ch, err := harness.WaitTask(context.Background(), handle.Config.ID) + require.NoError(err) + result := <-ch + require.NoError(harness.DestroyTask(task.ID, true)) + + stdout, err := ioutil.ReadFile(task.StdoutPath) + require.NoError(err) + require.Equal(`mounted device /inserted-random: 1:8 +reading from ro path: from-host +reading from rw path: from-host +overwriting file in rw succeeded +writing new file in rw succeeded`, strings.TrimSpace(string(stdout))) + + stderr, err := ioutil.ReadFile(task.StderrPath) + require.NoError(err) + require.Equal(`touch: cannot touch '/tmp/task-path-ro/testfile': Read-only file system +touch: cannot touch '/tmp/task-path-ro/testfile-from-ro': Read-only file system`, strings.TrimSpace(string(stderr))) + + // testing exit code last so we can inspect output first + require.Zero(result.ExitCode) + + fromRWContent, err := ioutil.ReadFile(filepath.Join(tmpDir, "testfile-from-rw")) + require.NoError(err) + require.Equal("from-exec", strings.TrimSpace(string(fromRWContent))) +} + func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) { evalCtx := &hcl.EvalContext{ Functions: shared.GetStdlibFuncs(), diff --git a/drivers/java/driver.go b/drivers/java/driver.go index a9ac5a0fb..68373f6c8 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -344,6 +344,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru TaskDir: cfg.TaskDir().Dir, StdoutPath: cfg.StdoutPath, StderrPath: cfg.StderrPath, + Mounts: cfg.Mounts, + Devices: cfg.Devices, } ps, err := exec.Launch(execCmd) diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index 85cd4cac4..1c5160fc0 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -22,6 +22,7 @@ import ( "github.com/hashicorp/nomad/client/stats" cstructs "github.com/hashicorp/nomad/client/structs" shelpers "github.com/hashicorp/nomad/helper/stats" + "github.com/hashicorp/nomad/plugins/drivers" ) const ( @@ -120,6 +121,12 @@ type ExecCommand struct { // doesn't enforce resource limits. To enforce limits, set ResourceLimits. // Using the cgroup does allow more precise cleanup of processes. BasicProcessCgroup bool + + // Mounts are the host paths to be be made available inside rootfs + Mounts []*drivers.MountConfig + + // Devices are the the device nodes to be created in isolation environment + Devices []*drivers.DeviceConfig } type nopCloser struct { diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index bb6da7890..92280059c 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -22,11 +22,14 @@ import ( "github.com/hashicorp/nomad/helper/discover" shelpers "github.com/hashicorp/nomad/helper/stats" "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/plugins/drivers" "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/cgroups" cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" lconfigs "github.com/opencontainers/runc/libcontainer/configs" + ldevices "github.com/opencontainers/runc/libcontainer/devices" "github.com/syndtr/gocapability/capability" + "golang.org/x/sys/unix" ) const ( @@ -125,7 +128,11 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro } // A container groups processes under the same isolation enforcement - container, err := factory.Create(l.id, newLibcontainerConfig(command)) + containerCfg, err := newLibcontainerConfig(command) + if err != nil { + return nil, fmt.Errorf("failed to configure container(%s): %v", l.id, err) + } + container, err := factory.Create(l.id, containerCfg) if err != nil { return nil, fmt.Errorf("failed to create container(%s): %v", l.id, err) } @@ -468,7 +475,7 @@ func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) { } -func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) { +func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error { defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV // set the new root directory for the container @@ -491,6 +498,14 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) { } cfg.Devices = lconfigs.DefaultAutoCreatedDevices + if len(command.Devices) > 0 { + devs, err := cmdDevices(command.Devices) + if err != nil { + return err + } + cfg.Devices = append(cfg.Devices, devs...) + } + cfg.Mounts = []*lconfigs.Mount{ { Source: "tmpfs", @@ -532,6 +547,12 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) { Flags: defaultMountFlags | syscall.MS_RDONLY, }, } + + if len(command.Mounts) > 0 { + cfg.Mounts = append(cfg.Mounts, cmdMounts(command.Mounts)...) + } + + return nil } func configureCgroups(cfg *lconfigs.Config, command *ExecCommand) error { @@ -592,7 +613,7 @@ func configureBasicCgroups(cfg *lconfigs.Config) error { return nil } -func newLibcontainerConfig(command *ExecCommand) *lconfigs.Config { +func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) { cfg := &lconfigs.Config{ Cgroups: &lconfigs.Cgroup{ Resources: &lconfigs.Resources{ @@ -605,9 +626,13 @@ func newLibcontainerConfig(command *ExecCommand) *lconfigs.Config { } configureCapabilities(cfg, command) - configureIsolation(cfg, command) - configureCgroups(cfg, command) - return cfg + if err := configureIsolation(cfg, command); err != nil { + return nil, err + } + if err := configureCgroups(cfg, command); err != nil { + return nil, err + } + return cfg, nil } // JoinRootCgroup moves the current process to the cgroups of the init process @@ -631,3 +656,47 @@ func JoinRootCgroup(subsystems []string) error { return mErrs.ErrorOrNil() } + +// cmdDevices converts a list of driver.DeviceConfigs into excutor.Devices. +func cmdDevices(devices []*drivers.DeviceConfig) ([]*lconfigs.Device, error) { + if len(devices) == 0 { + return nil, nil + } + + r := make([]*lconfigs.Device, len(devices)) + + for i, d := range devices { + ed, err := ldevices.DeviceFromPath(d.HostPath, d.Permissions) + if err != nil { + return nil, fmt.Errorf("failed to make device out for %s: %v", d.HostPath, err) + } + ed.Path = d.TaskPath + r[i] = ed + } + + return r, nil +} + +// cmdMounts converts a list of driver.MountConfigs into excutor.Mounts. +func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount { + if len(mounts) == 0 { + return nil + } + + r := make([]*lconfigs.Mount, len(mounts)) + + for i, m := range mounts { + flags := unix.MS_BIND + if m.Readonly { + flags |= unix.MS_RDONLY + } + r[i] = &lconfigs.Mount{ + Source: m.HostPath, + Destination: m.TaskPath, + Device: "bind", + Flags: flags, + } + } + + return r +} diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index e127825b3..041196ffc 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -17,8 +17,11 @@ import ( "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/plugins/drivers" tu "github.com/hashicorp/nomad/testutil" + lconfigs "github.com/opencontainers/runc/libcontainer/configs" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) func init() { @@ -194,3 +197,66 @@ func TestExecutor_ClientCleanup(t *testing.T) { output1 := execCmd.stdout.(*bufferCloser).String() require.Equal(len(output), len(output1)) } + +func TestExecutor_cmdDevices(t *testing.T) { + input := []*drivers.DeviceConfig{ + { + HostPath: "/dev/null", + TaskPath: "/task/dev/null", + Permissions: "rwm", + }, + } + + expected := &lconfigs.Device{ + Path: "/task/dev/null", + Type: 99, + Major: 1, + Minor: 3, + Permissions: "rwm", + } + + found, err := cmdDevices(input) + require.NoError(t, err) + require.Len(t, found, 1) + + // ignore file permission and ownership + // as they are host specific potentially + d := found[0] + d.FileMode = 0 + d.Uid = 0 + d.Gid = 0 + + require.EqualValues(t, expected, d) +} + +func TestExecutor_cmdMounts(t *testing.T) { + input := []*drivers.MountConfig{ + { + HostPath: "/host/path-ro", + TaskPath: "/task/path-ro", + Readonly: true, + }, + { + HostPath: "/host/path-rw", + TaskPath: "/task/path-rw", + Readonly: false, + }, + } + + expected := []*lconfigs.Mount{ + { + Source: "/host/path-ro", + Destination: "/task/path-ro", + Flags: unix.MS_BIND | unix.MS_RDONLY, + Device: "bind", + }, + { + Source: "/host/path-rw", + Destination: "/task/path-rw", + Flags: unix.MS_BIND, + Device: "bind", + }, + } + + require.EqualValues(t, expected, cmdMounts(input)) +}