luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

Merge pull request #4962 from hashicorp/f-exec-device-mounts

Browse source 
drivers/exec: Support devices mounts
This commit is contained in:
Mahmood Ali 2018-12-11 20:20:01 -05:00 committed by GitHub
parent f406e66ab8 74bd0be6ea
commit 567f1930fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 243 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
drivers/exec/driver.go
View file

@ -308,6 +308,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru
TaskDir: cfg.TaskDir().Dir,
StdoutPath: cfg.StdoutPath,
StderrPath: cfg.StderrPath,
Mounts: cfg.Mounts,
Devices: cfg.Devices,
}
ps, err := exec.Launch(execCmd)

91
drivers/exec/driver_test.go
View file

@ -490,6 +490,97 @@ func TestExecDriver_HandlerExec(t *testing.T) {
require.NoError(harness.DestroyTask(task.ID, true))
}
func TestExecDriver_DevicesAndMounts(t *testing.T) {
t.Parallel()
require := require.New(t)
ctestutils.ExecCompatible(t)
tmpDir, err := ioutil.TempDir("", "exec_binds_mounts")
require.NoError(err)
defer os.RemoveAll(tmpDir)
err = ioutil.WriteFile(filepath.Join(tmpDir, "testfile"), []byte("from-host"), 600)
require.NoError(err)
d := NewExecDriver(testlog.HCLogger(t))
harness := dtestutil.NewDriverHarness(t, d)
task := &drivers.TaskConfig{
ID: uuid.Generate(),
Name: "test",
StdoutPath: filepath.Join(tmpDir, "task-stdout"),
StderrPath: filepath.Join(tmpDir, "task-stderr"),
Devices: []*drivers.DeviceConfig{
{
TaskPath: "/dev/inserted-random",
HostPath: "/dev/random",
Permissions: "rw",
},
},
Mounts: []*drivers.MountConfig{
{
TaskPath: "/tmp/task-path-rw",
HostPath: tmpDir,
Readonly: false,
},
{
TaskPath: "/tmp/task-path-ro",
HostPath: tmpDir,
Readonly: true,
},
},
}
require.NoError(ioutil.WriteFile(task.StdoutPath, []byte{}, 660))
require.NoError(ioutil.WriteFile(task.StderrPath, []byte{}, 660))
taskConfig := map[string]interface{}{
"command": "/bin/bash",
"args": []string{"-c", `
export LANG=en.UTF-8
echo "mounted device /inserted-random: $(stat -c '%t:%T' /dev/inserted-random)"
echo "reading from ro path: $(cat /tmp/task-path-ro/testfile)"
echo "reading from rw path: $(cat /tmp/task-path-rw/testfile)"
touch /tmp/task-path-rw/testfile && echo 'overwriting file in rw succeeded'
touch /tmp/task-path-rw/testfile-from-rw && echo from-exec > /tmp/task-path-rw/testfile-from-rw && echo 'writing new file in rw succeeded'
touch /tmp/task-path-ro/testfile && echo 'overwriting file in ro succeeded'
touch /tmp/task-path-ro/testfile-from-ro && echo from-exec > /tmp/task-path-ro/testfile-from-ro && echo 'writing new file in ro succeeded'
exit 0
`},
}
encodeDriverHelper(require, task, taskConfig)
cleanup := harness.MkAllocDir(task, false)
defer cleanup()
handle, _, err := harness.StartTask(task)
require.NoError(err)
ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
require.NoError(err)
result := <-ch
require.NoError(harness.DestroyTask(task.ID, true))
stdout, err := ioutil.ReadFile(task.StdoutPath)
require.NoError(err)
require.Equal(`mounted device /inserted-random: 1:8
reading from ro path: from-host
reading from rw path: from-host
overwriting file in rw succeeded
writing new file in rw succeeded`, strings.TrimSpace(string(stdout)))
stderr, err := ioutil.ReadFile(task.StderrPath)
require.NoError(err)
require.Equal(`touch: cannot touch '/tmp/task-path-ro/testfile': Read-only file system
touch: cannot touch '/tmp/task-path-ro/testfile-from-ro': Read-only file system`, strings.TrimSpace(string(stderr)))
// testing exit code last so we can inspect output first
require.Zero(result.ExitCode)
fromRWContent, err := ioutil.ReadFile(filepath.Join(tmpDir, "testfile-from-rw"))
require.NoError(err)
require.Equal("from-exec", strings.TrimSpace(string(fromRWContent)))
}
func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) {
evalCtx := &hcl.EvalContext{
Functions: shared.GetStdlibFuncs(),

2
drivers/java/driver.go
View file

@ -344,6 +344,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru
TaskDir: cfg.TaskDir().Dir,
StdoutPath: cfg.StdoutPath,
StderrPath: cfg.StderrPath,
Mounts: cfg.Mounts,
Devices: cfg.Devices,
}
ps, err := exec.Launch(execCmd)

7
drivers/shared/executor/executor.go
View file

@ -22,6 +22,7 @@ import (
"github.com/hashicorp/nomad/client/stats"
cstructs "github.com/hashicorp/nomad/client/structs"
shelpers "github.com/hashicorp/nomad/helper/stats"
"github.com/hashicorp/nomad/plugins/drivers"
)
const (
@ -120,6 +121,12 @@ type ExecCommand struct {
// doesn't enforce resource limits. To enforce limits, set ResourceLimits.
// Using the cgroup does allow more precise cleanup of processes.
BasicProcessCgroup bool
// Mounts are the host paths to be be made available inside rootfs
Mounts []*drivers.MountConfig
// Devices are the the device nodes to be created in isolation environment
Devices []*drivers.DeviceConfig
}
type nopCloser struct {

81
drivers/shared/executor/executor_linux.go
View file

@ -22,11 +22,14 @@ import (
"github.com/hashicorp/nomad/helper/discover"
shelpers "github.com/hashicorp/nomad/helper/stats"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/plugins/drivers"
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/runc/libcontainer/cgroups"
cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
lconfigs "github.com/opencontainers/runc/libcontainer/configs"
ldevices "github.com/opencontainers/runc/libcontainer/devices"
"github.com/syndtr/gocapability/capability"
"golang.org/x/sys/unix"
)
const (
@ -125,7 +128,11 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro
}
// A container groups processes under the same isolation enforcement
container, err := factory.Create(l.id, newLibcontainerConfig(command))
containerCfg, err := newLibcontainerConfig(command)
if err != nil {
return nil, fmt.Errorf("failed to configure container(%s): %v", l.id, err)
}
container, err := factory.Create(l.id, containerCfg)
if err != nil {
return nil, fmt.Errorf("failed to create container(%s): %v", l.id, err)
}
@ -468,7 +475,7 @@ func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) {
}
func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) {
func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error {
defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
// set the new root directory for the container
@ -491,6 +498,14 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) {
}
cfg.Devices = lconfigs.DefaultAutoCreatedDevices
if len(command.Devices) > 0 {
devs, err := cmdDevices(command.Devices)
if err != nil {
return err
}
cfg.Devices = append(cfg.Devices, devs...)
}
cfg.Mounts = []*lconfigs.Mount{
{
Source: "tmpfs",
@ -532,6 +547,12 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) {
Flags: defaultMountFlags | syscall.MS_RDONLY,
},
}
if len(command.Mounts) > 0 {
cfg.Mounts = append(cfg.Mounts, cmdMounts(command.Mounts)...)
}
return nil
}
func configureCgroups(cfg *lconfigs.Config, command *ExecCommand) error {
@ -592,7 +613,7 @@ func configureBasicCgroups(cfg *lconfigs.Config) error {
return nil
}
func newLibcontainerConfig(command *ExecCommand) *lconfigs.Config {
func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) {
cfg := &lconfigs.Config{
Cgroups: &lconfigs.Cgroup{
Resources: &lconfigs.Resources{
@ -605,9 +626,13 @@ func newLibcontainerConfig(command *ExecCommand) *lconfigs.Config {
}
configureCapabilities(cfg, command)
configureIsolation(cfg, command)
configureCgroups(cfg, command)
return cfg
if err := configureIsolation(cfg, command); err != nil {
return nil, err
}
if err := configureCgroups(cfg, command); err != nil {
return nil, err
}
return cfg, nil
}
// JoinRootCgroup moves the current process to the cgroups of the init process
@ -631,3 +656,47 @@ func JoinRootCgroup(subsystems []string) error {
return mErrs.ErrorOrNil()
}
// cmdDevices converts a list of driver.DeviceConfigs into excutor.Devices.
func cmdDevices(devices []*drivers.DeviceConfig) ([]*lconfigs.Device, error) {
if len(devices) == 0 {
return nil, nil
}
r := make([]*lconfigs.Device, len(devices))
for i, d := range devices {
ed, err := ldevices.DeviceFromPath(d.HostPath, d.Permissions)
if err != nil {
return nil, fmt.Errorf("failed to make device out for %s: %v", d.HostPath, err)
}
ed.Path = d.TaskPath
r[i] = ed
}
return r, nil
}
// cmdMounts converts a list of driver.MountConfigs into excutor.Mounts.
func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount {
if len(mounts) == 0 {
return nil
}
r := make([]*lconfigs.Mount, len(mounts))
for i, m := range mounts {
flags := unix.MS_BIND
if m.Readonly {
flags |= unix.MS_RDONLY
}
r[i] = &lconfigs.Mount{
Source: m.HostPath,
Destination: m.TaskPath,
Device: "bind",
Flags: flags,
}
}
return r
}

66
drivers/shared/executor/executor_linux_test.go
View file

@ -17,8 +17,11 @@ import (
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/plugins/drivers"
tu "github.com/hashicorp/nomad/testutil"
lconfigs "github.com/opencontainers/runc/libcontainer/configs"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
func init() {
@ -194,3 +197,66 @@ func TestExecutor_ClientCleanup(t *testing.T) {
output1 := execCmd.stdout.(*bufferCloser).String()
require.Equal(len(output), len(output1))
}
func TestExecutor_cmdDevices(t *testing.T) {
input := []*drivers.DeviceConfig{
{
HostPath: "/dev/null",
TaskPath: "/task/dev/null",
Permissions: "rwm",
},
}
expected := &lconfigs.Device{
Path: "/task/dev/null",
Type: 99,
Major: 1,
Minor: 3,
Permissions: "rwm",
}
found, err := cmdDevices(input)
require.NoError(t, err)
require.Len(t, found, 1)
// ignore file permission and ownership
// as they are host specific potentially
d := found[0]
d.FileMode = 0
d.Uid = 0
d.Gid = 0
require.EqualValues(t, expected, d)
}
func TestExecutor_cmdMounts(t *testing.T) {
input := []*drivers.MountConfig{
{
HostPath: "/host/path-ro",
TaskPath: "/task/path-ro",
Readonly: true,
},
{
HostPath: "/host/path-rw",
TaskPath: "/task/path-rw",
Readonly: false,
},
}
expected := []*lconfigs.Mount{
{
Source: "/host/path-ro",
Destination: "/task/path-ro",
Flags: unix.MS_BIND | unix.MS_RDONLY,
Device: "bind",
},
{
Source: "/host/path-rw",
Destination: "/task/path-rw",
Flags: unix.MS_BIND,
Device: "bind",
},
}
require.EqualValues(t, expected, cmdMounts(input))
}