open-nomad/drivers/shared/executor/executor_linux_test.go
Mahmood Ali a89da9982d
raw_exec: don't use cgroups when no_cgroup is set (#9328)
When raw_exec is configured with [`no_cgroups`](https://www.nomadproject.io/docs/drivers/raw_exec#no_cgroups), raw_exec shouldn't attempt to create a cgroup.

Prior to this change, we accidentally always required freezer cgroup to do stats PID tracking. We already have the proper fallback in place for metrics, so only need to ensure that we don't create a cgroup for the task.

Fixes https://github.com/hashicorp/nomad/issues/8565
2020-11-11 16:20:34 -05:00

617 lines
16 KiB
Go

package executor
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
"time"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/plugins/drivers"
tu "github.com/hashicorp/nomad/testutil"
"github.com/opencontainers/runc/libcontainer/cgroups"
lconfigs "github.com/opencontainers/runc/libcontainer/configs"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
func init() {
executorFactories["LibcontainerExecutor"] = libcontainerFactory
}
var libcontainerFactory = executorFactory{
new: NewExecutorWithIsolation,
configureExecCmd: func(t *testing.T, cmd *ExecCommand) {
cmd.ResourceLimits = true
setupRootfs(t, cmd.TaskDir)
},
}
// testExecutorContextWithChroot returns an ExecutorContext and AllocDir with
// chroot. Use testExecutorContext if you don't need a chroot.
//
// The caller is responsible for calling AllocDir.Destroy() to cleanup.
func testExecutorCommandWithChroot(t *testing.T) *testExecCmd {
chrootEnv := map[string]string{
"/etc/ld.so.cache": "/etc/ld.so.cache",
"/etc/ld.so.conf": "/etc/ld.so.conf",
"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
"/etc/passwd": "/etc/passwd",
"/lib": "/lib",
"/lib64": "/lib64",
"/usr/lib": "/usr/lib",
"/bin/ls": "/bin/ls",
"/bin/cat": "/bin/cat",
"/bin/echo": "/bin/echo",
"/bin/bash": "/bin/bash",
"/bin/sleep": "/bin/sleep",
"/foobar": "/does/not/exist",
}
alloc := mock.Alloc()
task := alloc.Job.TaskGroups[0].Tasks[0]
taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build()
allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID))
if err := allocDir.Build(); err != nil {
t.Fatalf("AllocDir.Build() failed: %v", err)
}
if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil {
allocDir.Destroy()
t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err)
}
td := allocDir.TaskDirs[task.Name]
cmd := &ExecCommand{
Env: taskEnv.List(),
TaskDir: td.Dir,
Resources: &drivers.Resources{
NomadResources: alloc.AllocatedResources.Tasks[task.Name],
},
}
testCmd := &testExecCmd{
command: cmd,
allocDir: allocDir,
}
configureTLogging(t, testCmd)
return testCmd
}
func TestExecutor_IsolationAndConstraints(t *testing.T) {
t.Parallel()
require := require.New(t)
testutil.ExecCompatible(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "/bin/ls"
execCmd.Args = []string{"-F", "/", "/etc/"}
defer allocDir.Destroy()
execCmd.ResourceLimits = true
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
ps, err := executor.Launch(execCmd)
require.NoError(err)
require.NotZero(ps.Pid)
state, err := executor.Wait(context.Background())
require.NoError(err)
require.Zero(state.ExitCode)
// Check if the resource constraints were applied
if lexec, ok := executor.(*LibcontainerExecutor); ok {
state, err := lexec.container.State()
require.NoError(err)
memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes")
data, err := ioutil.ReadFile(memLimits)
require.NoError(err)
expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024))
actualMemLim := strings.TrimSpace(string(data))
require.Equal(actualMemLim, expectedMemLim)
require.NoError(executor.Shutdown("", 0))
executor.Wait(context.Background())
// Check if Nomad has actually removed the cgroups
tu.WaitForResult(func() (bool, error) {
_, err = os.Stat(memLimits)
if err == nil {
return false, fmt.Errorf("expected an error from os.Stat %s", memLimits)
}
return true, nil
}, func(err error) { t.Error(err) })
}
expected := `/:
alloc/
bin/
dev/
etc/
lib/
lib64/
local/
proc/
secrets/
sys/
tmp/
usr/
/etc/:
ld.so.cache
ld.so.conf
ld.so.conf.d/
passwd`
tu.WaitForResult(func() (bool, error) {
output := testExecCmd.stdout.String()
act := strings.TrimSpace(string(output))
if act != expected {
return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act)
}
return true, nil
}, func(err error) { t.Error(err) })
}
// TestExecutor_CgroupPaths asserts that process starts with independent cgroups
// hierarchy created for this process
func TestExecutor_CgroupPaths(t *testing.T) {
t.Parallel()
require := require.New(t)
testutil.ExecCompatible(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "/bin/bash"
execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"}
defer allocDir.Destroy()
execCmd.ResourceLimits = true
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
ps, err := executor.Launch(execCmd)
require.NoError(err)
require.NotZero(ps.Pid)
state, err := executor.Wait(context.Background())
require.NoError(err)
require.Zero(state.ExitCode)
tu.WaitForResult(func() (bool, error) {
output := strings.TrimSpace(testExecCmd.stdout.String())
// sanity check that we got some cgroups
if !strings.Contains(output, ":devices:") {
return false, fmt.Errorf("was expected cgroup files but found:\n%v", output)
}
lines := strings.Split(output, "\n")
for _, line := range lines {
// Every cgroup entry should be /nomad/$ALLOC_ID
if line == "" {
continue
}
// Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker
// don't isolate it by default.
// :: filters out odd empty cgroup found in latest Ubuntu lines, e.g. 0::/user.slice/user-1000.slice/session-17.scope
// that is also not used for isolation
if strings.Contains(line, ":rdma:") || strings.Contains(line, "::") {
continue
}
if !strings.Contains(line, ":/nomad/") {
return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line)
}
}
return true, nil
}, func(err error) { t.Error(err) })
}
// TestExecutor_CgroupPaths asserts that all cgroups created for a task
// are destroyed on shutdown
func TestExecutor_CgroupPathsAreDestroyed(t *testing.T) {
t.Parallel()
require := require.New(t)
testutil.ExecCompatible(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "/bin/bash"
execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"}
defer allocDir.Destroy()
execCmd.ResourceLimits = true
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
ps, err := executor.Launch(execCmd)
require.NoError(err)
require.NotZero(ps.Pid)
state, err := executor.Wait(context.Background())
require.NoError(err)
require.Zero(state.ExitCode)
var cgroupsPaths string
tu.WaitForResult(func() (bool, error) {
output := strings.TrimSpace(testExecCmd.stdout.String())
// sanity check that we got some cgroups
if !strings.Contains(output, ":devices:") {
return false, fmt.Errorf("was expected cgroup files but found:\n%v", output)
}
lines := strings.Split(output, "\n")
for _, line := range lines {
// Every cgroup entry should be /nomad/$ALLOC_ID
if line == "" {
continue
}
// Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker
// don't isolate it by default.
if strings.Contains(line, ":rdma:") || strings.Contains(line, "::") {
continue
}
if !strings.Contains(line, ":/nomad/") {
return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line)
}
}
cgroupsPaths = output
return true, nil
}, func(err error) { t.Error(err) })
// shutdown executor and test that cgroups are destroyed
executor.Shutdown("SIGKILL", 0)
// test that the cgroup paths are not visible
tmpFile, err := ioutil.TempFile("", "")
require.NoError(err)
defer os.Remove(tmpFile.Name())
_, err = tmpFile.WriteString(cgroupsPaths)
require.NoError(err)
tmpFile.Close()
subsystems, err := cgroups.ParseCgroupFile(tmpFile.Name())
require.NoError(err)
for subsystem, cgroup := range subsystems {
if !strings.Contains(cgroup, "nomad/") {
// this should only be rdma at this point
continue
}
p, err := getCgroupPathHelper(subsystem, cgroup)
require.NoError(err)
require.Falsef(cgroups.PathExists(p), "cgroup for %s %s still exists", subsystem, cgroup)
}
}
func TestUniversalExecutor_LookupTaskBin(t *testing.T) {
t.Parallel()
require := require.New(t)
// Create a temp dir
tmpDir, err := ioutil.TempDir("", "")
require.Nil(err)
defer os.Remove(tmpDir)
// Create the command
cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir}
// Make a foo subdir
os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700)
// Write a file under foo
filePath := filepath.Join(tmpDir, "foo", "tmp.txt")
err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend)
require.NoError(err)
// Lookout with an absolute path to the binary
cmd.Cmd = "/foo/tmp.txt"
_, err = lookupTaskBin(cmd)
require.NoError(err)
// Write a file under local subdir
os.MkdirAll(filepath.Join(tmpDir, "local"), 0700)
filePath2 := filepath.Join(tmpDir, "local", "tmp.txt")
ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend)
// Lookup with file name, should find the one we wrote above
cmd.Cmd = "tmp.txt"
_, err = lookupTaskBin(cmd)
require.NoError(err)
// Lookup a host absolute path
cmd.Cmd = "/bin/sh"
_, err = lookupTaskBin(cmd)
require.Error(err)
}
// Exec Launch looks for the binary only inside the chroot
func TestExecutor_EscapeContainer(t *testing.T) {
t.Parallel()
require := require.New(t)
testutil.ExecCompatible(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "/bin/kill" // missing from the chroot container
defer allocDir.Destroy()
execCmd.ResourceLimits = true
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
_, err := executor.Launch(execCmd)
require.Error(err)
require.Regexp("^file /bin/kill not found under path", err)
// Bare files are looked up using the system path, inside the container
allocDir.Destroy()
testExecCmd = testExecutorCommandWithChroot(t)
execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "kill"
_, err = executor.Launch(execCmd)
require.Error(err)
require.Regexp("^file kill not found under path", err)
allocDir.Destroy()
testExecCmd = testExecutorCommandWithChroot(t)
execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "echo"
_, err = executor.Launch(execCmd)
require.NoError(err)
}
func TestExecutor_Capabilities(t *testing.T) {
t.Parallel()
testutil.ExecCompatible(t)
cases := []struct {
user string
caps string
}{
{
user: "nobody",
caps: `
CapInh: 0000000000000000
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000`,
},
{
user: "root",
caps: `
CapInh: 0000000000000000
CapPrm: 0000003fffffffff
CapEff: 0000003fffffffff
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000`,
},
}
for _, c := range cases {
t.Run(c.user, func(t *testing.T) {
require := require.New(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
defer allocDir.Destroy()
execCmd.User = c.user
execCmd.ResourceLimits = true
execCmd.Cmd = "/bin/bash"
execCmd.Args = []string{"-c", "cat /proc/$$/status"}
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
_, err := executor.Launch(execCmd)
require.NoError(err)
ch := make(chan interface{})
go func() {
executor.Wait(context.Background())
close(ch)
}()
select {
case <-ch:
// all good
case <-time.After(5 * time.Second):
require.Fail("timeout waiting for exec to shutdown")
}
canonical := func(s string) string {
s = strings.TrimSpace(s)
s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ")
s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n")
return s
}
expected := canonical(c.caps)
tu.WaitForResult(func() (bool, error) {
output := canonical(testExecCmd.stdout.String())
if !strings.Contains(output, expected) {
return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output)
}
return true, nil
}, func(err error) { require.NoError(err) })
})
}
}
func TestExecutor_ClientCleanup(t *testing.T) {
t.Parallel()
testutil.ExecCompatible(t)
require := require.New(t)
testExecCmd := testExecutorCommandWithChroot(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
defer allocDir.Destroy()
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("", 0)
// Need to run a command which will produce continuous output but not
// too quickly to ensure executor.Exit() stops the process.
execCmd.Cmd = "/bin/bash"
execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"}
execCmd.ResourceLimits = true
ps, err := executor.Launch(execCmd)
require.NoError(err)
require.NotZero(ps.Pid)
time.Sleep(500 * time.Millisecond)
require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond))
ch := make(chan interface{})
go func() {
executor.Wait(context.Background())
close(ch)
}()
select {
case <-ch:
// all good
case <-time.After(5 * time.Second):
require.Fail("timeout waiting for exec to shutdown")
}
output := testExecCmd.stdout.String()
require.NotZero(len(output))
time.Sleep(2 * time.Second)
output1 := testExecCmd.stdout.String()
require.Equal(len(output), len(output1))
}
func TestExecutor_cmdDevices(t *testing.T) {
input := []*drivers.DeviceConfig{
{
HostPath: "/dev/null",
TaskPath: "/task/dev/null",
Permissions: "rwm",
},
}
expected := &lconfigs.Device{
DeviceRule: lconfigs.DeviceRule{
Type: 99,
Major: 1,
Minor: 3,
Permissions: "rwm",
},
Path: "/task/dev/null",
}
found, err := cmdDevices(input)
require.NoError(t, err)
require.Len(t, found, 1)
// ignore file permission and ownership
// as they are host specific potentially
d := found[0]
d.FileMode = 0
d.Uid = 0
d.Gid = 0
require.EqualValues(t, expected, d)
}
func TestExecutor_cmdMounts(t *testing.T) {
input := []*drivers.MountConfig{
{
HostPath: "/host/path-ro",
TaskPath: "/task/path-ro",
Readonly: true,
},
{
HostPath: "/host/path-rw",
TaskPath: "/task/path-rw",
Readonly: false,
},
}
expected := []*lconfigs.Mount{
{
Source: "/host/path-ro",
Destination: "/task/path-ro",
Flags: unix.MS_BIND | unix.MS_RDONLY,
Device: "bind",
PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC},
},
{
Source: "/host/path-rw",
Destination: "/task/path-rw",
Flags: unix.MS_BIND,
Device: "bind",
PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC},
},
}
require.EqualValues(t, expected, cmdMounts(input))
}
// TestUniversalExecutor_NoCgroup asserts that commands are executed in the
// same cgroup as parent process
func TestUniversalExecutor_NoCgroup(t *testing.T) {
t.Parallel()
testutil.ExecCompatible(t)
expectedBytes, err := ioutil.ReadFile("/proc/self/cgroup")
require.NoError(t, err)
expected := strings.TrimSpace(string(expectedBytes))
testExecCmd := testExecutorCommand(t)
execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
execCmd.Cmd = "/bin/cat"
execCmd.Args = []string{"/proc/self/cgroup"}
defer allocDir.Destroy()
execCmd.BasicProcessCgroup = false
execCmd.ResourceLimits = false
executor := NewExecutor(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
_, err = executor.Launch(execCmd)
require.NoError(t, err)
_, err = executor.Wait(context.Background())
require.NoError(t, err)
tu.WaitForResult(func() (bool, error) {
act := strings.TrimSpace(string(testExecCmd.stdout.String()))
if expected != act {
return false, fmt.Errorf("expected:\n%s actual:\n%s", expected, act)
}
return true, nil
}, func(err error) {
stderr := strings.TrimSpace(string(testExecCmd.stderr.String()))
t.Logf("stderr: %v", stderr)
require.NoError(t, err)
})
}