e2e: convert chroot env unit tests into e2e tests (#14710)
This PR translates two of our most flakey unit tests into e2e tests where they are fit much more naturally.
This commit is contained in:
Seth Hoenig
GitHub
parent
e6af1c0a14
commit
7235d9988b
|
|
@ -698,106 +698,6 @@ func TestTaskRunner_TaskEnv_Interpolated(t *testing.T) {
|
|||
assert.Equal(t, "global bar somebody", mockCfg.StdoutString)
|
||||
}
|
||||
|
||||
// TestTaskRunner_TaskEnv_Chroot asserts chroot drivers use chroot paths and
|
||||
// not host paths.
|
||||
func TestTaskRunner_TaskEnv_Chroot(t *testing.T) {
|
||||
ctestutil.ExecCompatible(t)
|
||||
ci.Parallel(t)
|
||||
|
||||
alloc := mock.BatchAlloc()
|
||||
task := alloc.Job.TaskGroups[0].Tasks[0]
|
||||
task.Driver = "exec"
|
||||
task.Config = map[string]interface{}{
|
||||
"command": "bash",
|
||||
"args": []string{"-c", "echo $NOMAD_ALLOC_DIR; " +
|
||||
"echo $NOMAD_TASK_DIR; " +
|
||||
"echo $NOMAD_SECRETS_DIR; " +
|
||||
"echo $PATH; ",
|
||||
},
|
||||
}
|
||||
|
||||
// Expect chroot paths and host $PATH
|
||||
exp := fmt.Sprintf(`/alloc
|
||||
/local
|
||||
/secrets
|
||||
%s
|
||||
`, os.Getenv("PATH"))
|
||||
|
||||
conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name)
|
||||
defer cleanup()
|
||||
|
||||
tr, err := NewTaskRunner(conf)
|
||||
require.NoError(t, err)
|
||||
go tr.Run()
|
||||
defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
|
||||
|
||||
// Wait for task to exit and kill the task runner to run the stop hooks.
|
||||
testWaitForTaskToDie(t, tr)
|
||||
tr.Kill(context.Background(), structs.NewTaskEvent("kill"))
|
||||
timeout := 15 * time.Second
|
||||
if testutil.IsCI() {
|
||||
timeout = 120 * time.Second
|
||||
}
|
||||
select {
|
||||
case <-tr.WaitCh():
|
||||
case <-time.After(timeout):
|
||||
require.Fail(t, "timeout waiting for task to exit")
|
||||
}
|
||||
|
||||
// Read stdout
|
||||
p := filepath.Join(conf.TaskDir.LogDir, task.Name+".stdout.0")
|
||||
stdout, err := ioutil.ReadFile(p)
|
||||
require.NoError(t, err)
|
||||
require.Equalf(t, exp, string(stdout), "expected: %s\n\nactual: %s\n", exp, stdout)
|
||||
}
|
||||
|
||||
// TestTaskRunner_TaskEnv_Image asserts image drivers use chroot paths and
|
||||
// not host paths. Host env vars should also be excluded.
|
||||
func TestTaskRunner_TaskEnv_Image(t *testing.T) {
|
||||
ctestutil.DockerCompatible(t)
|
||||
ci.Parallel(t)
|
||||
require := require.New(t)
|
||||
|
||||
alloc := mock.BatchAlloc()
|
||||
task := alloc.Job.TaskGroups[0].Tasks[0]
|
||||
task.Driver = "docker"
|
||||
task.Config = map[string]interface{}{
|
||||
"image": "redis:7-alpine",
|
||||
"network_mode": "none",
|
||||
"command": "sh",
|
||||
"args": []string{"-c", "echo $NOMAD_ALLOC_DIR; " +
|
||||
"echo $NOMAD_TASK_DIR; " +
|
||||
"echo $NOMAD_SECRETS_DIR; " +
|
||||
"echo $PATH",
|
||||
},
|
||||
}
|
||||
|
||||
// Expect chroot paths and image specific PATH
|
||||
exp := `/alloc
|
||||
/local
|
||||
/secrets
|
||||
/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
`
|
||||
|
||||
tr, conf, cleanup := runTestTaskRunner(t, alloc, task.Name)
|
||||
defer cleanup()
|
||||
|
||||
// Wait for task to exit and kill task runner to run the stop hooks.
|
||||
testWaitForTaskToDie(t, tr)
|
||||
tr.Kill(context.Background(), structs.NewTaskEvent("kill"))
|
||||
select {
|
||||
case <-tr.WaitCh():
|
||||
case <-time.After(15 * time.Second):
|
||||
require.Fail("timeout waiting for task to exit")
|
||||
}
|
||||
|
||||
// Read stdout
|
||||
p := filepath.Join(conf.TaskDir.LogDir, task.Name+".stdout.0")
|
||||
stdout, err := ioutil.ReadFile(p)
|
||||
require.NoError(err)
|
||||
require.Equalf(exp, string(stdout), "expected: %s\n\nactual: %s\n", exp, stdout)
|
||||
}
|
||||
|
||||
// TestTaskRunner_TaskEnv_None asserts raw_exec uses host paths and env vars.
|
||||
func TestTaskRunner_TaskEnv_None(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
package isolation
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/nomad/e2e/e2eutil"
|
||||
"github.com/hashicorp/nomad/helper/uuid"
|
||||
"github.com/shoenig/test/must"
|
||||
)
|
||||
|
||||
func TestChrootFS(t *testing.T) {
|
||||
nomad := e2eutil.NomadClient(t)
|
||||
|
||||
e2eutil.WaitForLeader(t, nomad)
|
||||
e2eutil.WaitForNodesReady(t, nomad, 1)
|
||||
|
||||
t.Run("testTaskEnvChroot", testExecUsesChroot)
|
||||
t.Run("testTaskImageChroot", testImageUsesChroot)
|
||||
}
|
||||
|
||||
func testExecUsesChroot(t *testing.T) {
|
||||
nomad := e2eutil.NomadClient(t)
|
||||
|
||||
jobID := "exec-chroot-" + uuid.Short()
|
||||
jobIDs := []string{jobID}
|
||||
t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))
|
||||
|
||||
// start job
|
||||
e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/chroot_exec.nomad", jobID, "")
|
||||
|
||||
// get allocation
|
||||
allocations, err := e2eutil.AllocsForJob(jobID, "")
|
||||
must.NoError(t, err)
|
||||
must.Len(t, 1, allocations)
|
||||
allocID := allocations[0]["ID"]
|
||||
|
||||
// wait for allocation stopped
|
||||
e2eutil.WaitForAllocsStopped(t, nomad, []string{allocID})
|
||||
|
||||
// assert log contents
|
||||
logs, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut)
|
||||
must.NoError(t, err)
|
||||
must.RegexMatch(t, regexp.MustCompile(`(?m:^/alloc\b)`), logs)
|
||||
must.RegexMatch(t, regexp.MustCompile(`(?m:^/local\b)`), logs)
|
||||
must.RegexMatch(t, regexp.MustCompile(`(?m:^/secrets\b)`), logs)
|
||||
must.StrContains(t, logs, "/bin")
|
||||
}
|
||||
|
||||
func testImageUsesChroot(t *testing.T) {
|
||||
nomad := e2eutil.NomadClient(t)
|
||||
|
||||
jobID := "docker-chroot-" + uuid.Short()
|
||||
jobIDs := []string{jobID}
|
||||
t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))
|
||||
|
||||
// start job
|
||||
e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/chroot_docker.nomad", jobID, "")
|
||||
|
||||
// get allocation
|
||||
allocations, err := e2eutil.AllocsForJob(jobID, "")
|
||||
must.NoError(t, err)
|
||||
must.Len(t, 1, allocations)
|
||||
allocID := allocations[0]["ID"]
|
||||
|
||||
// wait for allocation stopped
|
||||
e2eutil.WaitForAllocsStopped(t, nomad, []string{allocID})
|
||||
|
||||
// assert log contents
|
||||
logs, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut)
|
||||
must.NoError(t, err)
|
||||
must.RegexMatch(t, regexp.MustCompile(`(?m:^/alloc\b)`), logs)
|
||||
must.RegexMatch(t, regexp.MustCompile(`(?m:^/local\b)`), logs)
|
||||
must.RegexMatch(t, regexp.MustCompile(`(?m:^/secrets\b)`), logs)
|
||||
must.StrContains(t, logs, "/bin")
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
job "chroot_docker" {
|
||||
datacenters = ["dc1"]
|
||||
type = "batch"
|
||||
|
||||
constraint {
|
||||
attribute = "${attr.kernel.name}"
|
||||
value = "linux"
|
||||
}
|
||||
|
||||
group "print" {
|
||||
task "env" {
|
||||
driver = "docker"
|
||||
config {
|
||||
image = "busybox:1"
|
||||
args = [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"echo $NOMAD_ALLOC_DIR; echo $NOMAD_TASK_DIR; echo $NOMAD_SECRETS_DIR; echo $PATH"
|
||||
]
|
||||
}
|
||||
resources {
|
||||
cpu = 50
|
||||
memory = 32
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
job "chroot_exec" {
|
||||
datacenters = ["dc1"]
|
||||
type = "batch"
|
||||
constraint {
|
||||
attribute = "${attr.kernel.name}"
|
||||
value = "linux"
|
||||
}
|
||||
group "print" {
|
||||
task "env" {
|
||||
driver = "exec"
|
||||
config {
|
||||
command = "/bin/bash"
|
||||
args = [
|
||||
"-c",
|
||||
"echo $NOMAD_ALLOC_DIR; echo $NOMAD_TASK_DIR; echo $NOMAD_SECRETS_DIR; echo $PATH"
|
||||
]
|
||||
}
|
||||
resources {
|
||||
cpu = 10
|
||||
memory = 16
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type IsolationTest struct {
|
||||
type PIDsNamespacing struct {
|
||||
framework.TC
|
||||
|
||||
jobIDs []string
|
||||
|
|
@ -22,21 +22,21 @@ type IsolationTest struct {
|
|||
|
||||
func init() {
|
||||
framework.AddSuites(&framework.TestSuite{
|
||||
Component: "Isolation",
|
||||
Component: "PIDS",
|
||||
CanRunLocal: true,
|
||||
Cases: []framework.TestCase{
|
||||
new(IsolationTest),
|
||||
new(PIDsNamespacing),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) BeforeAll(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) BeforeAll(f *framework.F) {
|
||||
t := f.T()
|
||||
e2eutil.WaitForLeader(t, tc.Nomad())
|
||||
e2eutil.WaitForNodesReady(t, tc.Nomad(), 1)
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
|
|
@ -66,7 +66,7 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F)
|
|||
require.Contains(t, out, "my pid is 1\n")
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_host(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_ExecDriver_PIDNamespacing_host(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
|
|
@ -96,7 +96,7 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_host(f *framewo
|
|||
require.NotContains(t, out, "my pid is 1\n")
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
|
|
@ -151,7 +151,7 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *fr
|
|||
require.Len(t, lines, 3)
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
|
|
@ -181,7 +181,7 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F)
|
|||
require.Contains(t, out, "my pid is 1\n")
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_host(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_JavaDriver_PIDNamespacing_host(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
|
|
@ -211,7 +211,7 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_host(f *framewo
|
|||
require.NotContains(t, out, "my pid is 1\n")
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
|
|
@ -266,7 +266,7 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *fr
|
|||
require.Len(t, lines, 3)
|
||||
}
|
||||
|
||||
func (tc *IsolationTest) TestIsolation_RawExecDriver_NoPIDNamespacing(f *framework.F) {
|
||||
func (tc *PIDsNamespacing) TestIsolation_RawExecDriver_NoPIDNamespacing(f *framework.F) {
|
||||
t := f.T()
|
||||
|
||||
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
|
||||
Loading…
Reference in New Issue