luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

task lifecycle: e2e tests

Browse source
This commit is contained in:
Jasmine Dahilig 2020-08-18 10:49:50 -07:00
parent de08ae8083
commit ee522ab587
4 changed files with 403 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
e2e/e2e_test.go
View file

@ -17,6 +17,7 @@ import (
_ "github.com/hashicorp/nomad/e2e/deployment"
_ "github.com/hashicorp/nomad/e2e/example"
_ "github.com/hashicorp/nomad/e2e/hostvolumes"
_ "github.com/hashicorp/nomad/e2e/lifecycle"
_ "github.com/hashicorp/nomad/e2e/metrics"
_ "github.com/hashicorp/nomad/e2e/nomad09upgrade"
_ "github.com/hashicorp/nomad/e2e/nomadexec"

127
e2e/lifecycle/inputs/batch.nomad Normal file
View file

@ -0,0 +1,127 @@
# lifecycle hook test job for batch jobs. touches, removes, and tests
# for the existence of files to assert the order of running tasks.
# all tasks should exit 0 and the alloc dir should contain the following
# files: ./init-ran, ./main-ran, ./poststart-run
job "batch-lifecycle" {
datacenters = ["dc1"]
type = "batch"
group "test" {
task "init" {
lifecycle {
hook = "prestart"
}
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/prestart.sh"]
}
template {
data = <<EOT
#!/bin/sh
sleep 1
touch ${NOMAD_ALLOC_DIR}/init-ran
touch ${NOMAD_ALLOC_DIR}/init-running
sleep 5
if [ -f ${NOMAD_ALLOC_DIR}/main ]; then exit 7; fi
if [ -f ${NOMAD_ALLOC_DIR}/poststart-running ]; then exit 8; fi
rm ${NOMAD_ALLOC_DIR}/init-running
EOT
destination = "local/prestart.sh"
}
resources {
cpu = 64
memory = 64
}
}
task "main" {
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/main.sh"]
}
template {
data = <<EOT
#!/bin/sh
sleep 1
touch ${NOMAD_ALLOC_DIR}/main-running
touch ${NOMAD_ALLOC_DIR}/main-started
# NEED TO HANG AROUND TO GIVE POSTSTART TIME TO GET STARTED
sleep 10
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 9; fi
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 10; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/poststart-started ]; then exit 11; fi
touch ${NOMAD_ALLOC_DIR}/main-ran
rm ${NOMAD_ALLOC_DIR}/main-running
EOT
destination = "local/main.sh"
}
resources {
cpu = 64
memory = 64
}
}
task "poststart" {
lifecycle {
hook = "poststart"
}
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/poststart.sh"]
}
template {
data = <<EOT
#!/bin/sh
sleep 1
touch ${NOMAD_ALLOC_DIR}/poststart-ran
touch ${NOMAD_ALLOC_DIR}/poststart-running
touch ${NOMAD_ALLOC_DIR}/poststart-started
sleep 10
# THIS IS WHERE THE ACTUAL TESTING HAPPENS
# IF init-ran doesn't exist, then the init task hasn't run yet, so fail
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 12; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/main-started ]; then exit 15; fi
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 14; fi
rm ${NOMAD_ALLOC_DIR}/poststart-running
EOT
destination = "local/poststart.sh"
}
resources {
cpu = 64
memory = 64
}
}
}
}

160
e2e/lifecycle/inputs/service.nomad Normal file
View file

@ -0,0 +1,160 @@
# lifecycle hook test job for service jobs. touches, removes, and tests
# for the existence of files to assert the order of running tasks.
# after stopping, the alloc dir should contain the following files:
# files: ./init-ran, ./sidecar-ran, ./main-ran, ./poststart-run but not
# the ./main-running, ./sidecar-running, or ./poststart-running files
job "service-lifecycle" {
datacenters = ["dc1"]
type = "batch"
group "test" {
task "init" {
lifecycle {
hook = "prestart"
}
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/prestart.sh"]
}
template {
data = <<EOT
#!/bin/sh
sleep 1
touch ${NOMAD_ALLOC_DIR}/init-ran
touch ${NOMAD_ALLOC_DIR}/init-running
if [ -f ${NOMAD_ALLOC_DIR}/main ]; then exit 7; fi
if [ -f ${NOMAD_ALLOC_DIR}/poststart ]; then exit 8; fi
rm ${NOMAD_ALLOC_DIR}/init-running
EOT
destination = "local/prestart.sh"
}
resources {
cpu = 64
memory = 64
}
}
task "sidecar" {
lifecycle {
hook = "prestart"
sidecar = true
}
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/sidecar.sh"]
}
template {
data = <<EOT
#!/bin/sh
touch ${NOMAD_ALLOC_DIR}/sidecar-ran
touch ${NOMAD_ALLOC_DIR}/sidecar-running
sleep 5
if [ ! -f ${NOMAD_ALLOC_DIR}/main-running ]; then exit 9; fi
if [ -f ${NOMAD_ALLOC_DIR}/poststart-running ]; then exit 10; fi
sleep 300
EOT
destination = "local/sidecar.sh"
}
resources {
cpu = 64
memory = 64
}
}
task "main" {
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/main.sh"]
}
template {
data = <<EOT
#!/bin/sh
touch ${NOMAD_ALLOC_DIR}/main-ran
touch ${NOMAD_ALLOC_DIR}/main-running
touch ${NOMAD_ALLOC_DIR}/main-started
# NEED TO HANG AROUND TO GIVE POSTSTART TIME TO GET STARTED
sleep 10
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 11; fi
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 12; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-ran ]; then exit 13; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-running ]; then exit 14; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/poststart-started ]; then exit 15; fi
sleep 300
EOT
destination = "local/main.sh"
}
resources {
cpu = 64
memory = 64
}
}
task "poststart" {
lifecycle {
hook = "poststart"
}
driver = "docker"
config {
image = "busybox:1"
command = "/bin/sh"
args = ["local/poststart.sh"]
}
template {
data = <<EOT
#!/bin/sh
touch ${NOMAD_ALLOC_DIR}/poststart-ran
touch ${NOMAD_ALLOC_DIR}/poststart-running
touch ${NOMAD_ALLOC_DIR}/poststart-started
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 16; fi
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 17; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-ran ]; then exit 18; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-running ]; then exit 19; fi
if [ ! -f ${NOMAD_ALLOC_DIR}/main-started ]; then exit 20; fi
rm ${NOMAD_ALLOC_DIR}/poststart-running
EOT
destination = "local/poststart.sh"
}
resources {
cpu = 64
memory = 64
}
}
}
}

115
e2e/lifecycle/lifecycle.go Normal file
View file

@ -0,0 +1,115 @@
package lifecycle
import (
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/e2e/e2eutil"
"github.com/hashicorp/nomad/e2e/framework"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/stretchr/testify/require"
)
type LifecycleE2ETest struct {
framework.TC
jobIDs []string
}
func init() {
framework.AddSuites(&framework.TestSuite{
Component: "Lifecycle",
// YOU COULD RUN THIS LOCALLY BC DIS FLAG
CanRunLocal: true,
Cases: []framework.TestCase{new(LifecycleE2ETest)},
})
}
// Ensure cluster has leader and at least 1 client node
// in a ready state before running tests
func (tc *LifecycleE2ETest) BeforeAll(f *framework.F) {
e2eutil.WaitForLeader(f.T(), tc.Nomad())
e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
}
// TestBatchJob runs a batch job with prestart and poststop hooks
func (tc *LifecycleE2ETest) TestBatchJob(f *framework.F) {
t := f.T()
require := require.New(t)
nomadClient := tc.Nomad()
uuid := uuid.Generate()
jobID := "lifecycle-" + uuid[0:8]
tc.jobIDs = append(tc.jobIDs, jobID)
allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "lifecycle/inputs/batch.nomad", jobID, "")
require.Equal(1, len(allocs))
allocID := allocs[0].ID
// wait for the job to stop and assert we stopped successfully, not failed
e2eutil.WaitForAllocStopped(t, nomadClient, allocID)
alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
require.NoError(err)
require.Equal(structs.AllocClientStatusComplete, alloc.ClientStatus)
// assert the files were written as expected
afi, _, err := nomadClient.AllocFS().List(alloc, "alloc", nil)
require.NoError(err)
expected := map[string]bool{
"init-ran": true, "main-ran": true, "poststart-ran": true,
"init-running": false, "main-running": false, "poststart-running": false}
got := checkFiles(expected, afi)
require.Equal(expected, got)
}
// TODO: cleanup == poststop
// q: what is a good example for a poststart?
// a: notify(-slack)
// TestServiceJob runs a service job with prestart and poststop hooks
func (tc *LifecycleE2ETest) TestServiceJob(f *framework.F) {
t := f.T()
require := require.New(t)
nomadClient := tc.Nomad()
uuid := uuid.Generate()
jobID := "lifecycle-" + uuid[0:8]
tc.jobIDs = append(tc.jobIDs, jobID)
allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "lifecycle/inputs/service.nomad", jobID, "")
require.Equal(1, len(allocs))
allocID := allocs[0].ID
e2eutil.WaitForAllocRunning(t, nomadClient, allocID)
alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
require.NoError(err)
// stop the job
_, _, err = nomadClient.Jobs().Deregister(jobID, false, nil)
require.NoError(err)
e2eutil.WaitForAllocStopped(t, nomadClient, allocID)
// assert the files were written as expected
afi, _, err := nomadClient.AllocFS().List(alloc, "alloc", nil)
require.NoError(err)
expected := map[string]bool{
"init-ran": true, "sidecar-ran": true, "main-ran": true, "poststart-ran": true,
"poststart-started": true, "main-started": true,
"init-running": false, "sidecar-running": false,
"main-running": false, "poststart-running": false}
got := checkFiles(expected, afi)
require.Equal(expected, got)
}
// checkFiles returns a map of whether the expected files were found
// in the file info response
func checkFiles(expected map[string]bool, got []*api.AllocFileInfo) map[string]bool {
results := map[string]bool{}
for expect := range expected {
results[expect] = false
}
for _, file := range got {
// there will be files unrelated to the test, so ignore those
if _, ok := results[file.Name]; ok {
results[file.Name] = true
}
}
return results
}