236b4055a7
Ensure we pass the alloc ID to status. Otherwise, the test may fail if there is another spurious allocation running from another test.
110 lines
3.3 KiB
Go
110 lines
3.3 KiB
Go
package consul
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
e2e "github.com/hashicorp/nomad/e2e/e2eutil"
|
|
"github.com/hashicorp/nomad/e2e/framework"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
)
|
|
|
|
const ns = ""
|
|
|
|
type CheckRestartE2ETest struct {
|
|
framework.TC
|
|
jobIds []string
|
|
}
|
|
|
|
func (tc *CheckRestartE2ETest) BeforeAll(f *framework.F) {
|
|
e2e.WaitForLeader(f.T(), tc.Nomad())
|
|
e2e.WaitForNodesReady(f.T(), tc.Nomad(), 1)
|
|
}
|
|
|
|
func (tc *CheckRestartE2ETest) AfterEach(f *framework.F) {
|
|
if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
|
|
return
|
|
}
|
|
|
|
for _, id := range tc.jobIds {
|
|
_, err := e2e.Command("nomad", "job", "stop", "-purge", id)
|
|
f.Assert().NoError(err)
|
|
}
|
|
tc.jobIds = []string{}
|
|
_, err := e2e.Command("nomad", "system", "gc")
|
|
f.Assert().NoError(err)
|
|
}
|
|
|
|
// TestGroupCheckRestart runs a job with a group service that will never
|
|
// become healthy. Both tasks should be restarted up to the 'restart' limit.
|
|
func (tc *CheckRestartE2ETest) TestGroupCheckRestart(f *framework.F) {
|
|
|
|
jobID := "test-group-check-restart-" + uuid.Short()
|
|
f.NoError(e2e.Register(jobID, "consul/input/checks_group_restart.nomad"))
|
|
tc.jobIds = append(tc.jobIds, jobID)
|
|
|
|
f.NoError(
|
|
e2e.WaitForAllocStatusComparison(
|
|
func() ([]string, error) { return e2e.AllocStatuses(jobID, ns) },
|
|
func(got []string) bool { return reflect.DeepEqual(got, []string{"failed"}) },
|
|
&e2e.WaitConfig{Interval: time.Second * 10, Retries: 30},
|
|
))
|
|
|
|
allocs, err := e2e.AllocsForJob(jobID, ns)
|
|
f.NoError(err)
|
|
f.Len(allocs, 1)
|
|
|
|
allocID := allocs[0]["ID"]
|
|
expected := "Exceeded allowed attempts 2 in interval 5m0s and mode is \"fail\""
|
|
|
|
out, err := e2e.Command("nomad", "alloc", "status", allocID)
|
|
f.NoError(err, "could not get allocation status")
|
|
f.Contains(out, expected,
|
|
fmt.Errorf("expected '%s', got\n%v", expected, out))
|
|
|
|
re := regexp.MustCompile(`Total Restarts += (.*)\n`)
|
|
match := re.FindAllStringSubmatch(out, -1)
|
|
for _, m := range match {
|
|
f.Equal("2", strings.TrimSpace(m[1]),
|
|
fmt.Errorf("expected exactly 2 restarts for both tasks, got:\n%v", out))
|
|
}
|
|
}
|
|
|
|
// TestTaskCheckRestart runs a job with a task service that will never become
|
|
// healthy. Only the failed task should be restarted up to the 'restart'
|
|
// limit.
|
|
func (tc *CheckRestartE2ETest) TestTaskCheckRestart(f *framework.F) {
|
|
|
|
jobID := "test-task-check-restart-" + uuid.Short()
|
|
f.NoError(e2e.Register(jobID, "consul/input/checks_task_restart.nomad"))
|
|
tc.jobIds = append(tc.jobIds, jobID)
|
|
|
|
var allocID string
|
|
|
|
f.NoError(
|
|
e2e.WaitForAllocStatusComparison(
|
|
func() ([]string, error) { return e2e.AllocStatuses(jobID, ns) },
|
|
func(got []string) bool { return reflect.DeepEqual(got, []string{"failed"}) },
|
|
&e2e.WaitConfig{Interval: time.Second * 10, Retries: 30},
|
|
))
|
|
|
|
expected := "Exceeded allowed attempts 2 in interval 5m0s and mode is \"fail\""
|
|
|
|
out, err := e2e.Command("nomad", "alloc", "status", allocID)
|
|
f.NoError(err, "could not get allocation status")
|
|
f.Contains(out, expected,
|
|
fmt.Errorf("expected '%s', got\n%v", expected, out))
|
|
|
|
re := regexp.MustCompile(`Total Restarts += (.*)\n`)
|
|
match := re.FindAllStringSubmatch(out, -1)
|
|
f.Equal("2", strings.TrimSpace(match[0][1]),
|
|
fmt.Errorf("expected exactly 2 restarts for failed task, got:\n%v", out))
|
|
|
|
f.Equal("0", strings.TrimSpace(match[1][1]),
|
|
fmt.Errorf("expected exactly no restarts for healthy task, got:\n%v", out))
|
|
}
|