f87eb666c7
This is a followup to having tests run in serial in CI. The e2e package isn't in CI, but lets use the helper anyway so we can setup semgrep rules covering the entire repository.
474 lines
13 KiB
Go
474 lines
13 KiB
Go
package clientstate
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/hashicorp/nomad/api"
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/hashicorp/nomad/client/state"
|
|
"github.com/hashicorp/nomad/e2e/e2eutil"
|
|
"github.com/hashicorp/nomad/e2e/execagent"
|
|
"github.com/hashicorp/nomad/e2e/framework"
|
|
"github.com/hashicorp/nomad/helper/discover"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/testutil"
|
|
)
|
|
|
|
func init() {
|
|
framework.AddSuites(&framework.TestSuite{
|
|
Component: "clientstate",
|
|
CanRunLocal: true,
|
|
Cases: []framework.TestCase{
|
|
&ClientStateTC{},
|
|
},
|
|
})
|
|
}
|
|
|
|
type ClientStateTC struct {
|
|
framework.TC
|
|
|
|
// bin is the path to Nomad binary
|
|
bin string
|
|
}
|
|
|
|
func (tc *ClientStateTC) BeforeAll(f *framework.F) {
|
|
if os.Getenv("NOMAD_TEST_STATE") == "" {
|
|
f.T().Skip("Skipping very slow state corruption test unless NOMAD_TEST_STATE=1")
|
|
}
|
|
|
|
bin, err := discover.NomadExecutable()
|
|
f.NoError(err)
|
|
tc.bin = bin
|
|
}
|
|
|
|
func getPID(client *api.Client, alloc *api.Allocation, path string) (int, error) {
|
|
allocfs := client.AllocFS()
|
|
r, err := allocfs.Cat(alloc, path, nil)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer r.Close()
|
|
|
|
out, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
lines := bytes.SplitN(out, []byte{'\n'}, 2)
|
|
if len(lines) != 2 || len(lines[1]) > 0 {
|
|
return 0, fmt.Errorf("expected 1 line not %q", string(out))
|
|
}
|
|
|
|
// Capture pid
|
|
pid, err := strconv.Atoi(string(lines[0]))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return pid, nil
|
|
}
|
|
|
|
// TestClientState_Kill force kills Nomad agents and restarts them in a tight
|
|
// loop to assert Nomad is crash safe.
|
|
func (tc *ClientStateTC) TestClientState_Kill(f *framework.F) {
|
|
t := f.T()
|
|
ci.Parallel(t)
|
|
|
|
serverOut := testlog.NewPrefixWriter(t, "SERVER: ")
|
|
clientOut := testlog.NewPrefixWriter(t, "CLIENT: ")
|
|
serverAgent, clientAgent, err := execagent.NewClientServerPair(tc.bin, serverOut, clientOut)
|
|
f.NoError(err)
|
|
|
|
f.NoError(serverAgent.Start())
|
|
defer serverAgent.Destroy()
|
|
f.NoError(clientAgent.Start())
|
|
defer clientAgent.Destroy()
|
|
|
|
// Get a client for the server agent to use even while the client is
|
|
// down.
|
|
client, err := serverAgent.Client()
|
|
f.NoError(err)
|
|
|
|
jobID := "sleeper-" + uuid.Generate()[:8]
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(t, client, "clientstate/sleeper.nomad", jobID, "")
|
|
f.Len(allocs, 1)
|
|
|
|
alloc, _, err := client.Allocations().Info(allocs[0].ID, nil)
|
|
f.NoError(err)
|
|
|
|
defer func() {
|
|
if _, _, err := client.Jobs().Deregister(jobID, false, nil); err != nil {
|
|
t.Logf("error stopping job: %v", err)
|
|
}
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
sum, _, err := client.Jobs().Summary(jobID, nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if r := sum.Summary["sleeper"].Running; r > 0 {
|
|
return false, fmt.Errorf("still running: %d", r)
|
|
}
|
|
return true, nil
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
|
|
//XXX Must use client agent for gc'ing allocs?
|
|
clientAPI, err := clientAgent.Client()
|
|
f.NoError(err)
|
|
if err := clientAPI.Allocations().GC(alloc, nil); err != nil {
|
|
t.Logf("error garbage collecting alloc: %v", err)
|
|
}
|
|
|
|
if err := client.System().GarbageCollect(); err != nil {
|
|
t.Logf("error doing full gc: %v", err)
|
|
}
|
|
|
|
//HACK to wait until things have GC'd
|
|
time.Sleep(time.Second)
|
|
}()
|
|
|
|
assertHealthy := func() {
|
|
t.Helper()
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
alloc, _, err = client.Allocations().Info(alloc.ID, nil)
|
|
f.NoError(err) // should never error
|
|
|
|
if len(alloc.TaskStates) == 0 {
|
|
return false, fmt.Errorf("waiting for tasks to start")
|
|
}
|
|
|
|
if s := alloc.TaskStates["sleeper"].State; s != "running" {
|
|
return false, fmt.Errorf("task should be running: %q", s)
|
|
}
|
|
|
|
// Restarts should never happen
|
|
f.Zero(alloc.TaskStates["sleeper"].Restarts)
|
|
return true, nil
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
}
|
|
assertHealthy()
|
|
|
|
// Find pid
|
|
pid := 0
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
pid, err = getPID(client, alloc, "sleeper/pid")
|
|
return pid > 0, err
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
|
|
// Kill and restart a few times
|
|
tries := 10
|
|
for i := 0; i < tries; i++ {
|
|
t.Logf("TEST RUN %d/%d", i+1, tries)
|
|
|
|
// Kill -9 the Agent
|
|
agentPid := clientAgent.Cmd.Process.Pid
|
|
f.NoError(clientAgent.Cmd.Process.Signal(os.Kill))
|
|
|
|
state, err := clientAgent.Cmd.Process.Wait()
|
|
f.NoError(err)
|
|
f.False(state.Exited()) // kill signal != exited
|
|
f.False(state.Success())
|
|
|
|
// Assert sleeper is still running
|
|
f.NoError(syscall.Kill(pid, 0))
|
|
assertHealthy()
|
|
|
|
// Should not be able to reach its filesystem
|
|
_, err = getPID(client, alloc, "sleeper/pid")
|
|
f.Error(err)
|
|
|
|
// Restart the agent (have to create a new Cmd)
|
|
clientAgent.Cmd = exec.Command(clientAgent.BinPath, "agent",
|
|
"-config", clientAgent.ConfFile,
|
|
"-data-dir", clientAgent.DataDir,
|
|
"-servers", fmt.Sprintf("127.0.0.1:%d", serverAgent.Vars.RPC),
|
|
)
|
|
clientAgent.Cmd.Stdout = clientOut
|
|
clientAgent.Cmd.Stderr = clientOut
|
|
f.NoError(clientAgent.Start())
|
|
|
|
// Assert a new process did start
|
|
f.NotEqual(clientAgent.Cmd.Process.Pid, agentPid)
|
|
|
|
// Retrieving the pid should work once it restarts
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
newPid, err := getPID(client, alloc, "sleeper/pid")
|
|
return newPid == pid, err
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
|
|
// Alloc should still be running
|
|
assertHealthy()
|
|
}
|
|
}
|
|
|
|
// TestClientState_KillDuringRestart force kills Nomad agents and restarts them
|
|
// in a tight loop to assert Nomad is crash safe while a task is restarting.
|
|
func (tc *ClientStateTC) TestClientState_KillDuringRestart(f *framework.F) {
|
|
t := f.T()
|
|
ci.Parallel(t)
|
|
|
|
serverOut := testlog.NewPrefixWriter(t, "SERVER: ")
|
|
clientOut := testlog.NewPrefixWriter(t, "CLIENT: ")
|
|
serverAgent, clientAgent, err := execagent.NewClientServerPair(tc.bin, serverOut, clientOut)
|
|
f.NoError(err)
|
|
|
|
f.NoError(serverAgent.Start())
|
|
defer serverAgent.Destroy()
|
|
|
|
f.NoError(clientAgent.Start())
|
|
defer clientAgent.Destroy()
|
|
|
|
// Get a client for the server agent to use even while the client is
|
|
// down.
|
|
client, err := serverAgent.Client()
|
|
f.NoError(err)
|
|
|
|
jobID := "restarter-" + uuid.Generate()[:8]
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(t, client, "clientstate/restarter.nomad", jobID, "")
|
|
f.Len(allocs, 1)
|
|
|
|
alloc, _, err := client.Allocations().Info(allocs[0].ID, nil)
|
|
f.NoError(err)
|
|
|
|
defer func() {
|
|
//FIXME(schmichael): this cleanup is insufficient, but I can't
|
|
// figure out how to fix it
|
|
client.Jobs().Deregister(jobID, false, nil)
|
|
client.System().GarbageCollect()
|
|
time.Sleep(time.Second)
|
|
}()
|
|
|
|
var restarts uint64
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
alloc, _, err = client.Allocations().Info(alloc.ID, nil)
|
|
f.NoError(err) // should never error
|
|
|
|
if len(alloc.TaskStates) == 0 {
|
|
return false, fmt.Errorf("waiting for tasks to start")
|
|
}
|
|
|
|
n := alloc.TaskStates["restarter"].Restarts
|
|
if n < restarts {
|
|
// Restarts should never decrease; immediately fail
|
|
f.Failf("restarts decreased", "%d < %d", n, restarts)
|
|
}
|
|
|
|
// Capture current restarts
|
|
restarts = n
|
|
return true, nil
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
|
|
dice := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
|
|
// Kill and restart agent a few times
|
|
i := 0
|
|
for deadline := time.Now().Add(5 * time.Minute); time.Now().Before(deadline); {
|
|
i++
|
|
sleep := time.Duration(1500+dice.Int63n(6000)) * time.Millisecond
|
|
t.Logf("[TEST] ===> Run %d (pid: %d sleeping for %v; last restarts: %d)", i, clientAgent.Cmd.Process.Pid, sleep, restarts)
|
|
|
|
time.Sleep(sleep)
|
|
|
|
// Ensure restarts are progressing
|
|
alloc, _, err = client.Allocations().Info(alloc.ID, nil)
|
|
f.NoError(err) // should never error
|
|
n := alloc.TaskStates["restarter"].Restarts
|
|
if n < restarts {
|
|
// Restarts should never decrease; immediately fail
|
|
f.Failf("restarts decreased", "%d < %d", n, restarts)
|
|
}
|
|
if i > 5 && n == 0 {
|
|
// At least one restart should have happened by now
|
|
f.Failf("no restarts", "expected at least 1 restart after %d tries", i)
|
|
}
|
|
restarts = n
|
|
|
|
// Kill -9 Agent
|
|
agentPid := clientAgent.Cmd.Process.Pid
|
|
f.NoError(clientAgent.Cmd.Process.Signal(os.Kill))
|
|
t.Logf("[TEST] ===> Killed %d", agentPid)
|
|
|
|
state, err := clientAgent.Cmd.Process.Wait()
|
|
f.NoError(err)
|
|
f.False(state.Exited()) // kill signal != exited
|
|
f.False(state.Success())
|
|
|
|
// Restart the agent (have to create a new Cmd)
|
|
clientAgent.Cmd = exec.Command(clientAgent.BinPath, "agent",
|
|
"-config", clientAgent.ConfFile,
|
|
"-data-dir", clientAgent.DataDir,
|
|
"-servers", fmt.Sprintf("127.0.0.1:%d", serverAgent.Vars.RPC),
|
|
)
|
|
clientAgent.Cmd.Stdout = clientOut
|
|
clientAgent.Cmd.Stderr = clientOut
|
|
f.NoError(clientAgent.Start())
|
|
|
|
// Assert a new process did start
|
|
f.NotEqual(clientAgent.Cmd.Process.Pid, agentPid)
|
|
clientUrl := fmt.Sprintf("http://127.0.0.1:%d/v1/client/stats", clientAgent.Vars.HTTP)
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
resp, err := http.Get(clientUrl)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
resp.Body.Close()
|
|
return resp.StatusCode == 200, fmt.Errorf("%d != 200", resp.StatusCode)
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
}
|
|
|
|
t.Logf("[TEST] ===> Final restarts: %d", restarts)
|
|
}
|
|
|
|
// TestClientState_Corrupt removes task state from the client's state db to
|
|
// assert it recovers.
|
|
func (tc *ClientStateTC) TestClientState_Corrupt(f *framework.F) {
|
|
t := f.T()
|
|
ci.Parallel(t)
|
|
|
|
serverOut := testlog.NewPrefixWriter(t, "SERVER: ")
|
|
clientOut := testlog.NewPrefixWriter(t, "CLIENT: ")
|
|
serverAgent, clientAgent, err := execagent.NewClientServerPair(tc.bin, serverOut, clientOut)
|
|
f.NoError(err)
|
|
|
|
f.NoError(serverAgent.Start())
|
|
defer serverAgent.Destroy()
|
|
f.NoError(clientAgent.Start())
|
|
defer clientAgent.Destroy()
|
|
|
|
// Get a client for the server agent to use even while the client is
|
|
// down.
|
|
client, err := serverAgent.Client()
|
|
f.NoError(err)
|
|
|
|
jobID := "sleeper-" + uuid.Generate()[:8]
|
|
allocs := e2eutil.RegisterAndWaitForAllocs(t, client, "clientstate/sleeper.nomad", jobID, "")
|
|
f.Len(allocs, 1)
|
|
|
|
alloc, _, err := client.Allocations().Info(allocs[0].ID, nil)
|
|
f.NoError(err)
|
|
|
|
defer func() {
|
|
//FIXME(schmichael): this cleanup is insufficient, but I can't
|
|
// figure out how to fix it
|
|
client.Jobs().Deregister(jobID, false, nil)
|
|
client.System().GarbageCollect()
|
|
time.Sleep(time.Second)
|
|
}()
|
|
|
|
assertHealthy := func() {
|
|
t.Helper()
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
alloc, _, err = client.Allocations().Info(alloc.ID, nil)
|
|
f.NoError(err) // should never error
|
|
|
|
if len(alloc.TaskStates) == 0 {
|
|
return false, fmt.Errorf("waiting for tasks to start")
|
|
}
|
|
|
|
if s := alloc.TaskStates["sleeper"].State; s != "running" {
|
|
return false, fmt.Errorf("task should be running: %q", s)
|
|
}
|
|
|
|
// Restarts should never happen
|
|
f.Zero(alloc.TaskStates["sleeper"].Restarts)
|
|
return true, nil
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
}
|
|
assertHealthy()
|
|
|
|
// Find pid
|
|
pid := 0
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
pid, err = getPID(client, alloc, "sleeper/pid")
|
|
return pid > 0, err
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
|
|
// Kill and corrupt the state
|
|
agentPid := clientAgent.Cmd.Process.Pid
|
|
f.NoError(clientAgent.Cmd.Process.Signal(os.Interrupt))
|
|
|
|
procState, err := clientAgent.Cmd.Process.Wait()
|
|
f.NoError(err)
|
|
f.True(procState.Exited())
|
|
|
|
// Assert sleeper is still running
|
|
f.NoError(syscall.Kill(pid, 0))
|
|
assertHealthy()
|
|
|
|
// Remove task bucket from client state
|
|
db, err := state.NewBoltStateDB(testlog.HCLogger(t), filepath.Join(clientAgent.DataDir, "client"))
|
|
f.NoError(err)
|
|
|
|
f.NoError(db.DeleteTaskBucket(alloc.ID, "sleeper"))
|
|
f.NoError(db.Close())
|
|
|
|
// Restart the agent (have to create a new Cmd)
|
|
clientAgent.Cmd = exec.Command(clientAgent.BinPath, "agent",
|
|
"-config", clientAgent.ConfFile,
|
|
"-data-dir", clientAgent.DataDir,
|
|
"-servers", fmt.Sprintf("127.0.0.1:%d", serverAgent.Vars.RPC),
|
|
)
|
|
clientAgent.Cmd.Stdout = clientOut
|
|
clientAgent.Cmd.Stderr = clientOut
|
|
f.NoError(clientAgent.Start())
|
|
|
|
// Assert a new process did start
|
|
f.NotEqual(clientAgent.Cmd.Process.Pid, agentPid)
|
|
|
|
// Retrieving the pid should work once it restarts.
|
|
// Critically there are now 2 pids because the client task state was
|
|
// lost Nomad started a new copy.
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
allocfs := client.AllocFS()
|
|
r, err := allocfs.Cat(alloc, "sleeper/pid", nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer r.Close()
|
|
|
|
out, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
lines := bytes.SplitN(out, []byte{'\n'}, 3)
|
|
if len(lines) != 3 || len(lines[2]) > 0 {
|
|
return false, fmt.Errorf("expected 2 lines not %v", lines)
|
|
}
|
|
|
|
return true, nil
|
|
}, func(err error) {
|
|
f.NoError(err)
|
|
})
|
|
|
|
// Alloc should still be running
|
|
assertHealthy()
|
|
}
|