From c827e6e05a76b47c880d499d048813089b457f98 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Feb 2019 13:16:04 -0500 Subject: [PATCH 1/9] tests: port TestAllocRunner_MoveAllocDir test --- client/allocrunner/alloc_runner_test.go | 90 +++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/client/allocrunner/alloc_runner_test.go b/client/allocrunner/alloc_runner_test.go index 828002968..eae81306e 100644 --- a/client/allocrunner/alloc_runner_test.go +++ b/client/allocrunner/alloc_runner_test.go @@ -2,12 +2,15 @@ package allocrunner import ( "fmt" + "io/ioutil" "os" + "path/filepath" "testing" "time" "github.com/hashicorp/consul/api" "github.com/hashicorp/nomad/client/allochealth" + "github.com/hashicorp/nomad/client/allocwatcher" cconsul "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/command/agent/consul" @@ -625,3 +628,90 @@ func TestAllocRunner_Destroy(t *testing.T) { require.Failf(t, "expected NotExist error", "found %v", err) } } + +func TestAllocRunner_SimpleRun(t *testing.T) { + t.Parallel() + + alloc := mock.BatchAlloc() + + conf, cleanup := testAllocRunnerConfig(t, alloc) + defer cleanup() + ar, err := NewAllocRunner(conf) + require.NoError(t, err) + go ar.Run() + defer ar.Destroy() + + // Wait for alloc to be running + testutil.WaitForResult(func() (bool, error) { + state := ar.AllocState() + + if state.ClientStatus != structs.AllocClientStatusComplete { + return false, fmt.Errorf("got status %v; want %v", state.ClientStatus, structs.AllocClientStatusComplete) + } + + for t, s := range state.TaskStates { + if s.FinishedAt.IsZero() { + return false, fmt.Errorf("task %q has zero FinishedAt value", t) + } + } + + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + +} + +func TestAllocRunner_MoveAllocDir(t *testing.T) { + t.Parallel() + + // Step 1: start and run a task + alloc := mock.BatchAlloc() + conf, cleanup := testAllocRunnerConfig(t, alloc) + defer cleanup() + ar, err := NewAllocRunner(conf) + require.NoError(t, err) + ar.Run() + defer ar.Destroy() + + require.Equal(t, structs.AllocClientStatusComplete, ar.AllocState().ClientStatus) + + // Step 2. Modify its directory + task := alloc.Job.TaskGroups[0].Tasks[0] + dataFile := filepath.Join(ar.allocDir.SharedDir, "data", "data_file") + ioutil.WriteFile(dataFile, []byte("hello world"), os.ModePerm) + taskDir := ar.allocDir.TaskDirs[task.Name] + taskLocalFile := filepath.Join(taskDir.LocalDir, "local_file") + ioutil.WriteFile(taskLocalFile, []byte("good bye world"), os.ModePerm) + + // Step 3. Start a new alloc + alloc2 := mock.BatchAlloc() + alloc2.PreviousAllocation = alloc.ID + alloc2.Job.TaskGroups[0].EphemeralDisk.Sticky = true + + conf2, cleanup := testAllocRunnerConfig(t, alloc2) + conf2.PrevAllocWatcher, conf2.PrevAllocMigrator = allocwatcher.NewAllocWatcher(allocwatcher.Config{ + Alloc: alloc2, + PreviousRunner: ar, + Logger: conf2.Logger, + }) + defer cleanup() + ar2, err := NewAllocRunner(conf2) + require.NoError(t, err) + + ar2.Run() + defer ar2.Destroy() + + require.Equal(t, structs.AllocClientStatusComplete, ar2.AllocState().ClientStatus) + + // Ensure that data from ar was moved to ar2 + dataFile = filepath.Join(ar2.allocDir.SharedDir, "data", "data_file") + fileInfo, _ := os.Stat(dataFile) + require.NotNilf(t, fileInfo, "file %q not found", dataFile) + + taskDir = ar2.allocDir.TaskDirs[task.Name] + taskLocalFile = filepath.Join(taskDir.LocalDir, "local_file") + fileInfo, _ = os.Stat(taskLocalFile) + require.NotNilf(t, fileInfo, "file %q not found", dataFile) + +} From 1b14214a885a4bc7f32d1bd7ed2d06e7e79caa0c Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Feb 2019 13:37:59 -0500 Subject: [PATCH 2/9] tests: port TestAllocRunner_RetryArtifact Port TestAllocRunner_RetryArtifact from https://github.com/hashicorp/nomad/blob/v0.8.7/client/alloc_runner_test.go#L610-L672 I changed the test name because it doesn't actually test that artifact hooks is retried --- client/allocrunner/alloc_runner_test.go | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/client/allocrunner/alloc_runner_test.go b/client/allocrunner/alloc_runner_test.go index eae81306e..11e01f718 100644 --- a/client/allocrunner/alloc_runner_test.go +++ b/client/allocrunner/alloc_runner_test.go @@ -715,3 +715,63 @@ func TestAllocRunner_MoveAllocDir(t *testing.T) { require.NotNilf(t, fileInfo, "file %q not found", dataFile) } + +// TestAllocRuner_HandlesArtifactFailure ensures that if one task in a task group is +// retrying fetching an artifact, other tasks in the group should be able +// to proceed. +func TestAllocRunner_HandlesArtifactFailure(t *testing.T) { + t.Parallel() + + alloc := mock.BatchAlloc() + alloc.Job.TaskGroups[0].RestartPolicy = &structs.RestartPolicy{ + Mode: structs.RestartPolicyModeFail, + Attempts: 1, + Delay: time.Nanosecond, + Interval: time.Hour, + } + + // Create a new task with a bad artifact + badtask := alloc.Job.TaskGroups[0].Tasks[0].Copy() + badtask.Name = "bad" + badtask.Artifacts = []*structs.TaskArtifact{ + {GetterSource: "http://127.0.0.1:0/foo/bar/baz"}, + } + + alloc.Job.TaskGroups[0].Tasks = append(alloc.Job.TaskGroups[0].Tasks, badtask) + alloc.AllocatedResources.Tasks["bad"] = &structs.AllocatedTaskResources{ + Cpu: structs.AllocatedCpuResources{ + CpuShares: 500, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 256, + }, + } + + conf, cleanup := testAllocRunnerConfig(t, alloc) + defer cleanup() + ar, err := NewAllocRunner(conf) + require.NoError(t, err) + go ar.Run() + defer ar.Destroy() + + testutil.WaitForResult(func() (bool, error) { + state := ar.AllocState() + + switch state.ClientStatus { + case structs.AllocClientStatusComplete, structs.AllocClientStatusFailed: + return true, nil + default: + return false, fmt.Errorf("got status %v but want terminal", state.ClientStatus) + } + + }, func(err error) { + require.NoError(t, err) + }) + + state := ar.AllocState() + require.Equal(t, structs.AllocClientStatusFailed, state.ClientStatus) + require.Equal(t, structs.TaskStateDead, state.TaskStates["web"].State) + require.True(t, state.TaskStates["web"].Successful()) + require.Equal(t, structs.TaskStateDead, state.TaskStates["bad"].State) + require.True(t, state.TaskStates["bad"].Failed) +} From cfb80583afd424c068d064a528197d6d666234f8 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Feb 2019 22:02:50 -0500 Subject: [PATCH 3/9] tests: port TestTaskRunner_Template_Artifact From https://github.com/hashicorp/nomad/blob/v0.8.7/client/task_runner_test.go#L1195 --- .../taskrunner/task_runner_test.go | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 2dedc011f..bd4445a6b 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1238,6 +1238,64 @@ func TestTaskRunner_Run_RecoverableStartError(t *testing.T) { require.Equal(t, structs.TaskNotRestarting, state.Events[5].Type) } +// TestTaskRunner_Template_Artifact asserts that tasks can use artifacts as templates. +func TestTaskRunner_Template_Artifact(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.FileServer(http.Dir("."))) + defer ts.Close() + + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + f1 := "task_runner.go" + f2 := "test" + task.Artifacts = []*structs.TaskArtifact{ + {GetterSource: fmt.Sprintf("%s/%s", ts.URL, f1)}, + } + task.Templates = []*structs.Template{ + { + SourcePath: f1, + DestPath: "local/test", + ChangeMode: structs.TemplateChangeModeNoop, + }, + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + tr, err := NewTaskRunner(conf) + require.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + + // Wait for task to run and exit + select { + case <-tr.WaitCh(): + case <-time.After(15 * time.Second * time.Duration(testutil.TestMultiplier())): + require.Fail(t, "timed out waiting for task runner to exit") + } + + state := tr.TaskState() + require.Equal(t, structs.TaskStateDead, state.State) + require.True(t, state.Successful()) + require.False(t, state.Failed) + + artifactsDownloaded := false + for _, e := range state.Events { + if e.Type == structs.TaskDownloadingArtifacts { + artifactsDownloaded = true + } + } + assert.True(t, artifactsDownloaded, "expected artifacts downloaded events") + + // Check that both files exist. + _, err = os.Stat(filepath.Join(conf.TaskDir.Dir, f1)) + require.NoErrorf(t, err, "%v not downloaded", f1) + + _, err = os.Stat(filepath.Join(conf.TaskDir.LocalDir, f2)) + require.NoErrorf(t, err, "%v not rendered", f2) +} + // testWaitForTaskToStart waits for the task to be running or fails the test func testWaitForTaskToStart(t *testing.T, tr *TaskRunner) { testutil.WaitForResult(func() (bool, error) { From 0128b0ce7a505ab01a5ace2d507404a345dcd677 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Feb 2019 22:23:50 -0500 Subject: [PATCH 4/9] tests: port TestTaskRunner_Template_NewVaultToken From https://github.com/hashicorp/nomad/blob/v0.8.7/client/task_runner_test.go#L1275 --- .../taskrunner/task_runner_test.go | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index bd4445a6b..609121065 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1296,6 +1296,89 @@ func TestTaskRunner_Template_Artifact(t *testing.T) { require.NoErrorf(t, err, "%v not rendered", f2) } +func TestTaskRunner_Template_NewVaultToken(t *testing.T) { + t.Parallel() + + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Templates = []*structs.Template{ + { + EmbeddedTmpl: `{{key "foo"}}`, + DestPath: "local/test", + ChangeMode: structs.TemplateChangeModeNoop, + }, + } + task.Vault = &structs.Vault{Policies: []string{"default"}} + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + tr, err := NewTaskRunner(conf) + require.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + + // Wait for a Vault token + var token string + testutil.WaitForResult(func() (bool, error) { + tr.vaultTokenLock.Lock() + defer tr.vaultTokenLock.Unlock() + + token = tr.vaultToken + + if token == "" { + return false, fmt.Errorf("No Vault token") + } + + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + + vault := conf.Vault.(*vaultclient.MockVaultClient) + renewalCh, ok := vault.RenewTokens()[token] + require.True(t, ok, "no renewal channel for token") + + renewalCh <- fmt.Errorf("Test killing") + close(renewalCh) + + var token2 string + testutil.WaitForResult(func() (bool, error) { + tr.vaultTokenLock.Lock() + defer tr.vaultTokenLock.Unlock() + + token2 = tr.vaultToken + + if token2 == "" { + return false, fmt.Errorf("No Vault token") + } + + if token2 == token { + return false, fmt.Errorf("token wasn't recreated") + } + + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + + // Check the token was revoked + testutil.WaitForResult(func() (bool, error) { + if len(vault.StoppedTokens()) != 1 { + return false, fmt.Errorf("Expected a stopped token: %v", vault.StoppedTokens()) + } + + if a := vault.StoppedTokens()[0]; a != token { + return false, fmt.Errorf("got stopped token %q; want %q", a, token) + } + + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + +} + // testWaitForTaskToStart waits for the task to be running or fails the test func testWaitForTaskToStart(t *testing.T, tr *TaskRunner) { testutil.WaitForResult(func() (bool, error) { From 33122ca7c02c37c41d838365c05ce4c5b1ff471b Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Feb 2019 22:41:41 -0500 Subject: [PATCH 5/9] tests: port TestTaskRunner_UnregisterConsul_Retries From https://github.com/hashicorp/nomad/blob/v0.8.7/client/task_runner_test.go#L620 --- .../taskrunner/task_runner_test.go | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 609121065..3e7425698 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1379,6 +1379,49 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { } +// TestTaskRunner_UnregisterConsul_Retries asserts a task is unregistered from +// Consul when waiting to be retried. +func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) { + t.Parallel() + + alloc := mock.Alloc() + // Make the restart policy try one ctx.update + alloc.Job.TaskGroups[0].RestartPolicy = &structs.RestartPolicy{ + Attempts: 1, + Interval: 10 * time.Minute, + Delay: time.Nanosecond, + Mode: structs.RestartPolicyModeFail, + } + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Driver = "mock_driver" + task.Config = map[string]interface{}{ + "exit_code": "1", + "run_for": "1ns", + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + tr, err := NewTaskRunner(conf) + require.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + tr.Run() + + state := tr.TaskState() + require.Equal(t, structs.TaskStateDead, state.State) + + consul := conf.Consul.(*consulapi.MockConsulServiceClient) + consulOps := consul.GetOps() + require.Len(t, consulOps, 6) + // pattern: add followed by two removals + require.Equal(t, "add", consulOps[0].Op) + require.Equal(t, "remove", consulOps[1].Op) + require.Equal(t, "remove", consulOps[2].Op) + require.Equal(t, "add", consulOps[3].Op) + require.Equal(t, "remove", consulOps[4].Op) + require.Equal(t, "remove", consulOps[5].Op) +} + // testWaitForTaskToStart waits for the task to be running or fails the test func testWaitForTaskToStart(t *testing.T, tr *TaskRunner) { testutil.WaitForResult(func() (bool, error) { From 8e9e73231989394f701aec097be671e0b22d494a Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Feb 2019 09:20:57 -0500 Subject: [PATCH 6/9] tests: port TestTaskRunner_VaultManager_Restart From https://github.com/hashicorp/nomad/blob/v0.8.7/client/task_runner_test.go#L1352 --- .../taskrunner/task_runner_test.go | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 3e7425698..a04124afd 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1379,6 +1379,73 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { } +func TestTaskRunner_VaultManager_Restart(t *testing.T) { + t.Parallel() + + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Driver = "mock_driver" + task.Config = map[string]interface{}{ + "exit_code": "0", + "run_for": "10s", + } + task.Vault = &structs.Vault{ + Policies: []string{"default"}, + ChangeMode: structs.VaultChangeModeRestart, + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + tr, err := NewTaskRunner(conf) + require.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + + testWaitForTaskToStart(t, tr) + + tr.vaultTokenLock.Lock() + token := tr.vaultToken + tr.vaultTokenLock.Unlock() + + require.NotEmpty(t, token) + + vault := conf.Vault.(*vaultclient.MockVaultClient) + renewalCh, ok := vault.RenewTokens()[token] + require.True(t, ok, "no renewal channel for token") + + renewalCh <- fmt.Errorf("Test killing") + close(renewalCh) + + testutil.WaitForResult(func() (bool, error) { + state := tr.TaskState() + + if len(state.Events) == 0 { + return false, fmt.Errorf("no events yet") + } + + // TODO: check for RestartSignal too - 0.8 sent that event, but not 0.9 + foundRestarting := false + for _, e := range state.Events { + if e.Type == structs.TaskRestarting { + foundRestarting = true + } + } + + if !foundRestarting { + return false, fmt.Errorf("no restarting event yet: %#v", state.Events) + } + + lastEvent := state.Events[len(state.Events)-1] + if lastEvent.Type != structs.TaskStarted { + return false, fmt.Errorf("expected last event to be task restarting but was %#v", lastEvent) + } + return true, nil + }, func(err error) { + require.NoError(t, err) + }) +} + // TestTaskRunner_UnregisterConsul_Retries asserts a task is unregistered from // Consul when waiting to be retried. func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) { From 216eaa4843fe9f7339331554b240f3a3cd528975 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Feb 2019 09:31:02 -0500 Subject: [PATCH 7/9] tests: port TestTaskRunner_VaultManager_Signal From https://github.com/hashicorp/nomad/blob/v0.8.7/client/task_runner_test.go#L1427 --- .../taskrunner/task_runner_test.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index a04124afd..ca56921e1 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1446,6 +1446,70 @@ func TestTaskRunner_VaultManager_Restart(t *testing.T) { }) } +func TestTaskRunner_VaultManager_Signal(t *testing.T) { + t.Parallel() + + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Driver = "mock_driver" + task.Config = map[string]interface{}{ + "exit_code": "0", + "run_for": "10s", + } + task.Vault = &structs.Vault{ + Policies: []string{"default"}, + ChangeMode: structs.VaultChangeModeSignal, + ChangeSignal: "SIGUSR1", + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + tr, err := NewTaskRunner(conf) + require.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + + testWaitForTaskToStart(t, tr) + + tr.vaultTokenLock.Lock() + token := tr.vaultToken + tr.vaultTokenLock.Unlock() + + require.NotEmpty(t, token) + + vault := conf.Vault.(*vaultclient.MockVaultClient) + renewalCh, ok := vault.RenewTokens()[token] + require.True(t, ok, "no renewal channel for token") + + renewalCh <- fmt.Errorf("Test killing") + close(renewalCh) + + testutil.WaitForResult(func() (bool, error) { + state := tr.TaskState() + + if len(state.Events) == 0 { + return false, fmt.Errorf("no events yet") + } + + foundSignaling := false + for _, e := range state.Events { + if e.Type == structs.TaskSignaling { + foundSignaling = true + } + } + + if !foundSignaling { + return false, fmt.Errorf("no signaling event yet: %#v", state.Events) + } + + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + +} + // TestTaskRunner_UnregisterConsul_Retries asserts a task is unregistered from // Consul when waiting to be retried. func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) { From 8cb4bbcc08955518f7b6b445091412647184c8ed Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Feb 2019 14:35:21 -0500 Subject: [PATCH 8/9] address review comments --- client/allocrunner/alloc_runner_test.go | 10 ++++---- .../taskrunner/task_runner_test.go | 24 ++++++++----------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/client/allocrunner/alloc_runner_test.go b/client/allocrunner/alloc_runner_test.go index 11e01f718..139786b44 100644 --- a/client/allocrunner/alloc_runner_test.go +++ b/client/allocrunner/alloc_runner_test.go @@ -639,7 +639,7 @@ func TestAllocRunner_SimpleRun(t *testing.T) { ar, err := NewAllocRunner(conf) require.NoError(t, err) go ar.Run() - defer ar.Destroy() + defer destroy(ar) // Wait for alloc to be running testutil.WaitForResult(func() (bool, error) { @@ -662,6 +662,8 @@ func TestAllocRunner_SimpleRun(t *testing.T) { } +// TestAllocRunner_MoveAllocDir asserts that a rescheduled +// allocation copies ephemeral disk content from previous alloc run func TestAllocRunner_MoveAllocDir(t *testing.T) { t.Parallel() @@ -672,7 +674,7 @@ func TestAllocRunner_MoveAllocDir(t *testing.T) { ar, err := NewAllocRunner(conf) require.NoError(t, err) ar.Run() - defer ar.Destroy() + defer destroy(ar) require.Equal(t, structs.AllocClientStatusComplete, ar.AllocState().ClientStatus) @@ -700,7 +702,7 @@ func TestAllocRunner_MoveAllocDir(t *testing.T) { require.NoError(t, err) ar2.Run() - defer ar2.Destroy() + defer destroy(ar2) require.Equal(t, structs.AllocClientStatusComplete, ar2.AllocState().ClientStatus) @@ -752,7 +754,7 @@ func TestAllocRunner_HandlesArtifactFailure(t *testing.T) { ar, err := NewAllocRunner(conf) require.NoError(t, err) go ar.Run() - defer ar.Destroy() + defer destroy(ar) testutil.WaitForResult(func() (bool, error) { state := ar.AllocState() diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index ca56921e1..a0b92e3a2 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1296,6 +1296,8 @@ func TestTaskRunner_Template_Artifact(t *testing.T) { require.NoErrorf(t, err, "%v not rendered", f2) } +// TestTaskRunner_Template_NewVaultToken asserts that a new vault token is +// created when rendering template and that it is revoked on alloc completion func TestTaskRunner_Template_NewVaultToken(t *testing.T) { t.Parallel() @@ -1321,10 +1323,7 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { // Wait for a Vault token var token string testutil.WaitForResult(func() (bool, error) { - tr.vaultTokenLock.Lock() - defer tr.vaultTokenLock.Unlock() - - token = tr.vaultToken + token = tr.getVaultToken() if token == "" { return false, fmt.Errorf("No Vault token") @@ -1344,10 +1343,7 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { var token2 string testutil.WaitForResult(func() (bool, error) { - tr.vaultTokenLock.Lock() - defer tr.vaultTokenLock.Unlock() - - token2 = tr.vaultToken + token2 = tr.getVaultToken() if token2 == "" { return false, fmt.Errorf("No Vault token") @@ -1379,15 +1375,15 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { } +// TestTaskRunner_VaultManager_Restart asserts that the alloc is restarted when the alloc +// derived vault token expires, when task is configured with Restart change mode func TestTaskRunner_VaultManager_Restart(t *testing.T) { t.Parallel() alloc := mock.BatchAlloc() task := alloc.Job.TaskGroups[0].Tasks[0] - task.Driver = "mock_driver" task.Config = map[string]interface{}{ - "exit_code": "0", - "run_for": "10s", + "run_for": "10s", } task.Vault = &structs.Vault{ Policies: []string{"default"}, @@ -1446,15 +1442,15 @@ func TestTaskRunner_VaultManager_Restart(t *testing.T) { }) } +// TestTaskRunner_VaultManager_Signal asserts that the alloc is signalled when the alloc +// derived vault token expires, when task is configured with signal change mode func TestTaskRunner_VaultManager_Signal(t *testing.T) { t.Parallel() alloc := mock.BatchAlloc() task := alloc.Job.TaskGroups[0].Tasks[0] - task.Driver = "mock_driver" task.Config = map[string]interface{}{ - "exit_code": "0", - "run_for": "10s", + "run_for": "10s", } task.Vault = &structs.Vault{ Policies: []string{"default"}, From 32551fb0e5a661a888b33d6523f1c64a1eb5a89a Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Feb 2019 14:45:17 -0500 Subject: [PATCH 9/9] emit TaskRestartSignal event on vault restart When Vault token expires and task is restarted, emit `TaskRestartSignal` similar to v0.8.7 --- client/allocrunner/taskrunner/task_runner_test.go | 14 ++++++++++---- client/allocrunner/taskrunner/vault_hook.go | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index a0b92e3a2..7eed9742a 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1420,21 +1420,27 @@ func TestTaskRunner_VaultManager_Restart(t *testing.T) { return false, fmt.Errorf("no events yet") } - // TODO: check for RestartSignal too - 0.8 sent that event, but not 0.9 - foundRestarting := false + foundRestartSignal, foundRestarting := false, false for _, e := range state.Events { - if e.Type == structs.TaskRestarting { + switch e.Type { + case structs.TaskRestartSignal: + foundRestartSignal = true + case structs.TaskRestarting: foundRestarting = true } } + if !foundRestartSignal { + return false, fmt.Errorf("no restart signal event yet: %#v", state.Events) + } + if !foundRestarting { return false, fmt.Errorf("no restarting event yet: %#v", state.Events) } lastEvent := state.Events[len(state.Events)-1] if lastEvent.Type != structs.TaskStarted { - return false, fmt.Errorf("expected last event to be task restarting but was %#v", lastEvent) + return false, fmt.Errorf("expected last event to be task starting but was %#v", lastEvent) } return true, nil }, func(err error) { diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index e14446a74..016fbf610 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -254,7 +254,7 @@ OUTER: case structs.VaultChangeModeRestart: const noFailure = false h.lifecycle.Restart(h.ctx, - structs.NewTaskEvent(structs.TaskRestarting). + structs.NewTaskEvent(structs.TaskRestartSignal). SetDisplayMessage("Vault: new Vault token acquired"), false) case structs.VaultChangeModeNoop: fallthrough