luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge branch 'master' into b-extend-win-cpu-fingerprint-timeout

This commit is contained in:
Charlie Voiselle 2018-05-09 16:23:14 -04:00 committed by GitHub
parent 62f99cc629 0e79e1a46e
commit 6e58e1ff4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
71 changed files with 4687 additions and 1689 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@ -9,6 +9,16 @@ IMPROVEMENTS:
image pulls [[GH-4192](https://github.com/hashicorp/nomad/issues/4192)]
* env: Default interpolation of optional meta fields of parameterized jobs to
an empty string rather than the field key. [[GH-3720](https://github.com/hashicorp/nomad/issues/3720)]
* core: Add a new [progress_deadline](https://www.nomadproject.io/docs/job-specification/update.html#progress_deadline) parameter to
support rescheduling failed allocations during a deployment. This allows operators to specify a configurable deadline before which
a deployment should see healthy allocations [[GH-4259](https://github.com/hashicorp/nomad/issues/4259)]
* core: Canary allocations are tagged in Consul to enable
using service tags to isolate canary instances during deployments [[GH-4259](https://github.com/hashicorp/nomad/issues/4259)]
BUG FIXES:
* api/client: Fix potentially out of order logs and streamed file contents
[[GH-4234](https://github.com/hashicorp/nomad/issues/4234)]
## 0.8.3 (April 27, 2018)

6
api/allocations.go
View File

@ -143,6 +143,8 @@ type AllocationListStub struct {
// healthy.
type AllocDeploymentStatus struct {
Healthy *bool
Timestamp time.Time
Canary bool
ModifyIndex uint64
}
@ -214,6 +216,10 @@ type DesiredTransition struct {
// Migrate is used to indicate that this allocation should be stopped and
// migrated to another node.
Migrate *bool
// Reschedule is used to indicate that this allocation is eligible to be
// rescheduled.
Reschedule *bool
}
// ShouldMigrate returns whether the transition object dictates a migration.

19
api/deployments.go
View File

@ -2,6 +2,7 @@ package api
import (
"sort"
"time"
)
// Deployments is used to query the deployments endpoints.
@ -139,14 +140,16 @@ type Deployment struct {
// DeploymentState tracks the state of a deployment for a given task group.
type DeploymentState struct {
PlacedCanaries []string
AutoRevert bool
Promoted bool
DesiredCanaries int
DesiredTotal int
PlacedAllocs int
HealthyAllocs int
UnhealthyAllocs int
PlacedCanaries []string
AutoRevert bool
ProgressDeadline time.Duration
RequireProgressBy time.Time
Promoted bool
DesiredCanaries int
DesiredTotal int
PlacedAllocs int
HealthyAllocs int
UnhealthyAllocs int
}
// DeploymentIndexSort is a wrapper to sort deployments by CreateIndex. We

46
api/jobs.go
View File

@ -343,26 +343,28 @@ type periodicForceResponse struct {
// UpdateStrategy defines a task groups update strategy.
type UpdateStrategy struct {
Stagger *time.Duration `mapstructure:"stagger"`
MaxParallel *int `mapstructure:"max_parallel"`
HealthCheck *string `mapstructure:"health_check"`
MinHealthyTime *time.Duration `mapstructure:"min_healthy_time"`
HealthyDeadline *time.Duration `mapstructure:"healthy_deadline"`
AutoRevert *bool `mapstructure:"auto_revert"`
Canary *int `mapstructure:"canary"`
Stagger *time.Duration `mapstructure:"stagger"`
MaxParallel *int `mapstructure:"max_parallel"`
HealthCheck *string `mapstructure:"health_check"`
MinHealthyTime *time.Duration `mapstructure:"min_healthy_time"`
HealthyDeadline *time.Duration `mapstructure:"healthy_deadline"`
ProgressDeadline *time.Duration `mapstructure:"progress_deadline"`
AutoRevert *bool `mapstructure:"auto_revert"`
Canary *int `mapstructure:"canary"`
}
// DefaultUpdateStrategy provides a baseline that can be used to upgrade
// jobs with the old policy or for populating field defaults.
func DefaultUpdateStrategy() *UpdateStrategy {
return &UpdateStrategy{
Stagger: helper.TimeToPtr(30 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
Stagger: helper.TimeToPtr(30 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
}
}
@ -393,6 +395,10 @@ func (u *UpdateStrategy) Copy() *UpdateStrategy {
copy.HealthyDeadline = helper.TimeToPtr(*u.HealthyDeadline)
}
if u.ProgressDeadline != nil {
copy.ProgressDeadline = helper.TimeToPtr(*u.ProgressDeadline)
}
if u.AutoRevert != nil {
copy.AutoRevert = helper.BoolToPtr(*u.AutoRevert)
}
@ -429,6 +435,10 @@ func (u *UpdateStrategy) Merge(o *UpdateStrategy) {
u.HealthyDeadline = helper.TimeToPtr(*o.HealthyDeadline)
}
if o.ProgressDeadline != nil {
u.ProgressDeadline = helper.TimeToPtr(*o.ProgressDeadline)
}
if o.AutoRevert != nil {
u.AutoRevert = helper.BoolToPtr(*o.AutoRevert)
}
@ -457,6 +467,10 @@ func (u *UpdateStrategy) Canonicalize() {
u.HealthyDeadline = d.HealthyDeadline
}
if u.ProgressDeadline == nil {
u.ProgressDeadline = d.ProgressDeadline
}
if u.MinHealthyTime == nil {
u.MinHealthyTime = d.MinHealthyTime
}
@ -496,6 +510,10 @@ func (u *UpdateStrategy) Empty() bool {
return false
}
if u.ProgressDeadline != nil && *u.ProgressDeadline != 0 {
return false
}
if u.AutoRevert != nil && *u.AutoRevert {
return false
}

98
api/jobs_test.go
View File

@ -304,9 +304,10 @@ func TestJobs_Canonicalize(t *testing.T) {
},
Services: []*Service{
{
Name: "redis-cache",
Tags: []string{"global", "cache"},
PortLabel: "db",
Name: "redis-cache",
Tags: []string{"global", "cache"},
CanaryTags: []string{"canary", "global", "cache"},
PortLabel: "db",
Checks: []ServiceCheck{
{
Name: "alive",
@ -354,13 +355,14 @@ func TestJobs_Canonicalize(t *testing.T) {
JobModifyIndex: helper.Uint64ToPtr(0),
Datacenters: []string{"dc1"},
Update: &UpdateStrategy{
Stagger: helper.TimeToPtr(30 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
Stagger: helper.TimeToPtr(30 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
},
TaskGroups: []*TaskGroup{
{
@ -387,13 +389,14 @@ func TestJobs_Canonicalize(t *testing.T) {
},
Update: &UpdateStrategy{
Stagger: helper.TimeToPtr(30 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
Stagger: helper.TimeToPtr(30 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
},
Migrate: DefaultMigrateStrategy(),
Tasks: []*Task{
@ -425,6 +428,7 @@ func TestJobs_Canonicalize(t *testing.T) {
{
Name: "redis-cache",
Tags: []string{"global", "cache"},
CanaryTags: []string{"canary", "global", "cache"},
PortLabel: "db",
AddressMode: "auto",
Checks: []ServiceCheck{
@ -515,13 +519,14 @@ func TestJobs_Canonicalize(t *testing.T) {
ID: helper.StringToPtr("bar"),
ParentID: helper.StringToPtr("lol"),
Update: &UpdateStrategy{
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
},
TaskGroups: []*TaskGroup{
{
@ -569,13 +574,14 @@ func TestJobs_Canonicalize(t *testing.T) {
ModifyIndex: helper.Uint64ToPtr(0),
JobModifyIndex: helper.Uint64ToPtr(0),
Update: &UpdateStrategy{
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
},
TaskGroups: []*TaskGroup{
{
@ -601,13 +607,14 @@ func TestJobs_Canonicalize(t *testing.T) {
Unlimited: helper.BoolToPtr(true),
},
Update: &UpdateStrategy{
Stagger: helper.TimeToPtr(2 * time.Second),
MaxParallel: helper.IntToPtr(2),
HealthCheck: helper.StringToPtr("manual"),
MinHealthyTime: helper.TimeToPtr(1 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
AutoRevert: helper.BoolToPtr(true),
Canary: helper.IntToPtr(1),
Stagger: helper.TimeToPtr(2 * time.Second),
MaxParallel: helper.IntToPtr(2),
HealthCheck: helper.StringToPtr("manual"),
MinHealthyTime: helper.TimeToPtr(1 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
AutoRevert: helper.BoolToPtr(true),
Canary: helper.IntToPtr(1),
},
Migrate: DefaultMigrateStrategy(),
Tasks: []*Task{
@ -642,13 +649,14 @@ func TestJobs_Canonicalize(t *testing.T) {
Unlimited: helper.BoolToPtr(true),
},
Update: &UpdateStrategy{
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(1),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
},
Migrate: DefaultMigrateStrategy(),
Tasks: []*Task{

5
api/tasks.go
View File

@ -295,8 +295,9 @@ type Service struct {
Id string
Name string
Tags []string
PortLabel string `mapstructure:"port"`
AddressMode string `mapstructure:"address_mode"`
CanaryTags []string `mapstructure:"canary_tags"`
PortLabel string `mapstructure:"port"`
AddressMode string `mapstructure:"address_mode"`
Checks []ServiceCheck
CheckRestart *CheckRestart `mapstructure:"check_restart"`
}

15
api/tasks_test.go
View File

@ -252,13 +252,14 @@ func TestTaskGroup_Canonicalize_Update(t *testing.T) {
job := &Job{
ID: helper.StringToPtr("test"),
Update: &UpdateStrategy{
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
HealthCheck: helper.StringToPtr(""),
HealthyDeadline: helper.TimeToPtr(0),
MaxParallel: helper.IntToPtr(0),
MinHealthyTime: helper.TimeToPtr(0),
Stagger: helper.TimeToPtr(0),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(0),
HealthCheck: helper.StringToPtr(""),
HealthyDeadline: helper.TimeToPtr(0),
ProgressDeadline: helper.TimeToPtr(0),
MaxParallel: helper.IntToPtr(0),
MinHealthyTime: helper.TimeToPtr(0),
Stagger: helper.TimeToPtr(0),
},
}
job.Canonicalize()

5
client/alloc_runner.go
View File

@ -50,7 +50,8 @@ type AllocRunner struct {
alloc *structs.Allocation
allocClientStatus string // Explicit status of allocation. Set when there are failures
allocClientDescription string
allocHealth *bool // Whether the allocation is healthy
allocHealth *bool // Whether the allocation is healthy
allocHealthTime time.Time // Time at which allocation health has been set
allocBroadcast *cstructs.AllocBroadcaster
allocLock sync.Mutex
@ -580,6 +581,7 @@ func (r *AllocRunner) Alloc() *structs.Allocation {
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{}
}
alloc.DeploymentStatus.Healthy = helper.BoolToPtr(*r.allocHealth)
alloc.DeploymentStatus.Timestamp = r.allocHealthTime
}
r.allocLock.Unlock()
@ -943,6 +945,7 @@ OUTER:
// If the deployment ids have changed clear the health
if r.alloc.DeploymentID != update.DeploymentID {
r.allocHealth = nil
r.allocHealthTime = time.Time{}
}
r.alloc = update

1
client/alloc_runner_health_watcher.go
View File

@ -112,6 +112,7 @@ func (r *AllocRunner) watchHealth(ctx context.Context) {
r.allocLock.Lock()
r.allocHealth = helper.BoolToPtr(allocHealthy)
r.allocHealthTime = time.Now()
r.allocLock.Unlock()
// If deployment is unhealthy emit task events explaining why

9
client/consul.go
View File

@ -1,17 +1,14 @@
package client
import (
"github.com/hashicorp/nomad/client/driver"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/nomad/structs"
)
// ConsulServiceAPI is the interface the Nomad Client uses to register and
// remove services and checks from Consul.
type ConsulServiceAPI interface {
RegisterTask(allocID string, task *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error
RemoveTask(allocID string, task *structs.Task)
UpdateTask(allocID string, existing, newTask *structs.Task, restart consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error
RegisterTask(*consul.TaskServices) error
RemoveTask(*consul.TaskServices)
UpdateTask(old, newTask *consul.TaskServices) error
AllocRegistrations(allocID string) (*consul.AllocRegistration, error)
}

31
client/consul_testing.go
View File

@ -5,11 +5,8 @@ import (
"log"
"sync"
"github.com/hashicorp/nomad/client/driver"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/go-testing-interface"
)
@ -17,12 +14,10 @@ import (
type mockConsulOp struct {
op string // add, remove, or update
allocID string
task *structs.Task
exec driver.ScriptExecutor
net *cstructs.DriverNetwork
task string
}
func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) mockConsulOp {
func newMockConsulOp(op, allocID, task string) mockConsulOp {
if op != "add" && op != "remove" && op != "update" && op != "alloc_registrations" {
panic(fmt.Errorf("invalid consul op: %s", op))
}
@ -30,8 +25,6 @@ func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptE
op: op,
allocID: allocID,
task: task,
exec: exec,
net: net,
}
}
@ -56,34 +49,34 @@ func newMockConsulServiceClient(t testing.T) *mockConsulServiceClient {
return &m
}
func (m *mockConsulServiceClient) UpdateTask(allocID string, old, new *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error {
func (m *mockConsulServiceClient) UpdateTask(old, new *consul.TaskServices) error {
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Printf("[TEST] mock_consul: UpdateTask(%q, %v, %v, %T, %x)", allocID, old, new, exec, net.Hash())
m.ops = append(m.ops, newMockConsulOp("update", allocID, new, exec, net))
m.logger.Printf("[TEST] mock_consul: UpdateTask(alloc: %s, task: %s)", new.AllocID[:6], new.Name)
m.ops = append(m.ops, newMockConsulOp("update", new.AllocID, new.Name))
return nil
}
func (m *mockConsulServiceClient) RegisterTask(allocID string, task *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error {
func (m *mockConsulServiceClient) RegisterTask(task *consul.TaskServices) error {
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Printf("[TEST] mock_consul: RegisterTask(%q, %q, %T, %x)", allocID, task.Name, exec, net.Hash())
m.ops = append(m.ops, newMockConsulOp("add", allocID, task, exec, net))
m.logger.Printf("[TEST] mock_consul: RegisterTask(alloc: %s, task: %s)", task.AllocID, task.Name)
m.ops = append(m.ops, newMockConsulOp("add", task.AllocID, task.Name))
return nil
}
func (m *mockConsulServiceClient) RemoveTask(allocID string, task *structs.Task) {
func (m *mockConsulServiceClient) RemoveTask(task *consul.TaskServices) {
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Printf("[TEST] mock_consul: RemoveTask(%q, %q)", allocID, task.Name)
m.ops = append(m.ops, newMockConsulOp("remove", allocID, task, nil, nil))
m.logger.Printf("[TEST] mock_consul: RemoveTask(%q, %q)", task.AllocID, task.Name)
m.ops = append(m.ops, newMockConsulOp("remove", task.AllocID, task.Name))
}
func (m *mockConsulServiceClient) AllocRegistrations(allocID string) (*consul.AllocRegistration, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Printf("[TEST] mock_consul: AllocRegistrations(%q)", allocID)
m.ops = append(m.ops, newMockConsulOp("alloc_registrations", allocID, nil, nil, nil))
m.ops = append(m.ops, newMockConsulOp("alloc_registrations", allocID, ""))
if m.allocRegistrationsFn != nil {
return m.allocRegistrationsFn(allocID)

3
client/fs_endpoint.go
View File

@ -253,7 +253,8 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) {
go func() {
for {
if _, err := conn.Read(nil); err != nil {
if err == io.EOF {
if err == io.EOF || err == io.ErrClosedPipe {
// One end of the pipe was explicitly closed, exit cleanly
cancel()
return
}

39
client/task_runner.go
View File

@ -25,6 +25,7 @@ import (
"github.com/hashicorp/nomad/client/driver"
"github.com/hashicorp/nomad/client/getter"
"github.com/hashicorp/nomad/client/vaultclient"
"github.com/hashicorp/nomad/command/agent/consul"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/ugorji/go/codec"
@ -1218,8 +1219,7 @@ func (r *TaskRunner) run() {
// Remove from consul before killing the task so that traffic
// can be rerouted
interpTask := interpolateServices(r.envBuilder.Build(), r.task)
r.consul.RemoveTask(r.alloc.ID, interpTask)
r.removeServices()
// Delay actually killing the task if configured. See #244
if r.task.ShutdownDelay > 0 {
@ -1274,8 +1274,7 @@ func (r *TaskRunner) run() {
// stopping. Errors are logged.
func (r *TaskRunner) cleanup() {
// Remove from Consul
interpTask := interpolateServices(r.envBuilder.Build(), r.task)
r.consul.RemoveTask(r.alloc.ID, interpTask)
r.removeServices()
drv, err := r.createDriver()
if err != nil {
@ -1338,8 +1337,7 @@ func (r *TaskRunner) shouldRestart() bool {
}
// Unregister from Consul while waiting to restart.
interpTask := interpolateServices(r.envBuilder.Build(), r.task)
r.consul.RemoveTask(r.alloc.ID, interpTask)
r.removeServices()
// Sleep but watch for destroy events.
select {
@ -1498,7 +1496,8 @@ func (r *TaskRunner) registerServices(d driver.Driver, h driver.DriverHandle, n
exec = h
}
interpolatedTask := interpolateServices(r.envBuilder.Build(), r.task)
return r.consul.RegisterTask(r.alloc.ID, interpolatedTask, r, exec, n)
taskServices := consul.NewTaskServices(r.alloc, interpolatedTask, r, exec, n)
return r.consul.RegisterTask(taskServices)
}
// interpolateServices interpolates tags in a service and checks with values from the
@ -1532,6 +1531,7 @@ func interpolateServices(taskEnv *env.TaskEnv, task *structs.Task) *structs.Task
service.Name = taskEnv.ReplaceEnv(service.Name)
service.PortLabel = taskEnv.ReplaceEnv(service.PortLabel)
service.Tags = taskEnv.ParseAndReplace(service.Tags)
service.CanaryTags = taskEnv.ParseAndReplace(service.CanaryTags)
}
return taskCopy
}
@ -1679,7 +1679,7 @@ func (r *TaskRunner) handleUpdate(update *structs.Allocation) error {
// Update services in Consul
newInterpolatedTask := interpolateServices(r.envBuilder.Build(), updatedTask)
if err := r.updateServices(drv, r.handle, oldInterpolatedTask, newInterpolatedTask); err != nil {
if err := r.updateServices(drv, r.handle, r.alloc, oldInterpolatedTask, update, newInterpolatedTask); err != nil {
mErr.Errors = append(mErr.Errors, fmt.Errorf("error updating services and checks in Consul: %v", err))
}
}
@ -1697,7 +1697,10 @@ func (r *TaskRunner) handleUpdate(update *structs.Allocation) error {
}
// updateServices and checks with Consul. Tasks must be interpolated!
func (r *TaskRunner) updateServices(d driver.Driver, h driver.ScriptExecutor, oldTask, newTask *structs.Task) error {
func (r *TaskRunner) updateServices(d driver.Driver, h driver.ScriptExecutor,
oldAlloc *structs.Allocation, oldTask *structs.Task,
newAlloc *structs.Allocation, newTask *structs.Task) error {
var exec driver.ScriptExecutor
if d.Abilities().Exec {
// Allow set the script executor if the driver supports it
@ -1706,7 +1709,23 @@ func (r *TaskRunner) updateServices(d driver.Driver, h driver.ScriptExecutor, ol
r.driverNetLock.Lock()
net := r.driverNet.Copy()
r.driverNetLock.Unlock()
return r.consul.UpdateTask(r.alloc.ID, oldTask, newTask, r, exec, net)
oldTaskServices := consul.NewTaskServices(oldAlloc, oldTask, r, exec, net)
newTaskServices := consul.NewTaskServices(newAlloc, newTask, r, exec, net)
return r.consul.UpdateTask(oldTaskServices, newTaskServices)
}
// removeServices and checks from Consul. Handles interpolation and deleting
// Canary=true and Canary=false versions in case Canary=false is set at the
// same time as the alloc is stopped.
func (r *TaskRunner) removeServices() {
interpTask := interpolateServices(r.envBuilder.Build(), r.task)
taskServices := consul.NewTaskServices(r.alloc, interpTask, r, nil, nil)
r.consul.RemoveTask(taskServices)
// Flip Canary and remove again in case canary is getting flipped at
// the same time as the alloc is being destroyed
taskServices.Canary = !taskServices.Canary
r.consul.RemoveTask(taskServices)
}
// handleDestroy kills the task handle. In the case that killing fails,

16
client/task_runner_test.go
View File

@ -650,7 +650,7 @@ func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) {
defer ctx.Cleanup()
// Assert it is properly registered and unregistered
if expected := 4; len(consul.ops) != expected {
if expected := 6; len(consul.ops) != expected {
t.Errorf("expected %d consul ops but found: %d", expected, len(consul.ops))
}
if consul.ops[0].op != "add" {
@ -659,11 +659,17 @@ func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) {
if consul.ops[1].op != "remove" {
t.Errorf("expected second op to be remove but found: %q", consul.ops[1].op)
}
if consul.ops[2].op != "add" {
t.Errorf("expected third op to be add but found: %q", consul.ops[2].op)
if consul.ops[2].op != "remove" {
t.Errorf("expected third op to be remove but found: %q", consul.ops[2].op)
}
if consul.ops[3].op != "remove" {
t.Errorf("expected fourth/final op to be remove but found: %q", consul.ops[3].op)
if consul.ops[3].op != "add" {
t.Errorf("expected fourth op to be add but found: %q", consul.ops[3].op)
}
if consul.ops[4].op != "remove" {
t.Errorf("expected fifth op to be remove but found: %q", consul.ops[4].op)
}
if consul.ops[5].op != "remove" {
t.Errorf("expected sixth op to be remove but found: %q", consul.ops[5].op)
}
}

84
command/agent/consul/client.go
View File

@ -14,7 +14,6 @@ import (
metrics "github.com/armon/go-metrics"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/client/driver"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
@ -603,11 +602,11 @@ func (c *ServiceClient) RegisterAgent(role string, services []*structs.Service)
// serviceRegs creates service registrations, check registrations, and script
// checks from a service. It returns a service registration object with the
// service and check IDs populated.
func (c *ServiceClient) serviceRegs(ops *operations, allocID string, service *structs.Service,
task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) (*ServiceRegistration, error) {
func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, task *TaskServices) (
*ServiceRegistration, error) {
// Get the services ID
id := makeTaskServiceID(allocID, task.Name, service)
id := makeTaskServiceID(task.AllocID, task.Name, service, task.Canary)
sreg := &ServiceRegistration{
serviceID: id,
checkIDs: make(map[string]struct{}, len(service.Checks)),
@ -620,26 +619,33 @@ func (c *ServiceClient) serviceRegs(ops *operations, allocID string, service *st
}
// Determine the address to advertise based on the mode
ip, port, err := getAddress(addrMode, service.PortLabel, task.Resources.Networks, net)
ip, port, err := getAddress(addrMode, service.PortLabel, task.Networks, task.DriverNetwork)
if err != nil {
return nil, fmt.Errorf("unable to get address for service %q: %v", service.Name, err)
}
// Determine whether to use tags or canary_tags
var tags []string
if task.Canary {
tags = make([]string, len(service.CanaryTags))
copy(tags, service.CanaryTags)
} else {
tags = make([]string, len(service.Tags))
copy(tags, service.Tags)
}
// Build the Consul Service registration request
serviceReg := &api.AgentServiceRegistration{
ID: id,
Name: service.Name,
Tags: make([]string, len(service.Tags)),
Tags: tags,
Address: ip,
Port: port,
}
// copy isn't strictly necessary but can avoid bugs especially
// with tests that may reuse Tasks
copy(serviceReg.Tags, service.Tags)
ops.regServices = append(ops.regServices, serviceReg)
// Build the check registrations
checkIDs, err := c.checkRegs(ops, allocID, id, service, task, exec, net)
checkIDs, err := c.checkRegs(ops, id, service, task)
if err != nil {
return nil, err
}
@ -651,8 +657,8 @@ func (c *ServiceClient) serviceRegs(ops *operations, allocID string, service *st
// checkRegs registers the checks for the given service and returns the
// registered check ids.
func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, service *structs.Service,
task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) ([]string, error) {
func (c *ServiceClient) checkRegs(ops *operations, serviceID string, service *structs.Service,
task *TaskServices) ([]string, error) {
// Fast path
numChecks := len(service.Checks)
@ -665,11 +671,13 @@ func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, se
checkID := makeCheckID(serviceID, check)
checkIDs = append(checkIDs, checkID)
if check.Type == structs.ServiceCheckScript {
if exec == nil {
if task.DriverExec == nil {
return nil, fmt.Errorf("driver doesn't support script checks")
}
ops.scripts = append(ops.scripts, newScriptCheck(
allocID, task.Name, checkID, check, exec, c.client, c.logger, c.shutdownCh))
sc := newScriptCheck(task.AllocID, task.Name, checkID, check, task.DriverExec,
c.client, c.logger, c.shutdownCh)
ops.scripts = append(ops.scripts, sc)
// Skip getAddress for script checks
checkReg, err := createCheckReg(serviceID, checkID, check, "", 0)
@ -693,7 +701,7 @@ func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, se
addrMode = structs.AddressModeHost
}
ip, port, err := getAddress(addrMode, portLabel, task.Resources.Networks, net)
ip, port, err := getAddress(addrMode, portLabel, task.Networks, task.DriverNetwork)
if err != nil {
return nil, fmt.Errorf("error getting address for check %q: %v", check.Name, err)
}
@ -714,7 +722,7 @@ func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, se
// Checks will always use the IP from the Task struct (host's IP).
//
// Actual communication with Consul is done asynchronously (see Run).
func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restarter TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error {
func (c *ServiceClient) RegisterTask(task *TaskServices) error {
// Fast path
numServices := len(task.Services)
if numServices == 0 {
@ -726,7 +734,7 @@ func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restart
ops := &operations{}
for _, service := range task.Services {
sreg, err := c.serviceRegs(ops, allocID, service, task, exec, net)
sreg, err := c.serviceRegs(ops, service, task)
if err != nil {
return err
}
@ -734,18 +742,18 @@ func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restart
}
// Add the task to the allocation's registration
c.addTaskRegistration(allocID, task.Name, t)
c.addTaskRegistration(task.AllocID, task.Name, t)
c.commit(ops)
// Start watching checks. Done after service registrations are built
// since an error building them could leak watches.
for _, service := range task.Services {
serviceID := makeTaskServiceID(allocID, task.Name, service)
serviceID := makeTaskServiceID(task.AllocID, task.Name, service, task.Canary)
for _, check := range service.Checks {
if check.TriggersRestarts() {
checkID := makeCheckID(serviceID, check)
c.checkWatcher.Watch(allocID, task.Name, checkID, check, restarter)
c.checkWatcher.Watch(task.AllocID, task.Name, checkID, check, task.Restarter)
}
}
}
@ -756,19 +764,19 @@ func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restart
// changed.
//
// DriverNetwork must not change between invocations for the same allocation.
func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Task, restarter TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error {
func (c *ServiceClient) UpdateTask(old, newTask *TaskServices) error {
ops := &operations{}
taskReg := new(TaskRegistration)
taskReg.Services = make(map[string]*ServiceRegistration, len(newTask.Services))
existingIDs := make(map[string]*structs.Service, len(existing.Services))
for _, s := range existing.Services {
existingIDs[makeTaskServiceID(allocID, existing.Name, s)] = s
existingIDs := make(map[string]*structs.Service, len(old.Services))
for _, s := range old.Services {
existingIDs[makeTaskServiceID(old.AllocID, old.Name, s, old.Canary)] = s
}
newIDs := make(map[string]*structs.Service, len(newTask.Services))
for _, s := range newTask.Services {
newIDs[makeTaskServiceID(allocID, newTask.Name, s)] = s
newIDs[makeTaskServiceID(newTask.AllocID, newTask.Name, s, newTask.Canary)] = s
}
// Loop over existing Service IDs to see if they have been removed or
@ -816,7 +824,7 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta
}
// New check on an unchanged service; add them now
newCheckIDs, err := c.checkRegs(ops, allocID, existingID, newSvc, newTask, exec, net)
newCheckIDs, err := c.checkRegs(ops, existingID, newSvc, newTask)
if err != nil {
return err
}
@ -828,7 +836,7 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta
// Update all watched checks as CheckRestart fields aren't part of ID
if check.TriggersRestarts() {
c.checkWatcher.Watch(allocID, newTask.Name, checkID, check, restarter)
c.checkWatcher.Watch(newTask.AllocID, newTask.Name, checkID, check, newTask.Restarter)
}
}
@ -845,7 +853,7 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta
// Any remaining services should just be enqueued directly
for _, newSvc := range newIDs {
sreg, err := c.serviceRegs(ops, allocID, newSvc, newTask, exec, net)
sreg, err := c.serviceRegs(ops, newSvc, newTask)
if err != nil {
return err
}
@ -854,18 +862,18 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta
}
// Add the task to the allocation's registration
c.addTaskRegistration(allocID, newTask.Name, taskReg)
c.addTaskRegistration(newTask.AllocID, newTask.Name, taskReg)
c.commit(ops)
// Start watching checks. Done after service registrations are built
// since an error building them could leak watches.
for _, service := range newIDs {
serviceID := makeTaskServiceID(allocID, newTask.Name, service)
serviceID := makeTaskServiceID(newTask.AllocID, newTask.Name, service, newTask.Canary)
for _, check := range service.Checks {
if check.TriggersRestarts() {
checkID := makeCheckID(serviceID, check)
c.checkWatcher.Watch(allocID, newTask.Name, checkID, check, restarter)
c.checkWatcher.Watch(newTask.AllocID, newTask.Name, checkID, check, newTask.Restarter)
}
}
}
@ -875,11 +883,11 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta
// RemoveTask from Consul. Removes all service entries and checks.
//
// Actual communication with Consul is done asynchronously (see Run).
func (c *ServiceClient) RemoveTask(allocID string, task *structs.Task) {
func (c *ServiceClient) RemoveTask(task *TaskServices) {
ops := operations{}
for _, service := range task.Services {
id := makeTaskServiceID(allocID, task.Name, service)
id := makeTaskServiceID(task.AllocID, task.Name, service, task.Canary)
ops.deregServices = append(ops.deregServices, id)
for _, check := range service.Checks {
@ -893,7 +901,7 @@ func (c *ServiceClient) RemoveTask(allocID string, task *structs.Task) {
}
// Remove the task from the alloc's registrations
c.removeTaskRegistration(allocID, task.Name)
c.removeTaskRegistration(task.AllocID, task.Name)
// Now add them to the deregistration fields; main Run loop will update
c.commit(&ops)
@ -1037,7 +1045,7 @@ func (c *ServiceClient) removeTaskRegistration(allocID, taskName string) {
// Example Client ID: _nomad-client-ggnjpgl7yn7rgmvxzilmpvrzzvrszc7l
//
func makeAgentServiceID(role string, service *structs.Service) string {
return fmt.Sprintf("%s-%s-%s", nomadServicePrefix, role, service.Hash(role, ""))
return fmt.Sprintf("%s-%s-%s", nomadServicePrefix, role, service.Hash(role, "", false))
}
// makeTaskServiceID creates a unique ID for identifying a task service in
@ -1045,8 +1053,8 @@ func makeAgentServiceID(role string, service *structs.Service) string {
// Checks. This allows updates to merely compare IDs.
//
// Example Service ID: _nomad-task-TNM333JKJPM5AK4FAS3VXQLXFDWOF4VH
func makeTaskServiceID(allocID, taskName string, service *structs.Service) string {
return nomadTaskPrefix + service.Hash(allocID, taskName)
func makeTaskServiceID(allocID, taskName string, service *structs.Service, canary bool) string {
return nomadTaskPrefix + service.Hash(allocID, taskName, canary)
}
// makeCheckID creates a unique ID for a check.

67
command/agent/consul/structs.go Normal file
View File

@ -0,0 +1,67 @@
package consul
import (
"github.com/hashicorp/nomad/client/driver"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
type TaskServices struct {
AllocID string
// Name of the task
Name string
// Canary indicates whether or not the allocation is a canary
Canary bool
// Restarter allows restarting the task depending on the task's
// check_restart stanzas.
Restarter TaskRestarter
// Services and checks to register for the task.
Services []*structs.Service
// Networks from the task's resources stanza.
Networks structs.Networks
// DriverExec is the script executor for the task's driver.
DriverExec driver.ScriptExecutor
// DriverNetwork is the network specified by the driver and may be nil.
DriverNetwork *cstructs.DriverNetwork
}
func NewTaskServices(alloc *structs.Allocation, task *structs.Task, restarter TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) *TaskServices {
ts := TaskServices{
AllocID: alloc.ID,
Name: task.Name,
Restarter: restarter,
Services: task.Services,
DriverExec: exec,
DriverNetwork: net,
}
if task.Resources != nil {
ts.Networks = task.Resources.Networks
}
if alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Canary {
ts.Canary = true
}
return &ts
}
// Copy method for easing tests
func (t *TaskServices) Copy() *TaskServices {
newTS := new(TaskServices)
*newTS = *t
// Deep copy Services
newTS.Services = make([]*structs.Service, len(t.Services))
for i := range t.Services {
newTS.Services[i] = t.Services[i].Copy()
}
return newTS
}

228
command/agent/consul/unit_test.go
View File

@ -11,7 +11,9 @@ import (
"github.com/hashicorp/consul/api"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/kr/pretty"
@ -25,19 +27,11 @@ const (
yPort = 1235
)
func testTask() *structs.Task {
return &structs.Task{
Name: "taskname",
Resources: &structs.Resources{
Networks: []*structs.NetworkResource{
{
DynamicPorts: []structs.Port{
{Label: "x", Value: xPort},
{Label: "y", Value: yPort},
},
},
},
},
func testTask() *TaskServices {
return &TaskServices{
AllocID: uuid.Generate(),
Name: "taskname",
Restarter: &restartRecorder{},
Services: []*structs.Service{
{
Name: "taskname-service",
@ -45,9 +39,46 @@ func testTask() *structs.Task {
Tags: []string{"tag1", "tag2"},
},
},
Networks: []*structs.NetworkResource{
{
DynamicPorts: []structs.Port{
{Label: "x", Value: xPort},
{Label: "y", Value: yPort},
},
},
},
DriverExec: newMockExec(),
}
}
// mockExec implements the ScriptExecutor interface and will use an alternate
// implementation t.ExecFunc if non-nil.
type mockExec struct {
// Ticked whenever a script is called
execs chan int
// If non-nil will be called by script checks
ExecFunc func(ctx context.Context, cmd string, args []string) ([]byte, int, error)
}
func newMockExec() *mockExec {
return &mockExec{
execs: make(chan int, 100),
}
}
func (m *mockExec) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
select {
case m.execs <- 1:
default:
}
if m.ExecFunc == nil {
// Default impl is just "ok"
return []byte("ok"), 0, nil
}
return m.ExecFunc(ctx, cmd, args)
}
// restartRecorder is a minimal TaskRestarter implementation that simply
// counts how many restarts were triggered.
type restartRecorder struct {
@ -58,33 +89,12 @@ func (r *restartRecorder) Restart(source, reason string, failure bool) {
atomic.AddInt64(&r.restarts, 1)
}
// testFakeCtx contains a fake Consul AgentAPI and implements the Exec
// interface to allow testing without running Consul.
// testFakeCtx contains a fake Consul AgentAPI
type testFakeCtx struct {
ServiceClient *ServiceClient
FakeConsul *MockAgent
Task *structs.Task
Restarter *restartRecorder
// Ticked whenever a script is called
execs chan int
// If non-nil will be called by script checks
ExecFunc func(ctx context.Context, cmd string, args []string) ([]byte, int, error)
}
// Exec implements the ScriptExecutor interface and will use an alternate
// implementation t.ExecFunc if non-nil.
func (t *testFakeCtx) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
select {
case t.execs <- 1:
default:
}
if t.ExecFunc == nil {
// Default impl is just "ok"
return []byte("ok"), 0, nil
}
return t.ExecFunc(ctx, cmd, args)
Task *TaskServices
MockExec *mockExec
}
var errNoOps = fmt.Errorf("testing error: no pending operations")
@ -105,20 +115,19 @@ func (t *testFakeCtx) syncOnce() error {
// A test Task is also provided.
func setupFake(t *testing.T) *testFakeCtx {
fc := NewMockAgent()
tt := testTask()
return &testFakeCtx{
ServiceClient: NewServiceClient(fc, testlog.Logger(t)),
FakeConsul: fc,
Task: testTask(),
Restarter: &restartRecorder{},
execs: make(chan int, 100),
Task: tt,
MockExec: tt.DriverExec.(*mockExec),
}
}
func TestConsul_ChangeTags(t *testing.T) {
ctx := setupFake(t)
allocID := "allocid"
if err := ctx.ServiceClient.RegisterTask(allocID, ctx.Task, ctx.Restarter, nil, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -132,7 +141,7 @@ func TestConsul_ChangeTags(t *testing.T) {
// Query the allocs registrations and then again when we update. The IDs
// should change
reg1, err := ctx.ServiceClient.AllocRegistrations(allocID)
reg1, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID)
if err != nil {
t.Fatalf("Looking up alloc registration failed: %v", err)
}
@ -157,10 +166,9 @@ func TestConsul_ChangeTags(t *testing.T) {
}
}
origTask := ctx.Task
ctx.Task = testTask()
origTask := ctx.Task.Copy()
ctx.Task.Services[0].Tags[0] = "newtag"
if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, nil, nil); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
if err := ctx.syncOnce(); err != nil {
@ -184,7 +192,7 @@ func TestConsul_ChangeTags(t *testing.T) {
}
// Check again and ensure the IDs changed
reg2, err := ctx.ServiceClient.AllocRegistrations(allocID)
reg2, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID)
if err != nil {
t.Fatalf("Looking up alloc registration failed: %v", err)
}
@ -242,7 +250,7 @@ func TestConsul_ChangePorts(t *testing.T) {
},
}
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -285,8 +293,8 @@ func TestConsul_ChangePorts(t *testing.T) {
case "c2":
origScriptKey = k
select {
case <-ctx.execs:
if n := len(ctx.execs); n > 0 {
case <-ctx.MockExec.execs:
if n := len(ctx.MockExec.execs); n > 0 {
t.Errorf("expected 1 exec but found: %d", n+1)
}
case <-time.After(3 * time.Second):
@ -303,8 +311,7 @@ func TestConsul_ChangePorts(t *testing.T) {
}
// Now update the PortLabel on the Service and Check c3
origTask := ctx.Task
ctx.Task = testTask()
origTask := ctx.Task.Copy()
ctx.Task.Services[0].PortLabel = "y"
ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
{
@ -330,7 +337,7 @@ func TestConsul_ChangePorts(t *testing.T) {
// Removed PortLabel; should default to service's (y)
},
}
if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, ctx, nil); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
if err := ctx.syncOnce(); err != nil {
@ -374,8 +381,8 @@ func TestConsul_ChangePorts(t *testing.T) {
t.Errorf("expected key change for %s from %q", v.Name, origScriptKey)
}
select {
case <-ctx.execs:
if n := len(ctx.execs); n > 0 {
case <-ctx.MockExec.execs:
if n := len(ctx.MockExec.execs); n > 0 {
t.Errorf("expected 1 exec but found: %d", n+1)
}
case <-time.After(3 * time.Second):
@ -411,8 +418,7 @@ func TestConsul_ChangeChecks(t *testing.T) {
},
}
allocID := "allocid"
if err := ctx.ServiceClient.RegisterTask(allocID, ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -433,7 +439,7 @@ func TestConsul_ChangeChecks(t *testing.T) {
// Query the allocs registrations and then again when we update. The IDs
// should change
reg1, err := ctx.ServiceClient.AllocRegistrations(allocID)
reg1, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID)
if err != nil {
t.Fatalf("Looking up alloc registration failed: %v", err)
}
@ -489,7 +495,7 @@ func TestConsul_ChangeChecks(t *testing.T) {
PortLabel: "x",
},
}
if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, ctx, nil); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -547,7 +553,7 @@ func TestConsul_ChangeChecks(t *testing.T) {
}
// Check again and ensure the IDs changed
reg2, err := ctx.ServiceClient.AllocRegistrations(allocID)
reg2, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID)
if err != nil {
t.Fatalf("Looking up alloc registration failed: %v", err)
}
@ -603,7 +609,7 @@ func TestConsul_ChangeChecks(t *testing.T) {
PortLabel: "x",
},
}
if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, ctx, nil); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
if err := ctx.syncOnce(); err != nil {
@ -646,7 +652,7 @@ func TestConsul_RegServices(t *testing.T) {
},
}
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, nil, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -677,7 +683,7 @@ func TestConsul_RegServices(t *testing.T) {
// Assert the check update is properly formed
checkUpd := <-ctx.ServiceClient.checkWatcher.checkUpdateCh
if checkUpd.checkRestart.allocID != "allocid" {
if checkUpd.checkRestart.allocID != ctx.Task.AllocID {
t.Fatalf("expected check's allocid to be %q but found %q", "allocid", checkUpd.checkRestart.allocID)
}
if expected := 200 * time.Millisecond; checkUpd.checkRestart.timeLimit != expected {
@ -687,7 +693,7 @@ func TestConsul_RegServices(t *testing.T) {
// Make a change which will register a new service
ctx.Task.Services[0].Name = "taskname-service2"
ctx.Task.Services[0].Tags[0] = "tag3"
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, nil, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -737,7 +743,7 @@ func TestConsul_RegServices(t *testing.T) {
}
// Remove the new task
ctx.ServiceClient.RemoveTask("allocid", ctx.Task)
ctx.ServiceClient.RemoveTask(ctx.Task)
if err := ctx.syncOnce(); err != nil {
t.Fatalf("unexpected error syncing task: %v", err)
}
@ -787,7 +793,7 @@ func TestConsul_ShutdownOK(t *testing.T) {
go ctx.ServiceClient.Run()
// Register a task and agent
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -849,7 +855,7 @@ func TestConsul_ShutdownSlow(t *testing.T) {
// Make Exec slow, but not too slow
waiter := make(chan struct{})
ctx.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
ctx.MockExec.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
select {
case <-waiter:
default:
@ -865,7 +871,7 @@ func TestConsul_ShutdownSlow(t *testing.T) {
go ctx.ServiceClient.Run()
// Register a task and agent
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -924,7 +930,7 @@ func TestConsul_ShutdownBlocked(t *testing.T) {
// Make Exec block forever
waiter := make(chan struct{})
ctx.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
ctx.MockExec.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
close(waiter)
<-block
return []byte{}, 0, nil
@ -936,7 +942,7 @@ func TestConsul_ShutdownBlocked(t *testing.T) {
go ctx.ServiceClient.Run()
// Register a task and agent
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -988,7 +994,7 @@ func TestConsul_CancelScript(t *testing.T) {
},
}
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -1007,7 +1013,7 @@ func TestConsul_CancelScript(t *testing.T) {
for i := 0; i < 2; i++ {
select {
case <-ctx.execs:
case <-ctx.MockExec.execs:
// Script ran as expected!
case <-time.After(3 * time.Second):
t.Fatalf("timed out waiting for script check to run")
@ -1025,7 +1031,7 @@ func TestConsul_CancelScript(t *testing.T) {
},
}
if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, ctx.Restarter, ctx, nil); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -1044,7 +1050,7 @@ func TestConsul_CancelScript(t *testing.T) {
// Make sure exec wasn't called again
select {
case <-ctx.execs:
case <-ctx.MockExec.execs:
t.Errorf("unexpected execution of script; was goroutine not cancelled?")
case <-time.After(100 * time.Millisecond):
// No unexpected script execs
@ -1104,7 +1110,7 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) {
},
}
net := &cstructs.DriverNetwork{
ctx.Task.DriverNetwork = &cstructs.DriverNetwork{
PortMap: map[string]int{
"x": 8888,
"y": 9999,
@ -1113,7 +1119,7 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) {
AutoAdvertise: true,
}
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, net); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -1129,9 +1135,9 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) {
switch v.Name {
case ctx.Task.Services[0].Name: // x
// Since DriverNetwork.AutoAdvertise=true, driver ports should be used
if v.Port != net.PortMap["x"] {
if v.Port != ctx.Task.DriverNetwork.PortMap["x"] {
t.Errorf("expected service %s's port to be %d but found %d",
v.Name, net.PortMap["x"], v.Port)
v.Name, ctx.Task.DriverNetwork.PortMap["x"], v.Port)
}
// The order of checks in Consul is not guaranteed to
// be the same as their order in the Task definition,
@ -1159,13 +1165,13 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) {
}
case ctx.Task.Services[1].Name: // y
// Service should be container ip:port
if v.Address != net.IP {
if v.Address != ctx.Task.DriverNetwork.IP {
t.Errorf("expected service %s's address to be %s but found %s",
v.Name, net.IP, v.Address)
v.Name, ctx.Task.DriverNetwork.IP, v.Address)
}
if v.Port != net.PortMap["y"] {
if v.Port != ctx.Task.DriverNetwork.PortMap["y"] {
t.Errorf("expected service %s's port to be %d but found %d",
v.Name, net.PortMap["x"], v.Port)
v.Name, ctx.Task.DriverNetwork.PortMap["x"], v.Port)
}
// Check should be host ip:port
if v.Checks[0].TCP != ":1235" { // yPort
@ -1208,7 +1214,7 @@ func TestConsul_DriverNetwork_NoAutoUse(t *testing.T) {
},
}
net := &cstructs.DriverNetwork{
ctx.Task.DriverNetwork = &cstructs.DriverNetwork{
PortMap: map[string]int{
"x": 8888,
"y": 9999,
@ -1217,7 +1223,7 @@ func TestConsul_DriverNetwork_NoAutoUse(t *testing.T) {
AutoAdvertise: false,
}
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, net); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
@ -1239,13 +1245,13 @@ func TestConsul_DriverNetwork_NoAutoUse(t *testing.T) {
}
case ctx.Task.Services[1].Name: // y + driver mode
// Service should be container ip:port
if v.Address != net.IP {
if v.Address != ctx.Task.DriverNetwork.IP {
t.Errorf("expected service %s's address to be %s but found %s",
v.Name, net.IP, v.Address)
v.Name, ctx.Task.DriverNetwork.IP, v.Address)
}
if v.Port != net.PortMap["y"] {
if v.Port != ctx.Task.DriverNetwork.PortMap["y"] {
t.Errorf("expected service %s's port to be %d but found %d",
v.Name, net.PortMap["x"], v.Port)
v.Name, ctx.Task.DriverNetwork.PortMap["x"], v.Port)
}
case ctx.Task.Services[2].Name: // y + host mode
if v.Port != yPort {
@ -1272,7 +1278,7 @@ func TestConsul_DriverNetwork_Change(t *testing.T) {
},
}
net := &cstructs.DriverNetwork{
ctx.Task.DriverNetwork = &cstructs.DriverNetwork{
PortMap: map[string]int{
"x": 8888,
"y": 9999,
@ -1304,31 +1310,63 @@ func TestConsul_DriverNetwork_Change(t *testing.T) {
}
// Initial service should advertise host port x
if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, net); err != nil {
if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil {
t.Fatalf("unexpected error registering task: %v", err)
}
syncAndAssertPort(xPort)
// UpdateTask to use Host (shouldn't change anything)
orig := ctx.Task.Copy()
origTask := ctx.Task.Copy()
ctx.Task.Services[0].AddressMode = structs.AddressModeHost
if err := ctx.ServiceClient.UpdateTask("allocid", orig, ctx.Task, ctx.Restarter, ctx, net); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error updating task: %v", err)
}
syncAndAssertPort(xPort)
// UpdateTask to use Driver (*should* change IP and port)
orig = ctx.Task.Copy()
origTask = ctx.Task.Copy()
ctx.Task.Services[0].AddressMode = structs.AddressModeDriver
if err := ctx.ServiceClient.UpdateTask("allocid", orig, ctx.Task, ctx.Restarter, ctx, net); err != nil {
if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil {
t.Fatalf("unexpected error updating task: %v", err)
}
syncAndAssertPort(net.PortMap["x"])
syncAndAssertPort(ctx.Task.DriverNetwork.PortMap["x"])
}
// TestConsul_CanaryTags asserts CanaryTags are used when Canary=true
func TestConsul_CanaryTags(t *testing.T) {
t.Parallel()
require := require.New(t)
ctx := setupFake(t)
canaryTags := []string{"tag1", "canary"}
ctx.Task.Canary = true
ctx.Task.Services[0].CanaryTags = canaryTags
require.NoError(ctx.ServiceClient.RegisterTask(ctx.Task))
require.NoError(ctx.syncOnce())
require.Len(ctx.FakeConsul.services, 1)
for _, service := range ctx.FakeConsul.services {
require.Equal(canaryTags, service.Tags)
}
// Disable canary and assert tags are not the canary tags
origTask := ctx.Task.Copy()
ctx.Task.Canary = false
require.NoError(ctx.ServiceClient.UpdateTask(origTask, ctx.Task))
require.NoError(ctx.syncOnce())
require.Len(ctx.FakeConsul.services, 1)
for _, service := range ctx.FakeConsul.services {
require.NotEqual(canaryTags, service.Tags)
}
ctx.ServiceClient.RemoveTask(ctx.Task)
require.NoError(ctx.syncOnce())
require.Len(ctx.FakeConsul.services, 0)
}
// TestConsul_PeriodicSync asserts that Nomad periodically reconciles with

16
command/agent/job_endpoint.go
View File

@ -693,13 +693,14 @@ func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) {
if taskGroup.Update != nil {
tg.Update = &structs.UpdateStrategy{
Stagger: *taskGroup.Update.Stagger,
MaxParallel: *taskGroup.Update.MaxParallel,
HealthCheck: *taskGroup.Update.HealthCheck,
MinHealthyTime: *taskGroup.Update.MinHealthyTime,
HealthyDeadline: *taskGroup.Update.HealthyDeadline,
AutoRevert: *taskGroup.Update.AutoRevert,
Canary: *taskGroup.Update.Canary,
Stagger: *taskGroup.Update.Stagger,
MaxParallel: *taskGroup.Update.MaxParallel,
HealthCheck: *taskGroup.Update.HealthCheck,
MinHealthyTime: *taskGroup.Update.MinHealthyTime,
HealthyDeadline: *taskGroup.Update.HealthyDeadline,
ProgressDeadline: *taskGroup.Update.ProgressDeadline,
AutoRevert: *taskGroup.Update.AutoRevert,
Canary: *taskGroup.Update.Canary,
}
}
@ -743,6 +744,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) {
Name: service.Name,
PortLabel: service.PortLabel,
Tags: service.Tags,
CanaryTags: service.CanaryTags,
AddressMode: service.AddressMode,
}

49
command/agent/job_endpoint_test.go
View File

@ -1161,13 +1161,14 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
},
},
Update: &api.UpdateStrategy{
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(5),
HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual),
MinHealthyTime: helper.TimeToPtr(1 * time.Minute),
HealthyDeadline: helper.TimeToPtr(3 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(1),
Stagger: helper.TimeToPtr(1 * time.Second),
MaxParallel: helper.IntToPtr(5),
HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual),
MinHealthyTime: helper.TimeToPtr(1 * time.Minute),
HealthyDeadline: helper.TimeToPtr(3 * time.Minute),
ProgressDeadline: helper.TimeToPtr(3 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(1),
},
Periodic: &api.PeriodicConfig{
Enabled: helper.BoolToPtr(true),
@ -1222,10 +1223,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Migrate: helper.BoolToPtr(true),
},
Update: &api.UpdateStrategy{
HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Checks),
MinHealthyTime: helper.TimeToPtr(2 * time.Minute),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
AutoRevert: helper.BoolToPtr(true),
HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Checks),
MinHealthyTime: helper.TimeToPtr(2 * time.Minute),
HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
ProgressDeadline: helper.TimeToPtr(5 * time.Minute),
AutoRevert: helper.BoolToPtr(true),
},
Meta: map[string]string{
@ -1253,10 +1255,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Services: []*api.Service{
{
Id: "id",
Name: "serviceA",
Tags: []string{"1", "2"},
PortLabel: "foo",
Id: "id",
Name: "serviceA",
Tags: []string{"1", "2"},
CanaryTags: []string{"3", "4"},
PortLabel: "foo",
CheckRestart: &api.CheckRestart{
Limit: 4,
Grace: helper.TimeToPtr(11 * time.Second),
@ -1446,13 +1449,14 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
Migrate: true,
},
Update: &structs.UpdateStrategy{
Stagger: 1 * time.Second,
MaxParallel: 5,
HealthCheck: structs.UpdateStrategyHealthCheck_Checks,
MinHealthyTime: 2 * time.Minute,
HealthyDeadline: 5 * time.Minute,
AutoRevert: true,
Canary: 1,
Stagger: 1 * time.Second,
MaxParallel: 5,
HealthCheck: structs.UpdateStrategyHealthCheck_Checks,
MinHealthyTime: 2 * time.Minute,
HealthyDeadline: 5 * time.Minute,
ProgressDeadline: 5 * time.Minute,
AutoRevert: true,
Canary: 1,
},
Meta: map[string]string{
"key": "value",
@ -1480,6 +1484,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
{
Name: "serviceA",
Tags: []string{"1", "2"},
CanaryTags: []string{"3", "4"},
PortLabel: "foo",
AddressMode: "auto",
Checks: []*structs.ServiceCheck{

32
command/alloc_status.go
View File

@ -246,34 +246,22 @@ func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength
if alloc.DeploymentID != "" {
health := "unset"
if alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Healthy != nil {
if *alloc.DeploymentStatus.Healthy {
health = "healthy"
} else {
health = "unhealthy"
canary := false
if alloc.DeploymentStatus != nil {
if alloc.DeploymentStatus.Healthy != nil {
if *alloc.DeploymentStatus.Healthy {
health = "healthy"
} else {
health = "unhealthy"
}
}
canary = alloc.DeploymentStatus.Canary
}
basic = append(basic,
fmt.Sprintf("Deployment ID|%s", limit(alloc.DeploymentID, uuidLength)),
fmt.Sprintf("Deployment Health|%s", health))
// Check if this allocation is a canary
deployment, _, err := client.Deployments().Info(alloc.DeploymentID, nil)
if err != nil {
return "", fmt.Errorf("Error querying deployment %q: %s", alloc.DeploymentID, err)
}
canary := false
if state, ok := deployment.TaskGroups[alloc.TaskGroup]; ok {
for _, id := range state.PlacedCanaries {
if id == alloc.ID {
canary = true
break
}
}
}
if canary {
basic = append(basic, fmt.Sprintf("Canary|%v", true))
}

15
command/deployment_status.go
View File

@ -195,7 +195,7 @@ func formatDeployment(d *api.Deployment, uuidLength int) string {
func formatDeploymentGroups(d *api.Deployment, uuidLength int) string {
// Detect if we need to add these columns
canaries, autorevert := false, false
var canaries, autorevert, progressDeadline bool
tgNames := make([]string, 0, len(d.TaskGroups))
for name, state := range d.TaskGroups {
tgNames = append(tgNames, name)
@ -205,6 +205,9 @@ func formatDeploymentGroups(d *api.Deployment, uuidLength int) string {
if state.DesiredCanaries > 0 {
canaries = true
}
if state.ProgressDeadline != 0 {
progressDeadline = true
}
}
// Sort the task group names to get a reliable ordering
@ -223,6 +226,9 @@ func formatDeploymentGroups(d *api.Deployment, uuidLength int) string {
rowString += "Canaries|"
}
rowString += "Placed|Healthy|Unhealthy"
if progressDeadline {
rowString += "|Progress Deadline"
}
rows := make([]string, len(d.TaskGroups)+1)
rows[0] = rowString
@ -245,6 +251,13 @@ func formatDeploymentGroups(d *api.Deployment, uuidLength int) string {
row += fmt.Sprintf("%d|", state.DesiredCanaries)
}
row += fmt.Sprintf("%d|%d|%d", state.PlacedAllocs, state.HealthyAllocs, state.UnhealthyAllocs)
if progressDeadline {
if state.RequireProgressBy.IsZero() {
row += fmt.Sprintf("|%v", "N/A")
} else {
row += fmt.Sprintf("|%v", formatTime(state.RequireProgressBy))
}
}
rows[i] = row
i++
}

161
e2e/consul/canary_tags_test.go Normal file
View File

@ -0,0 +1,161 @@
package consul_test
import (
"flag"
"testing"
"time"
consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/jobspec"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var integration = flag.Bool("integration", false, "run integration tests")
func TestConsul(t *testing.T) {
if !*integration {
t.Skip("skipping test in non-integration mode.")
}
RegisterFailHandler(Fail)
RunSpecs(t, "Consul Canary Tags Test")
}
var _ = Describe("Consul Canary Tags Test", func() {
var (
agent *consulapi.Agent
allocations *api.Allocations
deployments *api.Deployments
jobs *api.Jobs
system *api.System
job *api.Job
specFile string
)
BeforeSuite(func() {
consulConf := consulapi.DefaultConfig()
consulClient, err := consulapi.NewClient(consulConf)
Expect(err).ShouldNot(HaveOccurred())
agent = consulClient.Agent()
conf := api.DefaultConfig()
client, err := api.NewClient(conf)
Expect(err).ShouldNot(HaveOccurred())
allocations = client.Allocations()
deployments = client.Deployments()
jobs = client.Jobs()
system = client.System()
})
JustBeforeEach(func() {
var err error
job, err = jobspec.ParseFile(specFile)
Expect(err).ShouldNot(HaveOccurred())
job.ID = helper.StringToPtr(*job.ID + uuid.Generate()[22:])
resp, _, err := jobs.Register(job, nil)
Expect(err).ShouldNot(HaveOccurred())
Expect(resp.EvalID).ShouldNot(BeEmpty())
})
AfterEach(func() {
jobs.Deregister(*job.ID, true, nil)
system.GarbageCollect()
})
Describe("Consul Canary Tags Test", func() {
Context("Canary Tags", func() {
BeforeEach(func() {
specFile = "input/canary_tags.hcl"
})
It("Should set and unset canary tags", func() {
// Eventually be running and healthy
Eventually(func() []string {
deploys, _, err := jobs.Deployments(*job.ID, nil)
Expect(err).ShouldNot(HaveOccurred())
healthyDeploys := make([]string, 0, len(deploys))
for _, d := range deploys {
if d.Status == "successful" {
healthyDeploys = append(healthyDeploys, d.ID)
}
}
return healthyDeploys
}, 5*time.Second, 20*time.Millisecond).Should(HaveLen(1))
// Start a deployment
job.Meta = map[string]string{"version": "2"}
resp, _, err := jobs.Register(job, nil)
Expect(err).ShouldNot(HaveOccurred())
Expect(resp.EvalID).ShouldNot(BeEmpty())
// Eventually have a canary
var deploys []*api.Deployment
Eventually(func() []*api.Deployment {
deploys, _, err = jobs.Deployments(*job.ID, nil)
Expect(err).ShouldNot(HaveOccurred())
return deploys
}, 2*time.Second, 20*time.Millisecond).Should(HaveLen(2))
var deploy *api.Deployment
Eventually(func() []string {
deploy, _, err = deployments.Info(deploys[0].ID, nil)
Expect(err).ShouldNot(HaveOccurred())
return deploy.TaskGroups["consul_canary_test"].PlacedCanaries
}, 2*time.Second, 20*time.Millisecond).Should(HaveLen(1))
Eventually(func() bool {
allocID := deploy.TaskGroups["consul_canary_test"].PlacedCanaries[0]
alloc, _, err := allocations.Info(allocID, nil)
Expect(err).ShouldNot(HaveOccurred())
return alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Healthy != nil && *alloc.DeploymentStatus.Healthy
}, 3*time.Second, 20*time.Millisecond).Should(BeTrue())
// Check Consul for canary tags
Eventually(func() []string {
services, err := agent.Services()
Expect(err).ShouldNot(HaveOccurred())
for _, v := range services {
if v.Service == "canarytest" {
return v.Tags
}
}
return nil
}, 2*time.Second, 20*time.Millisecond).Should(
Equal([]string{"foo", "canary"}))
// Manually promote
{
resp, _, err := deployments.PromoteAll(deploys[0].ID, nil)
Expect(err).ShouldNot(HaveOccurred())
Expect(resp.EvalID).ShouldNot(BeEmpty())
}
// Eventually canary is removed
Eventually(func() bool {
allocID := deploy.TaskGroups["consul_canary_test"].PlacedCanaries[0]
alloc, _, err := allocations.Info(allocID, nil)
Expect(err).ShouldNot(HaveOccurred())
return alloc.DeploymentStatus.Canary
}, 2*time.Second, 20*time.Millisecond).Should(BeFalse())
// Check Consul canary tags were removed
Eventually(func() []string {
services, err := agent.Services()
Expect(err).ShouldNot(HaveOccurred())
for _, v := range services {
if v.Service == "canarytest" {
return v.Tags
}
}
return nil
}, 2*time.Second, 20*time.Millisecond).Should(
Equal([]string{"foo", "bar"}))
})
})
})
})

36
e2e/consul/input/canary_tags.hcl Normal file
View File

@ -0,0 +1,36 @@
job "consul_canary_test" {
datacenters = ["dc1"]
group "consul_canary_test" {
count = 2
task "consul_canary_test" {
driver = "mock_driver"
config {
run_for = "10m"
exit_code = 9
}
service {
name = "canarytest"
tags = ["foo", "bar"]
canary_tags = ["foo", "canary"]
}
}
update {
max_parallel = 1
canary = 1
min_healthy_time = "1s"
health_check = "task_states"
auto_revert = false
}
restart {
attempts = 0
delay = "0s"
mode = "fail"
}
}
}

3
e2e/rescheduling/input/rescheduling_canary_autorevert.hcl
View File

@ -18,8 +18,9 @@ job "test" {
canary = 3
max_parallel = 1
min_healthy_time = "1s"
healthy_deadline = "1m"
auto_revert = true
healthy_deadline = "2s"
progress_deadline = "3s"
}
restart {

3
e2e/rescheduling/input/rescheduling_maxp.hcl
View File

@ -17,8 +17,9 @@ job "demo2" {
update {
max_parallel = 1
min_healthy_time = "1s"
healthy_deadline = "1m"
auto_revert = false
healthy_deadline = "2s"
progress_deadline = "5s"
}
restart {

5
e2e/rescheduling/input/rescheduling_maxp_autorevert.hcl
View File

@ -10,15 +10,16 @@ job "demo3" {
config {
command = "bash"
args = ["-c", "sleep 5000"]
args = ["-c", "sleep 15000"]
}
}
update {
max_parallel = 1
min_healthy_time = "1s"
healthy_deadline = "1m"
auto_revert = true
healthy_deadline = "2s"
progress_deadline = "3s"
}
restart {

4
e2e/rescheduling/input/rescheduling_update.hcl
View File

@ -16,8 +16,10 @@ job "test4" {
update {
max_parallel = 1
min_healthy_time = "10s"
min_healthy_time = "3s"
auto_revert = false
healthy_deadline = "5s"
progress_deadline = "10s"
}
restart {

26
e2e/rescheduling/server_side_restarts_test.go
View File

@ -168,11 +168,11 @@ var _ = Describe("Server Side Restart Tests", func() {
ConsistOf([]string{"running", "running", "running"}))
})
Context("Updating job to make allocs fail", func() {
It("Should have no rescheduled allocs", func() {
It("Should have rescheduled allocs until progress deadline", func() {
job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"}
_, _, err := jobs.Register(job, nil)
Expect(err).ShouldNot(HaveOccurred())
Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty())
Eventually(allocStatusesRescheduled, 5*time.Second, time.Second).ShouldNot(BeEmpty())
})
})
@ -192,22 +192,23 @@ var _ = Describe("Server Side Restart Tests", func() {
})
Context("Updating job to make allocs fail", func() {
It("Should have no rescheduled allocs", func() {
It("Should have rescheduled allocs until progress deadline", func() {
job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"}
_, _, err := jobs.Register(job, nil)
Expect(err).ShouldNot(HaveOccurred())
Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty())
Eventually(allocStatusesRescheduled, 5*time.Second, time.Second).ShouldNot(BeEmpty())
// Verify new deployment and its status
// Deployment status should be running (because of progress deadline)
time.Sleep(3 * time.Second) //TODO(preetha) figure out why this wasn't working with ginkgo constructs
Eventually(deploymentStatus(), 2*time.Second, time.Second).Should(
ContainElement(structs.DeploymentStatusFailed))
ContainElement(structs.DeploymentStatusRunning))
})
})
})
Context("Reschedule with canary and auto revert ", func() {
Context("Reschedule with canary, auto revert with short progress deadline ", func() {
BeforeEach(func() {
specFile = "input/rescheduling_canary_autorevert.hcl"
})
@ -228,11 +229,10 @@ var _ = Describe("Server Side Restart Tests", func() {
// Wait for the revert
Eventually(allocStatuses, 3*time.Second, time.Second).Should(
ConsistOf([]string{"failed", "failed", "failed", "running", "running", "running"}))
// Verify new deployment and its status
// There should be one successful, one failed, and one more successful (after revert)
time.Sleep(5 * time.Second) //TODO(preetha) figure out why this wasn't working with ginkgo constructs
Eventually(deploymentStatus(), 2*time.Second, time.Second).Should(
Eventually(deploymentStatus(), 5*time.Second, time.Second).Should(
ConsistOf(structs.DeploymentStatusSuccessful, structs.DeploymentStatusFailed, structs.DeploymentStatusSuccessful))
})
@ -252,11 +252,11 @@ var _ = Describe("Server Side Restart Tests", func() {
})
Context("Updating job to make allocs fail", func() {
It("Should have no rescheduled allocs", func() {
It("Should have rescheduled allocs till progress deadline", func() {
job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"}
_, _, err := jobs.Register(job, nil)
Expect(err).ShouldNot(HaveOccurred())
Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty())
Eventually(allocStatusesRescheduled, 3*time.Second, time.Second).ShouldNot(BeEmpty())
// Should have 1 failed from max_parallel
Eventually(allocStatuses, 3*time.Second, time.Second).Should(
@ -265,13 +265,13 @@ var _ = Describe("Server Side Restart Tests", func() {
// Verify new deployment and its status
time.Sleep(2 * time.Second)
Eventually(deploymentStatus(), 2*time.Second, time.Second).Should(
ContainElement(structs.DeploymentStatusFailed))
ContainElement(structs.DeploymentStatusRunning))
})
})
})
Context("Reschedule with max parallel and auto revert true ", func() {
Context("Reschedule with max parallel, auto revert true and short progress deadline", func() {
BeforeEach(func() {
specFile = "input/rescheduling_maxp_autorevert.hcl"
})
@ -290,7 +290,7 @@ var _ = Describe("Server Side Restart Tests", func() {
Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty())
// Wait for the revert
Eventually(allocStatuses, 3*time.Second, time.Second).Should(
Eventually(allocStatuses, 5*time.Second, time.Second).Should(
ConsistOf([]string{"complete", "failed", "running", "running", "running"}))
// Verify new deployment and its status

2
helper/stats/cpu.go
View File

@ -39,7 +39,7 @@ func Init() error {
var cpuInfo []cpu.InfoStat
ctx, _ := context.WithTimeout(context.Background(), cpuInfoTimeout)
if cpuInfo, err = cpu.InfoWithContext(ctx); err != nil {
merrs = multierror.Append(merrs, fmt.Errorf("Unable to obtain CPU information: %v", initErr))
merrs = multierror.Append(merrs, fmt.Errorf("Unable to obtain CPU information: %v", err))
}
for _, cpu := range cpuInfo {

2
jobspec/parse.go
View File

@ -987,6 +987,7 @@ func parseServices(jobName string, taskGroupName string, task *api.Task, service
valid := []string{
"name",
"tags",
"canary_tags",
"port",
"check",
"address_mode",
@ -1322,6 +1323,7 @@ func parseUpdate(result **api.UpdateStrategy, list *ast.ObjectList) error {
"health_check",
"min_healthy_time",
"healthy_deadline",
"progress_deadline",
"auto_revert",
"canary",
}

33
jobspec/parse_test.go
View File

@ -47,13 +47,14 @@ func TestParse(t *testing.T) {
},
Update: &api.UpdateStrategy{
Stagger: helper.TimeToPtr(60 * time.Second),
MaxParallel: helper.IntToPtr(2),
HealthCheck: helper.StringToPtr("manual"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(10 * time.Minute),
AutoRevert: helper.BoolToPtr(true),
Canary: helper.IntToPtr(1),
Stagger: helper.TimeToPtr(60 * time.Second),
MaxParallel: helper.IntToPtr(2),
HealthCheck: helper.StringToPtr("manual"),
MinHealthyTime: helper.TimeToPtr(10 * time.Second),
HealthyDeadline: helper.TimeToPtr(10 * time.Minute),
ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
AutoRevert: helper.BoolToPtr(true),
Canary: helper.IntToPtr(1),
},
TaskGroups: []*api.TaskGroup{
@ -103,12 +104,13 @@ func TestParse(t *testing.T) {
SizeMB: helper.IntToPtr(150),
},
Update: &api.UpdateStrategy{
MaxParallel: helper.IntToPtr(3),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(1 * time.Second),
HealthyDeadline: helper.TimeToPtr(1 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(2),
MaxParallel: helper.IntToPtr(3),
HealthCheck: helper.StringToPtr("checks"),
MinHealthyTime: helper.TimeToPtr(1 * time.Second),
HealthyDeadline: helper.TimeToPtr(1 * time.Minute),
ProgressDeadline: helper.TimeToPtr(1 * time.Minute),
AutoRevert: helper.BoolToPtr(false),
Canary: helper.IntToPtr(2),
},
Migrate: &api.MigrateStrategy{
MaxParallel: helper.IntToPtr(2),
@ -131,8 +133,9 @@ func TestParse(t *testing.T) {
},
Services: []*api.Service{
{
Tags: []string{"foo", "bar"},
PortLabel: "http",
Tags: []string{"foo", "bar"},
CanaryTags: []string{"canary", "bam"},
PortLabel: "http",
Checks: []api.ServiceCheck{
{
Name: "check-name",

3
jobspec/test-fixtures/basic.hcl
View File

@ -22,6 +22,7 @@ job "binstore-storagelocker" {
health_check = "manual"
min_healthy_time = "10s"
healthy_deadline = "10m"
progress_deadline = "10m"
auto_revert = true
canary = 1
}
@ -63,6 +64,7 @@ job "binstore-storagelocker" {
health_check = "checks"
min_healthy_time = "1s"
healthy_deadline = "1m"
progress_deadline = "1m"
auto_revert = false
canary = 2
}
@ -99,6 +101,7 @@ job "binstore-storagelocker" {
service {
tags = ["foo", "bar"]
canary_tags = ["canary", "bam"]
port = "http"
check {

8
nomad/deployment_endpoint_test.go
View File

@ -472,9 +472,9 @@ func TestDeploymentEndpoint_Promote(t *testing.T) {
j := mock.Job()
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.Canary = 2
j.TaskGroups[0].Update.Canary = 1
d := mock.Deployment()
d.TaskGroups["web"].DesiredCanaries = 2
d.TaskGroups["web"].DesiredCanaries = 1
d.JobID = j.ID
a := mock.Alloc()
d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID}
@ -536,9 +536,9 @@ func TestDeploymentEndpoint_Promote_ACL(t *testing.T) {
j := mock.Job()
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.Canary = 2
j.TaskGroups[0].Update.Canary = 1
d := mock.Deployment()
d.TaskGroups["web"].DesiredCanaries = 2
d.TaskGroups["web"].DesiredCanaries = 1
d.JobID = j.ID
a := mock.Alloc()
d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID}

13
nomad/deployment_watcher_shims.go
View File

@ -25,14 +25,6 @@ func (d *deploymentWatcherRaftShim) convertApplyErrors(applyResp interface{}, in
return index, err
}
func (d *deploymentWatcherRaftShim) UpsertEvals(evals []*structs.Evaluation) (uint64, error) {
update := &structs.EvalUpdateRequest{
Evals: evals,
}
fsmErrIntf, index, raftErr := d.apply(structs.EvalUpdateRequestType, update)
return d.convertApplyErrors(fsmErrIntf, index, raftErr)
}
func (d *deploymentWatcherRaftShim) UpsertJob(job *structs.Job) (uint64, error) {
job.SetSubmitTime()
update := &structs.JobRegisterRequest{
@ -56,3 +48,8 @@ func (d *deploymentWatcherRaftShim) UpdateDeploymentAllocHealth(req *structs.App
fsmErrIntf, index, raftErr := d.apply(structs.DeploymentAllocHealthRequestType, req)
return d.convertApplyErrors(fsmErrIntf, index, raftErr)
}
func (d *deploymentWatcherRaftShim) UpdateAllocDesiredTransition(req *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) {
fsmErrIntf, index, raftErr := d.apply(structs.AllocUpdateDesiredTransitionRequestType, req)
return d.convertApplyErrors(fsmErrIntf, index, raftErr)
}

81
nomad/deploymentwatcher/batcher.go
View File

@ -7,58 +7,62 @@ import (
"github.com/hashicorp/nomad/nomad/structs"
)
// EvalBatcher is used to batch the creation of evaluations
type EvalBatcher struct {
// AllocUpdateBatcher is used to batch the updates to the desired transitions
// of allocations and the creation of evals.
type AllocUpdateBatcher struct {
// batch is the batching duration
batch time.Duration
// raft is used to actually commit the evaluations
// raft is used to actually commit the updates
raft DeploymentRaftEndpoints
// workCh is used to pass evaluations to the daemon process
workCh chan *evalWrapper
workCh chan *updateWrapper
// ctx is used to exit the daemon batcher
ctx context.Context
}
// NewEvalBatcher returns an EvalBatcher that uses the passed raft endpoints to
// create the evaluations and exits the batcher when the passed exit channel is
// closed.
func NewEvalBatcher(batchDuration time.Duration, raft DeploymentRaftEndpoints, ctx context.Context) *EvalBatcher {
b := &EvalBatcher{
// NewAllocUpdateBatcher returns an AllocUpdateBatcher that uses the passed raft endpoints to
// create the allocation desired transition updates and new evaluations and
// exits the batcher when the passed exit channel is closed.
func NewAllocUpdateBatcher(batchDuration time.Duration, raft DeploymentRaftEndpoints, ctx context.Context) *AllocUpdateBatcher {
b := &AllocUpdateBatcher{
batch: batchDuration,
raft: raft,
ctx: ctx,
workCh: make(chan *evalWrapper, 10),
workCh: make(chan *updateWrapper, 10),
}
go b.batcher()
return b
}
// CreateEval batches the creation of the evaluation and returns a future that
// tracks the evaluations creation.
func (b *EvalBatcher) CreateEval(e *structs.Evaluation) *EvalFuture {
wrapper := &evalWrapper{
e: e,
f: make(chan *EvalFuture, 1),
// CreateUpdate batches the allocation desired transition update and returns a
// future that tracks the completion of the request.
func (b *AllocUpdateBatcher) CreateUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) *BatchFuture {
wrapper := &updateWrapper{
allocs: allocs,
e: eval,
f: make(chan *BatchFuture, 1),
}
b.workCh <- wrapper
return <-wrapper.f
}
type evalWrapper struct {
e *structs.Evaluation
f chan *EvalFuture
type updateWrapper struct {
allocs map[string]*structs.DesiredTransition
e *structs.Evaluation
f chan *BatchFuture
}
// batcher is the long lived batcher goroutine
func (b *EvalBatcher) batcher() {
func (b *AllocUpdateBatcher) batcher() {
var timerCh <-chan time.Time
allocs := make(map[string]*structs.DesiredTransition)
evals := make(map[string]*structs.Evaluation)
future := NewEvalFuture()
future := NewBatchFuture()
for {
select {
case <-b.ctx.Done():
@ -68,59 +72,68 @@ func (b *EvalBatcher) batcher() {
timerCh = time.After(b.batch)
}
// Store the eval and attach the future
// Store the eval and alloc updates, and attach the future
evals[w.e.DeploymentID] = w.e
for id, upd := range w.allocs {
allocs[id] = upd
}
w.f <- future
case <-timerCh:
// Capture the future and create a new one
f := future
future = NewEvalFuture()
future = NewBatchFuture()
// Shouldn't be possible
if f == nil {
panic("no future")
}
// Capture the evals
all := make([]*structs.Evaluation, 0, len(evals))
// Create the request
req := &structs.AllocUpdateDesiredTransitionRequest{
Allocs: allocs,
Evals: make([]*structs.Evaluation, 0, len(evals)),
}
for _, e := range evals {
all = append(all, e)
req.Evals = append(req.Evals, e)
}
// Upsert the evals in a go routine
go f.Set(b.raft.UpsertEvals(all))
go f.Set(b.raft.UpdateAllocDesiredTransition(req))
// Reset the evals list and timer
evals = make(map[string]*structs.Evaluation)
allocs = make(map[string]*structs.DesiredTransition)
timerCh = nil
}
}
}
// EvalFuture is a future that can be used to retrieve the index the eval was
// BatchFuture is a future that can be used to retrieve the index the eval was
// created at or any error in the creation process
type EvalFuture struct {
type BatchFuture struct {
index uint64
err error
waitCh chan struct{}
}
// NewEvalFuture returns a new EvalFuture
func NewEvalFuture() *EvalFuture {
return &EvalFuture{
// NewBatchFuture returns a new BatchFuture
func NewBatchFuture() *BatchFuture {
return &BatchFuture{
waitCh: make(chan struct{}),
}
}
// Set sets the results of the future, unblocking any client.
func (f *EvalFuture) Set(index uint64, err error) {
func (f *BatchFuture) Set(index uint64, err error) {
f.index = index
f.err = err
close(f.waitCh)
}
// Results returns the creation index and any error.
func (f *EvalFuture) Results() (uint64, error) {
func (f *BatchFuture) Results() (uint64, error) {
<-f.waitCh
return f.index, f.err
}

443
nomad/deploymentwatcher/deployment_watcher.go
View File

@ -2,6 +2,7 @@ package deploymentwatcher
import (
"context"
"fmt"
"log"
"sync"
"time"
@ -21,11 +22,21 @@ const (
perJobEvalBatchPeriod = 1 * time.Second
)
var (
// allowRescheduleTransition is the transition that allows failed
// allocations part of a deployment to be rescheduled. We create a one off
// variable to avoid creating a new object for every request.
allowRescheduleTransition = &structs.DesiredTransition{
Reschedule: helper.BoolToPtr(true),
}
)
// deploymentTriggers are the set of functions required to trigger changes on
// behalf of a deployment
type deploymentTriggers interface {
// createEvaluation is used to create an evaluation.
createEvaluation(eval *structs.Evaluation) (uint64, error)
// createUpdate is used to create allocation desired transition updates and
// an evaluation.
createUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) (uint64, error)
// upsertJob is used to roll back a job when autoreverting for a deployment
upsertJob(job *structs.Job) (uint64, error)
@ -55,6 +66,12 @@ type deploymentWatcher struct {
// state is the state that is watched for state changes.
state *state.StateStore
// deploymentID is the deployment's ID being watched
deploymentID string
// deploymentUpdateCh is triggered when there is an updated deployment
deploymentUpdateCh chan struct{}
// d is the deployment being watched
d *structs.Deployment
@ -62,9 +79,13 @@ type deploymentWatcher struct {
j *structs.Job
// outstandingBatch marks whether an outstanding function exists to create
// the evaluation. Access should be done through the lock
// the evaluation. Access should be done through the lock.
outstandingBatch bool
// outstandingAllowReplacements is the map of allocations that will be
// marked as allowing a replacement. Access should be done through the lock.
outstandingAllowReplacements map[string]*structs.DesiredTransition
// latestEval is the latest eval for the job. It is updated by the watch
// loop and any time an evaluation is created. The field should be accessed
// by holding the lock or using the setter and getter methods.
@ -85,6 +106,8 @@ func newDeploymentWatcher(parent context.Context, queryLimiter *rate.Limiter,
ctx, exitFn := context.WithCancel(parent)
w := &deploymentWatcher{
queryLimiter: queryLimiter,
deploymentID: d.ID,
deploymentUpdateCh: make(chan struct{}, 1),
d: d,
j: j,
state: state,
@ -100,6 +123,26 @@ func newDeploymentWatcher(parent context.Context, queryLimiter *rate.Limiter,
return w
}
// updateDeployment is used to update the tracked deployment.
func (w *deploymentWatcher) updateDeployment(d *structs.Deployment) {
w.l.Lock()
defer w.l.Unlock()
// Update and trigger
w.d = d
select {
case w.deploymentUpdateCh <- struct{}{}:
default:
}
}
// getDeployment returns the tracked deployment.
func (w *deploymentWatcher) getDeployment() *structs.Deployment {
w.l.RLock()
defer w.l.RUnlock()
return w.d
}
func (w *deploymentWatcher) SetAllocHealth(
req *structs.DeploymentAllocHealthRequest,
resp *structs.DeploymentUpdateResponse) error {
@ -137,7 +180,7 @@ func (w *deploymentWatcher) SetAllocHealth(
}
// Check if the group has autorevert set
group, ok := w.d.TaskGroups[alloc.TaskGroup]
group, ok := w.getDeployment().TaskGroups[alloc.TaskGroup]
if !ok || !group.AutoRevert {
continue
}
@ -163,9 +206,10 @@ func (w *deploymentWatcher) SetAllocHealth(
// Create the request
areq := &structs.ApplyDeploymentAllocHealthRequest{
DeploymentAllocHealthRequest: *req,
Eval: w.getEval(),
DeploymentUpdate: u,
Job: j,
Timestamp: time.Now(),
Eval: w.getEval(),
DeploymentUpdate: u,
Job: j,
}
index, err := w.upsertDeploymentAllocHealth(areq)
@ -264,7 +308,7 @@ func (w *deploymentWatcher) FailDeployment(
// Determine if we should rollback
rollback := false
for _, state := range w.d.TaskGroups {
for _, state := range w.getDeployment().TaskGroups {
if state.AutoRevert {
rollback = true
break
@ -312,100 +356,273 @@ func (w *deploymentWatcher) StopWatch() {
w.exitFn()
}
// watch is the long running watcher that takes actions upon allocation changes
// watch is the long running watcher that watches for both allocation and
// deployment changes. Its function is to create evaluations to trigger the
// scheduler when more progress can be made, to fail the deployment if it has
// failed and potentially rolling back the job. Progress can be made when an
// allocation transitions to healthy, so we create an eval.
func (w *deploymentWatcher) watch() {
// Get the deadline. This is likely a zero time to begin with but we need to
// handle the case that the deployment has already progressed and we are now
// just starting to watch it. This must likely would occur if there was a
// leader transition and we are now starting our watcher.
currentDeadline := getDeploymentProgressCutoff(w.getDeployment())
var deadlineTimer *time.Timer
if currentDeadline.IsZero() {
deadlineTimer = time.NewTimer(0)
if !deadlineTimer.Stop() {
<-deadlineTimer.C
}
} else {
deadlineTimer = time.NewTimer(currentDeadline.Sub(time.Now()))
}
allocIndex := uint64(1)
var updates *allocUpdates
rollback, deadlineHit := false, false
FAIL:
for {
// Block getting all allocations that are part of the deployment using
// the last evaluation index. This will have us block waiting for
// something to change past what the scheduler has evaluated.
allocs, index, err := w.getAllocs(allocIndex)
if err != nil {
if err == context.Canceled || w.ctx.Err() == context.Canceled {
return
}
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to retrieve allocations for deployment %q: %v", w.d.ID, err)
select {
case <-w.ctx.Done():
return
}
allocIndex = index
// Get the latest evaluation index
latestEval, err := w.latestEvalIndex()
if err != nil {
if err == context.Canceled || w.ctx.Err() == context.Canceled {
return
case <-deadlineTimer.C:
// We have hit the progress deadline so fail the deployment. We need
// to determine whether we should roll back the job by inspecting
// which allocs as part of the deployment are healthy and which
// aren't.
deadlineHit = true
fail, rback, err := w.shouldFail()
if err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to determine whether to rollback job for deployment %q: %v", w.deploymentID, err)
}
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to determine last evaluation index for job %q: %v", w.d.JobID, err)
return
}
// Create an evaluation trigger if there is any allocation whose
// deployment status has been updated past the latest eval index.
createEval, failDeployment, rollback := false, false, false
for _, alloc := range allocs {
if alloc.DeploymentStatus == nil || alloc.DeploymentStatus.ModifyIndex <= latestEval {
if !fail {
w.logger.Printf("[DEBUG] nomad.deployment_watcher: skipping deadline for deployment %q", w.deploymentID)
continue
}
// We need to create an eval
createEval = true
w.logger.Printf("[DEBUG] nomad.deployment_watcher: deadline for deployment %q hit and rollback is %v", w.deploymentID, rback)
rollback = rback
break FAIL
case <-w.deploymentUpdateCh:
// Get the updated deployment and check if we should change the
// deadline timer
next := getDeploymentProgressCutoff(w.getDeployment())
if !next.Equal(currentDeadline) {
prevDeadlineZero := currentDeadline.IsZero()
currentDeadline = next
// The most recent deadline can be zero if no allocs were created for this deployment.
// The deadline timer would have already been stopped once in that case. To prevent
// deadlocking on the already stopped deadline timer, we only drain the channel if
// the previous deadline was not zero.
if !prevDeadlineZero && !deadlineTimer.Stop() {
select {
case <-deadlineTimer.C:
default:
}
}
deadlineTimer.Reset(next.Sub(time.Now()))
}
if alloc.DeploymentStatus.IsUnhealthy() {
// Check if the group has autorevert set
group, ok := w.d.TaskGroups[alloc.TaskGroup]
if ok && group.AutoRevert {
rollback = true
case updates = <-w.getAllocsCh(allocIndex):
if err := updates.err; err != nil {
if err == context.Canceled || w.ctx.Err() == context.Canceled {
return
}
// Since we have an unhealthy allocation, fail the deployment
failDeployment = true
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to retrieve allocations for deployment %q: %v", w.deploymentID, err)
return
}
allocIndex = updates.index
// All conditions have been hit so we can break
if createEval && failDeployment && rollback {
break
}
}
// Change the deployments status to failed
if failDeployment {
// Default description
desc := structs.DeploymentStatusDescriptionFailedAllocations
// Rollback to the old job if necessary
var j *structs.Job
if rollback {
var err error
j, err = w.latestStableJob()
if err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to lookup latest stable job for %q: %v", w.d.JobID, err)
// We have allocation changes for this deployment so determine the
// steps to take.
res, err := w.handleAllocUpdate(updates.allocs)
if err != nil {
if err == context.Canceled || w.ctx.Err() == context.Canceled {
return
}
// Description should include that the job is being rolled back to
// version N
if j != nil {
j, desc = w.handleRollbackValidity(j, desc)
} else {
desc = structs.DeploymentStatusDescriptionNoRollbackTarget(desc)
}
w.logger.Printf("[ERR] nomad.deployment_watcher: failed handling allocation updates: %v", err)
return
}
// Update the status of the deployment to failed and create an
// evaluation.
e := w.getEval()
u := w.getDeploymentStatusUpdate(structs.DeploymentStatusFailed, desc)
if index, err := w.upsertDeploymentStatusUpdate(u, e, j); err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to update deployment %q status: %v", w.d.ID, err)
} else {
w.setLatestEval(index)
// The deployment has failed, so break out of the watch loop and
// handle the failure
if res.failDeployment {
rollback = res.rollback
break FAIL
}
} else if createEval {
// Create an eval to push the deployment along
w.createEvalBatched(index)
if res.createEval || len(res.allowReplacements) != 0 {
w.createBatchedUpdate(res.allowReplacements, allocIndex)
}
}
}
// Change the deployments status to failed
desc := structs.DeploymentStatusDescriptionFailedAllocations
if deadlineHit {
desc = structs.DeploymentStatusDescriptionProgressDeadline
}
// Rollback to the old job if necessary
var j *structs.Job
if rollback {
var err error
j, err = w.latestStableJob()
if err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to lookup latest stable job for %q: %v", w.j.ID, err)
}
// Description should include that the job is being rolled back to
// version N
if j != nil {
j, desc = w.handleRollbackValidity(j, desc)
} else {
desc = structs.DeploymentStatusDescriptionNoRollbackTarget(desc)
}
}
// Update the status of the deployment to failed and create an evaluation.
e := w.getEval()
u := w.getDeploymentStatusUpdate(structs.DeploymentStatusFailed, desc)
if index, err := w.upsertDeploymentStatusUpdate(u, e, j); err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to update deployment %q status: %v", w.deploymentID, err)
} else {
w.setLatestEval(index)
}
}
// allocUpdateResult is used to return the desired actions given the newest set
// of allocations for the deployment.
type allocUpdateResult struct {
createEval bool
failDeployment bool
rollback bool
allowReplacements []string
}
// handleAllocUpdate is used to compute the set of actions to take based on the
// updated allocations for the deployment.
func (w *deploymentWatcher) handleAllocUpdate(allocs []*structs.AllocListStub) (allocUpdateResult, error) {
var res allocUpdateResult
// Get the latest evaluation index
latestEval, err := w.latestEvalIndex()
if err != nil {
if err == context.Canceled || w.ctx.Err() == context.Canceled {
return res, err
}
return res, fmt.Errorf("failed to determine last evaluation index for job %q: %v", w.j.ID, err)
}
deployment := w.getDeployment()
for _, alloc := range allocs {
dstate, ok := deployment.TaskGroups[alloc.TaskGroup]
if !ok {
continue
}
// Nothing to do for this allocation
if alloc.DeploymentStatus == nil || alloc.DeploymentStatus.ModifyIndex <= latestEval {
continue
}
// Determine if the update stanza for this group is progress based
progressBased := dstate.ProgressDeadline != 0
// We need to create an eval so the job can progress.
if alloc.DeploymentStatus.IsHealthy() {
res.createEval = true
} else if progressBased && alloc.DeploymentStatus.IsUnhealthy() && deployment.Active() && !alloc.DesiredTransition.ShouldReschedule() {
res.allowReplacements = append(res.allowReplacements, alloc.ID)
}
// If the group is using a progress deadline, we don't have to do anything.
if progressBased {
continue
}
// Fail on the first bad allocation
if alloc.DeploymentStatus.IsUnhealthy() {
// Check if the group has autorevert set
if dstate.AutoRevert {
res.rollback = true
}
// Since we have an unhealthy allocation, fail the deployment
res.failDeployment = true
}
// All conditions have been hit so we can break
if res.createEval && res.failDeployment && res.rollback {
break
}
}
return res, nil
}
// shouldFail returns whether the job should be failed and whether it should
// rolled back to an earlier stable version by examining the allocations in the
// deployment.
func (w *deploymentWatcher) shouldFail() (fail, rollback bool, err error) {
snap, err := w.state.Snapshot()
if err != nil {
return false, false, err
}
d, err := snap.DeploymentByID(nil, w.deploymentID)
if err != nil {
return false, false, err
}
if d == nil {
// The deployment wasn't in the state store, possibly due to a system gc
return false, false, fmt.Errorf("deployment id not found: %q", w.deploymentID)
}
fail = false
for tg, state := range d.TaskGroups {
// If we are in a canary state we fail if there aren't enough healthy
// allocs to satisfy DesiredCanaries
if state.DesiredCanaries > 0 && !state.Promoted {
if state.HealthyAllocs >= state.DesiredCanaries {
continue
}
} else if state.HealthyAllocs >= state.DesiredTotal {
continue
}
// We have failed this TG
fail = true
// We don't need to autorevert this group
upd := w.j.LookupTaskGroup(tg).Update
if upd == nil || !upd.AutoRevert {
continue
}
// Unhealthy allocs and we need to autorevert
return true, true, nil
}
return fail, false, nil
}
// getDeploymentProgressCutoff returns the progress cutoff for the given
// deployment
func getDeploymentProgressCutoff(d *structs.Deployment) time.Time {
var next time.Time
for _, state := range d.TaskGroups {
if next.IsZero() || state.RequireProgressBy.Before(next) {
next = state.RequireProgressBy
}
}
return next
}
// latestStableJob returns the latest stable job. It may be nil if none exist
@ -415,7 +632,7 @@ func (w *deploymentWatcher) latestStableJob() (*structs.Job, error) {
return nil, err
}
versions, err := snap.JobVersionsByID(nil, w.d.Namespace, w.d.JobID)
versions, err := snap.JobVersionsByID(nil, w.j.Namespace, w.j.ID)
if err != nil {
return nil, err
}
@ -431,12 +648,21 @@ func (w *deploymentWatcher) latestStableJob() (*structs.Job, error) {
return stable, nil
}
// createEvalBatched creates an eval but batches calls together
func (w *deploymentWatcher) createEvalBatched(forIndex uint64) {
// createBatchedUpdate creates an eval for the given index as well as updating
// the given allocations to allow them to reschedule.
func (w *deploymentWatcher) createBatchedUpdate(allowReplacements []string, forIndex uint64) {
w.l.Lock()
defer w.l.Unlock()
if w.outstandingBatch || forIndex < w.latestEval {
// Store the allocations that can be replaced
for _, allocID := range allowReplacements {
if w.outstandingAllowReplacements == nil {
w.outstandingAllowReplacements = make(map[string]*structs.DesiredTransition, len(allowReplacements))
}
w.outstandingAllowReplacements[allocID] = allowRescheduleTransition
}
if w.outstandingBatch || (forIndex < w.latestEval && len(allowReplacements) == 0) {
return
}
@ -451,18 +677,18 @@ func (w *deploymentWatcher) createEvalBatched(forIndex uint64) {
default:
}
// Create the eval
evalCreateIndex, err := w.createEvaluation(w.getEval())
if err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to create evaluation for deployment %q: %v", w.d.ID, err)
} else {
w.setLatestEval(evalCreateIndex)
}
w.l.Lock()
replacements := w.outstandingAllowReplacements
w.outstandingAllowReplacements = nil
w.outstandingBatch = false
w.l.Unlock()
// Create the eval
if index, err := w.createUpdate(replacements, w.getEval()); err != nil {
w.logger.Printf("[ERR] nomad.deployment_watcher: failed to create evaluation for deployment %q: %v", w.deploymentID, err)
} else {
w.setLatestEval(index)
}
})
}
@ -475,7 +701,7 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation {
Type: w.j.Type,
TriggeredBy: structs.EvalTriggerDeploymentWatcher,
JobID: w.j.ID,
DeploymentID: w.d.ID,
DeploymentID: w.deploymentID,
Status: structs.EvalStatusPending,
}
}
@ -483,12 +709,34 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation {
// getDeploymentStatusUpdate returns a deployment status update
func (w *deploymentWatcher) getDeploymentStatusUpdate(status, desc string) *structs.DeploymentStatusUpdate {
return &structs.DeploymentStatusUpdate{
DeploymentID: w.d.ID,
DeploymentID: w.deploymentID,
Status: status,
StatusDescription: desc,
}
}
type allocUpdates struct {
allocs []*structs.AllocListStub
index uint64
err error
}
// getAllocsCh retrieves the allocations that are part of the deployment blocking
// at the given index.
func (w *deploymentWatcher) getAllocsCh(index uint64) <-chan *allocUpdates {
out := make(chan *allocUpdates, 1)
go func() {
allocs, index, err := w.getAllocs(index)
out <- &allocUpdates{
allocs: allocs,
index: index,
err: err,
}
}()
return out
}
// getAllocs retrieves the allocations that are part of the deployment blocking
// at the given index.
func (w *deploymentWatcher) getAllocs(index uint64) ([]*structs.AllocListStub, uint64, error) {
@ -510,7 +758,7 @@ func (w *deploymentWatcher) getAllocsImpl(ws memdb.WatchSet, state *state.StateS
}
// Capture all the allocations
allocs, err := state.AllocsByDeployment(ws, w.d.ID)
allocs, err := state.AllocsByDeployment(ws, w.deploymentID)
if err != nil {
return nil, 0, err
}
@ -542,7 +790,7 @@ func (w *deploymentWatcher) latestEvalIndex() (uint64, error) {
return 0, err
}
evals, err := snap.EvalsByJob(nil, w.d.Namespace, w.d.JobID)
evals, err := snap.EvalsByJob(nil, w.j.Namespace, w.j.ID)
if err != nil {
return 0, err
}
@ -552,6 +800,7 @@ func (w *deploymentWatcher) latestEvalIndex() (uint64, error) {
if err != nil {
w.setLatestEval(idx)
}
return idx, err
}

50
nomad/deploymentwatcher/deployments_watcher.go
View File

@ -19,9 +19,10 @@ const (
// second
LimitStateQueriesPerSecond = 100.0
// CrossDeploymentEvalBatchDuration is the duration in which evaluations are
// batched across all deployment watchers before committing to Raft.
CrossDeploymentEvalBatchDuration = 250 * time.Millisecond
// CrossDeploymentUpdateBatchDuration is the duration in which allocation
// desired transition and evaluation creation updates are batched across
// all deployment watchers before committing to Raft.
CrossDeploymentUpdateBatchDuration = 250 * time.Millisecond
)
var (
@ -33,9 +34,6 @@ var (
// DeploymentRaftEndpoints exposes the deployment watcher to a set of functions
// to apply data transforms via Raft.
type DeploymentRaftEndpoints interface {
// UpsertEvals is used to upsert a set of evaluations
UpsertEvals([]*structs.Evaluation) (uint64, error)
// UpsertJob is used to upsert a job
UpsertJob(job *structs.Job) (uint64, error)
@ -49,6 +47,10 @@ type DeploymentRaftEndpoints interface {
// UpdateDeploymentAllocHealth is used to set the health of allocations in a
// deployment
UpdateDeploymentAllocHealth(req *structs.ApplyDeploymentAllocHealthRequest) (uint64, error)
// UpdateAllocDesiredTransition is used to update the desired transition
// for allocations.
UpdateAllocDesiredTransition(req *structs.AllocUpdateDesiredTransitionRequest) (uint64, error)
}
// Watcher is used to watch deployments and their allocations created
@ -61,9 +63,9 @@ type Watcher struct {
// queryLimiter is used to limit the rate of blocking queries
queryLimiter *rate.Limiter
// evalBatchDuration is the duration to batch eval creation across all
// deployment watchers
evalBatchDuration time.Duration
// updateBatchDuration is the duration to batch allocation desired
// transition and eval creation across all deployment watchers
updateBatchDuration time.Duration
// raft contains the set of Raft endpoints that can be used by the
// deployments watcher
@ -75,8 +77,9 @@ type Watcher struct {
// watchers is the set of active watchers, one per deployment
watchers map[string]*deploymentWatcher
// evalBatcher is used to batch the creation of evaluations
evalBatcher *EvalBatcher
// allocUpdateBatcher is used to batch the creation of evaluations and
// allocation desired transition updates
allocUpdateBatcher *AllocUpdateBatcher
// ctx and exitFn are used to cancel the watcher
ctx context.Context
@ -89,13 +92,13 @@ type Watcher struct {
// deployments and trigger the scheduler as needed.
func NewDeploymentsWatcher(logger *log.Logger,
raft DeploymentRaftEndpoints, stateQueriesPerSecond float64,
evalBatchDuration time.Duration) *Watcher {
updateBatchDuration time.Duration) *Watcher {
return &Watcher{
raft: raft,
queryLimiter: rate.NewLimiter(rate.Limit(stateQueriesPerSecond), 100),
evalBatchDuration: evalBatchDuration,
logger: logger,
raft: raft,
queryLimiter: rate.NewLimiter(rate.Limit(stateQueriesPerSecond), 100),
updateBatchDuration: updateBatchDuration,
logger: logger,
}
}
@ -136,7 +139,7 @@ func (w *Watcher) flush() {
w.watchers = make(map[string]*deploymentWatcher, 32)
w.ctx, w.exitFn = context.WithCancel(context.Background())
w.evalBatcher = NewEvalBatcher(w.evalBatchDuration, w.raft, w.ctx)
w.allocUpdateBatcher = NewAllocUpdateBatcher(w.updateBatchDuration, w.raft, w.ctx)
}
// watchDeployments is the long lived go-routine that watches for deployments to
@ -228,8 +231,9 @@ func (w *Watcher) addLocked(d *structs.Deployment) (*deploymentWatcher, error) {
return nil, fmt.Errorf("deployment %q is terminal", d.ID)
}
// Already watched so no-op
if _, ok := w.watchers[d.ID]; ok {
// Already watched so just update the deployment
if w, ok := w.watchers[d.ID]; ok {
w.updateDeployment(d)
return nil, nil
}
@ -353,10 +357,10 @@ func (w *Watcher) FailDeployment(req *structs.DeploymentFailRequest, resp *struc
return watcher.FailDeployment(req, resp)
}
// createEvaluation commits the given evaluation to Raft but batches the commit
// with other calls.
func (w *Watcher) createEvaluation(eval *structs.Evaluation) (uint64, error) {
return w.evalBatcher.CreateEval(eval).Results()
// createUpdate commits the given allocation desired transition and evaluation
// to Raft but batches the commit with other calls.
func (w *Watcher) createUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) (uint64, error) {
return w.allocUpdateBatcher.CreateUpdate(allocs, eval).Results()
}
// upsertJob commits the given job to Raft

514
nomad/deploymentwatcher/deployments_watcher_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/assert"
mocker "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func testDeploymentWatcher(t *testing.T, qps float64, batchDur time.Duration) (*Watcher, *mockBackend) {
@ -22,7 +23,7 @@ func testDeploymentWatcher(t *testing.T, qps float64, batchDur time.Duration) (*
}
func defaultTestDeploymentWatcher(t *testing.T) (*Watcher, *mockBackend) {
return testDeploymentWatcher(t, LimitStateQueriesPerSecond, CrossDeploymentEvalBatchDuration)
return testDeploymentWatcher(t, LimitStateQueriesPerSecond, CrossDeploymentUpdateBatchDuration)
}
// Tests that the watcher properly watches for deployments and reconciles them
@ -141,10 +142,6 @@ func TestWatcher_SetAllocHealth_Unknown(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentAllocHealth
a := mock.Alloc()
matchConfig := &matchDeploymentAllocHealthRequestConfig{
@ -155,6 +152,10 @@ func TestWatcher_SetAllocHealth_Unknown(t *testing.T) {
matcher := matchDeploymentAllocHealthRequest(matchConfig)
m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call SetAllocHealth
req := &structs.DeploymentAllocHealthRequest{
DeploymentID: d.ID,
@ -184,10 +185,6 @@ func TestWatcher_SetAllocHealth_Healthy(t *testing.T) {
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentAllocHealth
matchConfig := &matchDeploymentAllocHealthRequestConfig{
DeploymentID: d.ID,
@ -197,6 +194,10 @@ func TestWatcher_SetAllocHealth_Healthy(t *testing.T) {
matcher := matchDeploymentAllocHealthRequest(matchConfig)
m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call SetAllocHealth
req := &structs.DeploymentAllocHealthRequest{
DeploymentID: d.ID,
@ -225,10 +226,6 @@ func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) {
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentAllocHealth
matchConfig := &matchDeploymentAllocHealthRequestConfig{
DeploymentID: d.ID,
@ -243,6 +240,10 @@ func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) {
matcher := matchDeploymentAllocHealthRequest(matchConfig)
m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call SetAllocHealth
req := &structs.DeploymentAllocHealthRequest{
DeploymentID: d.ID,
@ -268,6 +269,7 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) {
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.AutoRevert = true
j.TaskGroups[0].Update.ProgressDeadline = 0
j.Stable = true
d := mock.Deployment()
d.JobID = j.ID
@ -286,10 +288,6 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentAllocHealth
matchConfig := &matchDeploymentAllocHealthRequestConfig{
DeploymentID: d.ID,
@ -305,6 +303,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) {
matcher := matchDeploymentAllocHealthRequest(matchConfig)
m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call SetAllocHealth
req := &structs.DeploymentAllocHealthRequest{
DeploymentID: d.ID,
@ -330,6 +332,7 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) {
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.AutoRevert = true
j.TaskGroups[0].Update.ProgressDeadline = 0
j.Stable = true
d := mock.Deployment()
d.JobID = j.ID
@ -346,10 +349,6 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentAllocHealth
matchConfig := &matchDeploymentAllocHealthRequestConfig{
DeploymentID: d.ID,
@ -365,6 +364,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) {
matcher := matchDeploymentAllocHealthRequest(matchConfig)
m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call SetAllocHealth
req := &structs.DeploymentAllocHealthRequest{
DeploymentID: d.ID,
@ -389,10 +392,12 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) {
j := mock.Job()
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.Canary = 2
j.TaskGroups[0].Update.Canary = 1
j.TaskGroups[0].Update.ProgressDeadline = 0
d := mock.Deployment()
d.JobID = j.ID
a := mock.Alloc()
d.TaskGroups[a.TaskGroup].DesiredCanaries = 1
d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID}
a.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
@ -402,10 +407,6 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) {
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentPromotion
matchConfig := &matchDeploymentPromoteRequestConfig{
Promotion: &structs.DeploymentPromoteRequest{
@ -417,6 +418,14 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) {
matcher := matchDeploymentPromoteRequest(matchConfig)
m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil)
// We may get an update for the desired transition.
m1 := matchUpdateAllocDesiredTransitions([]string{d.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once()
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call PromoteDeployment
req := &structs.DeploymentPromoteRequest{
DeploymentID: d.ID,
@ -440,19 +449,17 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) {
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.Canary = 2
j.TaskGroups[0].Update.ProgressDeadline = 0
d := mock.Deployment()
d.JobID = j.ID
a := mock.Alloc()
d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID}
d.TaskGroups[a.TaskGroup].DesiredCanaries = 2
a.DeploymentID = d.ID
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentPromotion
matchConfig := &matchDeploymentPromoteRequestConfig{
Promotion: &structs.DeploymentPromoteRequest{
@ -464,6 +471,10 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) {
matcher := matchDeploymentPromoteRequest(matchConfig)
m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call SetAllocHealth
req := &structs.DeploymentPromoteRequest{
DeploymentID: d.ID,
@ -472,7 +483,7 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) {
var resp structs.DeploymentUpdateResponse
err := w.PromoteDeployment(req, &resp)
if assert.NotNil(err, "PromoteDeployment") {
assert.Contains(err.Error(), "is not healthy", "Should error because canary isn't marked healthy")
assert.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`, "Should error because canary isn't marked healthy")
}
assert.Equal(1, len(w.watchers), "Deployment should still be active")
@ -492,10 +503,6 @@ func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentStatusUpdate
matchConfig := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
@ -505,6 +512,10 @@ func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) {
matcher := matchDeploymentStatusUpdateRequest(matchConfig)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call PauseDeployment
req := &structs.DeploymentPauseRequest{
DeploymentID: d.ID,
@ -532,10 +543,6 @@ func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentStatusUpdate
matchConfig := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
@ -545,6 +552,10 @@ func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) {
matcher := matchDeploymentStatusUpdateRequest(matchConfig)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call PauseDeployment
req := &structs.DeploymentPauseRequest{
DeploymentID: d.ID,
@ -572,10 +583,6 @@ func TestWatcher_PauseDeployment_Unpause_Paused(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentStatusUpdate
matchConfig := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
@ -586,6 +593,10 @@ func TestWatcher_PauseDeployment_Unpause_Paused(t *testing.T) {
matcher := matchDeploymentStatusUpdateRequest(matchConfig)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call PauseDeployment
req := &structs.DeploymentPauseRequest{
DeploymentID: d.ID,
@ -612,10 +623,6 @@ func TestWatcher_PauseDeployment_Unpause_Running(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentStatusUpdate
matchConfig := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
@ -626,6 +633,10 @@ func TestWatcher_PauseDeployment_Unpause_Running(t *testing.T) {
matcher := matchDeploymentStatusUpdateRequest(matchConfig)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call PauseDeployment
req := &structs.DeploymentPauseRequest{
DeploymentID: d.ID,
@ -652,10 +663,6 @@ func TestWatcher_FailDeployment_Running(t *testing.T) {
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we get a call to UpsertDeploymentStatusUpdate
matchConfig := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
@ -666,6 +673,10 @@ func TestWatcher_FailDeployment_Running(t *testing.T) {
matcher := matchDeploymentStatusUpdateRequest(matchConfig)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Call PauseDeployment
req := &structs.DeploymentFailRequest{
DeploymentID: d.ID,
@ -680,7 +691,7 @@ func TestWatcher_FailDeployment_Running(t *testing.T) {
// Tests that the watcher properly watches for allocation changes and takes the
// proper actions
func TestDeploymentWatcher_Watch(t *testing.T) {
func TestDeploymentWatcher_Watch_NoProgressDeadline(t *testing.T) {
t.Parallel()
assert := assert.New(t)
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
@ -690,6 +701,7 @@ func TestDeploymentWatcher_Watch(t *testing.T) {
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.AutoRevert = true
j.TaskGroups[0].Update.ProgressDeadline = 0
j.Stable = true
d := mock.Deployment()
d.JobID = j.ID
@ -707,15 +719,26 @@ func TestDeploymentWatcher_Watch(t *testing.T) {
j2.Stable = false
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2")
// Assert that we will get a update allocation call only once. This will
// verify that the watcher is batching allocation changes
m1 := matchUpdateAllocDesiredTransitions([]string{d.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once()
// Assert that we get a call to UpsertDeploymentStatusUpdate
c := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
Status: structs.DeploymentStatusFailed,
StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0),
JobVersion: helper.Uint64ToPtr(0),
Eval: true,
}
m2 := matchDeploymentStatusUpdateRequest(c)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we will get a createEvaluation call only once. This will
// verify that the watcher is batching allocation changes
m1 := matchUpsertEvals([]string{d.ID})
m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once()
// Update the allocs health to healthy which should create an evaluation
for i := 0; i < 5; i++ {
req := &structs.ApplyDeploymentAllocHealthRequest{
@ -744,17 +767,6 @@ func TestDeploymentWatcher_Watch(t *testing.T) {
t.Fatal(err)
})
// Assert that we get a call to UpsertDeploymentStatusUpdate
c := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
Status: structs.DeploymentStatusFailed,
StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0),
JobVersion: helper.Uint64ToPtr(0),
Eval: true,
}
m2 := matchDeploymentStatusUpdateRequest(c)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil)
// Update the allocs health to unhealthy which should create a job rollback,
// status update and eval
req2 := &structs.ApplyDeploymentAllocHealthRequest{
@ -782,7 +794,7 @@ func TestDeploymentWatcher_Watch(t *testing.T) {
t.Fatal(err)
})
m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1))
m.AssertCalled(t, "UpdateAllocDesiredTransition", mocker.MatchedBy(m1))
// After we upsert the job version will go to 2. So use this to assert the
// original call happened.
@ -799,6 +811,305 @@ func TestDeploymentWatcher_Watch(t *testing.T) {
func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") })
}
func TestDeploymentWatcher_Watch_ProgressDeadline(t *testing.T) {
t.Parallel()
assert := assert.New(t)
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
// Create a job, alloc, and a deployment
j := mock.Job()
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.ProgressDeadline = 500 * time.Millisecond
j.Stable = true
d := mock.Deployment()
d.JobID = j.ID
d.TaskGroups["web"].ProgressDeadline = 500 * time.Millisecond
a := mock.Alloc()
now := time.Now()
a.CreateTime = now.UnixNano()
a.ModifyTime = now.UnixNano()
a.DeploymentID = d.ID
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
// Assert that we get a call to UpsertDeploymentStatusUpdate
c := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
Status: structs.DeploymentStatusFailed,
StatusDescription: structs.DeploymentStatusDescriptionProgressDeadline,
Eval: true,
}
m2 := matchDeploymentStatusUpdateRequest(c)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Update the alloc to be unhealthy and assert that nothing happens.
a2 := a.Copy()
a2.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(false),
Timestamp: now,
}
assert.Nil(m.state.UpdateAllocsFromClient(100, []*structs.Allocation{a2}))
// Wait for the deployment to be failed
testutil.WaitForResult(func() (bool, error) {
d, err := m.state.DeploymentByID(nil, d.ID)
if err != nil {
return false, err
}
return d.Status == structs.DeploymentStatusFailed, fmt.Errorf("bad status %q", d.Status)
}, func(err error) {
t.Fatal(err)
})
// Assert there are is only one evaluation
testutil.WaitForResult(func() (bool, error) {
ws := memdb.NewWatchSet()
evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID)
if err != nil {
return false, err
}
if l := len(evals); l != 1 {
return false, fmt.Errorf("Got %d evals; want 1", l)
}
return true, nil
}, func(err error) {
t.Fatal(err)
})
}
// Test that we will allow the progress deadline to be reached when the canaries
// are healthy but we haven't promoted
func TestDeploymentWatcher_Watch_ProgressDeadline_Canaries(t *testing.T) {
t.Parallel()
require := require.New(t)
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
// Create a job, alloc, and a deployment
j := mock.Job()
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.Canary = 1
j.TaskGroups[0].Update.MaxParallel = 1
j.TaskGroups[0].Update.ProgressDeadline = 500 * time.Millisecond
j.Stable = true
d := mock.Deployment()
d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
d.JobID = j.ID
d.TaskGroups["web"].ProgressDeadline = 500 * time.Millisecond
d.TaskGroups["web"].DesiredCanaries = 1
a := mock.Alloc()
now := time.Now()
a.CreateTime = now.UnixNano()
a.ModifyTime = now.UnixNano()
a.DeploymentID = d.ID
require.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
require.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
require.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
// Assert that we will get a createEvaluation call only once. This will
// verify that the watcher is batching allocation changes
m1 := matchUpdateAllocDesiredTransitions([]string{d.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once()
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { require.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Update the alloc to be unhealthy and require that nothing happens.
a2 := a.Copy()
a2.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Timestamp: now,
}
require.Nil(m.state.UpdateAllocsFromClient(m.nextIndex(), []*structs.Allocation{a2}))
// Wait for the deployment to cross the deadline
dout, err := m.state.DeploymentByID(nil, d.ID)
require.NoError(err)
require.NotNil(dout)
state := dout.TaskGroups["web"]
require.NotNil(state)
time.Sleep(state.RequireProgressBy.Add(time.Second).Sub(now))
// Require the deployment is still running
dout, err = m.state.DeploymentByID(nil, d.ID)
require.NoError(err)
require.NotNil(dout)
require.Equal(structs.DeploymentStatusRunning, dout.Status)
require.Equal(structs.DeploymentStatusDescriptionRunningNeedsPromotion, dout.StatusDescription)
// require there are is only one evaluation
testutil.WaitForResult(func() (bool, error) {
ws := memdb.NewWatchSet()
evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID)
if err != nil {
return false, err
}
if l := len(evals); l != 1 {
return false, fmt.Errorf("Got %d evals; want 1", l)
}
return true, nil
}, func(err error) {
t.Fatal(err)
})
}
// Test that a promoted deployment with alloc healthy updates create
// evals to move the deployment forward
func TestDeploymentWatcher_PromotedCanary_UpdatedAllocs(t *testing.T) {
t.Parallel()
require := require.New(t)
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
// Create a job, alloc, and a deployment
j := mock.Job()
j.TaskGroups[0].Count = 2
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.Canary = 1
j.TaskGroups[0].Update.MaxParallel = 1
j.TaskGroups[0].Update.ProgressDeadline = 50 * time.Millisecond
j.Stable = true
d := mock.Deployment()
d.TaskGroups["web"].DesiredTotal = 2
d.TaskGroups["web"].DesiredCanaries = 1
d.TaskGroups["web"].HealthyAllocs = 1
d.StatusDescription = structs.DeploymentStatusDescriptionRunning
d.JobID = j.ID
d.TaskGroups["web"].ProgressDeadline = 50 * time.Millisecond
d.TaskGroups["web"].RequireProgressBy = time.Now().Add(50 * time.Millisecond)
a := mock.Alloc()
now := time.Now()
a.CreateTime = now.UnixNano()
a.ModifyTime = now.UnixNano()
a.DeploymentID = d.ID
a.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Timestamp: now,
}
require.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
require.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
require.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { require.Equal(1, len(w.watchers), "Should have 1 deployment") })
m1 := matchUpdateAllocDesiredTransitions([]string{d.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Twice()
// Create another alloc
a2 := a.Copy()
a2.ID = uuid.Generate()
now = time.Now()
a2.CreateTime = now.UnixNano()
a2.ModifyTime = now.UnixNano()
a2.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Timestamp: now,
}
d.TaskGroups["web"].RequireProgressBy = time.Now().Add(2 * time.Second)
require.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
// Wait until batch eval period passes before updating another alloc
time.Sleep(1 * time.Second)
require.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a2}), "UpsertAllocs")
// Wait for the deployment to cross the deadline
dout, err := m.state.DeploymentByID(nil, d.ID)
require.NoError(err)
require.NotNil(dout)
state := dout.TaskGroups["web"]
require.NotNil(state)
time.Sleep(state.RequireProgressBy.Add(time.Second).Sub(now))
// There should be two evals
testutil.WaitForResult(func() (bool, error) {
ws := memdb.NewWatchSet()
evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID)
if err != nil {
return false, err
}
if l := len(evals); l != 2 {
return false, fmt.Errorf("Got %d evals; want 2", l)
}
return true, nil
}, func(err error) {
t.Fatal(err)
})
}
// Test scenario where deployment initially has no progress deadline
// After the deployment is updated, a failed alloc's DesiredTransition should be set
func TestDeploymentWatcher_Watch_StartWithoutProgressDeadline(t *testing.T) {
t.Parallel()
assert := assert.New(t)
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
// Create a job, and a deployment
j := mock.Job()
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.ProgressDeadline = 500 * time.Millisecond
j.Stable = true
d := mock.Deployment()
d.JobID = j.ID
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
a := mock.Alloc()
a.CreateTime = time.Now().UnixNano()
a.DeploymentID = d.ID
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
d.TaskGroups["web"].ProgressDeadline = 500 * time.Millisecond
// Update the deployment with a progress deadline
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
// Match on DesiredTransition set to Reschedule for the failed alloc
m1 := matchUpdateAllocDesiredTransitionReschedule([]string{a.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once()
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Update the alloc to be unhealthy
a2 := a.Copy()
a2.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(false),
Timestamp: time.Now(),
}
assert.Nil(m.state.UpdateAllocsFromClient(m.nextIndex(), []*structs.Allocation{a2}))
// Wait for the alloc's DesiredState to set reschedule
testutil.WaitForResult(func() (bool, error) {
a, err := m.state.AllocByID(nil, a.ID)
if err != nil {
return false, err
}
dt := a.DesiredTransition
shouldReschedule := dt.Reschedule != nil && *dt.Reschedule
return shouldReschedule, fmt.Errorf("Desired Transition Reschedule should be set but got %v", shouldReschedule)
}, func(err error) {
t.Fatal(err)
})
}
// Tests that the watcher fails rollback when the spec hasn't changed
func TestDeploymentWatcher_RollbackFailed(t *testing.T) {
t.Parallel()
@ -810,6 +1121,7 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) {
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j.TaskGroups[0].Update.MaxParallel = 2
j.TaskGroups[0].Update.AutoRevert = true
j.TaskGroups[0].Update.ProgressDeadline = 0
j.Stable = true
d := mock.Deployment()
d.JobID = j.ID
@ -826,15 +1138,26 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) {
j2.Stable = false
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2")
// Assert that we will get a createEvaluation call only once. This will
// verify that the watcher is batching allocation changes
m1 := matchUpdateAllocDesiredTransitions([]string{d.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once()
// Assert that we get a call to UpsertDeploymentStatusUpdate with roll back failed as the status
c := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
Status: structs.DeploymentStatusFailed,
StatusDescription: structs.DeploymentStatusDescriptionRollbackNoop(structs.DeploymentStatusDescriptionFailedAllocations, 0),
JobVersion: nil,
Eval: true,
}
m2 := matchDeploymentStatusUpdateRequest(c)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil)
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
// Assert that we will get a createEvaluation call only once. This will
// verify that the watcher is batching allocation changes
m1 := matchUpsertEvals([]string{d.ID})
m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once()
// Update the allocs health to healthy which should create an evaluation
for i := 0; i < 5; i++ {
req := &structs.ApplyDeploymentAllocHealthRequest{
@ -863,17 +1186,6 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) {
t.Fatal(err)
})
// Assert that we get a call to UpsertDeploymentStatusUpdate with roll back failed as the status
c := &matchDeploymentStatusUpdateConfig{
DeploymentID: d.ID,
Status: structs.DeploymentStatusFailed,
StatusDescription: structs.DeploymentStatusDescriptionRollbackNoop(structs.DeploymentStatusDescriptionFailedAllocations, 0),
JobVersion: nil,
Eval: true,
}
m2 := matchDeploymentStatusUpdateRequest(c)
m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil)
// Update the allocs health to unhealthy which will cause attempting a rollback,
// fail in that step, do status update and eval
req2 := &structs.ApplyDeploymentAllocHealthRequest{
@ -901,30 +1213,38 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) {
t.Fatal(err)
})
m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1))
m.AssertCalled(t, "UpdateAllocDesiredTransition", mocker.MatchedBy(m1))
// verify that the job version hasn't changed after upsert
m.state.JobByID(nil, structs.DefaultNamespace, j.ID)
assert.Equal(uint64(0), j.Version, "Expected job version 0 but got ", j.Version)
}
// Test evaluations are batched between watchers
func TestWatcher_BatchEvals(t *testing.T) {
// Test allocation updates and evaluation creation is batched between watchers
func TestWatcher_BatchAllocUpdates(t *testing.T) {
t.Parallel()
assert := assert.New(t)
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Second)
// Create a job, alloc, for two deployments
j1 := mock.Job()
j1.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j1.TaskGroups[0].Update.ProgressDeadline = 0
d1 := mock.Deployment()
d1.JobID = j1.ID
a1 := mock.Alloc()
a1.Job = j1
a1.JobID = j1.ID
a1.DeploymentID = d1.ID
j2 := mock.Job()
j2.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
j2.TaskGroups[0].Update.ProgressDeadline = 0
d2 := mock.Deployment()
d2.JobID = j2.ID
a2 := mock.Alloc()
a2.Job = j2
a2.JobID = j2.ID
a2.DeploymentID = d2.ID
assert.Nil(m.state.UpsertJob(m.nextIndex(), j1), "UpsertJob")
@ -934,15 +1254,15 @@ func TestWatcher_BatchEvals(t *testing.T) {
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a1}), "UpsertAllocs")
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a2}), "UpsertAllocs")
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil },
func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") })
// Assert that we will get a createEvaluation call only once and it contains
// both deployments. This will verify that the watcher is batching
// allocation changes
m1 := matchUpsertEvals([]string{d1.ID, d2.ID})
m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once()
m1 := matchUpdateAllocDesiredTransitions([]string{d1.ID, d2.ID})
m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once()
w.SetEnabled(true, m.state)
testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil },
func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") })
// Update the allocs health to healthy which should create an evaluation
req := &structs.ApplyDeploymentAllocHealthRequest{
@ -975,11 +1295,11 @@ func TestWatcher_BatchEvals(t *testing.T) {
}
if l := len(evals1); l != 1 {
return false, fmt.Errorf("Got %d evals; want 1", l)
return false, fmt.Errorf("Got %d evals for job %v; want 1", l, j1.ID)
}
if l := len(evals2); l != 1 {
return false, fmt.Errorf("Got %d evals; want 1", l)
return false, fmt.Errorf("Got %d evals for job 2; want 1", l)
}
return true, nil
@ -987,7 +1307,7 @@ func TestWatcher_BatchEvals(t *testing.T) {
t.Fatal(err)
})
m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1))
m.AssertCalled(t, "UpdateAllocDesiredTransition", mocker.MatchedBy(m1))
testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil },
func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") })
}

42
nomad/deploymentwatcher/testutil_test.go
View File

@ -39,16 +39,16 @@ func (m *mockBackend) nextIndex() uint64 {
return i
}
func (m *mockBackend) UpsertEvals(evals []*structs.Evaluation) (uint64, error) {
m.Called(evals)
func (m *mockBackend) UpdateAllocDesiredTransition(u *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) {
m.Called(u)
i := m.nextIndex()
return i, m.state.UpsertEvals(i, evals)
return i, m.state.UpdateAllocsDesiredTransitions(i, u.Allocs, u.Evals)
}
// matchUpsertEvals is used to match an upsert request
func matchUpsertEvals(deploymentIDs []string) func(evals []*structs.Evaluation) bool {
return func(evals []*structs.Evaluation) bool {
if len(evals) != len(deploymentIDs) {
// matchUpdateAllocDesiredTransitions is used to match an upsert request
func matchUpdateAllocDesiredTransitions(deploymentIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
return func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
if len(update.Evals) != len(deploymentIDs) {
return false
}
@ -57,7 +57,7 @@ func matchUpsertEvals(deploymentIDs []string) func(evals []*structs.Evaluation)
dmap[d] = struct{}{}
}
for _, e := range evals {
for _, e := range update.Evals {
if _, ok := dmap[e.DeploymentID]; !ok {
return false
}
@ -69,6 +69,27 @@ func matchUpsertEvals(deploymentIDs []string) func(evals []*structs.Evaluation)
}
}
// matchUpdateAllocDesiredTransitionReschedule is used to match allocs that have their DesiredTransition set to Reschedule
func matchUpdateAllocDesiredTransitionReschedule(allocIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
return func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
amap := make(map[string]struct{}, len(allocIDs))
for _, d := range allocIDs {
amap[d] = struct{}{}
}
for allocID, dt := range update.Allocs {
if _, ok := amap[allocID]; !ok {
return false
}
if !*dt.Reschedule {
return false
}
}
return true
}
}
func (m *mockBackend) UpsertJob(job *structs.Job) (uint64, error) {
m.Called(job)
i := m.nextIndex()
@ -196,6 +217,11 @@ func matchDeploymentAllocHealthRequest(c *matchDeploymentAllocHealthRequestConfi
return false
}
// Require a timestamp
if args.Timestamp.IsZero() {
return false
}
if len(c.Healthy) != len(args.HealthyAllocationIDs) {
return false
}

10
nomad/drainer/drainer.go
View File

@ -254,7 +254,7 @@ func (n *NodeDrainer) handleDeadlinedNodes(nodes []string) {
n.l.RUnlock()
n.batchDrainAllocs(forceStop)
// Submit the node transistions in a sharded form to ensure a reasonable
// Submit the node transitions in a sharded form to ensure a reasonable
// Raft transaction size.
for _, nodes := range partitionIds(defaultMaxIdsPerTxn, nodes) {
if _, err := n.raft.NodesDrainComplete(nodes); err != nil {
@ -324,7 +324,7 @@ func (n *NodeDrainer) handleMigratedAllocs(allocs []*structs.Allocation) {
}
}
// Submit the node transistions in a sharded form to ensure a reasonable
// Submit the node transitions in a sharded form to ensure a reasonable
// Raft transaction size.
for _, nodes := range partitionIds(defaultMaxIdsPerTxn, done) {
if _, err := n.raft.NodesDrainComplete(nodes); err != nil {
@ -374,9 +374,9 @@ func (n *NodeDrainer) batchDrainAllocs(allocs []*structs.Allocation) (uint64, er
func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs.Allocation) {
// Compute the effected jobs and make the transition map
jobs := make(map[string]*structs.Allocation, 4)
transistions := make(map[string]*structs.DesiredTransition, len(allocs))
transitions := make(map[string]*structs.DesiredTransition, len(allocs))
for _, alloc := range allocs {
transistions[alloc.ID] = &structs.DesiredTransition{
transitions[alloc.ID] = &structs.DesiredTransition{
Migrate: helper.BoolToPtr(true),
}
jobs[alloc.JobID] = alloc
@ -397,7 +397,7 @@ func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs
// Commit this update via Raft
var finalIndex uint64
for _, u := range partitionAllocDrain(defaultMaxIdsPerTxn, transistions, evals) {
for _, u := range partitionAllocDrain(defaultMaxIdsPerTxn, transitions, evals) {
index, err := n.raft.AllocUpdateDesiredTransition(u.Transitions, u.Evals)
if err != nil {
future.Respond(0, err)

4
nomad/drainer/drainer_util_test.go
View File

@ -13,9 +13,9 @@ func TestDrainer_PartitionAllocDrain(t *testing.T) {
maxIdsPerTxn := 2
require := require.New(t)
transistions := map[string]*structs.DesiredTransition{"a": nil, "b": nil, "c": nil}
transitions := map[string]*structs.DesiredTransition{"a": nil, "b": nil, "c": nil}
evals := []*structs.Evaluation{nil, nil, nil}
requests := partitionAllocDrain(maxIdsPerTxn, transistions, evals)
requests := partitionAllocDrain(maxIdsPerTxn, transitions, evals)
require.Len(requests, 3)
first := requests[0]

6
nomad/drainer_int_test.go
View File

@ -38,6 +38,7 @@ func allocPromoter(errCh chan<- error, ctx context.Context,
// For each alloc that doesn't have its deployment status set, set it
var updates []*structs.Allocation
now := time.Now()
for _, alloc := range allocs {
if alloc.Job.Type != structs.JobTypeService {
continue
@ -48,7 +49,8 @@ func allocPromoter(errCh chan<- error, ctx context.Context,
}
newAlloc := alloc.Copy()
newAlloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Healthy: helper.BoolToPtr(true),
Timestamp: now,
}
updates = append(updates, newAlloc)
logger.Printf("Marked deployment health for alloc %q", alloc.ID)
@ -824,7 +826,7 @@ func TestDrainer_AllTypes_Deadline_GarbageCollectedNode(t *testing.T) {
})
}
// Test that transistions to force drain work.
// Test that transitions to force drain work.
func TestDrainer_Batch_TransitionToForce(t *testing.T) {
t.Parallel()
require := require.New(t)

59
nomad/fsm.go
View File

@ -582,21 +582,36 @@ func (n *nomadFSM) upsertEvals(index uint64, evals []*structs.Evaluation) error
return err
}
for _, eval := range evals {
if eval.ShouldEnqueue() {
n.evalBroker.Enqueue(eval)
} else if eval.ShouldBlock() {
n.blockedEvals.Block(eval)
} else if eval.Status == structs.EvalStatusComplete &&
len(eval.FailedTGAllocs) == 0 {
// If we have a successful evaluation for a node, untrack any
// blocked evaluation
n.blockedEvals.Untrack(eval.JobID)
}
}
n.handleUpsertedEvals(evals)
return nil
}
// handleUpsertingEval is a helper for taking action after upserting
// evaluations.
func (n *nomadFSM) handleUpsertedEvals(evals []*structs.Evaluation) {
for _, eval := range evals {
n.handleUpsertedEval(eval)
}
}
// handleUpsertingEval is a helper for taking action after upserting an eval.
func (n *nomadFSM) handleUpsertedEval(eval *structs.Evaluation) {
if eval == nil {
return
}
if eval.ShouldEnqueue() {
n.evalBroker.Enqueue(eval)
} else if eval.ShouldBlock() {
n.blockedEvals.Block(eval)
} else if eval.Status == structs.EvalStatusComplete &&
len(eval.FailedTGAllocs) == 0 {
// If we have a successful evaluation for a node, untrack any
// blocked evaluation
n.blockedEvals.Untrack(eval.JobID)
}
}
func (n *nomadFSM) applyDeleteEval(buf []byte, index uint64) interface{} {
defer metrics.MeasureSince([]string{"nomad", "fsm", "delete_eval"}, time.Now())
var req structs.EvalDeleteRequest
@ -731,10 +746,7 @@ func (n *nomadFSM) applyAllocUpdateDesiredTransition(buf []byte, index uint64) i
return err
}
if err := n.upsertEvals(index, req.Evals); err != nil {
n.logger.Printf("[ERR] nomad.fsm: AllocUpdateDesiredTransition failed to upsert %d eval(s): %v", len(req.Evals), err)
return err
}
n.handleUpsertedEvals(req.Evals)
return nil
}
@ -826,10 +838,7 @@ func (n *nomadFSM) applyDeploymentStatusUpdate(buf []byte, index uint64) interfa
return err
}
if req.Eval != nil && req.Eval.ShouldEnqueue() {
n.evalBroker.Enqueue(req.Eval)
}
n.handleUpsertedEval(req.Eval)
return nil
}
@ -846,10 +855,7 @@ func (n *nomadFSM) applyDeploymentPromotion(buf []byte, index uint64) interface{
return err
}
if req.Eval != nil && req.Eval.ShouldEnqueue() {
n.evalBroker.Enqueue(req.Eval)
}
n.handleUpsertedEval(req.Eval)
return nil
}
@ -867,10 +873,7 @@ func (n *nomadFSM) applyDeploymentAllocHealth(buf []byte, index uint64) interfac
return err
}
if req.Eval != nil && req.Eval.ShouldEnqueue() {
n.evalBroker.Enqueue(req.Eval)
}
n.handleUpsertedEval(req.Eval)
return nil
}

4
nomad/node_endpoint.go
View File

@ -476,7 +476,7 @@ func (n *Node) UpdateDrain(args *structs.NodeUpdateDrainRequest,
}
reply.NodeModifyIndex = index
// If the node is transistioning to be eligible, create Node evaluations
// If the node is transitioning to be eligible, create Node evaluations
// because there may be a System job registered that should be evaluated.
if node.SchedulingEligibility == structs.NodeSchedulingIneligible && args.MarkEligible && args.DrainStrategy == nil {
evalIDs, evalIndex, err := n.createNodeEvals(args.NodeID, index)
@ -556,7 +556,7 @@ func (n *Node) UpdateEligibility(args *structs.NodeUpdateEligibilityRequest,
}
}
// If the node is transistioning to be eligible, create Node evaluations
// If the node is transitioning to be eligible, create Node evaluations
// because there may be a System job registered that should be evaluated.
if node.SchedulingEligibility == structs.NodeSchedulingIneligible && args.Eligibility == structs.NodeSchedulingEligible {
evalIDs, evalIndex, err := n.createNodeEvals(args.NodeID, index)

2
nomad/server.go
View File

@ -893,7 +893,7 @@ func (s *Server) setupDeploymentWatcher() error {
s.deploymentWatcher = deploymentwatcher.NewDeploymentsWatcher(
s.logger, raftShim,
deploymentwatcher.LimitStateQueriesPerSecond,
deploymentwatcher.CrossDeploymentEvalBatchDuration)
deploymentwatcher.CrossDeploymentUpdateBatchDuration)
return nil
}

90
nomad/state/state_store.go
View File

@ -1890,7 +1890,13 @@ func (s *StateStore) nestedUpdateAllocFromClient(txn *memdb.Txn, index uint64, a
copyAlloc.ClientStatus = alloc.ClientStatus
copyAlloc.ClientDescription = alloc.ClientDescription
copyAlloc.TaskStates = alloc.TaskStates
// Merge the deployment status taking only what the client should set
oldDeploymentStatus := copyAlloc.DeploymentStatus
copyAlloc.DeploymentStatus = alloc.DeploymentStatus
if oldDeploymentStatus != nil && oldDeploymentStatus.Canary {
copyAlloc.DeploymentStatus.Canary = true
}
// Update the modify index
copyAlloc.ModifyIndex = index
@ -1961,6 +1967,9 @@ func (s *StateStore) upsertAllocsImpl(index uint64, allocs []*structs.Allocation
alloc.CreateIndex = index
alloc.ModifyIndex = index
alloc.AllocModifyIndex = index
if alloc.DeploymentStatus != nil {
alloc.DeploymentStatus.ModifyIndex = index
}
// Issue https://github.com/hashicorp/nomad/issues/2583 uncovered
// the a race between a forced garbage collection and the scheduler
@ -2085,6 +2094,12 @@ func (s *StateStore) UpdateAllocsDesiredTransitions(index uint64, allocs map[str
}
}
for _, eval := range evals {
if err := s.nestedUpsertEval(txn, index, eval); err != nil {
return err
}
}
// Update the indexes
if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil {
return fmt.Errorf("index update failed: %v", err)
@ -2614,11 +2629,13 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD
return err
}
// groupIndex is a map of groups being promoted
groupIndex := make(map[string]struct{}, len(req.Groups))
for _, g := range req.Groups {
groupIndex[g] = struct{}{}
}
// canaryIndex is the set of placed canaries in the deployment
canaryIndex := make(map[string]struct{}, len(deployment.TaskGroups))
for _, state := range deployment.TaskGroups {
for _, c := range state.PlacedCanaries {
@ -2626,8 +2643,13 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD
}
}
haveCanaries := false
var unhealthyErr multierror.Error
// healthyCounts is a mapping of group to the number of healthy canaries
healthyCounts := make(map[string]int, len(deployment.TaskGroups))
// promotable is the set of allocations that we can move from canary to
// non-canary
var promotable []*structs.Allocation
for {
raw := iter.Next()
if raw == nil {
@ -2648,21 +2670,34 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD
// Ensure the canaries are healthy
if !alloc.DeploymentStatus.IsHealthy() {
multierror.Append(&unhealthyErr, fmt.Errorf("Canary allocation %q for group %q is not healthy", alloc.ID, alloc.TaskGroup))
continue
}
haveCanaries = true
healthyCounts[alloc.TaskGroup]++
promotable = append(promotable, alloc)
}
// Determine if we have enough healthy allocations
var unhealthyErr multierror.Error
for tg, state := range deployment.TaskGroups {
if _, ok := groupIndex[tg]; !req.All && !ok {
continue
}
need := state.DesiredCanaries
if need == 0 {
continue
}
if have := healthyCounts[tg]; have < need {
multierror.Append(&unhealthyErr, fmt.Errorf("Task group %q has %d/%d healthy allocations", tg, have, need))
}
}
if err := unhealthyErr.ErrorOrNil(); err != nil {
return err
}
if !haveCanaries {
return fmt.Errorf("no canaries to promote")
}
// Update deployment
copy := deployment.Copy()
copy.ModifyIndex = index
@ -2692,6 +2727,24 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD
}
}
// For each promotable allocation remoce the canary field
for _, alloc := range promotable {
promoted := alloc.Copy()
promoted.DeploymentStatus.Canary = false
promoted.DeploymentStatus.ModifyIndex = index
promoted.ModifyIndex = index
promoted.AllocModifyIndex = index
if err := txn.Insert("allocs", promoted); err != nil {
return fmt.Errorf("alloc insert failed: %v", err)
}
}
// Update the alloc index
if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil {
return fmt.Errorf("index update failed: %v", err)
}
txn.Commit()
return nil
}
@ -2715,7 +2768,7 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl
// Update the health status of each allocation
if total := len(req.HealthyAllocationIDs) + len(req.UnhealthyAllocationIDs); total != 0 {
setAllocHealth := func(id string, healthy bool) error {
setAllocHealth := func(id string, healthy bool, ts time.Time) error {
existing, err := txn.First("allocs", "id", id)
if err != nil {
return fmt.Errorf("alloc %q lookup failed: %v", id, err)
@ -2735,6 +2788,7 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl
copy.DeploymentStatus = &structs.AllocDeploymentStatus{}
}
copy.DeploymentStatus.Healthy = helper.BoolToPtr(healthy)
copy.DeploymentStatus.Timestamp = ts
copy.DeploymentStatus.ModifyIndex = index
if err := s.updateDeploymentWithAlloc(index, copy, old, txn); err != nil {
@ -2749,12 +2803,12 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl
}
for _, id := range req.HealthyAllocationIDs {
if err := setAllocHealth(id, true); err != nil {
if err := setAllocHealth(id, true, req.Timestamp); err != nil {
return err
}
}
for _, id := range req.UnhealthyAllocationIDs {
if err := setAllocHealth(id, false); err != nil {
if err := setAllocHealth(id, false, req.Timestamp); err != nil {
return err
}
}
@ -3284,6 +3338,20 @@ func (s *StateStore) updateDeploymentWithAlloc(index uint64, alloc, existing *st
state.HealthyAllocs += healthy
state.UnhealthyAllocs += unhealthy
// Update the progress deadline
if pd := state.ProgressDeadline; pd != 0 {
// If we are the first placed allocation for the deployment start the progress deadline.
if placed != 0 && state.RequireProgressBy.IsZero() {
// Use modify time instead of create time because we may in-place
// update the allocation to be part of a new deployment.
state.RequireProgressBy = time.Unix(0, alloc.ModifyTime).Add(pd)
} else if healthy != 0 {
if d := alloc.DeploymentStatus.Timestamp.Add(pd); d.After(state.RequireProgressBy) {
state.RequireProgressBy = d
}
}
}
// Upsert the deployment
if err := s.upsertDeploymentImpl(index, deploymentCopy, txn); err != nil {
return err

269
nomad/state/state_store_test.go
View File

@ -3536,6 +3536,86 @@ func TestStateStore_UpdateMultipleAllocsFromClient(t *testing.T) {
}
}
func TestStateStore_UpdateAllocsFromClient_Deployment(t *testing.T) {
require := require.New(t)
state := testStateStore(t)
alloc := mock.Alloc()
now := time.Now()
alloc.CreateTime = now.UnixNano()
pdeadline := 5 * time.Minute
deployment := mock.Deployment()
deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline
alloc.DeploymentID = deployment.ID
require.Nil(state.UpsertJob(999, alloc.Job))
require.Nil(state.UpsertDeployment(1000, deployment))
require.Nil(state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
healthy := now.Add(time.Second)
update := &structs.Allocation{
ID: alloc.ID,
ClientStatus: structs.AllocClientStatusRunning,
JobID: alloc.JobID,
TaskGroup: alloc.TaskGroup,
DeploymentStatus: &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Timestamp: healthy,
},
}
require.Nil(state.UpdateAllocsFromClient(1001, []*structs.Allocation{update}))
// Check that the deployment state was updated because the healthy
// deployment
dout, err := state.DeploymentByID(nil, deployment.ID)
require.Nil(err)
require.NotNil(dout)
require.Len(dout.TaskGroups, 1)
dstate := dout.TaskGroups[alloc.TaskGroup]
require.NotNil(dstate)
require.Equal(1, dstate.PlacedAllocs)
require.True(healthy.Add(pdeadline).Equal(dstate.RequireProgressBy))
}
// This tests that the deployment state is merged correctly
func TestStateStore_UpdateAllocsFromClient_DeploymentStateMerges(t *testing.T) {
require := require.New(t)
state := testStateStore(t)
alloc := mock.Alloc()
now := time.Now()
alloc.CreateTime = now.UnixNano()
pdeadline := 5 * time.Minute
deployment := mock.Deployment()
deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline
alloc.DeploymentID = deployment.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
}
require.Nil(state.UpsertJob(999, alloc.Job))
require.Nil(state.UpsertDeployment(1000, deployment))
require.Nil(state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
update := &structs.Allocation{
ID: alloc.ID,
ClientStatus: structs.AllocClientStatusRunning,
JobID: alloc.JobID,
TaskGroup: alloc.TaskGroup,
DeploymentStatus: &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Canary: false,
},
}
require.Nil(state.UpdateAllocsFromClient(1001, []*structs.Allocation{update}))
// Check that the merging of the deployment status was correct
out, err := state.AllocByID(nil, alloc.ID)
require.Nil(err)
require.NotNil(out)
require.True(out.DeploymentStatus.Canary)
}
func TestStateStore_UpsertAlloc_Alloc(t *testing.T) {
state := testStateStore(t)
alloc := mock.Alloc()
@ -3610,28 +3690,26 @@ func TestStateStore_UpsertAlloc_Alloc(t *testing.T) {
}
func TestStateStore_UpsertAlloc_Deployment(t *testing.T) {
require := require.New(t)
state := testStateStore(t)
deployment := mock.Deployment()
alloc := mock.Alloc()
now := time.Now()
alloc.CreateTime = now.UnixNano()
alloc.ModifyTime = now.UnixNano()
pdeadline := 5 * time.Minute
deployment := mock.Deployment()
deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline
alloc.DeploymentID = deployment.ID
if err := state.UpsertJob(999, alloc.Job); err != nil {
t.Fatalf("err: %v", err)
}
if err := state.UpsertDeployment(1000, deployment); err != nil {
t.Fatalf("err: %v", err)
}
require.Nil(state.UpsertJob(999, alloc.Job))
require.Nil(state.UpsertDeployment(1000, deployment))
// Create a watch set so we can test that update fires the watch
ws := memdb.NewWatchSet()
if _, err := state.AllocsByDeployment(ws, alloc.DeploymentID); err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.AllocsByDeployment(ws, alloc.DeploymentID))
err := state.UpsertAllocs(1001, []*structs.Allocation{alloc})
if err != nil {
t.Fatalf("err: %v", err)
}
require.Nil(err)
if !watchFired(ws) {
t.Fatalf("watch not fired")
@ -3639,29 +3717,26 @@ func TestStateStore_UpsertAlloc_Deployment(t *testing.T) {
ws = memdb.NewWatchSet()
allocs, err := state.AllocsByDeployment(ws, alloc.DeploymentID)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(allocs) != 1 {
t.Fatalf("bad: %#v", allocs)
}
if !reflect.DeepEqual(alloc, allocs[0]) {
t.Fatalf("bad: %#v %#v", alloc, allocs[0])
}
require.Nil(err)
require.Len(allocs, 1)
require.EqualValues(alloc, allocs[0])
index, err := state.Index("allocs")
if err != nil {
t.Fatalf("err: %v", err)
}
if index != 1001 {
t.Fatalf("bad: %d", index)
}
require.Nil(err)
require.EqualValues(1001, index)
if watchFired(ws) {
t.Fatalf("bad")
}
// Check that the deployment state was updated
dout, err := state.DeploymentByID(nil, deployment.ID)
require.Nil(err)
require.NotNil(dout)
require.Len(dout.TaskGroups, 1)
dstate := dout.TaskGroups[alloc.TaskGroup]
require.NotNil(dstate)
require.Equal(1, dstate.PlacedAllocs)
require.True(now.Add(pdeadline).Equal(dstate.RequireProgressBy))
}
// Testing to ensure we keep issue
@ -3981,6 +4056,11 @@ func TestStateStore_UpdateAllocDesiredTransition(t *testing.T) {
require.Nil(err)
require.EqualValues(1001, index)
// Check the eval is created
eout, err := state.EvalByID(nil, eval.ID)
require.Nil(err)
require.NotNil(eout)
m = map[string]*structs.DesiredTransition{alloc.ID: t2}
require.Nil(state.UpdateAllocsDesiredTransitions(1002, m, evals))
@ -5494,19 +5574,17 @@ func TestStateStore_UpsertDeploymentPromotion_Terminal(t *testing.T) {
// Test promoting unhealthy canaries in a deployment.
func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) {
state := testStateStore(t)
require := require.New(t)
// Create a job
j := mock.Job()
if err := state.UpsertJob(1, j); err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.UpsertJob(1, j))
// Create a deployment
d := mock.Deployment()
d.JobID = j.ID
if err := state.UpsertDeployment(2, d); err != nil {
t.Fatalf("bad: %v", err)
}
d.TaskGroups["web"].DesiredCanaries = 2
require.Nil(state.UpsertDeployment(2, d))
// Create a set of allocations
c1 := mock.Alloc()
@ -5518,9 +5596,7 @@ func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) {
c2.DeploymentID = d.ID
d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID)
if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil {
t.Fatalf("err: %v", err)
}
require.Nil(state.UpsertAllocs(3, []*structs.Allocation{c1, c2}))
// Promote the canaries
req := &structs.ApplyDeploymentPromoteRequest{
@ -5530,33 +5606,24 @@ func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) {
},
}
err := state.UpdateDeploymentPromotion(4, req)
if err == nil {
t.Fatalf("bad: %v", err)
}
if !strings.Contains(err.Error(), c1.ID) {
t.Fatalf("expect canary %q to be listed as unhealth: %v", c1.ID, err)
}
if !strings.Contains(err.Error(), c2.ID) {
t.Fatalf("expect canary %q to be listed as unhealth: %v", c2.ID, err)
}
require.NotNil(err)
require.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`)
}
// Test promoting a deployment with no canaries
func TestStateStore_UpsertDeploymentPromotion_NoCanaries(t *testing.T) {
state := testStateStore(t)
require := require.New(t)
// Create a job
j := mock.Job()
if err := state.UpsertJob(1, j); err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.UpsertJob(1, j))
// Create a deployment
d := mock.Deployment()
d.TaskGroups["web"].DesiredCanaries = 2
d.JobID = j.ID
if err := state.UpsertDeployment(2, d); err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.UpsertDeployment(2, d))
// Promote the canaries
req := &structs.ApplyDeploymentPromoteRequest{
@ -5566,12 +5633,8 @@ func TestStateStore_UpsertDeploymentPromotion_NoCanaries(t *testing.T) {
},
}
err := state.UpdateDeploymentPromotion(4, req)
if err == nil {
t.Fatalf("bad: %v", err)
}
if !strings.Contains(err.Error(), "no canaries to promote") {
t.Fatalf("expect error promoting nonexistent canaries: %v", err)
}
require.NotNil(err)
require.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`)
}
// Test promoting all canaries in a deployment.
@ -5674,6 +5737,7 @@ func TestStateStore_UpsertDeploymentPromotion_All(t *testing.T) {
// Test promoting a subset of canaries in a deployment.
func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) {
state := testStateStore(t)
require := require.New(t)
// Create a job with two task groups
j := mock.Job()
@ -5681,9 +5745,7 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) {
tg2 := tg1.Copy()
tg2.Name = "foo"
j.TaskGroups = append(j.TaskGroups, tg2)
if err := state.UpsertJob(1, j); err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.UpsertJob(1, j))
// Create a deployment
d := mock.Deployment()
@ -5698,18 +5760,19 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) {
DesiredCanaries: 1,
},
}
if err := state.UpsertDeployment(2, d); err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.UpsertDeployment(2, d))
// Create a set of allocations
// Create a set of allocations for both groups, including an unhealthy one
c1 := mock.Alloc()
c1.JobID = j.ID
c1.DeploymentID = d.ID
d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID)
c1.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Canary: true,
}
// Should still be a canary
c2 := mock.Alloc()
c2.JobID = j.ID
c2.DeploymentID = d.ID
@ -5717,12 +5780,20 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) {
c2.TaskGroup = tg2.Name
c2.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(true),
Canary: true,
}
if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil {
t.Fatalf("err: %v", err)
c3 := mock.Alloc()
c3.JobID = j.ID
c3.DeploymentID = d.ID
d.TaskGroups[c3.TaskGroup].PlacedCanaries = append(d.TaskGroups[c3.TaskGroup].PlacedCanaries, c3.ID)
c3.DeploymentStatus = &structs.AllocDeploymentStatus{
Healthy: helper.BoolToPtr(false),
Canary: true,
}
require.Nil(state.UpsertAllocs(3, []*structs.Allocation{c1, c2, c3}))
// Create an eval
e := mock.Eval()
@ -5734,36 +5805,34 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) {
},
Eval: e,
}
err := state.UpdateDeploymentPromotion(4, req)
if err != nil {
t.Fatalf("bad: %v", err)
}
require.Nil(state.UpdateDeploymentPromotion(4, req))
// Check that the status per task group was updated properly
ws := memdb.NewWatchSet()
dout, err := state.DeploymentByID(ws, d.ID)
if err != nil {
t.Fatalf("bad: %v", err)
}
if len(dout.TaskGroups) != 2 {
t.Fatalf("bad: %#v", dout.TaskGroups)
}
stateout, ok := dout.TaskGroups["web"]
if !ok {
t.Fatalf("bad: no state for task group web")
}
if !stateout.Promoted {
t.Fatalf("bad: task group web not promoted: %#v", stateout)
}
require.Nil(err)
require.Len(dout.TaskGroups, 2)
require.Contains(dout.TaskGroups, "web")
require.True(dout.TaskGroups["web"].Promoted)
// Check that the evaluation was created
eout, _ := state.EvalByID(ws, e.ID)
if err != nil {
t.Fatalf("bad: %v", err)
}
if eout == nil {
t.Fatalf("bad: %#v", eout)
}
eout, err := state.EvalByID(ws, e.ID)
require.Nil(err)
require.NotNil(eout)
// Check the canary field was set properly
aout1, err1 := state.AllocByID(ws, c1.ID)
aout2, err2 := state.AllocByID(ws, c2.ID)
aout3, err3 := state.AllocByID(ws, c3.ID)
require.Nil(err1)
require.Nil(err2)
require.Nil(err3)
require.NotNil(aout1)
require.NotNil(aout2)
require.NotNil(aout3)
require.False(aout1.DeploymentStatus.Canary)
require.True(aout2.DeploymentStatus.Canary)
require.True(aout3.DeploymentStatus.Canary)
}
// Test that allocation health can't be set against a nonexistent deployment
@ -5872,6 +5941,7 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) {
// Insert a deployment
d := mock.Deployment()
d.TaskGroups["web"].ProgressDeadline = 5 * time.Minute
if err := state.UpsertDeployment(1, d); err != nil {
t.Fatalf("bad: %v", err)
}
@ -5899,6 +5969,9 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) {
StatusDescription: desc,
}
// Capture the time for the update
ts := time.Now()
// Set health against the deployment
req := &structs.ApplyDeploymentAllocHealthRequest{
DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{
@ -5909,6 +5982,7 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) {
Job: j,
Eval: e,
DeploymentUpdate: u,
Timestamp: ts,
}
err := state.UpdateDeploymentAllocHealth(3, req)
if err != nil {
@ -5959,6 +6033,13 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) {
if !out2.DeploymentStatus.IsUnhealthy() {
t.Fatalf("bad: alloc %q not unhealthy", out2.ID)
}
if !out1.DeploymentStatus.Timestamp.Equal(ts) {
t.Fatalf("bad: alloc %q had timestamp %v; want %v", out1.ID, out1.DeploymentStatus.Timestamp, ts)
}
if !out2.DeploymentStatus.Timestamp.Equal(ts) {
t.Fatalf("bad: alloc %q had timestamp %v; want %v", out2.ID, out2.DeploymentStatus.Timestamp, ts)
}
}
func TestStateStore_UpsertVaultAccessors(t *testing.T) {

9
nomad/structs/diff.go
View File

@ -533,6 +533,15 @@ func serviceDiff(old, new *Service, contextual bool) *ObjectDiff {
// Diff the primitive fields.
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
if setDiff := stringSetDiff(old.CanaryTags, new.CanaryTags, "CanaryTags", contextual); setDiff != nil {
diff.Objects = append(diff.Objects, setDiff)
}
// Tag diffs
if setDiff := stringSetDiff(old.Tags, new.Tags, "Tags", contextual); setDiff != nil {
diff.Objects = append(diff.Objects, setDiff)
}
// Checks diffs
if cDiffs := serviceCheckDiffs(old.Checks, new.Checks, contextual); cDiffs != nil {
diff.Objects = append(diff.Objects, cDiffs...)

169
nomad/structs/diff_test.go
View File

@ -1787,6 +1787,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "0",
New: "",
},
{
Type: DiffTypeDeleted,
Name: "ProgressDeadline",
Old: "0",
New: "",
},
},
},
},
@ -1837,6 +1843,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "",
New: "0",
},
{
Type: DiffTypeAdded,
Name: "ProgressDeadline",
Old: "",
New: "0",
},
},
},
},
@ -1846,22 +1858,24 @@ func TestTaskGroupDiff(t *testing.T) {
// Update strategy edited
Old: &TaskGroup{
Update: &UpdateStrategy{
MaxParallel: 5,
HealthCheck: "foo",
MinHealthyTime: 1 * time.Second,
HealthyDeadline: 30 * time.Second,
AutoRevert: true,
Canary: 2,
MaxParallel: 5,
HealthCheck: "foo",
MinHealthyTime: 1 * time.Second,
HealthyDeadline: 30 * time.Second,
ProgressDeadline: 29 * time.Second,
AutoRevert: true,
Canary: 2,
},
},
New: &TaskGroup{
Update: &UpdateStrategy{
MaxParallel: 7,
HealthCheck: "bar",
MinHealthyTime: 2 * time.Second,
HealthyDeadline: 31 * time.Second,
AutoRevert: false,
Canary: 1,
MaxParallel: 7,
HealthCheck: "bar",
MinHealthyTime: 2 * time.Second,
HealthyDeadline: 31 * time.Second,
ProgressDeadline: 32 * time.Second,
AutoRevert: false,
Canary: 1,
},
},
Expected: &TaskGroupDiff{
@ -1907,6 +1921,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "1000000000",
New: "2000000000",
},
{
Type: DiffTypeEdited,
Name: "ProgressDeadline",
Old: "29000000000",
New: "32000000000",
},
},
},
},
@ -1917,22 +1937,24 @@ func TestTaskGroupDiff(t *testing.T) {
Contextual: true,
Old: &TaskGroup{
Update: &UpdateStrategy{
MaxParallel: 5,
HealthCheck: "foo",
MinHealthyTime: 1 * time.Second,
HealthyDeadline: 30 * time.Second,
AutoRevert: true,
Canary: 2,
MaxParallel: 5,
HealthCheck: "foo",
MinHealthyTime: 1 * time.Second,
HealthyDeadline: 30 * time.Second,
ProgressDeadline: 30 * time.Second,
AutoRevert: true,
Canary: 2,
},
},
New: &TaskGroup{
Update: &UpdateStrategy{
MaxParallel: 7,
HealthCheck: "foo",
MinHealthyTime: 1 * time.Second,
HealthyDeadline: 30 * time.Second,
AutoRevert: true,
Canary: 2,
MaxParallel: 7,
HealthCheck: "foo",
MinHealthyTime: 1 * time.Second,
HealthyDeadline: 30 * time.Second,
ProgressDeadline: 30 * time.Second,
AutoRevert: true,
Canary: 2,
},
},
Expected: &TaskGroupDiff{
@ -1978,6 +2000,12 @@ func TestTaskGroupDiff(t *testing.T) {
Old: "1000000000",
New: "1000000000",
},
{
Type: DiffTypeNone,
Name: "ProgressDeadline",
Old: "30000000000",
New: "30000000000",
},
},
},
},
@ -3428,6 +3456,99 @@ func TestTaskDiff(t *testing.T) {
},
},
},
{
Name: "Services tags edited (no checks) with context",
Contextual: true,
Old: &Task{
Services: []*Service{
{
Tags: []string{"foo", "bar"},
CanaryTags: []string{"foo", "bar"},
},
},
},
New: &Task{
Services: []*Service{
{
Tags: []string{"bar", "bam"},
CanaryTags: []string{"bar", "bam"},
},
},
},
Expected: &TaskDiff{
Type: DiffTypeEdited,
Objects: []*ObjectDiff{
{
Type: DiffTypeEdited,
Name: "Service",
Objects: []*ObjectDiff{
{
Type: DiffTypeEdited,
Name: "CanaryTags",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "CanaryTags",
Old: "",
New: "bam",
},
{
Type: DiffTypeNone,
Name: "CanaryTags",
Old: "bar",
New: "bar",
},
{
Type: DiffTypeDeleted,
Name: "CanaryTags",
Old: "foo",
New: "",
},
},
},
{
Type: DiffTypeEdited,
Name: "Tags",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "Tags",
Old: "",
New: "bam",
},
{
Type: DiffTypeNone,
Name: "Tags",
Old: "bar",
New: "bar",
},
{
Type: DiffTypeDeleted,
Name: "Tags",
Old: "foo",
New: "",
},
},
},
},
Fields: []*FieldDiff{
{
Type: DiffTypeNone,
Name: "AddressMode",
},
{
Type: DiffTypeNone,
Name: "Name",
},
{
Type: DiffTypeNone,
Name: "PortLabel",
},
},
},
},
},
},
{
Name: "Service Checks edited",
Old: &Task{

93
nomad/structs/structs.go
View File

@ -788,6 +788,9 @@ type DeploymentAllocHealthRequest struct {
type ApplyDeploymentAllocHealthRequest struct {
DeploymentAllocHealthRequest
// Timestamp is the timestamp to use when setting the allocations health.
Timestamp time.Time
// An optional field to update the status of a deployment
DeploymentUpdate *DeploymentStatusUpdate
@ -2478,13 +2481,14 @@ var (
// DefaultUpdateStrategy provides a baseline that can be used to upgrade
// jobs with the old policy or for populating field defaults.
DefaultUpdateStrategy = &UpdateStrategy{
Stagger: 30 * time.Second,
MaxParallel: 1,
HealthCheck: UpdateStrategyHealthCheck_Checks,
MinHealthyTime: 10 * time.Second,
HealthyDeadline: 5 * time.Minute,
AutoRevert: false,
Canary: 0,
Stagger: 30 * time.Second,
MaxParallel: 1,
HealthCheck: UpdateStrategyHealthCheck_Checks,
MinHealthyTime: 10 * time.Second,
HealthyDeadline: 5 * time.Minute,
ProgressDeadline: 10 * time.Minute,
AutoRevert: false,
Canary: 0,
}
)
@ -2511,6 +2515,12 @@ type UpdateStrategy struct {
// period doesn't count against the MinHealthyTime.
HealthyDeadline time.Duration
// ProgressDeadline is the time in which an allocation as part of the
// deployment must transition to healthy. If no allocation becomes healthy
// after the deadline, the deployment is marked as failed. If the deadline
// is zero, the first failure causes the deployment to fail.
ProgressDeadline time.Duration
// AutoRevert declares that if a deployment fails because of unhealthy
// allocations, there should be an attempt to auto-revert the job to a
// stable version.
@ -2555,9 +2565,15 @@ func (u *UpdateStrategy) Validate() error {
if u.HealthyDeadline <= 0 {
multierror.Append(&mErr, fmt.Errorf("Healthy deadline must be greater than zero: %v", u.HealthyDeadline))
}
if u.ProgressDeadline < 0 {
multierror.Append(&mErr, fmt.Errorf("Progress deadline must be zero or greater: %v", u.ProgressDeadline))
}
if u.MinHealthyTime >= u.HealthyDeadline {
multierror.Append(&mErr, fmt.Errorf("Minimum healthy time must be less than healthy deadline: %v > %v", u.MinHealthyTime, u.HealthyDeadline))
}
if u.ProgressDeadline != 0 && u.HealthyDeadline >= u.ProgressDeadline {
multierror.Append(&mErr, fmt.Errorf("Healthy deadline must be less than progress deadline: %v > %v", u.HealthyDeadline, u.ProgressDeadline))
}
if u.Stagger <= 0 {
multierror.Append(&mErr, fmt.Errorf("Stagger must be greater than zero: %v", u.Stagger))
}
@ -3734,8 +3750,9 @@ type Service struct {
// this service.
AddressMode string
Tags []string // List of tags for the service
Checks []*ServiceCheck // List of checks associated with the service
Tags []string // List of tags for the service
CanaryTags []string // List of tags for the service when it is a canary
Checks []*ServiceCheck // List of checks associated with the service
}
func (s *Service) Copy() *Service {
@ -3745,6 +3762,7 @@ func (s *Service) Copy() *Service {
ns := new(Service)
*ns = *s
ns.Tags = helper.CopySliceString(ns.Tags)
ns.CanaryTags = helper.CopySliceString(ns.CanaryTags)
if s.Checks != nil {
checks := make([]*ServiceCheck, len(ns.Checks))
@ -3765,6 +3783,9 @@ func (s *Service) Canonicalize(job string, taskGroup string, task string) {
if len(s.Tags) == 0 {
s.Tags = nil
}
if len(s.CanaryTags) == 0 {
s.CanaryTags = nil
}
if len(s.Checks) == 0 {
s.Checks = nil
}
@ -3832,7 +3853,7 @@ func (s *Service) ValidateName(name string) error {
// Hash returns a base32 encoded hash of a Service's contents excluding checks
// as they're hashed independently.
func (s *Service) Hash(allocID, taskName string) string {
func (s *Service) Hash(allocID, taskName string, canary bool) string {
h := sha1.New()
io.WriteString(h, allocID)
io.WriteString(h, taskName)
@ -3842,6 +3863,14 @@ func (s *Service) Hash(allocID, taskName string) string {
for _, tag := range s.Tags {
io.WriteString(h, tag)
}
for _, tag := range s.CanaryTags {
io.WriteString(h, tag)
}
// Vary ID on whether or not CanaryTags will be used
if canary {
h.Write([]byte("Canary"))
}
// Base32 is used for encoding the hash as sha1 hashes can always be
// encoded without padding, only 4 bytes larger than base64, and saves
@ -5300,6 +5329,7 @@ const (
DeploymentStatusDescriptionStoppedJob = "Cancelled because job is stopped"
DeploymentStatusDescriptionNewerJob = "Cancelled due to newer version of job"
DeploymentStatusDescriptionFailedAllocations = "Failed due to unhealthy allocations"
DeploymentStatusDescriptionProgressDeadline = "Failed due to progress deadline"
DeploymentStatusDescriptionFailedByUser = "Deployment marked as failed"
)
@ -5452,6 +5482,14 @@ type DeploymentState struct {
// reverted on failure
AutoRevert bool
// ProgressDeadline is the deadline by which an allocation must transition
// to healthy before the deployment is considered failed.
ProgressDeadline time.Duration
// RequireProgressBy is the time by which an allocation must transition
// to healthy before the deployment is considered failed.
RequireProgressBy time.Time
// Promoted marks whether the canaries have been promoted
Promoted bool
@ -5563,6 +5601,13 @@ type DesiredTransition struct {
// Migrate is used to indicate that this allocation should be stopped and
// migrated to another node.
Migrate *bool
// Reschedule is used to indicate that this allocation is eligible to be
// rescheduled. Most allocations are automatically eligible for
// rescheduling, so this field is only required when an allocation is not
// automatically eligible. An example is an allocation that is part of a
// deployment.
Reschedule *bool
}
// Merge merges the two desired transitions, preferring the values from the
@ -5571,6 +5616,10 @@ func (d *DesiredTransition) Merge(o *DesiredTransition) {
if o.Migrate != nil {
d.Migrate = o.Migrate
}
if o.Reschedule != nil {
d.Reschedule = o.Reschedule
}
}
// ShouldMigrate returns whether the transition object dictates a migration.
@ -5578,6 +5627,12 @@ func (d *DesiredTransition) ShouldMigrate() bool {
return d.Migrate != nil && *d.Migrate
}
// ShouldReschedule returns whether the transition object dictates a
// rescheduling.
func (d *DesiredTransition) ShouldReschedule() bool {
return d.Reschedule != nil && *d.Reschedule
}
const (
AllocDesiredStatusRun = "run" // Allocation should run
AllocDesiredStatusStop = "stop" // Allocation should stop
@ -5997,6 +6052,7 @@ func (a *Allocation) Stub() *AllocListStub {
DesiredDescription: a.DesiredDescription,
ClientStatus: a.ClientStatus,
ClientDescription: a.ClientDescription,
DesiredTransition: a.DesiredTransition,
TaskStates: a.TaskStates,
DeploymentStatus: a.DeploymentStatus,
FollowupEvalID: a.FollowupEvalID,
@ -6021,6 +6077,7 @@ type AllocListStub struct {
DesiredDescription string
ClientStatus string
ClientDescription string
DesiredTransition DesiredTransition
TaskStates map[string]*TaskState
DeploymentStatus *AllocDeploymentStatus
FollowupEvalID string
@ -6172,6 +6229,13 @@ type AllocDeploymentStatus struct {
// healthy or unhealthy.
Healthy *bool
// Timestamp is the time at which the health status was set.
Timestamp time.Time
// Canary marks whether the allocation is a canary or not. A canary that has
// been promoted will have this field set to false.
Canary bool
// ModifyIndex is the raft index in which the deployment status was last
// changed.
ModifyIndex uint64
@ -6202,6 +6266,15 @@ func (a *AllocDeploymentStatus) IsUnhealthy() bool {
return a.Healthy != nil && !*a.Healthy
}
// IsCanary returns if the allocation is marked as a canary
func (a *AllocDeploymentStatus) IsCanary() bool {
if a == nil {
return false
}
return a.Canary
}
func (a *AllocDeploymentStatus) Copy() *AllocDeploymentStatus {
if a == nil {
return nil

21
nomad/structs/structs_test.go
View File

@ -1565,12 +1565,13 @@ func TestConstraint_Validate(t *testing.T) {
func TestUpdateStrategy_Validate(t *testing.T) {
u := &UpdateStrategy{
MaxParallel: 0,
HealthCheck: "foo",
MinHealthyTime: -10,
HealthyDeadline: -15,
AutoRevert: false,
Canary: -1,
MaxParallel: 0,
HealthCheck: "foo",
MinHealthyTime: -10,
HealthyDeadline: -15,
ProgressDeadline: -25,
AutoRevert: false,
Canary: -1,
}
err := u.Validate()
@ -1590,7 +1591,13 @@ func TestUpdateStrategy_Validate(t *testing.T) {
if !strings.Contains(mErr.Errors[4].Error(), "Healthy deadline must be greater than zero") {
t.Fatalf("err: %s", err)
}
if !strings.Contains(mErr.Errors[5].Error(), "Minimum healthy time must be less than healthy deadline") {
if !strings.Contains(mErr.Errors[5].Error(), "Progress deadline must be zero or greater") {
t.Fatalf("err: %s", err)
}
if !strings.Contains(mErr.Errors[6].Error(), "Minimum healthy time must be less than healthy deadline") {
t.Fatalf("err: %s", err)
}
if !strings.Contains(mErr.Errors[7].Error(), "Healthy deadline must be less than progress deadline") {
t.Fatalf("err: %s", err)
}
}

6
scheduler/generic_sched.go
View File

@ -499,11 +499,15 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul
}
// If we are placing a canary and we found a match, add the canary
// to the deployment state object.
// to the deployment state object and mark it as a canary.
if missing.Canary() {
if state, ok := s.deployment.TaskGroups[tg.Name]; ok {
state.PlacedCanaries = append(state.PlacedCanaries, alloc.ID)
}
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
}
}
// Track the placement

150
scheduler/generic_sched_test.go
View File

@ -1814,6 +1814,11 @@ func TestServiceSched_JobModify_Canaries(t *testing.T) {
if len(planned) != desiredUpdates {
t.Fatalf("bad: %#v", plan)
}
for _, canary := range planned {
if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary {
t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID)
}
}
h.AssertEvalStatus(t, structs.EvalStatusComplete)
@ -3108,67 +3113,92 @@ func TestServiceSched_Reschedule_PruneEvents(t *testing.T) {
}
// Tests that deployments with failed allocs don't result in placements
func TestDeployment_FailedAllocs_NoReschedule(t *testing.T) {
h := NewHarness(t)
require := require.New(t)
// Create some nodes
var nodes []*structs.Node
for i := 0; i < 10; i++ {
node := mock.Node()
nodes = append(nodes, node)
noErr(t, h.State.UpsertNode(h.NextIndex(), node))
// Tests that deployments with failed allocs result in placements as long as the
// deployment is running.
func TestDeployment_FailedAllocs_Reschedule(t *testing.T) {
for _, failedDeployment := range []bool{false, true} {
t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) {
h := NewHarness(t)
require := require.New(t)
// Create some nodes
var nodes []*structs.Node
for i := 0; i < 10; i++ {
node := mock.Node()
nodes = append(nodes, node)
noErr(t, h.State.UpsertNode(h.NextIndex(), node))
}
// Generate a fake job with allocations and a reschedule policy.
job := mock.Job()
job.TaskGroups[0].Count = 2
job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{
Attempts: 1,
Interval: 15 * time.Minute,
}
jobIndex := h.NextIndex()
require.Nil(h.State.UpsertJob(jobIndex, job))
deployment := mock.Deployment()
deployment.JobID = job.ID
deployment.JobCreateIndex = jobIndex
deployment.JobVersion = job.Version
if failedDeployment {
deployment.Status = structs.DeploymentStatusFailed
}
require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment))
var allocs []*structs.Allocation
for i := 0; i < 2; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = nodes[i].ID
alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
alloc.DeploymentID = deployment.ID
allocs = append(allocs, alloc)
}
// Mark one of the allocations as failed in the past
allocs[1].ClientStatus = structs.AllocClientStatusFailed
allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start",
StartedAt: time.Now().Add(-12 * time.Hour),
FinishedAt: time.Now().Add(-10 * time.Hour)}}
allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true)
require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs))
// Create a mock evaluation
eval := &structs.Evaluation{
Namespace: structs.DefaultNamespace,
ID: uuid.Generate(),
Priority: 50,
TriggeredBy: structs.EvalTriggerNodeUpdate,
JobID: job.ID,
Status: structs.EvalStatusPending,
}
require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval}))
// Process the evaluation
require.Nil(h.Process(NewServiceScheduler, eval))
if failedDeployment {
// Verify no plan created
require.Len(h.Plans, 0)
} else {
require.Len(h.Plans, 1)
plan := h.Plans[0]
// Ensure the plan allocated
var planned []*structs.Allocation
for _, allocList := range plan.NodeAllocation {
planned = append(planned, allocList...)
}
if len(planned) != 1 {
t.Fatalf("bad: %#v", plan)
}
}
})
}
// Generate a fake job with allocations and a reschedule policy.
job := mock.Job()
job.TaskGroups[0].Count = 2
job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{
Attempts: 1,
Interval: 15 * time.Minute,
}
jobIndex := h.NextIndex()
require.Nil(h.State.UpsertJob(jobIndex, job))
deployment := mock.Deployment()
deployment.JobID = job.ID
deployment.JobCreateIndex = jobIndex
deployment.JobVersion = job.Version
require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment))
var allocs []*structs.Allocation
for i := 0; i < 2; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = nodes[i].ID
alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
alloc.DeploymentID = deployment.ID
allocs = append(allocs, alloc)
}
// Mark one of the allocations as failed
allocs[1].ClientStatus = structs.AllocClientStatusFailed
require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs))
// Create a mock evaluation
eval := &structs.Evaluation{
Namespace: structs.DefaultNamespace,
ID: uuid.Generate(),
Priority: 50,
TriggeredBy: structs.EvalTriggerNodeUpdate,
JobID: job.ID,
Status: structs.EvalStatusPending,
}
require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval}))
// Process the evaluation
require.Nil(h.Process(NewServiceScheduler, eval))
// Verify no plan created
require.Equal(0, len(h.Plans))
}
func TestBatchSched_Run_CompleteAlloc(t *testing.T) {

105
scheduler/reconcile.go
View File

@ -194,20 +194,8 @@ func (a *allocReconciler) Compute() *reconcileResults {
// Detect if the deployment is paused
if a.deployment != nil {
// Detect if any allocs associated with this deploy have failed
// Failed allocations could edge trigger an evaluation before the deployment watcher
// runs and marks the deploy as failed. This block makes sure that is still
// considered a failed deploy
failedAllocsInDeploy := false
for _, as := range m {
for _, alloc := range as {
if alloc.DeploymentID == a.deployment.ID && alloc.ClientStatus == structs.AllocClientStatusFailed {
failedAllocsInDeploy = true
}
}
}
a.deploymentPaused = a.deployment.Status == structs.DeploymentStatusPaused
a.deploymentFailed = a.deployment.Status == structs.DeploymentStatusFailed || failedAllocsInDeploy
a.deploymentFailed = a.deployment.Status == structs.DeploymentStatusFailed
}
// Reconcile each group
@ -334,12 +322,10 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
dstate, existingDeployment = a.deployment.TaskGroups[group]
}
if !existingDeployment {
autorevert := false
if tg.Update != nil && tg.Update.AutoRevert {
autorevert = true
}
dstate = &structs.DeploymentState{
AutoRevert: autorevert,
dstate = &structs.DeploymentState{}
if tg.Update != nil {
dstate.AutoRevert = tg.Update.AutoRevert
dstate.ProgressDeadline = tg.Update.ProgressDeadline
}
}
@ -348,13 +334,15 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
all, ignore := a.filterOldTerminalAllocs(all)
desiredChanges.Ignore += uint64(len(ignore))
// canaries is the set of canaries for the current deployment and all is all
// allocs including the canaries
canaries, all := a.handleGroupCanaries(all, desiredChanges)
// Determine what set of allocations are on tainted nodes
untainted, migrate, lost := all.filterByTainted(a.taintedNodes)
// Determine what set of terminal allocations need to be rescheduled
untainted, rescheduleNow, rescheduleLater := untainted.filterByRescheduleable(a.batch, a.now, a.evalID)
untainted, rescheduleNow, rescheduleLater := untainted.filterByRescheduleable(a.batch, a.now, a.evalID, a.deployment)
// Create batched follow up evaluations for allocations that are
// reschedulable later and mark the allocations for in place updating
@ -371,14 +359,6 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
desiredChanges.Stop += uint64(len(stop))
untainted = untainted.difference(stop)
// Having stopped un-needed allocations, append the canaries to the existing
// set of untainted because they are promoted. This will cause them to be
// treated like non-canaries
if !canaryState {
untainted = untainted.union(canaries)
nameIndex.Set(canaries)
}
// Do inplace upgrades where possible and capture the set of upgrades that
// need to be done destructively.
ignore, inplace, destructive := a.computeUpdates(tg, untainted)
@ -388,6 +368,12 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
dstate.DesiredTotal += len(destructive) + len(inplace)
}
// Remove the canaries now that we have handled rescheduling so that we do
// not consider them when making placement decisions.
if canaryState {
untainted = untainted.difference(canaries)
}
// The fact that we have destructive updates and have less canaries than is
// desired means we need to create canaries
numDestructive := len(destructive)
@ -396,7 +382,6 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
requireCanary := numDestructive != 0 && strategy != nil && len(canaries) < strategy.Canary && !canariesPromoted
if requireCanary && !a.deploymentPaused && !a.deploymentFailed {
number := strategy.Canary - len(canaries)
number = helper.IntMin(numDestructive, number)
desiredChanges.Canary += uint64(number)
if !existingDeployment {
dstate.DesiredCanaries = strategy.Canary
@ -436,16 +421,29 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
min := helper.IntMin(len(place), limit)
limit -= min
} else if !deploymentPlaceReady && len(lost) != 0 {
// We are in a situation where we shouldn't be placing more than we need
// to but we have lost allocations. It is a very weird user experience
// if you have a node go down and Nomad doesn't replace the allocations
// because the deployment is paused/failed so we only place to recover
// the lost allocations.
allowed := helper.IntMin(len(lost), len(place))
desiredChanges.Place += uint64(allowed)
for _, p := range place[:allowed] {
a.result.place = append(a.result.place, p)
} else if !deploymentPlaceReady {
// We do not want to place additional allocations but in the case we
// have lost allocations or allocations that require rescheduling now,
// we do so regardless to avoid odd user experiences.
if len(lost) != 0 {
allowed := helper.IntMin(len(lost), len(place))
desiredChanges.Place += uint64(allowed)
for _, p := range place[:allowed] {
a.result.place = append(a.result.place, p)
}
}
// Handle rescheduling of failed allocations even if the deployment is
// failed. We do not reschedule if the allocation is part of the failed
// deployment.
if now := len(rescheduleNow); now != 0 {
for _, p := range place {
prev := p.PreviousAllocation()
if p.IsRescheduling() && !(a.deploymentFailed && prev != nil && a.deployment.ID == prev.DeploymentID) {
a.result.place = append(a.result.place, p)
desiredChanges.Place++
}
}
}
}
@ -522,16 +520,15 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool {
// deploymentComplete is whether the deployment is complete which largely
// means that no placements were made or desired to be made
deploymentComplete := len(destructive)+len(inplace)+len(place)+len(migrate) == 0 && !requireCanary
deploymentComplete := len(destructive)+len(inplace)+len(place)+len(migrate)+len(rescheduleNow)+len(rescheduleLater) == 0 && !requireCanary
// Final check to see if the deployment is complete is to ensure everything
// is healthy
if deploymentComplete && a.deployment != nil {
partOf, _ := untainted.filterByDeployment(a.deployment.ID)
for _, alloc := range partOf {
if !alloc.DeploymentStatus.IsHealthy() {
if dstate, ok := a.deployment.TaskGroups[group]; ok {
if dstate.HealthyAllocs < helper.IntMax(dstate.DesiredTotal, dstate.DesiredCanaries) || // Make sure we have enough healthy allocs
(dstate.DesiredCanaries > 0 && !dstate.Promoted) { // Make sure we are promoted if we have canaries
deploymentComplete = false
break
}
}
}
@ -663,26 +660,24 @@ func (a *allocReconciler) computeLimit(group *structs.TaskGroup, untainted, dest
func (a *allocReconciler) computePlacements(group *structs.TaskGroup,
nameIndex *allocNameIndex, untainted, migrate allocSet, reschedule allocSet) []allocPlaceResult {
// Hot path the nothing to do case
existing := len(untainted) + len(migrate)
if existing >= group.Count {
return nil
}
var place []allocPlaceResult
// Add rescheduled placement results
// Any allocations being rescheduled will remain at DesiredStatusRun ClientStatusFailed
var place []allocPlaceResult
for _, alloc := range reschedule {
place = append(place, allocPlaceResult{
name: alloc.Name,
taskGroup: group,
previousAlloc: alloc,
reschedule: true,
canary: alloc.DeploymentStatus.IsCanary(),
})
existing += 1
if existing == group.Count {
break
}
}
// Hot path the nothing to do case
existing := len(untainted) + len(migrate) + len(reschedule)
if existing >= group.Count {
return place
}
// Add remaining placement results
if existing < group.Count {
for _, name := range nameIndex.Next(uint(group.Count - existing)) {

682
scheduler/reconcile_test.go
View File

@ -19,69 +19,6 @@ import (
"github.com/stretchr/testify/require"
)
/*
Basic Tests:
√ Place when there is nothing in the cluster
√ Place remainder when there is some in the cluster
√ Scale down from n to n-m where n != m
√ Scale down from n to zero
√ Inplace upgrade test
√ Inplace upgrade and scale up test
√ Inplace upgrade and scale down test
√ Destructive upgrade
√ Destructive upgrade and scale up test
√ Destructive upgrade and scale down test
√ Handle lost nodes
√ Handle lost nodes and scale up
√ Handle lost nodes and scale down
√ Handle draining nodes
√ Handle draining nodes and scale up
√ Handle draining nodes and scale down
√ Handle task group being removed
√ Handle job being stopped both as .Stopped and nil
√ Place more that one group
√ Handle delayed rescheduling failed allocs for batch jobs
√ Handle delayed rescheduling failed allocs for service jobs
√ Handle eligible now rescheduling failed allocs for batch jobs
√ Handle eligible now rescheduling failed allocs for service jobs
√ Previously rescheduled allocs should not be rescheduled again
√ Aggregated evaluations for allocations that fail close together
Update stanza Tests:
√ Stopped job cancels any active deployment
√ Stopped job doesn't cancel terminal deployment
√ JobIndex change cancels any active deployment
√ JobIndex change doesn't cancels any terminal deployment
√ Destructive changes create deployment and get rolled out via max_parallelism
√ Don't create a deployment if there are no changes
√ Deployment created by all inplace updates
√ Paused or failed deployment doesn't create any more canaries
√ Paused or failed deployment doesn't do any placements unless replacing lost allocs
√ Paused or failed deployment doesn't do destructive updates
√ Paused does do migrations
√ Failed deployment doesn't do migrations
√ Canary that is on a draining node
√ Canary that is on a lost node
√ Stop old canaries
√ Create new canaries on job change
√ Create new canaries on job change while scaling up
√ Create new canaries on job change while scaling down
√ Fill canaries if partial placement
√ Promote canaries unblocks max_parallel
√ Promote canaries when canaries == count
√ Only place as many as are healthy in deployment
√ Limit calculation accounts for healthy allocs on migrating/lost nodes
√ Failed deployment should not place anything
√ Run after canaries have been promoted, new allocs have been rolled out and there is no deployment
√ Failed deployment cancels non-promoted task groups
√ Failed deployment and updated job works
√ Finished deployment gets marked as complete
√ Change job change while scaling up
√ Update the job when all allocations from the previous job haven't been placed yet.
√ Paused or failed deployment doesn't do any rescheduling of failed allocs
√ Running deployment with failed allocs doesn't do any rescheduling of failed allocs
*/
var (
canaryUpdate = &structs.UpdateStrategy{
Canary: 2,
@ -321,14 +258,14 @@ func assertResults(t *testing.T, r *reconcileResults, exp *resultExpectation) {
assert := assert.New(t)
if exp.createDeployment != nil && r.deployment == nil {
t.Fatalf("Expect a created deployment got none")
t.Errorf("Expect a created deployment got none")
} else if exp.createDeployment == nil && r.deployment != nil {
t.Fatalf("Expect no created deployment; got %#v", r.deployment)
t.Errorf("Expect no created deployment; got %#v", r.deployment)
} else if exp.createDeployment != nil && r.deployment != nil {
// Clear the deployment ID
r.deployment.ID, exp.createDeployment.ID = "", ""
if !reflect.DeepEqual(r.deployment, exp.createDeployment) {
t.Fatalf("Unexpected createdDeployment; got\n %#v\nwant\n%#v\nDiff: %v",
t.Errorf("Unexpected createdDeployment; got\n %#v\nwant\n%#v\nDiff: %v",
r.deployment, exp.createDeployment, pretty.Diff(r.deployment, exp.createDeployment))
}
}
@ -1193,6 +1130,55 @@ func TestReconciler_MultiTG(t *testing.T) {
assertNamesHaveIndexes(t, intRange(2, 9, 0, 9), placeResultsToNames(r.place))
}
// Tests the reconciler properly handles jobs with multiple task groups with
// only one having an update stanza and a deployment already being created
func TestReconciler_MultiTG_SingleUpdateStanza(t *testing.T) {
job := mock.Job()
tg2 := job.TaskGroups[0].Copy()
tg2.Name = "foo"
job.TaskGroups = append(job.TaskGroups, tg2)
job.TaskGroups[0].Update = noCanaryUpdate
// Create all the allocs
var allocs []*structs.Allocation
for i := 0; i < 2; i++ {
for j := 0; j < 10; j++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[i].Name, uint(j))
alloc.TaskGroup = job.TaskGroups[i].Name
allocs = append(allocs, alloc)
}
}
d := structs.NewDeployment(job)
d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{
DesiredTotal: 10,
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil, "")
r := reconciler.Compute()
// Assert the correct results
assertResults(t, r, &resultExpectation{
createDeployment: nil,
deploymentUpdates: nil,
place: 0,
inplace: 0,
stop: 0,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Ignore: 10,
},
tg2.Name: {
Ignore: 10,
},
},
})
}
// Tests delayed rescheduling of failed batch allocations
func TestReconciler_RescheduleLater_Batch(t *testing.T) {
require := require.New(t)
@ -1881,6 +1867,362 @@ func TestReconciler_RescheduleNow_EvalIDMatch(t *testing.T) {
assertPlacementsAreRescheduled(t, 1, r.place)
}
// Tests rescheduling failed service allocations when there are canaries
func TestReconciler_RescheduleNow_Service_WithCanaries(t *testing.T) {
require := require.New(t)
// Set desired 5
job := mock.Job()
job.TaskGroups[0].Count = 5
tgName := job.TaskGroups[0].Name
now := time.Now()
// Set up reschedule policy and update stanza
job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{
Attempts: 1,
Interval: 24 * time.Hour,
Delay: 5 * time.Second,
DelayFunction: "",
MaxDelay: 1 * time.Hour,
Unlimited: false,
}
job.TaskGroups[0].Update = canaryUpdate
job2 := job.Copy()
job2.Version++
d := structs.NewDeployment(job2)
d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
s := &structs.DeploymentState{
DesiredCanaries: 2,
DesiredTotal: 5,
}
d.TaskGroups[job.TaskGroups[0].Name] = s
// Create 5 existing allocations
var allocs []*structs.Allocation
for i := 0; i < 5; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
allocs = append(allocs, alloc)
alloc.ClientStatus = structs.AllocClientStatusRunning
}
// Mark three as failed
allocs[0].ClientStatus = structs.AllocClientStatusFailed
// Mark one of them as already rescheduled once
allocs[0].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{
{RescheduleTime: time.Now().Add(-1 * time.Hour).UTC().UnixNano(),
PrevAllocID: uuid.Generate(),
PrevNodeID: uuid.Generate(),
},
}}
allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
allocs[1].ClientStatus = structs.AllocClientStatusFailed
// Mark one as desired state stop
allocs[4].ClientStatus = structs.AllocClientStatusFailed
// Create 2 canary allocations
for i := 0; i < 2; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.ClientStatus = structs.AllocClientStatusRunning
alloc.DeploymentID = d.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
Healthy: helper.BoolToPtr(false),
}
s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID)
allocs = append(allocs, alloc)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job2, d, allocs, nil, "")
r := reconciler.Compute()
// Verify that no follow up evals were created
evals := r.desiredFollowupEvals[tgName]
require.Nil(evals)
// Verify that one rescheduled alloc and one replacement for terminal alloc were placed
assertResults(t, r, &resultExpectation{
createDeployment: nil,
deploymentUpdates: nil,
place: 2,
inplace: 0,
stop: 0,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Place: 2,
Ignore: 5,
},
},
})
// Rescheduled allocs should have previous allocs
assertNamesHaveIndexes(t, intRange(1, 1, 4, 4), placeResultsToNames(r.place))
assertPlaceResultsHavePreviousAllocs(t, 2, r.place)
assertPlacementsAreRescheduled(t, 2, r.place)
}
// Tests rescheduling failed canary service allocations
func TestReconciler_RescheduleNow_Service_Canaries(t *testing.T) {
require := require.New(t)
// Set desired 5
job := mock.Job()
job.TaskGroups[0].Count = 5
tgName := job.TaskGroups[0].Name
now := time.Now()
// Set up reschedule policy and update stanza
job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{
Delay: 5 * time.Second,
DelayFunction: "constant",
MaxDelay: 1 * time.Hour,
Unlimited: true,
}
job.TaskGroups[0].Update = canaryUpdate
job2 := job.Copy()
job2.Version++
d := structs.NewDeployment(job2)
d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
s := &structs.DeploymentState{
DesiredCanaries: 2,
DesiredTotal: 5,
}
d.TaskGroups[job.TaskGroups[0].Name] = s
// Create 5 existing allocations
var allocs []*structs.Allocation
for i := 0; i < 5; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
allocs = append(allocs, alloc)
alloc.ClientStatus = structs.AllocClientStatusRunning
}
// Create 2 healthy canary allocations
for i := 0; i < 2; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.ClientStatus = structs.AllocClientStatusRunning
alloc.DeploymentID = d.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
Healthy: helper.BoolToPtr(false),
}
s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID)
allocs = append(allocs, alloc)
}
// Mark the canaries as failed
allocs[5].ClientStatus = structs.AllocClientStatusFailed
allocs[5].DesiredTransition.Reschedule = helper.BoolToPtr(true)
// Mark one of them as already rescheduled once
allocs[5].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{
{RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(),
PrevAllocID: uuid.Generate(),
PrevNodeID: uuid.Generate(),
},
}}
allocs[6].TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
allocs[6].ClientStatus = structs.AllocClientStatusFailed
allocs[6].DesiredTransition.Reschedule = helper.BoolToPtr(true)
// Create 4 unhealthy canary allocations that have already been replaced
for i := 0; i < 4; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i%2))
alloc.ClientStatus = structs.AllocClientStatusFailed
alloc.DeploymentID = d.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
Healthy: helper.BoolToPtr(false),
}
s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID)
allocs = append(allocs, alloc)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job2, d, allocs, nil, "")
reconciler.now = now
r := reconciler.Compute()
// Verify that no follow up evals were created
evals := r.desiredFollowupEvals[tgName]
require.Nil(evals)
// Verify that one rescheduled alloc and one replacement for terminal alloc were placed
assertResults(t, r, &resultExpectation{
createDeployment: nil,
deploymentUpdates: nil,
place: 2,
inplace: 0,
stop: 0,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Place: 2,
Ignore: 9,
},
},
})
// Rescheduled allocs should have previous allocs
assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place))
assertPlaceResultsHavePreviousAllocs(t, 2, r.place)
assertPlacementsAreRescheduled(t, 2, r.place)
}
// Tests rescheduling failed canary service allocations when one has reached its
// reschedule limit
func TestReconciler_RescheduleNow_Service_Canaries_Limit(t *testing.T) {
require := require.New(t)
// Set desired 5
job := mock.Job()
job.TaskGroups[0].Count = 5
tgName := job.TaskGroups[0].Name
now := time.Now()
// Set up reschedule policy and update stanza
job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{
Attempts: 1,
Interval: 24 * time.Hour,
Delay: 5 * time.Second,
DelayFunction: "",
MaxDelay: 1 * time.Hour,
Unlimited: false,
}
job.TaskGroups[0].Update = canaryUpdate
job2 := job.Copy()
job2.Version++
d := structs.NewDeployment(job2)
d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
s := &structs.DeploymentState{
DesiredCanaries: 2,
DesiredTotal: 5,
}
d.TaskGroups[job.TaskGroups[0].Name] = s
// Create 5 existing allocations
var allocs []*structs.Allocation
for i := 0; i < 5; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
allocs = append(allocs, alloc)
alloc.ClientStatus = structs.AllocClientStatusRunning
}
// Create 2 healthy canary allocations
for i := 0; i < 2; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.ClientStatus = structs.AllocClientStatusRunning
alloc.DeploymentID = d.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
Healthy: helper.BoolToPtr(false),
}
s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID)
allocs = append(allocs, alloc)
}
// Mark the canaries as failed
allocs[5].ClientStatus = structs.AllocClientStatusFailed
allocs[5].DesiredTransition.Reschedule = helper.BoolToPtr(true)
// Mark one of them as already rescheduled once
allocs[5].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{
{RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(),
PrevAllocID: uuid.Generate(),
PrevNodeID: uuid.Generate(),
},
}}
allocs[6].TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
allocs[6].ClientStatus = structs.AllocClientStatusFailed
allocs[6].DesiredTransition.Reschedule = helper.BoolToPtr(true)
// Create 4 unhealthy canary allocations that have already been replaced
for i := 0; i < 4; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i%2))
alloc.ClientStatus = structs.AllocClientStatusFailed
alloc.DeploymentID = d.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{
Canary: true,
Healthy: helper.BoolToPtr(false),
}
s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID)
allocs = append(allocs, alloc)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job2, d, allocs, nil, "")
reconciler.now = now
r := reconciler.Compute()
// Verify that no follow up evals were created
evals := r.desiredFollowupEvals[tgName]
require.Nil(evals)
// Verify that one rescheduled alloc and one replacement for terminal alloc were placed
assertResults(t, r, &resultExpectation{
createDeployment: nil,
deploymentUpdates: nil,
place: 1,
inplace: 0,
stop: 0,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Place: 1,
Ignore: 10,
},
},
})
// Rescheduled allocs should have previous allocs
assertNamesHaveIndexes(t, intRange(1, 1), placeResultsToNames(r.place))
assertPlaceResultsHavePreviousAllocs(t, 1, r.place)
assertPlacementsAreRescheduled(t, 1, r.place)
}
// Tests failed service allocations that were already rescheduled won't be rescheduled again
func TestReconciler_DontReschedule_PreviouslyRescheduled(t *testing.T) {
// Set desired 5
@ -2810,6 +3152,55 @@ func TestReconciler_NewCanaries(t *testing.T) {
assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place))
}
// Tests the reconciler creates new canaries when the job changes and the
// canary count is greater than the task group count
func TestReconciler_NewCanaries_CountGreater(t *testing.T) {
job := mock.Job()
job.TaskGroups[0].Count = 3
job.TaskGroups[0].Update = canaryUpdate.Copy()
job.TaskGroups[0].Update.Canary = 7
// Create 3 allocations from the old job
var allocs []*structs.Allocation
for i := 0; i < 3; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.TaskGroup = job.TaskGroups[0].Name
allocs = append(allocs, alloc)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil, "")
r := reconciler.Compute()
newD := structs.NewDeployment(job)
newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
state := &structs.DeploymentState{
DesiredCanaries: 7,
DesiredTotal: 3,
}
newD.TaskGroups[job.TaskGroups[0].Name] = state
// Assert the correct results
assertResults(t, r, &resultExpectation{
createDeployment: newD,
deploymentUpdates: nil,
place: 7,
inplace: 0,
stop: 0,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Canary: 7,
Ignore: 3,
},
},
})
assertNamesHaveIndexes(t, intRange(0, 2, 3, 6), placeResultsToNames(r.place))
}
// Tests the reconciler creates new canaries when the job changes for multiple
// task groups
func TestReconciler_NewCanaries_MultiTG(t *testing.T) {
@ -3117,6 +3508,7 @@ func TestReconciler_PromoteCanaries_CanariesEqualCount(t *testing.T) {
DesiredTotal: 2,
DesiredCanaries: 2,
PlacedAllocs: 2,
HealthyAllocs: 2,
}
d.TaskGroups[job.TaskGroups[0].Name] = s
@ -3490,6 +3882,69 @@ func TestReconciler_CompleteDeployment(t *testing.T) {
})
}
// Tests that the reconciler marks a deployment as complete once there is
// nothing left to place even if there are failed allocations that are part of
// the deployment.
func TestReconciler_MarkDeploymentComplete_FailedAllocations(t *testing.T) {
job := mock.Job()
job.TaskGroups[0].Update = noCanaryUpdate
d := structs.NewDeployment(job)
d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{
DesiredTotal: 10,
PlacedAllocs: 20,
HealthyAllocs: 10,
}
// Create 10 healthy allocs and 10 allocs that are failed
var allocs []*structs.Allocation
for i := 0; i < 20; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i%10))
alloc.TaskGroup = job.TaskGroups[0].Name
alloc.DeploymentID = d.ID
alloc.DeploymentStatus = &structs.AllocDeploymentStatus{}
if i < 10 {
alloc.ClientStatus = structs.AllocClientStatusRunning
alloc.DeploymentStatus.Healthy = helper.BoolToPtr(true)
} else {
alloc.DesiredStatus = structs.AllocDesiredStatusStop
alloc.ClientStatus = structs.AllocClientStatusFailed
alloc.DeploymentStatus.Healthy = helper.BoolToPtr(false)
}
allocs = append(allocs, alloc)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil, "")
r := reconciler.Compute()
updates := []*structs.DeploymentStatusUpdate{
{
DeploymentID: d.ID,
Status: structs.DeploymentStatusSuccessful,
StatusDescription: structs.DeploymentStatusDescriptionSuccessful,
},
}
// Assert the correct results
assertResults(t, r, &resultExpectation{
createDeployment: nil,
deploymentUpdates: updates,
place: 0,
inplace: 0,
stop: 0,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Ignore: 10,
},
},
})
}
// Test that a failed deployment cancels non-promoted canaries
func TestReconciler_FailedDeployment_CancelCanaries(t *testing.T) {
// Create a job with two task groups
@ -3883,6 +4338,7 @@ func TestReconciler_FailedDeployment_DontReschedule(t *testing.T) {
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.TaskGroup = job.TaskGroups[0].Name
alloc.DeploymentID = d.ID
allocs = append(allocs, alloc)
}
@ -3913,7 +4369,8 @@ func TestReconciler_FailedDeployment_DontReschedule(t *testing.T) {
})
}
// Test that a running deployment with failed allocs will not result in rescheduling failed allocations
// Test that a running deployment with failed allocs will not result in
// rescheduling failed allocations unless they are marked as reschedulable.
func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) {
job := mock.Job()
job.TaskGroups[0].Update = noCanaryUpdate
@ -3925,13 +4382,13 @@ func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) {
d.Status = structs.DeploymentStatusRunning
d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{
Promoted: false,
DesiredTotal: 5,
PlacedAllocs: 4,
DesiredTotal: 10,
PlacedAllocs: 10,
}
// Create 4 allocations and mark two as failed
// Create 10 allocations
var allocs []*structs.Allocation
for i := 0; i < 4; i++ {
for i := 0; i < 10; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
@ -3939,31 +4396,30 @@ func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) {
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.TaskGroup = job.TaskGroups[0].Name
alloc.DeploymentID = d.ID
alloc.ClientStatus = structs.AllocClientStatusFailed
alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
allocs = append(allocs, alloc)
}
// Create allocs that are reschedulable now
allocs[2].ClientStatus = structs.AllocClientStatusFailed
allocs[2].TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
allocs[3].ClientStatus = structs.AllocClientStatusFailed
allocs[3].TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
// Mark half of them as reschedulable
for i := 0; i < 5; i++ {
allocs[i].DesiredTransition.Reschedule = helper.BoolToPtr(true)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, d, allocs, nil, "")
r := reconciler.Compute()
// Assert that no rescheduled placements were created
assertResults(t, r, &resultExpectation{
place: 0,
place: 5,
createDeployment: nil,
deploymentUpdates: nil,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Ignore: 2,
Place: 5,
Ignore: 5,
},
},
})
@ -3993,12 +4449,12 @@ func TestReconciler_FailedDeployment_AutoRevert_CancelCanaries(t *testing.T) {
jobv2.Version = 2
jobv2.TaskGroups[0].Meta = map[string]string{"version": "2"}
// Create an existing failed deployment that has promoted one task group
d := structs.NewDeployment(jobv2)
state := &structs.DeploymentState{
Promoted: false,
DesiredTotal: 3,
PlacedAllocs: 3,
Promoted: true,
DesiredTotal: 3,
PlacedAllocs: 3,
HealthyAllocs: 3,
}
d.TaskGroups[job.TaskGroups[0].Name] = state
@ -4062,3 +4518,55 @@ func TestReconciler_FailedDeployment_AutoRevert_CancelCanaries(t *testing.T) {
},
})
}
// Test that a successful deployment with failed allocs will result in
// rescheduling failed allocations
func TestReconciler_SuccessfulDeploymentWithFailedAllocs_Reschedule(t *testing.T) {
job := mock.Job()
job.TaskGroups[0].Update = noCanaryUpdate
tgName := job.TaskGroups[0].Name
now := time.Now()
// Mock deployment with failed allocs, but deployment watcher hasn't marked it as failed yet
d := structs.NewDeployment(job)
d.Status = structs.DeploymentStatusSuccessful
d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{
Promoted: false,
DesiredTotal: 10,
PlacedAllocs: 10,
}
// Create 10 allocations
var allocs []*structs.Allocation
for i := 0; i < 10; i++ {
alloc := mock.Alloc()
alloc.Job = job
alloc.JobID = job.ID
alloc.NodeID = uuid.Generate()
alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i))
alloc.TaskGroup = job.TaskGroups[0].Name
alloc.DeploymentID = d.ID
alloc.ClientStatus = structs.AllocClientStatusFailed
alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "start",
StartedAt: now.Add(-1 * time.Hour),
FinishedAt: now.Add(-10 * time.Second)}}
allocs = append(allocs, alloc)
}
reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, d, allocs, nil, "")
r := reconciler.Compute()
// Assert that rescheduled placements were created
assertResults(t, r, &resultExpectation{
place: 10,
createDeployment: nil,
deploymentUpdates: nil,
desiredTGUpdates: map[string]*structs.DesiredUpdates{
job.TaskGroups[0].Name: {
Place: 10,
Ignore: 0,
},
},
})
assertPlaceResultsHavePreviousAllocs(t, 10, r.place)
}

40
scheduler/reconcile_util.go
View File

@ -234,7 +234,7 @@ func (a allocSet) filterByTainted(nodes map[string]*structs.Node) (untainted, mi
// untainted or a set of allocations that must be rescheduled now. Allocations that can be rescheduled
// at a future time are also returned so that we can create follow up evaluations for them. Allocs are
// skipped or considered untainted according to logic defined in shouldFilter method.
func (a allocSet) filterByRescheduleable(isBatch bool, now time.Time, evalID string) (untainted, rescheduleNow allocSet, rescheduleLater []*delayedRescheduleInfo) {
func (a allocSet) filterByRescheduleable(isBatch bool, now time.Time, evalID string, deployment *structs.Deployment) (untainted, rescheduleNow allocSet, rescheduleLater []*delayedRescheduleInfo) {
untainted = make(map[string]*structs.Allocation)
rescheduleNow = make(map[string]*structs.Allocation)
@ -257,7 +257,7 @@ func (a allocSet) filterByRescheduleable(isBatch bool, now time.Time, evalID str
// Only failed allocs with desired state run get to this point
// If the failed alloc is not eligible for rescheduling now we add it to the untainted set
eligibleNow, eligibleLater, rescheduleTime = updateByReschedulable(alloc, now, evalID)
eligibleNow, eligibleLater, rescheduleTime = updateByReschedulable(alloc, now, evalID, deployment)
if !eligibleNow {
untainted[alloc.ID] = alloc
if eligibleLater {
@ -320,9 +320,15 @@ func shouldFilter(alloc *structs.Allocation, isBatch bool) (untainted, ignore bo
// updateByReschedulable is a helper method that encapsulates logic for whether a failed allocation
// should be rescheduled now, later or left in the untainted set
func updateByReschedulable(alloc *structs.Allocation, now time.Time, evalID string) (rescheduleNow, rescheduleLater bool, rescheduleTime time.Time) {
rescheduleTime, eligible := alloc.NextRescheduleTime()
func updateByReschedulable(alloc *structs.Allocation, now time.Time, evalID string, d *structs.Deployment) (rescheduleNow, rescheduleLater bool, rescheduleTime time.Time) {
// If the allocation is part of an ongoing active deployment, we only allow it to reschedule
// if it has been marked eligible
if d != nil && alloc.DeploymentID == d.ID && d.Active() && !alloc.DesiredTransition.ShouldReschedule() {
return
}
// Reschedule if the eval ID matches the alloc's followup evalID or if its close to its reschedule time
rescheduleTime, eligible := alloc.NextRescheduleTime()
if eligible && (alloc.FollowupEvalID == evalID || rescheduleTime.Sub(now) <= rescheduleWindowSize) {
rescheduleNow = true
return
@ -470,7 +476,7 @@ func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) []
// First select indexes from the allocations that are undergoing destructive
// updates. This way we avoid duplicate names as they will get replaced.
dmap := bitmapFrom(destructive, uint(a.count))
var remainder uint
remainder := n
for _, idx := range dmap.IndexesInRange(true, uint(0), uint(a.count)-1) {
name := structs.AllocName(a.job, a.taskGroup, uint(idx))
if _, used := existingNames[name]; !used {
@ -478,7 +484,7 @@ func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) []
a.b.Set(uint(idx))
// If we have enough, return
remainder := n - uint(len(next))
remainder = n - uint(len(next))
if remainder == 0 {
return next
}
@ -500,21 +506,15 @@ func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) []
}
}
// We have exhausted the preferred and free set, now just pick overlapping
// indexes
var i uint
for i = 0; i < remainder; i++ {
// We have exhausted the preferred and free set. Pick starting from n to
// n+remainder, to avoid overlapping where possible. An example is the
// desired count is 3 and we want 5 canaries. The first 3 canaries can use
// index [0, 1, 2] but after that we prefer picking indexes [4, 5] so that
// we do not overlap. Once the canaries are promoted, these would be the
// allocations that would be shut down as well.
for i := uint(a.count); i < uint(a.count)+remainder; i++ {
name := structs.AllocName(a.job, a.taskGroup, i)
if _, used := existingNames[name]; !used {
next = append(next, name)
a.b.Set(i)
// If we have enough, return
remainder = n - uint(len(next))
if remainder == 0 {
return next
}
}
next = append(next, name)
}
return next

239
vendor/github.com/stretchr/testify/assert/assertion_format.go generated vendored
View File

@ -13,6 +13,9 @@ import (
// Conditionf uses a Comparison to assert a complex condition.
func Conditionf(t TestingT, comp Comparison, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Condition(t, comp, append([]interface{}{msg}, args...)...)
}
@ -22,19 +25,41 @@ func Conditionf(t TestingT, comp Comparison, msg string, args ...interface{}) bo
// assert.Containsf(t, "Hello World", "World", "error message %s", "formatted")
// assert.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted")
// assert.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Contains(t, s, contains, append([]interface{}{msg}, args...)...)
}
// DirExistsf checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists.
func DirExistsf(t TestingT, path string, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return DirExists(t, path, append([]interface{}{msg}, args...)...)
}
// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified
// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements,
// the number of appearances of each of them in both lists should match.
//
// assert.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted")
func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return ElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...)
}
// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either
// a slice or a channel with len == 0.
//
// assert.Emptyf(t, obj, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Empty(t, object, append([]interface{}{msg}, args...)...)
}
@ -42,12 +67,13 @@ func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) boo
//
// assert.Equalf(t, 123, 123, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
//
// Pointer variable equality is determined based on the equality of the
// referenced values (as opposed to the memory addresses). Function equality
// cannot be determined and will always fail.
func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Equal(t, expected, actual, append([]interface{}{msg}, args...)...)
}
@ -56,9 +82,10 @@ func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, ar
//
// actualObj, err := SomeFunction()
// assert.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return EqualError(t, theError, errString, append([]interface{}{msg}, args...)...)
}
@ -66,9 +93,10 @@ func EqualErrorf(t TestingT, theError error, errString string, msg string, args
// and equal.
//
// assert.EqualValuesf(t, uint32(123, "error message %s", "formatted"), int32(123))
//
// Returns whether the assertion was successful (true) or not (false).
func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return EqualValues(t, expected, actual, append([]interface{}{msg}, args...)...)
}
@ -78,48 +106,68 @@ func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg stri
// if assert.Errorf(t, err, "error message %s", "formatted") {
// assert.Equal(t, expectedErrorf, err)
// }
//
// Returns whether the assertion was successful (true) or not (false).
func Errorf(t TestingT, err error, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Error(t, err, append([]interface{}{msg}, args...)...)
}
// Exactlyf asserts that two objects are equal is value and type.
// Exactlyf asserts that two objects are equal in value and type.
//
// assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123))
//
// Returns whether the assertion was successful (true) or not (false).
func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Exactly(t, expected, actual, append([]interface{}{msg}, args...)...)
}
// Failf reports a failure through
func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Fail(t, failureMessage, append([]interface{}{msg}, args...)...)
}
// FailNowf fails test
func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return FailNow(t, failureMessage, append([]interface{}{msg}, args...)...)
}
// Falsef asserts that the specified value is false.
//
// assert.Falsef(t, myBool, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Falsef(t TestingT, value bool, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return False(t, value, append([]interface{}{msg}, args...)...)
}
// FileExistsf checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file.
func FileExistsf(t TestingT, path string, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return FileExists(t, path, append([]interface{}{msg}, args...)...)
}
// HTTPBodyContainsf asserts that a specified handler returns a
// body that contains a string.
//
// assert.HTTPBodyContainsf(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool {
return HTTPBodyContains(t, handler, method, url, values, str)
func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return HTTPBodyContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...)
}
// HTTPBodyNotContainsf asserts that a specified handler returns a
@ -128,8 +176,11 @@ func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url
// assert.HTTPBodyNotContainsf(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool {
return HTTPBodyNotContains(t, handler, method, url, values, str)
func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return HTTPBodyNotContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...)
}
// HTTPErrorf asserts that a specified handler returns an error status code.
@ -137,8 +188,11 @@ func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, u
// assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}}
//
// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false).
func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) bool {
return HTTPError(t, handler, method, url, values)
func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return HTTPError(t, handler, method, url, values, append([]interface{}{msg}, args...)...)
}
// HTTPRedirectf asserts that a specified handler returns a redirect status code.
@ -146,8 +200,11 @@ func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string,
// assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}}
//
// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false).
func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) bool {
return HTTPRedirect(t, handler, method, url, values)
func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return HTTPRedirect(t, handler, method, url, values, append([]interface{}{msg}, args...)...)
}
// HTTPSuccessf asserts that a specified handler returns a success status code.
@ -155,54 +212,80 @@ func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url stri
// assert.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) bool {
return HTTPSuccess(t, handler, method, url, values)
func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return HTTPSuccess(t, handler, method, url, values, append([]interface{}{msg}, args...)...)
}
// Implementsf asserts that an object is implemented by the specified interface.
//
// assert.Implementsf(t, (*MyInterface, "error message %s", "formatted")(nil), new(MyObject))
func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Implements(t, interfaceObject, object, append([]interface{}{msg}, args...)...)
}
// InDeltaf asserts that the two numerals are within delta of each other.
//
// assert.InDeltaf(t, math.Pi, (22 / 7.0, "error message %s", "formatted"), 0.01)
//
// Returns whether the assertion was successful (true) or not (false).
func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return InDelta(t, expected, actual, delta, append([]interface{}{msg}, args...)...)
}
// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys.
func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return InDeltaMapValues(t, expected, actual, delta, append([]interface{}{msg}, args...)...)
}
// InDeltaSlicef is the same as InDelta, except it compares two slices.
func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return InDeltaSlice(t, expected, actual, delta, append([]interface{}{msg}, args...)...)
}
// InEpsilonf asserts that expected and actual have a relative error less than epsilon
//
// Returns whether the assertion was successful (true) or not (false).
func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return InEpsilon(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...)
}
// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices.
func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return InEpsilonSlice(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...)
}
// IsTypef asserts that the specified objects are of the same type.
func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return IsType(t, expectedType, object, append([]interface{}{msg}, args...)...)
}
// JSONEqf asserts that two JSON strings are equivalent.
//
// assert.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return JSONEq(t, expected, actual, append([]interface{}{msg}, args...)...)
}
@ -210,18 +293,20 @@ func JSONEqf(t TestingT, expected string, actual string, msg string, args ...int
// Lenf also fails if the object has a type that len() not accept.
//
// assert.Lenf(t, mySlice, 3, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Len(t, object, length, append([]interface{}{msg}, args...)...)
}
// Nilf asserts that the specified object is nil.
//
// assert.Nilf(t, err, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Nil(t, object, append([]interface{}{msg}, args...)...)
}
@ -231,9 +316,10 @@ func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) bool
// if assert.NoErrorf(t, err, "error message %s", "formatted") {
// assert.Equal(t, expectedObj, actualObj)
// }
//
// Returns whether the assertion was successful (true) or not (false).
func NoErrorf(t TestingT, err error, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NoError(t, err, append([]interface{}{msg}, args...)...)
}
@ -243,9 +329,10 @@ func NoErrorf(t TestingT, err error, msg string, args ...interface{}) bool {
// assert.NotContainsf(t, "Hello World", "Earth", "error message %s", "formatted")
// assert.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted")
// assert.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotContains(t, s, contains, append([]interface{}{msg}, args...)...)
}
@ -255,9 +342,10 @@ func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, a
// if assert.NotEmptyf(t, obj, "error message %s", "formatted") {
// assert.Equal(t, "two", obj[1])
// }
//
// Returns whether the assertion was successful (true) or not (false).
func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotEmpty(t, object, append([]interface{}{msg}, args...)...)
}
@ -265,29 +353,32 @@ func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{})
//
// assert.NotEqualf(t, obj1, obj2, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
//
// Pointer variable equality is determined based on the equality of the
// referenced values (as opposed to the memory addresses).
func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotEqual(t, expected, actual, append([]interface{}{msg}, args...)...)
}
// NotNilf asserts that the specified object is not nil.
//
// assert.NotNilf(t, err, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotNil(t, object, append([]interface{}{msg}, args...)...)
}
// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic.
//
// assert.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotPanics(t, f, append([]interface{}{msg}, args...)...)
}
@ -295,9 +386,10 @@ func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bo
//
// assert.NotRegexpf(t, regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting")
// assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotRegexp(t, rx, str, append([]interface{}{msg}, args...)...)
}
@ -305,23 +397,28 @@ func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ..
// elements given in the specified subset(array, slice...).
//
// assert.NotSubsetf(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotSubset(t, list, subset, append([]interface{}{msg}, args...)...)
}
// NotZerof asserts that i is not the zero value for its type and returns the truth.
// NotZerof asserts that i is not the zero value for its type.
func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return NotZero(t, i, append([]interface{}{msg}, args...)...)
}
// Panicsf asserts that the code inside the specified PanicTestFunc panics.
//
// assert.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Panicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Panics(t, f, append([]interface{}{msg}, args...)...)
}
@ -329,9 +426,10 @@ func Panicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool
// the recovered panic value equals the expected panic value.
//
// assert.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return PanicsWithValue(t, expected, f, append([]interface{}{msg}, args...)...)
}
@ -339,9 +437,10 @@ func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg str
//
// assert.Regexpf(t, regexp.MustCompile("start", "error message %s", "formatted"), "it's starting")
// assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Regexp(t, rx, str, append([]interface{}{msg}, args...)...)
}
@ -349,31 +448,37 @@ func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...in
// elements given in the specified subset(array, slice...).
//
// assert.Subsetf(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]", "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Subset(t, list, subset, append([]interface{}{msg}, args...)...)
}
// Truef asserts that the specified value is true.
//
// assert.Truef(t, myBool, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func Truef(t TestingT, value bool, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return True(t, value, append([]interface{}{msg}, args...)...)
}
// WithinDurationf asserts that the two times are within duration delta of each other.
//
// assert.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted")
//
// Returns whether the assertion was successful (true) or not (false).
func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return WithinDuration(t, expected, actual, delta, append([]interface{}{msg}, args...)...)
}
// Zerof asserts that i is the zero value for its type and returns the truth.
// Zerof asserts that i is the zero value for its type.
func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
return Zero(t, i, append([]interface{}{msg}, args...)...)
}

1
vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl generated vendored
View File

@ -1,4 +1,5 @@
{{.CommentFormat}}
func {{.DocInfo.Name}}f(t TestingT, {{.ParamsFormat}}) bool {
if h, ok := t.(tHelper); ok { h.Helper() }
return {{.DocInfo.Name}}(t, {{.ForwardedParamsFormat}})
}

478
vendor/github.com/stretchr/testify/assert/assertion_forward.go generated vendored
View File

File diff suppressed because it is too large Load Diff

1
vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl generated vendored
View File

@ -1,4 +1,5 @@
{{.CommentWithoutT "a"}}
func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool {
if h, ok := a.t.(tHelper); ok { h.Helper() }
return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}})
}

454
vendor/github.com/stretchr/testify/assert/assertions.go generated vendored
View File

@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"math"
"os"
"reflect"
"regexp"
"runtime"
@ -26,6 +27,22 @@ type TestingT interface {
Errorf(format string, args ...interface{})
}
// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful
// for table driven tests.
type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) bool
// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful
// for table driven tests.
type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool
// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful
// for table driven tests.
type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool
// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful
// for table driven tests.
type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool
// Comparison a custom function that returns true on success and false on failure
type Comparison func() (success bool)
@ -155,21 +172,6 @@ func isTest(name, prefix string) bool {
return !unicode.IsLower(rune)
}
// getWhitespaceString returns a string that is long enough to overwrite the default
// output from the go testing framework.
func getWhitespaceString() string {
_, file, line, ok := runtime.Caller(1)
if !ok {
return ""
}
parts := strings.Split(file, "/")
file = parts[len(parts)-1]
return strings.Repeat(" ", len(fmt.Sprintf("%s:%d: ", file, line)))
}
func messageFromMsgAndArgs(msgAndArgs ...interface{}) string {
if len(msgAndArgs) == 0 || msgAndArgs == nil {
return ""
@ -194,7 +196,7 @@ func indentMessageLines(message string, longestLabelLen int) string {
// no need to align first line because it starts at the correct location (after the label)
if i != 0 {
// append alignLen+1 spaces to align with "{{longestLabel}}:" before adding tab
outBuf.WriteString("\n\r\t" + strings.Repeat(" ", longestLabelLen+1) + "\t")
outBuf.WriteString("\n\t" + strings.Repeat(" ", longestLabelLen+1) + "\t")
}
outBuf.WriteString(scanner.Text())
}
@ -208,6 +210,9 @@ type failNower interface {
// FailNow fails test
func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
Fail(t, failureMessage, msgAndArgs...)
// We cannot extend TestingT with FailNow() and
@ -226,17 +231,27 @@ func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool
// Fail reports a failure through
func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
content := []labeledContent{
{"Error Trace", strings.Join(CallerInfo(), "\n\r\t\t\t")},
{"Error Trace", strings.Join(CallerInfo(), "\n\t\t\t")},
{"Error", failureMessage},
}
// Add test name if the Go version supports it
if n, ok := t.(interface {
Name() string
}); ok {
content = append(content, labeledContent{"Test", n.Name()})
}
message := messageFromMsgAndArgs(msgAndArgs...)
if len(message) > 0 {
content = append(content, labeledContent{"Messages", message})
}
t.Errorf("%s", "\r"+getWhitespaceString()+labeledOutput(content...))
t.Errorf("\n%s", ""+labeledOutput(content...))
return false
}
@ -248,7 +263,7 @@ type labeledContent struct {
// labeledOutput returns a string consisting of the provided labeledContent. Each labeled output is appended in the following manner:
//
// \r\t{{label}}:{{align_spaces}}\t{{content}}\n
// \t{{label}}:{{align_spaces}}\t{{content}}\n
//
// The initial carriage return is required to undo/erase any padding added by testing.T.Errorf. The "\t{{label}}:" is for the label.
// If a label is shorter than the longest label provided, padding spaces are added to make all the labels match in length. Once this
@ -264,7 +279,7 @@ func labeledOutput(content ...labeledContent) string {
}
var output string
for _, v := range content {
output += "\r\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n"
output += "\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n"
}
return output
}
@ -273,19 +288,26 @@ func labeledOutput(content ...labeledContent) string {
//
// assert.Implements(t, (*MyInterface)(nil), new(MyObject))
func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
interfaceType := reflect.TypeOf(interfaceObject).Elem()
if object == nil {
return Fail(t, fmt.Sprintf("Cannot check if nil implements %v", interfaceType), msgAndArgs...)
}
if !reflect.TypeOf(object).Implements(interfaceType) {
return Fail(t, fmt.Sprintf("%T must implement %v", object, interfaceType), msgAndArgs...)
}
return true
}
// IsType asserts that the specified objects are of the same type.
func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if !ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) {
return Fail(t, fmt.Sprintf("Object expected to be of type %v, but was %v", reflect.TypeOf(expectedType), reflect.TypeOf(object)), msgAndArgs...)
@ -298,12 +320,13 @@ func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs
//
// assert.Equal(t, 123, 123)
//
// Returns whether the assertion was successful (true) or not (false).
//
// Pointer variable equality is determined based on the equality of the
// referenced values (as opposed to the memory addresses). Function equality
// cannot be determined and will always fail.
func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if err := validateEqualArgs(expected, actual); err != nil {
return Fail(t, fmt.Sprintf("Invalid operation: %#v == %#v (%s)",
expected, actual, err), msgAndArgs...)
@ -314,7 +337,7 @@ func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{})
expected, actual = formatUnequalValues(expected, actual)
return Fail(t, fmt.Sprintf("Not equal: \n"+
"expected: %s\n"+
"actual: %s%s", expected, actual, diff), msgAndArgs...)
"actual : %s%s", expected, actual, diff), msgAndArgs...)
}
return true
@ -341,34 +364,36 @@ func formatUnequalValues(expected, actual interface{}) (e string, a string) {
// and equal.
//
// assert.EqualValues(t, uint32(123), int32(123))
//
// Returns whether the assertion was successful (true) or not (false).
func EqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if !ObjectsAreEqualValues(expected, actual) {
diff := diff(expected, actual)
expected, actual = formatUnequalValues(expected, actual)
return Fail(t, fmt.Sprintf("Not equal: \n"+
"expected: %s\n"+
"actual: %s%s", expected, actual, diff), msgAndArgs...)
"actual : %s%s", expected, actual, diff), msgAndArgs...)
}
return true
}
// Exactly asserts that two objects are equal is value and type.
// Exactly asserts that two objects are equal in value and type.
//
// assert.Exactly(t, int32(123), int64(123))
//
// Returns whether the assertion was successful (true) or not (false).
func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
aType := reflect.TypeOf(expected)
bType := reflect.TypeOf(actual)
if aType != bType {
return Fail(t, fmt.Sprintf("Types expected to match exactly\n\r\t%v != %v", aType, bType), msgAndArgs...)
return Fail(t, fmt.Sprintf("Types expected to match exactly\n\t%v != %v", aType, bType), msgAndArgs...)
}
return Equal(t, expected, actual, msgAndArgs...)
@ -378,9 +403,10 @@ func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}
// NotNil asserts that the specified object is not nil.
//
// assert.NotNil(t, err)
//
// Returns whether the assertion was successful (true) or not (false).
func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if !isNil(object) {
return true
}
@ -405,84 +431,52 @@ func isNil(object interface{}) bool {
// Nil asserts that the specified object is nil.
//
// assert.Nil(t, err)
//
// Returns whether the assertion was successful (true) or not (false).
func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if isNil(object) {
return true
}
return Fail(t, fmt.Sprintf("Expected nil, but got: %#v", object), msgAndArgs...)
}
var numericZeros = []interface{}{
int(0),
int8(0),
int16(0),
int32(0),
int64(0),
uint(0),
uint8(0),
uint16(0),
uint32(0),
uint64(0),
float32(0),
float64(0),
}
// isEmpty gets whether the specified object is considered empty or not.
func isEmpty(object interface{}) bool {
// get nil case out of the way
if object == nil {
return true
} else if object == "" {
return true
} else if object == false {
return true
}
for _, v := range numericZeros {
if object == v {
return true
}
}
objValue := reflect.ValueOf(object)
switch objValue.Kind() {
case reflect.Map:
fallthrough
case reflect.Slice, reflect.Chan:
{
return (objValue.Len() == 0)
}
case reflect.Struct:
switch object.(type) {
case time.Time:
return object.(time.Time).IsZero()
}
// collection types are empty when they have no element
case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice:
return objValue.Len() == 0
// pointers are empty if nil or if the value they point to is empty
case reflect.Ptr:
{
if objValue.IsNil() {
return true
}
switch object.(type) {
case *time.Time:
return object.(*time.Time).IsZero()
default:
return false
}
if objValue.IsNil() {
return true
}
deref := objValue.Elem().Interface()
return isEmpty(deref)
// for all other types, compare against the zero value
default:
zero := reflect.Zero(objValue.Type())
return reflect.DeepEqual(object, zero.Interface())
}
return false
}
// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either
// a slice or a channel with len == 0.
//
// assert.Empty(t, obj)
//
// Returns whether the assertion was successful (true) or not (false).
func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
pass := isEmpty(object)
if !pass {
@ -499,9 +493,10 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
// if assert.NotEmpty(t, obj) {
// assert.Equal(t, "two", obj[1])
// }
//
// Returns whether the assertion was successful (true) or not (false).
func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
pass := !isEmpty(object)
if !pass {
@ -528,9 +523,10 @@ func getLen(x interface{}) (ok bool, length int) {
// Len also fails if the object has a type that len() not accept.
//
// assert.Len(t, mySlice, 3)
//
// Returns whether the assertion was successful (true) or not (false).
func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
ok, l := getLen(object)
if !ok {
return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", object), msgAndArgs...)
@ -545,9 +541,15 @@ func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{})
// True asserts that the specified value is true.
//
// assert.True(t, myBool)
//
// Returns whether the assertion was successful (true) or not (false).
func True(t TestingT, value bool, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if h, ok := t.(interface {
Helper()
}); ok {
h.Helper()
}
if value != true {
return Fail(t, "Should be true", msgAndArgs...)
@ -560,9 +562,10 @@ func True(t TestingT, value bool, msgAndArgs ...interface{}) bool {
// False asserts that the specified value is false.
//
// assert.False(t, myBool)
//
// Returns whether the assertion was successful (true) or not (false).
func False(t TestingT, value bool, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if value != false {
return Fail(t, "Should be false", msgAndArgs...)
@ -576,11 +579,12 @@ func False(t TestingT, value bool, msgAndArgs ...interface{}) bool {
//
// assert.NotEqual(t, obj1, obj2)
//
// Returns whether the assertion was successful (true) or not (false).
//
// Pointer variable equality is determined based on the equality of the
// referenced values (as opposed to the memory addresses).
func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if err := validateEqualArgs(expected, actual); err != nil {
return Fail(t, fmt.Sprintf("Invalid operation: %#v != %#v (%s)",
expected, actual, err), msgAndArgs...)
@ -638,9 +642,10 @@ func includeElement(list interface{}, element interface{}) (ok, found bool) {
// assert.Contains(t, "Hello World", "World")
// assert.Contains(t, ["Hello", "World"], "World")
// assert.Contains(t, {"Hello": "World"}, "Hello")
//
// Returns whether the assertion was successful (true) or not (false).
func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
ok, found := includeElement(s, contains)
if !ok {
@ -660,9 +665,10 @@ func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bo
// assert.NotContains(t, "Hello World", "Earth")
// assert.NotContains(t, ["Hello", "World"], "Earth")
// assert.NotContains(t, {"Hello": "World"}, "Earth")
//
// Returns whether the assertion was successful (true) or not (false).
func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
ok, found := includeElement(s, contains)
if !ok {
@ -680,9 +686,10 @@ func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{})
// elements given in the specified subset(array, slice...).
//
// assert.Subset(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]")
//
// Returns whether the assertion was successful (true) or not (false).
func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if subset == nil {
return true // we consider nil to be equal to the nil set
}
@ -723,11 +730,12 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok
// elements given in the specified subset(array, slice...).
//
// assert.NotSubset(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]")
//
// Returns whether the assertion was successful (true) or not (false).
func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if subset == nil {
return false // we consider nil to be equal to the nil set
return Fail(t, fmt.Sprintf("nil is the empty set which is a subset of every set"), msgAndArgs...)
}
subsetValue := reflect.ValueOf(subset)
@ -762,8 +770,68 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{})
return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...)
}
// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified
// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements,
// the number of appearances of each of them in both lists should match.
//
// assert.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2])
func ElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if isEmpty(listA) && isEmpty(listB) {
return true
}
aKind := reflect.TypeOf(listA).Kind()
bKind := reflect.TypeOf(listB).Kind()
if aKind != reflect.Array && aKind != reflect.Slice {
return Fail(t, fmt.Sprintf("%q has an unsupported type %s", listA, aKind), msgAndArgs...)
}
if bKind != reflect.Array && bKind != reflect.Slice {
return Fail(t, fmt.Sprintf("%q has an unsupported type %s", listB, bKind), msgAndArgs...)
}
aValue := reflect.ValueOf(listA)
bValue := reflect.ValueOf(listB)
aLen := aValue.Len()
bLen := bValue.Len()
if aLen != bLen {
return Fail(t, fmt.Sprintf("lengths don't match: %d != %d", aLen, bLen), msgAndArgs...)
}
// Mark indexes in bValue that we already used
visited := make([]bool, bLen)
for i := 0; i < aLen; i++ {
element := aValue.Index(i).Interface()
found := false
for j := 0; j < bLen; j++ {
if visited[j] {
continue
}
if ObjectsAreEqual(bValue.Index(j).Interface(), element) {
visited[j] = true
found = true
break
}
}
if !found {
return Fail(t, fmt.Sprintf("element %s appears more times in %s than in %s", element, aValue, bValue), msgAndArgs...)
}
}
return true
}
// Condition uses a Comparison to assert a complex condition.
func Condition(t TestingT, comp Comparison, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
result := comp()
if !result {
Fail(t, "Condition failed!", msgAndArgs...)
@ -800,12 +868,13 @@ func didPanic(f PanicTestFunc) (bool, interface{}) {
// Panics asserts that the code inside the specified PanicTestFunc panics.
//
// assert.Panics(t, func(){ GoCrazy() })
//
// Returns whether the assertion was successful (true) or not (false).
func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if funcDidPanic, panicValue := didPanic(f); !funcDidPanic {
return Fail(t, fmt.Sprintf("func %#v should panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...)
return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%v", f, panicValue), msgAndArgs...)
}
return true
@ -815,16 +884,17 @@ func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool {
// the recovered panic value equals the expected panic value.
//
// assert.PanicsWithValue(t, "crazy error", func(){ GoCrazy() })
//
// Returns whether the assertion was successful (true) or not (false).
func PanicsWithValue(t TestingT, expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
funcDidPanic, panicValue := didPanic(f)
if !funcDidPanic {
return Fail(t, fmt.Sprintf("func %#v should panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...)
return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%v", f, panicValue), msgAndArgs...)
}
if panicValue != expected {
return Fail(t, fmt.Sprintf("func %#v should panic with value:\t%v\n\r\tPanic value:\t%v", f, expected, panicValue), msgAndArgs...)
return Fail(t, fmt.Sprintf("func %#v should panic with value:\t%v\n\tPanic value:\t%v", f, expected, panicValue), msgAndArgs...)
}
return true
@ -833,12 +903,13 @@ func PanicsWithValue(t TestingT, expected interface{}, f PanicTestFunc, msgAndAr
// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic.
//
// assert.NotPanics(t, func(){ RemainCalm() })
//
// Returns whether the assertion was successful (true) or not (false).
func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if funcDidPanic, panicValue := didPanic(f); funcDidPanic {
return Fail(t, fmt.Sprintf("func %#v should not panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...)
return Fail(t, fmt.Sprintf("func %#v should not panic\n\tPanic value:\t%v", f, panicValue), msgAndArgs...)
}
return true
@ -847,9 +918,10 @@ func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool {
// WithinDuration asserts that the two times are within duration delta of each other.
//
// assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second)
//
// Returns whether the assertion was successful (true) or not (false).
func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
dt := expected.Sub(actual)
if dt < -delta || dt > delta {
@ -886,6 +958,8 @@ func toFloat(x interface{}) (float64, bool) {
xf = float64(xn)
case float64:
xf = float64(xn)
case time.Duration:
xf = float64(xn)
default:
xok = false
}
@ -896,9 +970,10 @@ func toFloat(x interface{}) (float64, bool) {
// InDelta asserts that the two numerals are within delta of each other.
//
// assert.InDelta(t, math.Pi, (22 / 7.0), 0.01)
//
// Returns whether the assertion was successful (true) or not (false).
func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
af, aok := toFloat(expected)
bf, bok := toFloat(actual)
@ -908,7 +983,7 @@ func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs
}
if math.IsNaN(af) {
return Fail(t, fmt.Sprintf("Actual must not be NaN"), msgAndArgs...)
return Fail(t, fmt.Sprintf("Expected must not be NaN"), msgAndArgs...)
}
if math.IsNaN(bf) {
@ -925,6 +1000,9 @@ func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs
// InDeltaSlice is the same as InDelta, except it compares two slices.
func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if expected == nil || actual == nil ||
reflect.TypeOf(actual).Kind() != reflect.Slice ||
reflect.TypeOf(expected).Kind() != reflect.Slice {
@ -944,6 +1022,50 @@ func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAn
return true
}
// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys.
func InDeltaMapValues(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if expected == nil || actual == nil ||
reflect.TypeOf(actual).Kind() != reflect.Map ||
reflect.TypeOf(expected).Kind() != reflect.Map {
return Fail(t, "Arguments must be maps", msgAndArgs...)
}
expectedMap := reflect.ValueOf(expected)
actualMap := reflect.ValueOf(actual)
if expectedMap.Len() != actualMap.Len() {
return Fail(t, "Arguments must have the same number of keys", msgAndArgs...)
}
for _, k := range expectedMap.MapKeys() {
ev := expectedMap.MapIndex(k)
av := actualMap.MapIndex(k)
if !ev.IsValid() {
return Fail(t, fmt.Sprintf("missing key %q in expected map", k), msgAndArgs...)
}
if !av.IsValid() {
return Fail(t, fmt.Sprintf("missing key %q in actual map", k), msgAndArgs...)
}
if !InDelta(
t,
ev.Interface(),
av.Interface(),
delta,
msgAndArgs...,
) {
return false
}
}
return true
}
func calcRelativeError(expected, actual interface{}) (float64, error) {
af, aok := toFloat(expected)
if !aok {
@ -954,23 +1076,24 @@ func calcRelativeError(expected, actual interface{}) (float64, error) {
}
bf, bok := toFloat(actual)
if !bok {
return 0, fmt.Errorf("expected value %q cannot be converted to float", actual)
return 0, fmt.Errorf("actual value %q cannot be converted to float", actual)
}
return math.Abs(af-bf) / math.Abs(af), nil
}
// InEpsilon asserts that expected and actual have a relative error less than epsilon
//
// Returns whether the assertion was successful (true) or not (false).
func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
actualEpsilon, err := calcRelativeError(expected, actual)
if err != nil {
return Fail(t, err.Error(), msgAndArgs...)
}
if actualEpsilon > epsilon {
return Fail(t, fmt.Sprintf("Relative error is too high: %#v (expected)\n"+
" < %#v (actual)", actualEpsilon, epsilon), msgAndArgs...)
" < %#v (actual)", epsilon, actualEpsilon), msgAndArgs...)
}
return true
@ -978,6 +1101,9 @@ func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAnd
// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices.
func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if expected == nil || actual == nil ||
reflect.TypeOf(actual).Kind() != reflect.Slice ||
reflect.TypeOf(expected).Kind() != reflect.Slice {
@ -1007,9 +1133,10 @@ func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, m
// if assert.NoError(t, err) {
// assert.Equal(t, expectedObj, actualObj)
// }
//
// Returns whether the assertion was successful (true) or not (false).
func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if err != nil {
return Fail(t, fmt.Sprintf("Received unexpected error:\n%+v", err), msgAndArgs...)
}
@ -1023,9 +1150,10 @@ func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool {
// if assert.Error(t, err) {
// assert.Equal(t, expectedError, err)
// }
//
// Returns whether the assertion was successful (true) or not (false).
func Error(t TestingT, err error, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if err == nil {
return Fail(t, "An error is expected but got nil.", msgAndArgs...)
@ -1039,9 +1167,10 @@ func Error(t TestingT, err error, msgAndArgs ...interface{}) bool {
//
// actualObj, err := SomeFunction()
// assert.EqualError(t, err, expectedErrorString)
//
// Returns whether the assertion was successful (true) or not (false).
func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if !Error(t, theError, msgAndArgs...) {
return false
}
@ -1051,7 +1180,7 @@ func EqualError(t TestingT, theError error, errString string, msgAndArgs ...inte
if expected != actual {
return Fail(t, fmt.Sprintf("Error message not equal:\n"+
"expected: %q\n"+
"actual: %q", expected, actual), msgAndArgs...)
"actual : %q", expected, actual), msgAndArgs...)
}
return true
}
@ -1074,9 +1203,10 @@ func matchRegexp(rx interface{}, str interface{}) bool {
//
// assert.Regexp(t, regexp.MustCompile("start"), "it's starting")
// assert.Regexp(t, "start...$", "it's not starting")
//
// Returns whether the assertion was successful (true) or not (false).
func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
match := matchRegexp(rx, str)
@ -1091,9 +1221,10 @@ func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface
//
// assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting")
// assert.NotRegexp(t, "^start", "it's not starting")
//
// Returns whether the assertion was successful (true) or not (false).
func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
match := matchRegexp(rx, str)
if match {
@ -1104,28 +1235,71 @@ func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interf
}
// Zero asserts that i is the zero value for its type and returns the truth.
// Zero asserts that i is the zero value for its type.
func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if i != nil && !reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) {
return Fail(t, fmt.Sprintf("Should be zero, but was %v", i), msgAndArgs...)
}
return true
}
// NotZero asserts that i is not the zero value for its type and returns the truth.
// NotZero asserts that i is not the zero value for its type.
func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
if i == nil || reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) {
return Fail(t, fmt.Sprintf("Should not be zero, but was %v", i), msgAndArgs...)
}
return true
}
// FileExists checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file.
func FileExists(t TestingT, path string, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
info, err := os.Lstat(path)
if err != nil {
if os.IsNotExist(err) {
return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...)
}
return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...)
}
if info.IsDir() {
return Fail(t, fmt.Sprintf("%q is a directory", path), msgAndArgs...)
}
return true
}
// DirExists checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists.
func DirExists(t TestingT, path string, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
info, err := os.Lstat(path)
if err != nil {
if os.IsNotExist(err) {
return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...)
}
return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...)
}
if !info.IsDir() {
return Fail(t, fmt.Sprintf("%q is a file", path), msgAndArgs...)
}
return true
}
// JSONEq asserts that two JSON strings are equivalent.
//
// assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`)
//
// Returns whether the assertion was successful (true) or not (false).
func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
var expectedJSONAsInterface, actualJSONAsInterface interface{}
if err := json.Unmarshal([]byte(expected), &expectedJSONAsInterface); err != nil {
@ -1206,3 +1380,7 @@ var spewConfig = spew.ConfigState{
DisableCapacities: true,
SortKeys: true,
}
type tHelper interface {
Helper()
}

28
vendor/github.com/stretchr/testify/assert/http_assertions.go generated vendored
View File

@ -12,10 +12,11 @@ import (
// an error if building a new request fails.
func httpCode(handler http.HandlerFunc, method, url string, values url.Values) (int, error) {
w := httptest.NewRecorder()
req, err := http.NewRequest(method, url+"?"+values.Encode(), nil)
req, err := http.NewRequest(method, url, nil)
if err != nil {
return -1, err
}
req.URL.RawQuery = values.Encode()
handler(w, req)
return w.Code, nil
}
@ -25,7 +26,10 @@ func httpCode(handler http.HandlerFunc, method, url string, values url.Values) (
// assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil)
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool {
func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
code, err := httpCode(handler, method, url, values)
if err != nil {
Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err))
@ -45,7 +49,10 @@ func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, value
// assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}}
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool {
func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
code, err := httpCode(handler, method, url, values)
if err != nil {
Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err))
@ -65,7 +72,10 @@ func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, valu
// assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}}
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool {
func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
code, err := httpCode(handler, method, url, values)
if err != nil {
Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err))
@ -98,7 +108,10 @@ func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) s
// assert.HTTPBodyContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky")
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}) bool {
func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
body := HTTPBody(handler, method, url, values)
contains := strings.Contains(body, fmt.Sprint(str))
@ -115,7 +128,10 @@ func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string,
// assert.HTTPBodyNotContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky")
//
// Returns whether the assertion was successful (true) or not (false).
func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}) bool {
func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
body := HTTPBody(handler, method, url, values)
contains := strings.Contains(body, fmt.Sprint(str))

194
vendor/github.com/stretchr/testify/mock/mock.go generated vendored
View File

@ -1,6 +1,7 @@
package mock
import (
"errors"
"fmt"
"reflect"
"regexp"
@ -41,6 +42,9 @@ type Call struct {
// this method is called.
ReturnArguments Arguments
// Holds the caller info for the On() call
callerInfo []string
// The number of times to return the return arguments when setting
// expectations. 0 means to always return the value.
Repeatability int
@ -48,22 +52,28 @@ type Call struct {
// Amount of times this call has been called
totalCalls int
// Call to this method can be optional
optional bool
// Holds a channel that will be used to block the Return until it either
// receives a message or is closed. nil means it returns immediately.
WaitFor <-chan time.Time
waitTime time.Duration
// Holds a handler used to manipulate arguments content that are passed by
// reference. It's useful when mocking methods such as unmarshalers or
// decoders.
RunFn func(Arguments)
}
func newCall(parent *Mock, methodName string, methodArguments ...interface{}) *Call {
func newCall(parent *Mock, methodName string, callerInfo []string, methodArguments ...interface{}) *Call {
return &Call{
Parent: parent,
Method: methodName,
Arguments: methodArguments,
ReturnArguments: make([]interface{}, 0),
callerInfo: callerInfo,
Repeatability: 0,
WaitFor: nil,
RunFn: nil,
@ -130,7 +140,10 @@ func (c *Call) WaitUntil(w <-chan time.Time) *Call {
//
// Mock.On("MyMethod", arg1, arg2).After(time.Second)
func (c *Call) After(d time.Duration) *Call {
return c.WaitUntil(time.After(d))
c.lock()
defer c.unlock()
c.waitTime = d
return c
}
// Run sets a handler to be called before returning. It can be used when
@ -148,6 +161,15 @@ func (c *Call) Run(fn func(args Arguments)) *Call {
return c
}
// Maybe allows the method call to be optional. Not calling an optional method
// will not cause an error while asserting expectations
func (c *Call) Maybe() *Call {
c.lock()
defer c.unlock()
c.optional = true
return c
}
// On chains a new expectation description onto the mocked interface. This
// allows syntax like.
//
@ -169,6 +191,10 @@ type Mock struct {
// Holds the calls that were made to this mocked object.
Calls []Call
// test is An optional variable that holds the test struct, to be used when an
// invalid mock call was made.
test TestingT
// TestData holds any data that might be useful for testing. Testify ignores
// this data completely allowing you to do whatever you like with it.
testData objx.Map
@ -191,6 +217,27 @@ func (m *Mock) TestData() objx.Map {
Setting expectations
*/
// Test sets the test struct variable of the mock object
func (m *Mock) Test(t TestingT) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.test = t
}
// fail fails the current test with the given formatted format and args.
// In case that a test was defined, it uses the test APIs for failing a test,
// otherwise it uses panic.
func (m *Mock) fail(format string, args ...interface{}) {
m.mutex.Lock()
defer m.mutex.Unlock()
if m.test == nil {
panic(fmt.Sprintf(format, args...))
}
m.test.Errorf(format, args...)
m.test.FailNow()
}
// On starts a description of an expectation of the specified method
// being called.
//
@ -204,7 +251,7 @@ func (m *Mock) On(methodName string, arguments ...interface{}) *Call {
m.mutex.Lock()
defer m.mutex.Unlock()
c := newCall(m, methodName, arguments...)
c := newCall(m, methodName, assert.CallerInfo(), arguments...)
m.ExpectedCalls = append(m.ExpectedCalls, c)
return c
}
@ -227,27 +274,25 @@ func (m *Mock) findExpectedCall(method string, arguments ...interface{}) (int, *
return -1, nil
}
func (m *Mock) findClosestCall(method string, arguments ...interface{}) (bool, *Call) {
diffCount := 0
func (m *Mock) findClosestCall(method string, arguments ...interface{}) (*Call, string) {
var diffCount int
var closestCall *Call
var err string
for _, call := range m.expectedCalls() {
if call.Method == method {
_, tempDiffCount := call.Arguments.Diff(arguments)
errInfo, tempDiffCount := call.Arguments.Diff(arguments)
if tempDiffCount < diffCount || diffCount == 0 {
diffCount = tempDiffCount
closestCall = call
err = errInfo
}
}
}
if closestCall == nil {
return false, nil
}
return true, closestCall
return closestCall, err
}
func callString(method string, arguments Arguments, includeArgumentValues bool) string {
@ -294,6 +339,7 @@ func (m *Mock) Called(arguments ...interface{}) Arguments {
// If Call.WaitFor is set, blocks until the channel is closed or receives a message.
func (m *Mock) MethodCalled(methodName string, arguments ...interface{}) Arguments {
m.mutex.Lock()
//TODO: could combine expected and closes in single loop
found, call := m.findExpectedCall(methodName, arguments...)
if found < 0 {
@ -304,43 +350,52 @@ func (m *Mock) MethodCalled(methodName string, arguments ...interface{}) Argumen
// b) the arguments are not what was expected, or
// c) the developer has forgotten to add an accompanying On...Return pair.
closestFound, closestCall := m.findClosestCall(methodName, arguments...)
closestCall, mismatch := m.findClosestCall(methodName, arguments...)
m.mutex.Unlock()
if closestFound {
panic(fmt.Sprintf("\n\nmock: Unexpected Method Call\n-----------------------------\n\n%s\n\nThe closest call I have is: \n\n%s\n\n%s\n", callString(methodName, arguments, true), callString(methodName, closestCall.Arguments, true), diffArguments(arguments, closestCall.Arguments)))
if closestCall != nil {
m.fail("\n\nmock: Unexpected Method Call\n-----------------------------\n\n%s\n\nThe closest call I have is: \n\n%s\n\n%s\nDiff: %s",
callString(methodName, arguments, true),
callString(methodName, closestCall.Arguments, true),
diffArguments(closestCall.Arguments, arguments),
strings.TrimSpace(mismatch),
)
} else {
panic(fmt.Sprintf("\nassert: mock: I don't know what to return because the method call was unexpected.\n\tEither do Mock.On(\"%s\").Return(...) first, or remove the %s() call.\n\tThis method was unexpected:\n\t\t%s\n\tat: %s", methodName, methodName, callString(methodName, arguments, true), assert.CallerInfo()))
m.fail("\nassert: mock: I don't know what to return because the method call was unexpected.\n\tEither do Mock.On(\"%s\").Return(...) first, or remove the %s() call.\n\tThis method was unexpected:\n\t\t%s\n\tat: %s", methodName, methodName, callString(methodName, arguments, true), assert.CallerInfo())
}
}
switch {
case call.Repeatability == 1:
if call.Repeatability == 1 {
call.Repeatability = -1
call.totalCalls++
case call.Repeatability > 1:
} else if call.Repeatability > 1 {
call.Repeatability--
call.totalCalls++
case call.Repeatability == 0:
call.totalCalls++
}
call.totalCalls++
// add the call
m.Calls = append(m.Calls, *newCall(m, methodName, arguments...))
m.Calls = append(m.Calls, *newCall(m, methodName, assert.CallerInfo(), arguments...))
m.mutex.Unlock()
// block if specified
if call.WaitFor != nil {
<-call.WaitFor
} else {
time.Sleep(call.waitTime)
}
if call.RunFn != nil {
call.RunFn(arguments)
m.mutex.Lock()
runFn := call.RunFn
m.mutex.Unlock()
if runFn != nil {
runFn(arguments)
}
return call.ReturnArguments
m.mutex.Lock()
returnArgs := call.ReturnArguments
m.mutex.Unlock()
return returnArgs
}
/*
@ -356,6 +411,9 @@ type assertExpectationser interface {
//
// Calls may have occurred in any order.
func AssertExpectationsForObjects(t TestingT, testObjects ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
for _, obj := range testObjects {
if m, ok := obj.(Mock); ok {
t.Logf("Deprecated mock.AssertExpectationsForObjects(myMock.Mock) use mock.AssertExpectationsForObjects(myMock)")
@ -363,6 +421,7 @@ func AssertExpectationsForObjects(t TestingT, testObjects ...interface{}) bool {
}
m := obj.(assertExpectationser)
if !m.AssertExpectations(t) {
t.Logf("Expectations didn't match for Mock: %+v", reflect.TypeOf(m))
return false
}
}
@ -372,6 +431,9 @@ func AssertExpectationsForObjects(t TestingT, testObjects ...interface{}) bool {
// AssertExpectations asserts that everything specified with On and Return was
// in fact called as expected. Calls may have occurred in any order.
func (m *Mock) AssertExpectations(t TestingT) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
m.mutex.Lock()
defer m.mutex.Unlock()
var somethingMissing bool
@ -380,16 +442,17 @@ func (m *Mock) AssertExpectations(t TestingT) bool {
// iterate through each expectation
expectedCalls := m.expectedCalls()
for _, expectedCall := range expectedCalls {
if !m.methodWasCalled(expectedCall.Method, expectedCall.Arguments) && expectedCall.totalCalls == 0 {
if !expectedCall.optional && !m.methodWasCalled(expectedCall.Method, expectedCall.Arguments) && expectedCall.totalCalls == 0 {
somethingMissing = true
failedExpectations++
t.Logf("\u274C\t%s(%s)", expectedCall.Method, expectedCall.Arguments.String())
t.Logf("FAIL:\t%s(%s)\n\t\tat: %s", expectedCall.Method, expectedCall.Arguments.String(), expectedCall.callerInfo)
} else {
if expectedCall.Repeatability > 0 {
somethingMissing = true
failedExpectations++
t.Logf("FAIL:\t%s(%s)\n\t\tat: %s", expectedCall.Method, expectedCall.Arguments.String(), expectedCall.callerInfo)
} else {
t.Logf("\u2705\t%s(%s)", expectedCall.Method, expectedCall.Arguments.String())
t.Logf("PASS:\t%s(%s)", expectedCall.Method, expectedCall.Arguments.String())
}
}
}
@ -403,6 +466,9 @@ func (m *Mock) AssertExpectations(t TestingT) bool {
// AssertNumberOfCalls asserts that the method was called expectedCalls times.
func (m *Mock) AssertNumberOfCalls(t TestingT, methodName string, expectedCalls int) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
m.mutex.Lock()
defer m.mutex.Unlock()
var actualCalls int
@ -417,11 +483,22 @@ func (m *Mock) AssertNumberOfCalls(t TestingT, methodName string, expectedCalls
// AssertCalled asserts that the method was called.
// It can produce a false result when an argument is a pointer type and the underlying value changed after calling the mocked method.
func (m *Mock) AssertCalled(t TestingT, methodName string, arguments ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
m.mutex.Lock()
defer m.mutex.Unlock()
if !assert.True(t, m.methodWasCalled(methodName, arguments), fmt.Sprintf("The \"%s\" method should have been called with %d argument(s), but was not.", methodName, len(arguments))) {
t.Logf("%v", m.expectedCalls())
return false
if !m.methodWasCalled(methodName, arguments) {
var calledWithArgs []string
for _, call := range m.calls() {
calledWithArgs = append(calledWithArgs, fmt.Sprintf("%v", call.Arguments))
}
if len(calledWithArgs) == 0 {
return assert.Fail(t, "Should have called with given arguments",
fmt.Sprintf("Expected %q to have been called with:\n%v\nbut no actual calls happened", methodName, arguments))
}
return assert.Fail(t, "Should have called with given arguments",
fmt.Sprintf("Expected %q to have been called with:\n%v\nbut actual calls were:\n %v", methodName, arguments, strings.Join(calledWithArgs, "\n")))
}
return true
}
@ -429,11 +506,14 @@ func (m *Mock) AssertCalled(t TestingT, methodName string, arguments ...interfac
// AssertNotCalled asserts that the method was not called.
// It can produce a false result when an argument is a pointer type and the underlying value changed after calling the mocked method.
func (m *Mock) AssertNotCalled(t TestingT, methodName string, arguments ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
m.mutex.Lock()
defer m.mutex.Unlock()
if !assert.False(t, m.methodWasCalled(methodName, arguments), fmt.Sprintf("The \"%s\" method was called with %d argument(s), but should NOT have been.", methodName, len(arguments))) {
t.Logf("%v", m.expectedCalls())
return false
if m.methodWasCalled(methodName, arguments) {
return assert.Fail(t, "Should not have called with given arguments",
fmt.Sprintf("Expected %q to not have been called with:\n%v\nbut actually it was.", methodName, arguments))
}
return true
}
@ -473,7 +553,7 @@ type Arguments []interface{}
const (
// Anything is used in Diff and Assert when the argument being tested
// shouldn't be taken into consideration.
Anything string = "mock.Anything"
Anything = "mock.Anything"
)
// AnythingOfTypeArgument is a string that contains the type of an argument
@ -498,9 +578,25 @@ type argumentMatcher struct {
func (f argumentMatcher) Matches(argument interface{}) bool {
expectType := f.fn.Type().In(0)
expectTypeNilSupported := false
switch expectType.Kind() {
case reflect.Interface, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice, reflect.Ptr:
expectTypeNilSupported = true
}
if reflect.TypeOf(argument).AssignableTo(expectType) {
result := f.fn.Call([]reflect.Value{reflect.ValueOf(argument)})
argType := reflect.TypeOf(argument)
var arg reflect.Value
if argType == nil {
arg = reflect.New(expectType).Elem()
} else {
arg = reflect.ValueOf(argument)
}
if argType == nil && !expectTypeNilSupported {
panic(errors.New("attempting to call matcher with nil for non-nil expected type"))
}
if argType == nil || argType.AssignableTo(expectType) {
result := f.fn.Call([]reflect.Value{arg})
return result[0].Bool()
}
return false
@ -560,6 +656,7 @@ func (args Arguments) Is(objects ...interface{}) bool {
//
// Returns the diff string and number of differences found.
func (args Arguments) Diff(objects []interface{}) (string, int) {
//TODO: could return string as error and nil for No difference
var output = "\n"
var differences int
@ -586,10 +683,10 @@ func (args Arguments) Diff(objects []interface{}) (string, int) {
if matcher, ok := expected.(argumentMatcher); ok {
if matcher.Matches(actual) {
output = fmt.Sprintf("%s\t%d: \u2705 %s matched by %s\n", output, i, actual, matcher)
output = fmt.Sprintf("%s\t%d: PASS: %s matched by %s\n", output, i, actual, matcher)
} else {
differences++
output = fmt.Sprintf("%s\t%d: \u2705 %s not matched by %s\n", output, i, actual, matcher)
output = fmt.Sprintf("%s\t%d: PASS: %s not matched by %s\n", output, i, actual, matcher)
}
} else if reflect.TypeOf(expected) == reflect.TypeOf((*AnythingOfTypeArgument)(nil)).Elem() {
@ -597,7 +694,7 @@ func (args Arguments) Diff(objects []interface{}) (string, int) {
if reflect.TypeOf(actual).Name() != string(expected.(AnythingOfTypeArgument)) && reflect.TypeOf(actual).String() != string(expected.(AnythingOfTypeArgument)) {
// not match
differences++
output = fmt.Sprintf("%s\t%d: \u274C type %s != type %s - %s\n", output, i, expected, reflect.TypeOf(actual).Name(), actual)
output = fmt.Sprintf("%s\t%d: FAIL: type %s != type %s - %s\n", output, i, expected, reflect.TypeOf(actual).Name(), actual)
}
} else {
@ -606,11 +703,11 @@ func (args Arguments) Diff(objects []interface{}) (string, int) {
if assert.ObjectsAreEqual(expected, Anything) || assert.ObjectsAreEqual(actual, Anything) || assert.ObjectsAreEqual(actual, expected) {
// match
output = fmt.Sprintf("%s\t%d: \u2705 %s == %s\n", output, i, actual, expected)
output = fmt.Sprintf("%s\t%d: PASS: %s == %s\n", output, i, actual, expected)
} else {
// not match
differences++
output = fmt.Sprintf("%s\t%d: \u274C %s != %s\n", output, i, actual, expected)
output = fmt.Sprintf("%s\t%d: FAIL: %s != %s\n", output, i, actual, expected)
}
}
@ -627,6 +724,9 @@ func (args Arguments) Diff(objects []interface{}) (string, int) {
// Assert compares the arguments with the specified objects and fails if
// they do not exactly match.
func (args Arguments) Assert(t TestingT, objects ...interface{}) bool {
if h, ok := t.(tHelper); ok {
h.Helper()
}
// get the differences
diff, diffCount := args.Diff(objects)
@ -774,3 +874,7 @@ var spewConfig = spew.ConfigState{
DisableCapacities: true,
SortKeys: true,
}
type tHelper interface {
Helper()
}

494
vendor/github.com/stretchr/testify/require/require.go generated vendored
View File

File diff suppressed because it is too large Load Diff

1
vendor/github.com/stretchr/testify/require/require.go.tmpl generated vendored
View File

@ -1,5 +1,6 @@
{{.Comment}}
func {{.DocInfo.Name}}(t TestingT, {{.Params}}) {
if h, ok := t.(tHelper); ok { h.Helper() }
if !assert.{{.DocInfo.Name}}(t, {{.ForwardedParams}}) {
t.FailNow()
}

478
vendor/github.com/stretchr/testify/require/require_forward.go generated vendored
View File

File diff suppressed because it is too large Load Diff

1
vendor/github.com/stretchr/testify/require/require_forward.go.tmpl generated vendored
View File

@ -1,4 +1,5 @@
{{.CommentWithoutT "a"}}
func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) {
if h, ok := a.t.(tHelper); ok { h.Helper() }
{{.DocInfo.Name}}(a.t, {{.ForwardedParams}})
}

20
vendor/github.com/stretchr/testify/require/requirements.go generated vendored
View File

@ -6,4 +6,24 @@ type TestingT interface {
FailNow()
}
type tHelper interface {
Helper()
}
// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful
// for table driven tests.
type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{})
// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful
// for table driven tests.
type ValueAssertionFunc func(TestingT, interface{}, ...interface{})
// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful
// for table driven tests.
type BoolAssertionFunc func(TestingT, bool, ...interface{})
// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful
// for table driven tests.
type ErrorAssertionFunc func(TestingT, error, ...interface{})
//go:generate go run ../_codegen/main.go -output-package=require -template=require.go.tmpl -include-format-funcs

6
vendor/vendor.json vendored
View File

@ -288,9 +288,9 @@
{"path":"github.com/skratchdot/open-golang/open","checksumSHA1":"h/HMhokbQHTdLUbruoBBTee+NYw=","revision":"75fb7ed4208cf72d323d7d02fd1a5964a7a9073c","revisionTime":"2016-03-02T14:40:31Z"},
{"path":"github.com/spf13/pflag","checksumSHA1":"Q52Y7t0lEtk/wcDn5q7tS7B+jqs=","revision":"7aff26db30c1be810f9de5038ec5ef96ac41fd7c","revisionTime":"2017-08-24T17:57:12Z"},
{"path":"github.com/stretchr/objx","checksumSHA1":"K0crHygPTP42i1nLKWphSlvOQJw=","revision":"1a9d0bb9f541897e62256577b352fdbc1fb4fd94","revisionTime":"2015-09-28T12:21:52Z"},
{"path":"github.com/stretchr/testify/assert","checksumSHA1":"5NBHAe3S15q3L9hOLThnMZjIZRE=","revision":"f6abca593680b2315d2075e0f5e2a9751e3f431a","revisionTime":"2017-06-01T20:57:54Z"},
{"path":"github.com/stretchr/testify/mock","checksumSHA1":"o+jsS/rxceTym4M3reSPfrPxaio=","revision":"f6abca593680b2315d2075e0f5e2a9751e3f431a","revisionTime":"2017-06-01T20:57:54Z"},
{"path":"github.com/stretchr/testify/require","checksumSHA1":"7vs6dSc1PPGBKyzb/SCIyeMJPLQ=","revision":"f6abca593680b2315d2075e0f5e2a9751e3f431a","revisionTime":"2017-06-01T20:57:54Z"},
{"path":"github.com/stretchr/testify/assert","checksumSHA1":"6LwXZI7kXm1C0h4Ui0Y52p9uQhk=","revision":"c679ae2cc0cb27ec3293fea7e254e47386f05d69","revisionTime":"2018-03-14T08:05:35Z"},
{"path":"github.com/stretchr/testify/mock","checksumSHA1":"Qloi2PTvZv+D9FDHXM/banCoaFY=","revision":"c679ae2cc0cb27ec3293fea7e254e47386f05d69","revisionTime":"2018-03-14T08:05:35Z"},
{"path":"github.com/stretchr/testify/require","checksumSHA1":"KqYmXUcuGwsvBL6XVsQnXsFb3LI=","revision":"c679ae2cc0cb27ec3293fea7e254e47386f05d69","revisionTime":"2018-03-14T08:05:35Z"},
{"path":"github.com/syndtr/gocapability/capability","checksumSHA1":"PgEklGW56c5RLHqQhORxt6jS3fY=","revision":"db04d3cc01c8b54962a58ec7e491717d06cfcc16","revisionTime":"2017-07-04T07:02:18Z"},
{"path":"github.com/tonnerre/golang-text","checksumSHA1":"t24KnvC9jRxiANVhpw2pqFpmEu8=","revision":"048ed3d792f7104850acbc8cfc01e5a6070f4c04","revisionTime":"2013-09-25T19:58:46Z"},
{"path":"github.com/ugorji/go/codec","checksumSHA1":"8G1zvpE4gTtWQRuP/x2HPVDmflo=","revision":"0053ebfd9d0ee06ccefbfe17072021e1d4acebee","revisionTime":"2017-06-20T06:01:02Z"},

15
website/source/api/json-jobs.html.md
View File

@ -620,12 +620,19 @@ determined. The potential values are:
- `MinHealthyTime` - Specifies the minimum time the allocation must be in the
healthy state before it is marked as healthy and unblocks further allocations
from being updated. This is specified using a label suffix like "30s" or
"15m".
from being updated.
- `HealthyDeadline` - Specifies the deadline in which the allocation must be
marked as healthy after which the allocation is automatically transitioned to
unhealthy. This is specified using a label suffix like "2m" or "1h".
unhealthy.
- `ProgressDeadline` - Specifies the deadline in which an allocation must be
marked as healthy. The deadline begins when the first allocation for the
deployment is created and is reset whenever an allocation as part of the
deployment transitions to a healthy state. If no allocation transitions to the
healthy state before the progress deadline, the deployment is marked as
failed. If the `progress_deadline` is set to `0`, the first allocation to be
marked as unhealthy causes the deployment to fail.
- `AutoRevert` - Specifies if the job should auto-revert to the last stable job
on deployment failure. A job is marked as stable if all the allocations as
@ -638,7 +645,7 @@ determined. The potential values are:
allocations at a rate of `max_parallel`.
- `Stagger` - Specifies the delay between migrating allocations off nodes marked
for draining. This is specified using a label suffix like "30s" or "1h".
for draining.
An example `Update` block:

24
website/source/docs/job-specification/update.html.md
View File

@ -31,13 +31,14 @@ highest precedence and then the job.
```hcl
job "docs" {
update {
max_parallel = 3
health_check = "checks"
min_healthy_time = "10s"
healthy_deadline = "10m"
auto_revert = true
canary = 1
stagger = "30s"
max_parallel = 3
health_check = "checks"
min_healthy_time = "10s"
healthy_deadline = "5m"
progress_deadline = "10m"
auto_revert = true
canary = 1
stagger = "30s"
}
}
```
@ -77,6 +78,15 @@ set of updates. The `system` scheduler will be updated to support the new
automatically transitioned to unhealthy. This is specified using a label
suffix like "2m" or "1h".
- `progress_deadline` `(string: "10m")` - Specifies the deadline in which an
allocation must be marked as healthy. The deadline begins when the first
allocation for the deployment is created and is reset whenever an allocation
as part of the deployment transitions to a healthy state. If no allocation
transitions to the healthy state before the progress deadline, the deployment
is marked as failed. If the `progress_deadline` is set to `0`, the first
allocation to be marked as unhealthy causes the deployment to fail. This is
specified using a label suffix like "2m" or "1h".
- `auto_revert` `(bool: false)` - Specifies if the job should auto-revert to the
last stable job on deployment failure. A job is marked as stable if all the
allocations as part of its deployment were marked healthy.