open-nomad/nomad/deploymentwatcher/testutil_test.go
Mahmood Ali 9d88f1d568 tests: deflake deploymentwatcher package
This deflake the tests in the deploymentwatcher package.  The package
uses a mock deployment watcher backend, where the watcher in a
background goroutine calls UpdateDeploymentStatus .  If the mock isn't
configured to expect the call, the background goroutine will fail.  One
UpdateDeploymentStatus call is made at the end of the background
goroutine, which may occur after the test completes, thus explaining the
flakiness.
2020-03-12 15:42:01 -04:00

270 lines
7 KiB
Go

package deploymentwatcher
import (
"reflect"
"strings"
"sync"
"testing"
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/structs"
mocker "github.com/stretchr/testify/mock"
)
type mockBackend struct {
mocker.Mock
index uint64
state *state.StateStore
l sync.Mutex
}
func newMockBackend(t *testing.T) *mockBackend {
m := &mockBackend{
index: 10000,
state: state.TestStateStore(t),
}
m.Test(t)
return m
}
func (m *mockBackend) nextIndex() uint64 {
m.l.Lock()
defer m.l.Unlock()
i := m.index
m.index++
return i
}
func (m *mockBackend) UpdateAllocDesiredTransition(u *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) {
m.Called(u)
i := m.nextIndex()
return i, m.state.UpdateAllocsDesiredTransitions(i, u.Allocs, u.Evals)
}
// matchUpdateAllocDesiredTransitions is used to match an upsert request
func matchUpdateAllocDesiredTransitions(deploymentIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
return func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
if len(update.Evals) != len(deploymentIDs) {
return false
}
dmap := make(map[string]struct{}, len(deploymentIDs))
for _, d := range deploymentIDs {
dmap[d] = struct{}{}
}
for _, e := range update.Evals {
if _, ok := dmap[e.DeploymentID]; !ok {
return false
}
delete(dmap, e.DeploymentID)
}
return true
}
}
// matchUpdateAllocDesiredTransitionReschedule is used to match allocs that have their DesiredTransition set to Reschedule
func matchUpdateAllocDesiredTransitionReschedule(allocIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
return func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
amap := make(map[string]struct{}, len(allocIDs))
for _, d := range allocIDs {
amap[d] = struct{}{}
}
for allocID, dt := range update.Allocs {
if _, ok := amap[allocID]; !ok {
return false
}
if !*dt.Reschedule {
return false
}
}
return true
}
}
func (m *mockBackend) UpsertJob(job *structs.Job) (uint64, error) {
m.Called(job)
i := m.nextIndex()
return i, m.state.UpsertJob(i, job)
}
func (m *mockBackend) UpdateDeploymentStatus(u *structs.DeploymentStatusUpdateRequest) (uint64, error) {
m.Called(u)
i := m.nextIndex()
return i, m.state.UpdateDeploymentStatus(i, u)
}
// matchDeploymentStatusUpdateConfig is used to configure the matching
// function
type matchDeploymentStatusUpdateConfig struct {
// DeploymentID is the expected ID
DeploymentID string
// Status is the desired status
Status string
// StatusDescription is the desired status description
StatusDescription string
// JobVersion marks whether we expect a roll back job at the given version
JobVersion *uint64
// Eval marks whether we expect an evaluation.
Eval bool
}
// matchDeploymentStatusUpdateRequest is used to match an update request
func matchDeploymentStatusUpdateRequest(c *matchDeploymentStatusUpdateConfig) func(args *structs.DeploymentStatusUpdateRequest) bool {
return func(args *structs.DeploymentStatusUpdateRequest) bool {
if args.DeploymentUpdate.DeploymentID != c.DeploymentID {
return false
}
if args.DeploymentUpdate.Status != c.Status && args.DeploymentUpdate.StatusDescription != c.StatusDescription {
return false
}
if c.Eval && args.Eval == nil || !c.Eval && args.Eval != nil {
return false
}
if c.JobVersion != nil {
if args.Job == nil {
return false
} else if args.Job.Version != *c.JobVersion {
return false
}
} else if c.JobVersion == nil && args.Job != nil {
return false
}
return true
}
}
func (m *mockBackend) UpdateDeploymentPromotion(req *structs.ApplyDeploymentPromoteRequest) (uint64, error) {
m.Called(req)
i := m.nextIndex()
return i, m.state.UpdateDeploymentPromotion(i, req)
}
// matchDeploymentPromoteRequestConfig is used to configure the matching
// function
type matchDeploymentPromoteRequestConfig struct {
// Promotion holds the expected promote request
Promotion *structs.DeploymentPromoteRequest
// Eval marks whether we expect an evaluation.
Eval bool
}
// matchDeploymentPromoteRequest is used to match a promote request
func matchDeploymentPromoteRequest(c *matchDeploymentPromoteRequestConfig) func(args *structs.ApplyDeploymentPromoteRequest) bool {
return func(args *structs.ApplyDeploymentPromoteRequest) bool {
if !reflect.DeepEqual(*c.Promotion, args.DeploymentPromoteRequest) {
return false
}
if c.Eval && args.Eval == nil || !c.Eval && args.Eval != nil {
return false
}
return true
}
}
func (m *mockBackend) UpdateDeploymentAllocHealth(req *structs.ApplyDeploymentAllocHealthRequest) (uint64, error) {
m.Called(req)
i := m.nextIndex()
return i, m.state.UpdateDeploymentAllocHealth(i, req)
}
// matchDeploymentAllocHealthRequestConfig is used to configure the matching
// function
type matchDeploymentAllocHealthRequestConfig struct {
// DeploymentID is the expected ID
DeploymentID string
// Healthy and Unhealthy contain the expected allocation IDs that are having
// their health set
Healthy, Unhealthy []string
// DeploymentUpdate holds the expected values of status and description. We
// don't check for exact match but string contains
DeploymentUpdate *structs.DeploymentStatusUpdate
// JobVersion marks whether we expect a roll back job at the given version
JobVersion *uint64
// Eval marks whether we expect an evaluation.
Eval bool
}
// matchDeploymentAllocHealthRequest is used to match an update request
func matchDeploymentAllocHealthRequest(c *matchDeploymentAllocHealthRequestConfig) func(args *structs.ApplyDeploymentAllocHealthRequest) bool {
return func(args *structs.ApplyDeploymentAllocHealthRequest) bool {
if args.DeploymentID != c.DeploymentID {
return false
}
// Require a timestamp
if args.Timestamp.IsZero() {
return false
}
if len(c.Healthy) != len(args.HealthyAllocationIDs) {
return false
}
if len(c.Unhealthy) != len(args.UnhealthyAllocationIDs) {
return false
}
hmap, umap := make(map[string]struct{}, len(c.Healthy)), make(map[string]struct{}, len(c.Unhealthy))
for _, h := range c.Healthy {
hmap[h] = struct{}{}
}
for _, u := range c.Unhealthy {
umap[u] = struct{}{}
}
for _, h := range args.HealthyAllocationIDs {
if _, ok := hmap[h]; !ok {
return false
}
}
for _, u := range args.UnhealthyAllocationIDs {
if _, ok := umap[u]; !ok {
return false
}
}
if c.DeploymentUpdate != nil {
if args.DeploymentUpdate == nil {
return false
}
if !strings.Contains(args.DeploymentUpdate.Status, c.DeploymentUpdate.Status) {
return false
}
if !strings.Contains(args.DeploymentUpdate.StatusDescription, c.DeploymentUpdate.StatusDescription) {
return false
}
} else if args.DeploymentUpdate != nil {
return false
}
if c.Eval && args.Eval == nil || !c.Eval && args.Eval != nil {
return false
}
if (c.JobVersion != nil && (args.Job == nil || args.Job.Version != *c.JobVersion)) || c.JobVersion == nil && args.Job != nil {
return false
}
return true
}
}