open-nomad/nomad/deploymentwatcher/testutil_test.go
Seth Hoenig ba728f8f97
api: enable support for setting original job source (#16763)
* api: enable support for setting original source alongside job

This PR adds support for setting job source material along with
the registration of a job.

This includes a new HTTP endpoint and a new RPC endpoint for
making queries for the original source of a job. The
HTTP endpoint is /v1/job/<id>/submission?version=<version> and
the RPC method is Job.GetJobSubmission.

The job source (if submitted, and doing so is always optional), is
stored in the job_submission memdb table, separately from the
actual job. This way we do not incur overhead of reading the large
string field throughout normal job operations.

The server config now includes job_max_source_size for configuring
the maximum size the job source may be, before the server simply
drops the source material. This should help prevent Bad Things from
happening when huge jobs are submitted. If the value is set to 0,
all job source material will be dropped.

* api: avoid writing var content to disk for parsing

* api: move submission validation into RPC layer

* api: return an error if updating a job submission without namespace or job id

* api: be exact about the job index we associate a submission with (modify)

* api: reword api docs scheduling

* api: prune all but the last 6 job submissions

* api: protect against nil job submission in job validation

* api: set max job source size in test server

* api: fixups from pr
2023-04-11 08:45:08 -05:00

273 lines
7.2 KiB
Go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package deploymentwatcher
import (
"reflect"
"strings"
"sync"
"testing"
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/structs"
mocker "github.com/stretchr/testify/mock"
)
type mockBackend struct {
mocker.Mock
index uint64
state *state.StateStore
l sync.Mutex
}
func newMockBackend(t *testing.T) *mockBackend {
m := &mockBackend{
index: 10000,
state: state.TestStateStore(t),
}
m.Test(t)
return m
}
func (m *mockBackend) nextIndex() uint64 {
m.l.Lock()
defer m.l.Unlock()
i := m.index
m.index++
return i
}
func (m *mockBackend) UpdateAllocDesiredTransition(u *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) {
m.Called(u)
i := m.nextIndex()
return i, m.state.UpdateAllocsDesiredTransitions(structs.MsgTypeTestSetup, i, u.Allocs, u.Evals)
}
// matchUpdateAllocDesiredTransitions is used to match an upsert request
func matchUpdateAllocDesiredTransitions(deploymentIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
return func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
if len(update.Evals) != len(deploymentIDs) {
return false
}
dmap := make(map[string]struct{}, len(deploymentIDs))
for _, d := range deploymentIDs {
dmap[d] = struct{}{}
}
for _, e := range update.Evals {
if _, ok := dmap[e.DeploymentID]; !ok {
return false
}
delete(dmap, e.DeploymentID)
}
return true
}
}
// matchUpdateAllocDesiredTransitionReschedule is used to match allocs that have their DesiredTransition set to Reschedule
func matchUpdateAllocDesiredTransitionReschedule(allocIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
return func(update *structs.AllocUpdateDesiredTransitionRequest) bool {
amap := make(map[string]struct{}, len(allocIDs))
for _, d := range allocIDs {
amap[d] = struct{}{}
}
for allocID, dt := range update.Allocs {
if _, ok := amap[allocID]; !ok {
return false
}
if !*dt.Reschedule {
return false
}
}
return true
}
}
func (m *mockBackend) UpsertJob(job *structs.Job) (uint64, error) {
m.Called(job)
i := m.nextIndex()
return i, m.state.UpsertJob(structs.MsgTypeTestSetup, i, nil, job)
}
func (m *mockBackend) UpdateDeploymentStatus(u *structs.DeploymentStatusUpdateRequest) (uint64, error) {
m.Called(u)
i := m.nextIndex()
return i, m.state.UpdateDeploymentStatus(structs.MsgTypeTestSetup, i, u)
}
// matchDeploymentStatusUpdateConfig is used to configure the matching
// function
type matchDeploymentStatusUpdateConfig struct {
// DeploymentID is the expected ID
DeploymentID string
// Status is the desired status
Status string
// StatusDescription is the desired status description
StatusDescription string
// JobVersion marks whether we expect a roll back job at the given version
JobVersion *uint64
// Eval marks whether we expect an evaluation.
Eval bool
}
// matchDeploymentStatusUpdateRequest is used to match an update request
func matchDeploymentStatusUpdateRequest(c *matchDeploymentStatusUpdateConfig) func(args *structs.DeploymentStatusUpdateRequest) bool {
return func(args *structs.DeploymentStatusUpdateRequest) bool {
if args.DeploymentUpdate.DeploymentID != c.DeploymentID {
return false
}
if args.DeploymentUpdate.Status != c.Status && args.DeploymentUpdate.StatusDescription != c.StatusDescription {
return false
}
if c.Eval && args.Eval == nil || !c.Eval && args.Eval != nil {
return false
}
if c.JobVersion != nil {
if args.Job == nil {
return false
} else if args.Job.Version != *c.JobVersion {
return false
}
} else if c.JobVersion == nil && args.Job != nil {
return false
}
return true
}
}
func (m *mockBackend) UpdateDeploymentPromotion(req *structs.ApplyDeploymentPromoteRequest) (uint64, error) {
m.Called(req)
i := m.nextIndex()
return i, m.state.UpdateDeploymentPromotion(structs.MsgTypeTestSetup, i, req)
}
// matchDeploymentPromoteRequestConfig is used to configure the matching
// function
type matchDeploymentPromoteRequestConfig struct {
// Promotion holds the expected promote request
Promotion *structs.DeploymentPromoteRequest
// Eval marks whether we expect an evaluation.
Eval bool
}
// matchDeploymentPromoteRequest is used to match a promote request
func matchDeploymentPromoteRequest(c *matchDeploymentPromoteRequestConfig) func(args *structs.ApplyDeploymentPromoteRequest) bool {
return func(args *structs.ApplyDeploymentPromoteRequest) bool {
if !reflect.DeepEqual(*c.Promotion, args.DeploymentPromoteRequest) {
return false
}
if c.Eval && args.Eval == nil || !c.Eval && args.Eval != nil {
return false
}
return true
}
}
func (m *mockBackend) UpdateDeploymentAllocHealth(req *structs.ApplyDeploymentAllocHealthRequest) (uint64, error) {
m.Called(req)
i := m.nextIndex()
return i, m.state.UpdateDeploymentAllocHealth(structs.MsgTypeTestSetup, i, req)
}
// matchDeploymentAllocHealthRequestConfig is used to configure the matching
// function
type matchDeploymentAllocHealthRequestConfig struct {
// DeploymentID is the expected ID
DeploymentID string
// Healthy and Unhealthy contain the expected allocation IDs that are having
// their health set
Healthy, Unhealthy []string
// DeploymentUpdate holds the expected values of status and description. We
// don't check for exact match but string contains
DeploymentUpdate *structs.DeploymentStatusUpdate
// JobVersion marks whether we expect a roll back job at the given version
JobVersion *uint64
// Eval marks whether we expect an evaluation.
Eval bool
}
// matchDeploymentAllocHealthRequest is used to match an update request
func matchDeploymentAllocHealthRequest(c *matchDeploymentAllocHealthRequestConfig) func(args *structs.ApplyDeploymentAllocHealthRequest) bool {
return func(args *structs.ApplyDeploymentAllocHealthRequest) bool {
if args.DeploymentID != c.DeploymentID {
return false
}
// Require a timestamp
if args.Timestamp.IsZero() {
return false
}
if len(c.Healthy) != len(args.HealthyAllocationIDs) {
return false
}
if len(c.Unhealthy) != len(args.UnhealthyAllocationIDs) {
return false
}
hmap, umap := make(map[string]struct{}, len(c.Healthy)), make(map[string]struct{}, len(c.Unhealthy))
for _, h := range c.Healthy {
hmap[h] = struct{}{}
}
for _, u := range c.Unhealthy {
umap[u] = struct{}{}
}
for _, h := range args.HealthyAllocationIDs {
if _, ok := hmap[h]; !ok {
return false
}
}
for _, u := range args.UnhealthyAllocationIDs {
if _, ok := umap[u]; !ok {
return false
}
}
if c.DeploymentUpdate != nil {
if args.DeploymentUpdate == nil {
return false
}
if !strings.Contains(args.DeploymentUpdate.Status, c.DeploymentUpdate.Status) {
return false
}
if !strings.Contains(args.DeploymentUpdate.StatusDescription, c.DeploymentUpdate.StatusDescription) {
return false
}
} else if args.DeploymentUpdate != nil {
return false
}
if c.Eval && args.Eval == nil || !c.Eval && args.Eval != nil {
return false
}
if (c.JobVersion != nil && (args.Job == nil || args.Job.Version != *c.JobVersion)) || c.JobVersion == nil && args.Job != nil {
return false
}
return true
}
}