ba728f8f97
* api: enable support for setting original source alongside job This PR adds support for setting job source material along with the registration of a job. This includes a new HTTP endpoint and a new RPC endpoint for making queries for the original source of a job. The HTTP endpoint is /v1/job/<id>/submission?version=<version> and the RPC method is Job.GetJobSubmission. The job source (if submitted, and doing so is always optional), is stored in the job_submission memdb table, separately from the actual job. This way we do not incur overhead of reading the large string field throughout normal job operations. The server config now includes job_max_source_size for configuring the maximum size the job source may be, before the server simply drops the source material. This should help prevent Bad Things from happening when huge jobs are submitted. If the value is set to 0, all job source material will be dropped. * api: avoid writing var content to disk for parsing * api: move submission validation into RPC layer * api: return an error if updating a job submission without namespace or job id * api: be exact about the job index we associate a submission with (modify) * api: reword api docs scheduling * api: prune all but the last 6 job submissions * api: protect against nil job submission in job validation * api: set max job source size in test server * api: fixups from pr
2051 lines
59 KiB
Go
2051 lines
59 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package nomad
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
memdb "github.com/hashicorp/go-memdb"
|
|
"github.com/hashicorp/go-set"
|
|
msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
|
|
"github.com/hashicorp/nomad/acl"
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/scheduler"
|
|
"github.com/hashicorp/nomad/testutil"
|
|
"github.com/shoenig/test/must"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestEvalEndpoint_GetEval(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
eval2 := mock.Eval()
|
|
|
|
// Link the evals
|
|
eval1.NextEval = eval2.ID
|
|
eval2.PreviousEval = eval1.ID
|
|
|
|
err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2})
|
|
require.NoError(t, err)
|
|
|
|
t.Run("lookup eval", func(t *testing.T) {
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: eval1.ID,
|
|
QueryOptions: structs.QueryOptions{Region: "global"},
|
|
}
|
|
var resp structs.SingleEvalResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1000, resp.Index, "bad index")
|
|
require.Equal(t, eval1, resp.Eval)
|
|
})
|
|
|
|
t.Run("lookup non-existing eval", func(t *testing.T) {
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: uuid.Generate(),
|
|
QueryOptions: structs.QueryOptions{Region: "global"},
|
|
}
|
|
var resp structs.SingleEvalResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1000, resp.Index, "bad index")
|
|
require.Nil(t, resp.Eval, "unexpected eval")
|
|
})
|
|
|
|
t.Run("lookup related evals", func(t *testing.T) {
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: eval1.ID,
|
|
QueryOptions: structs.QueryOptions{Region: "global"},
|
|
IncludeRelated: true,
|
|
}
|
|
var resp structs.SingleEvalResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1000, resp.Index, "bad index")
|
|
require.Equal(t, eval1.ID, resp.Eval.ID)
|
|
|
|
// Make sure we didn't modify the eval on a read request.
|
|
require.Nil(t, eval1.RelatedEvals)
|
|
|
|
// Check for the related evals
|
|
expected := []*structs.EvaluationStub{
|
|
eval2.Stub(),
|
|
}
|
|
require.Equal(t, expected, resp.Eval.RelatedEvals)
|
|
})
|
|
}
|
|
|
|
func TestEvalEndpoint_GetEval_ACL(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, root, cleanupS1 := TestACLServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
assert := assert.New(t)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
state := s1.fsm.State()
|
|
state.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1})
|
|
|
|
// Create ACL tokens
|
|
validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}))
|
|
invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs}))
|
|
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: eval1.ID,
|
|
QueryOptions: structs.QueryOptions{Region: "global"},
|
|
}
|
|
|
|
// Try with no token and expect permission denied
|
|
{
|
|
var resp structs.SingleEvalResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)
|
|
assert.NotNil(err)
|
|
assert.Contains(err.Error(), structs.ErrPermissionDenied.Error())
|
|
}
|
|
|
|
// Try with an invalid token and expect permission denied
|
|
{
|
|
get.AuthToken = invalidToken.SecretID
|
|
var resp structs.SingleEvalResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)
|
|
assert.NotNil(err)
|
|
assert.Contains(err.Error(), structs.ErrPermissionDenied.Error())
|
|
}
|
|
|
|
// Lookup the eval using a valid token
|
|
{
|
|
get.AuthToken = validToken.SecretID
|
|
var resp structs.SingleEvalResponse
|
|
assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp))
|
|
assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000)
|
|
assert.Equal(eval1, resp.Eval)
|
|
}
|
|
|
|
// Lookup the eval using a root token
|
|
{
|
|
get.AuthToken = root.SecretID
|
|
var resp structs.SingleEvalResponse
|
|
assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp))
|
|
assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000)
|
|
assert.Equal(eval1, resp.Eval)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_GetEval_Blocking(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
state := s1.fsm.State()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the evals
|
|
eval1 := mock.Eval()
|
|
eval2 := mock.Eval()
|
|
|
|
// First create an unrelated eval
|
|
time.AfterFunc(100*time.Millisecond, func() {
|
|
err := state.UpsertEvals(structs.MsgTypeTestSetup, 100, []*structs.Evaluation{eval1})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
// Upsert the eval we are watching later
|
|
time.AfterFunc(200*time.Millisecond, func() {
|
|
err := state.UpsertEvals(structs.MsgTypeTestSetup, 200, []*structs.Evaluation{eval2})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
// Lookup the eval
|
|
req := &structs.EvalSpecificRequest{
|
|
EvalID: eval2.ID,
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
MinQueryIndex: 150,
|
|
},
|
|
}
|
|
var resp structs.SingleEvalResponse
|
|
start := time.Now()
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
|
|
t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
|
|
}
|
|
if resp.Index != 200 {
|
|
t.Fatalf("Bad index: %d %d", resp.Index, 200)
|
|
}
|
|
if resp.Eval == nil || resp.Eval.ID != eval2.ID {
|
|
t.Fatalf("bad: %#v", resp.Eval)
|
|
}
|
|
|
|
// Eval delete triggers watches
|
|
time.AfterFunc(100*time.Millisecond, func() {
|
|
err := state.DeleteEval(300, []string{eval2.ID}, []string{}, false)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
req.QueryOptions.MinQueryIndex = 250
|
|
var resp2 structs.SingleEvalResponse
|
|
start = time.Now()
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp2); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
|
|
t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
|
|
}
|
|
if resp2.Index != 300 {
|
|
t.Fatalf("Bad index: %d %d", resp2.Index, 300)
|
|
}
|
|
if resp2.Eval != nil {
|
|
t.Fatalf("bad: %#v", resp2.Eval)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Dequeue(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
|
|
// Dequeue the eval
|
|
get := &structs.EvalDequeueRequest{
|
|
Schedulers: defaultSched,
|
|
SchedulerVersion: scheduler.SchedulerVersion,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDequeueResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(eval1, resp.Eval) {
|
|
t.Fatalf("bad: %v %v", eval1, resp.Eval)
|
|
}
|
|
|
|
// Ensure outstanding
|
|
token, ok := s1.evalBroker.Outstanding(eval1.ID)
|
|
if !ok {
|
|
t.Fatalf("should be outstanding")
|
|
}
|
|
if token != resp.Token {
|
|
t.Fatalf("bad token: %#v %#v", token, resp.Token)
|
|
}
|
|
|
|
if resp.WaitIndex != eval1.ModifyIndex {
|
|
t.Fatalf("bad wait index; got %d; want %d", resp.WaitIndex, eval1.ModifyIndex)
|
|
}
|
|
}
|
|
|
|
// TestEvalEndpoint_Dequeue_WaitIndex_Snapshot asserts that an eval's wait
|
|
// index will be equal to the highest eval modify index in the state store.
|
|
func TestEvalEndpoint_Dequeue_WaitIndex_Snapshot(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
eval2 := mock.Eval()
|
|
eval2.JobID = eval1.JobID
|
|
s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1})
|
|
s1.evalBroker.Enqueue(eval1)
|
|
s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval2})
|
|
|
|
// Dequeue the eval
|
|
get := &structs.EvalDequeueRequest{
|
|
Schedulers: defaultSched,
|
|
SchedulerVersion: scheduler.SchedulerVersion,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDequeueResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(eval1, resp.Eval) {
|
|
t.Fatalf("bad: %v %v", eval1, resp.Eval)
|
|
}
|
|
|
|
// Ensure outstanding
|
|
token, ok := s1.evalBroker.Outstanding(eval1.ID)
|
|
if !ok {
|
|
t.Fatalf("should be outstanding")
|
|
}
|
|
if token != resp.Token {
|
|
t.Fatalf("bad token: %#v %#v", token, resp.Token)
|
|
}
|
|
|
|
if resp.WaitIndex != 1001 {
|
|
t.Fatalf("bad wait index; got %d; want %d", resp.WaitIndex, 1001)
|
|
}
|
|
}
|
|
|
|
// TestEvalEndpoint_Dequeue_WaitIndex_Eval asserts that an eval's wait index
|
|
// will be its own modify index if its modify index is greater than all of the
|
|
// indexes in the state store. This can happen if Dequeue receives an eval that
|
|
// has not yet been applied from the Raft log to the local node's state store.
|
|
func TestEvalEndpoint_Dequeue_WaitIndex_Eval(t *testing.T) {
|
|
ci.Parallel(t)
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request but only upsert 1 into the state store
|
|
eval1 := mock.Eval()
|
|
eval2 := mock.Eval()
|
|
eval2.JobID = eval1.JobID
|
|
s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1})
|
|
eval2.ModifyIndex = 1001
|
|
s1.evalBroker.Enqueue(eval2)
|
|
|
|
// Dequeue the eval
|
|
get := &structs.EvalDequeueRequest{
|
|
Schedulers: defaultSched,
|
|
SchedulerVersion: scheduler.SchedulerVersion,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDequeueResponse
|
|
require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp))
|
|
require.Equal(t, eval2, resp.Eval)
|
|
|
|
// Ensure outstanding
|
|
token, ok := s1.evalBroker.Outstanding(eval2.ID)
|
|
require.True(t, ok)
|
|
require.Equal(t, resp.Token, token)
|
|
|
|
// WaitIndex should be equal to the max ModifyIndex - even when that
|
|
// modify index is of the dequeued eval which has yet to be applied to
|
|
// the state store.
|
|
require.Equal(t, eval2.ModifyIndex, resp.WaitIndex)
|
|
}
|
|
|
|
func TestEvalEndpoint_Dequeue_UpdateWaitIndex(t *testing.T) {
|
|
// test enqueuing an eval, updating a plan result for the same eval and de-queueing the eval
|
|
ci.Parallel(t)
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
alloc := mock.Alloc()
|
|
job := alloc.Job
|
|
alloc.Job = nil
|
|
|
|
state := s1.fsm.State()
|
|
|
|
if err := state.UpsertJob(structs.MsgTypeTestSetup, 999, nil, job); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
eval := mock.Eval()
|
|
eval.JobID = job.ID
|
|
|
|
// Create an eval
|
|
if err := state.UpsertEvals(structs.MsgTypeTestSetup, 1, []*structs.Evaluation{eval}); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
s1.evalBroker.Enqueue(eval)
|
|
|
|
// Create a plan result and apply it with a later index
|
|
res := structs.ApplyPlanResultsRequest{
|
|
AllocUpdateRequest: structs.AllocUpdateRequest{
|
|
Alloc: []*structs.Allocation{alloc},
|
|
Job: job,
|
|
},
|
|
EvalID: eval.ID,
|
|
}
|
|
assert := assert.New(t)
|
|
err := state.UpsertPlanResults(structs.MsgTypeTestSetup, 1000, &res)
|
|
assert.Nil(err)
|
|
|
|
// Dequeue the eval
|
|
get := &structs.EvalDequeueRequest{
|
|
Schedulers: defaultSched,
|
|
SchedulerVersion: scheduler.SchedulerVersion,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDequeueResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Ensure outstanding
|
|
token, ok := s1.evalBroker.Outstanding(eval.ID)
|
|
if !ok {
|
|
t.Fatalf("should be outstanding")
|
|
}
|
|
if token != resp.Token {
|
|
t.Fatalf("bad token: %#v %#v", token, resp.Token)
|
|
}
|
|
|
|
if resp.WaitIndex != 1000 {
|
|
t.Fatalf("bad wait index; got %d; want %d", resp.WaitIndex, 1000)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Dequeue_Version_Mismatch(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
|
|
// Dequeue the eval
|
|
get := &structs.EvalDequeueRequest{
|
|
Schedulers: defaultSched,
|
|
SchedulerVersion: 0,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDequeueResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp)
|
|
if err == nil || !strings.Contains(err.Error(), "scheduler version is 0") {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Dequeue_BrokerDisabled(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue.
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register a request.
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
|
|
// Disable the eval broker and try to dequeue.
|
|
s1.evalBroker.SetEnabled(false)
|
|
|
|
get := &structs.EvalDequeueRequest{
|
|
Schedulers: defaultSched,
|
|
SchedulerVersion: scheduler.SchedulerVersion,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDequeueResponse
|
|
require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp))
|
|
require.Empty(t, resp.Eval)
|
|
}
|
|
|
|
func TestEvalEndpoint_Ack(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("missing eval")
|
|
}
|
|
|
|
// Ack the eval
|
|
get := &structs.EvalAckRequest{
|
|
EvalID: out.ID,
|
|
Token: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Ack", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Ensure outstanding
|
|
if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok {
|
|
t.Fatalf("should not be outstanding")
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Nack(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
// Disable all of the schedulers so we can manually dequeue
|
|
// evals and check the queue status
|
|
c.NumSchedulers = 0
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
out, token, _ := s1.evalBroker.Dequeue(defaultSched, time.Second)
|
|
if out == nil {
|
|
t.Fatalf("missing eval")
|
|
}
|
|
|
|
// Nack the eval
|
|
get := &structs.EvalAckRequest{
|
|
EvalID: out.ID,
|
|
Token: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Nack", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Ensure outstanding
|
|
if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok {
|
|
t.Fatalf("should not be outstanding")
|
|
}
|
|
|
|
// Should get it back
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
out2, _, _ := s1.evalBroker.Dequeue(defaultSched, time.Second)
|
|
if out2 != out {
|
|
return false, fmt.Errorf("nack failed")
|
|
}
|
|
|
|
return true, nil
|
|
}, func(err error) {
|
|
t.Fatal(err)
|
|
})
|
|
}
|
|
|
|
func TestEvalEndpoint_Update(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("missing eval")
|
|
}
|
|
|
|
// Update the eval
|
|
eval2 := eval1.Copy()
|
|
eval2.Status = structs.EvalStatusComplete
|
|
|
|
get := &structs.EvalUpdateRequest{
|
|
Evals: []*structs.Evaluation{eval2},
|
|
EvalToken: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Update", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Ensure updated
|
|
ws := memdb.NewWatchSet()
|
|
outE, err := s1.fsm.State().EvalByID(ws, eval2.ID)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if outE.Status != structs.EvalStatusComplete {
|
|
t.Fatalf("Bad: %#v", out)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Create(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the register request
|
|
prev := mock.Eval()
|
|
s1.evalBroker.Enqueue(prev)
|
|
out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("missing eval")
|
|
}
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
eval1.PreviousEval = prev.ID
|
|
get := &structs.EvalUpdateRequest{
|
|
Evals: []*structs.Evaluation{eval1},
|
|
EvalToken: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Create", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Ensure created
|
|
ws := memdb.NewWatchSet()
|
|
outE, err := s1.fsm.State().EvalByID(ws, eval1.ID)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
eval1.CreateIndex = resp.Index
|
|
eval1.ModifyIndex = resp.Index
|
|
if !reflect.DeepEqual(eval1, outE) {
|
|
t.Fatalf("Bad: %#v %#v", outE, eval1)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Reap(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1})
|
|
|
|
// Reap the eval
|
|
get := &structs.EvalReapRequest{
|
|
Evals: []string{eval1.ID},
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Reap", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if resp.Index == 0 {
|
|
t.Fatalf("Bad index: %d", resp.Index)
|
|
}
|
|
|
|
// Ensure deleted
|
|
ws := memdb.NewWatchSet()
|
|
outE, err := s1.fsm.State().EvalByID(ws, eval1.ID)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if outE != nil {
|
|
t.Fatalf("Bad: %#v", outE)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Delete(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
setup := func(t *testing.T) (*Server, *structs.ACLToken, func()) {
|
|
t.Helper()
|
|
testServer, rootToken, cleanupFn := TestACLServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0
|
|
})
|
|
testutil.WaitForLeader(t, testServer.RPC)
|
|
return testServer, rootToken, cleanupFn
|
|
}
|
|
|
|
// Set the expected eval broker state and scheduler config
|
|
setBrokerEnabled := func(t *testing.T, testServer *Server, enabled bool) {
|
|
t.Helper()
|
|
testServer.evalBroker.SetEnabled(enabled)
|
|
|
|
_, schedulerConfig, err := testServer.fsm.State().SchedulerConfig()
|
|
must.NoError(t, err)
|
|
must.NotNil(t, schedulerConfig)
|
|
|
|
schedulerConfig.PauseEvalBroker = !enabled
|
|
must.NoError(t, testServer.fsm.State().SchedulerSetConfig(10, schedulerConfig))
|
|
}
|
|
|
|
t.Run("unsuccessful delete broker enabled", func(t *testing.T) {
|
|
|
|
testServer, rootToken, cleanup := setup(t)
|
|
defer cleanup()
|
|
codec := rpcClient(t, testServer)
|
|
|
|
// Ensure broker is enabled
|
|
setBrokerEnabled(t, testServer, true)
|
|
|
|
// Create and upsert an evaluation.
|
|
mockEval := mock.Eval()
|
|
must.NoError(t, testServer.fsm.State().UpsertEvals(
|
|
structs.MsgTypeTestSetup, 20, []*structs.Evaluation{mockEval}))
|
|
|
|
// Attempt to delete the eval, which should fail because the
|
|
// eval broker is not paused.
|
|
get := &structs.EvalDeleteRequest{
|
|
EvalIDs: []string{mockEval.ID},
|
|
WriteRequest: structs.WriteRequest{
|
|
Region: "global",
|
|
AuthToken: rootToken.SecretID},
|
|
}
|
|
var resp structs.EvalDeleteResponse
|
|
err := msgpackrpc.CallWithCodec(codec, structs.EvalDeleteRPCMethod, get, &resp)
|
|
must.EqError(t, err, "eval broker is enabled; eval broker must be paused to delete evals")
|
|
})
|
|
|
|
t.Run("successful delete without ACLs", func(t *testing.T) {
|
|
testServer, testServerCleanup := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0
|
|
})
|
|
defer testServerCleanup()
|
|
|
|
codec := rpcClient(t, testServer)
|
|
testutil.WaitForLeader(t, testServer.RPC)
|
|
|
|
// Ensure broker is disabled
|
|
setBrokerEnabled(t, testServer, false)
|
|
|
|
// Create and upsert an evaluation.
|
|
mockEval := mock.Eval()
|
|
must.NoError(t, testServer.fsm.State().UpsertEvals(
|
|
structs.MsgTypeTestSetup, 10, []*structs.Evaluation{mockEval}))
|
|
|
|
// Attempt to delete the eval, which should succeed as the eval
|
|
// broker is disabled.
|
|
get := &structs.EvalDeleteRequest{
|
|
EvalIDs: []string{mockEval.ID},
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.EvalDeleteResponse
|
|
must.NoError(t, msgpackrpc.CallWithCodec(codec, structs.EvalDeleteRPCMethod, get, &resp))
|
|
|
|
// Attempt to read the eval from state; this should not be found.
|
|
ws := memdb.NewWatchSet()
|
|
respEval, err := testServer.fsm.State().EvalByID(ws, mockEval.ID)
|
|
must.Nil(t, err)
|
|
must.Nil(t, respEval)
|
|
})
|
|
|
|
t.Run("successful delete with ACLs", func(t *testing.T) {
|
|
|
|
testServer, rootToken, cleanup := setup(t)
|
|
defer cleanup()
|
|
codec := rpcClient(t, testServer)
|
|
|
|
// Ensure broker is disabled
|
|
setBrokerEnabled(t, testServer, false)
|
|
|
|
// Create and upsert an evaluation.
|
|
mockEval := mock.Eval()
|
|
must.NoError(t, testServer.fsm.State().UpsertEvals(
|
|
structs.MsgTypeTestSetup, 20, []*structs.Evaluation{mockEval}))
|
|
|
|
// Attempt to delete the eval, which should succeed as the eval
|
|
// broker is disabled, and we are using a management token.
|
|
get := &structs.EvalDeleteRequest{
|
|
EvalIDs: []string{mockEval.ID},
|
|
WriteRequest: structs.WriteRequest{
|
|
AuthToken: rootToken.SecretID,
|
|
Region: "global",
|
|
},
|
|
}
|
|
var resp structs.EvalDeleteResponse
|
|
must.NoError(t, msgpackrpc.CallWithCodec(codec, structs.EvalDeleteRPCMethod, get, &resp))
|
|
|
|
// Attempt to read the eval from state; this should not be found.
|
|
ws := memdb.NewWatchSet()
|
|
respEval, err := testServer.fsm.State().EvalByID(ws, mockEval.ID)
|
|
must.Nil(t, err)
|
|
must.Nil(t, respEval)
|
|
})
|
|
|
|
t.Run("unsuccessful delete with ACLs incorrect token permissions", func(t *testing.T) {
|
|
|
|
testServer, _, cleanup := setup(t)
|
|
defer cleanup()
|
|
codec := rpcClient(t, testServer)
|
|
|
|
// Ensure broker is disabled
|
|
setBrokerEnabled(t, testServer, false)
|
|
|
|
// Create and upsert an evaluation.
|
|
mockEval := mock.Eval()
|
|
must.NoError(t, testServer.fsm.State().UpsertEvals(
|
|
structs.MsgTypeTestSetup, 10, []*structs.Evaluation{mockEval}))
|
|
|
|
nonMgntToken := mock.CreatePolicyAndToken(t, testServer.State(), 20, "test-valid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob}))
|
|
|
|
// Attempt to delete the eval, which should not succeed as we
|
|
// are using a non-management token.
|
|
get := &structs.EvalDeleteRequest{
|
|
EvalIDs: []string{mockEval.ID},
|
|
WriteRequest: structs.WriteRequest{
|
|
AuthToken: nonMgntToken.SecretID,
|
|
Region: "global",
|
|
},
|
|
}
|
|
var resp structs.EvalDeleteResponse
|
|
err := msgpackrpc.CallWithCodec(codec, structs.EvalDeleteRPCMethod, get, &resp)
|
|
must.EqError(t, err, structs.ErrPermissionDenied.Error())
|
|
})
|
|
|
|
t.Run("successful delete by filter", func(t *testing.T) {
|
|
|
|
testServer, rootToken, cleanup := setup(t)
|
|
defer cleanup()
|
|
codec := rpcClient(t, testServer)
|
|
|
|
// Ensure broker is disabled
|
|
setBrokerEnabled(t, testServer, false)
|
|
|
|
evalCount := 10000
|
|
index := uint64(100)
|
|
|
|
store := testServer.fsm.State()
|
|
|
|
// Create a large set of pending evaluations
|
|
|
|
evals := []*structs.Evaluation{}
|
|
for i := 0; i < evalCount; i++ {
|
|
mockEval := mock.Eval()
|
|
evals = append(evals, mockEval)
|
|
}
|
|
must.NoError(t, store.UpsertEvals(
|
|
structs.MsgTypeTestSetup, index, evals))
|
|
|
|
// Create some evaluations we don't want to delete
|
|
|
|
evalsToKeep := []*structs.Evaluation{}
|
|
for i := 0; i < 3; i++ {
|
|
mockEval := mock.Eval()
|
|
mockEval.JobID = "keepme"
|
|
evalsToKeep = append(evalsToKeep, mockEval)
|
|
}
|
|
index++
|
|
must.NoError(t, store.UpsertEvals(
|
|
structs.MsgTypeTestSetup, index, evalsToKeep))
|
|
|
|
// Create a job with running allocs and evaluations those allocs reference
|
|
|
|
job := mock.Job()
|
|
job.ID = "notsafetodelete"
|
|
job.Status = structs.JobStatusRunning
|
|
index++
|
|
must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, index, nil, job))
|
|
|
|
evalsNotSafeToDelete := []*structs.Evaluation{}
|
|
for i := 0; i < 3; i++ {
|
|
mockEval := mock.Eval()
|
|
mockEval.JobID = job.ID
|
|
evalsNotSafeToDelete = append(evalsNotSafeToDelete, mockEval)
|
|
}
|
|
index++
|
|
must.NoError(t, store.UpsertEvals(
|
|
structs.MsgTypeTestSetup, index, evalsNotSafeToDelete))
|
|
|
|
allocs := []*structs.Allocation{}
|
|
for i := 0; i < 3; i++ {
|
|
alloc := mock.Alloc()
|
|
alloc.ClientStatus = structs.AllocClientStatusRunning
|
|
alloc.EvalID = evalsNotSafeToDelete[i].ID
|
|
allocs = append(allocs, alloc)
|
|
}
|
|
index++
|
|
must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, index, allocs))
|
|
|
|
// Delete all the unwanted evals
|
|
|
|
get := &structs.EvalDeleteRequest{
|
|
Filter: "JobID != \"keepme\"",
|
|
WriteRequest: structs.WriteRequest{AuthToken: rootToken.SecretID, Region: "global"},
|
|
}
|
|
var resp structs.EvalDeleteResponse
|
|
must.NoError(t, msgpackrpc.CallWithCodec(codec, structs.EvalDeleteRPCMethod, get, &resp))
|
|
must.Eq(t, resp.Count, evalCount)
|
|
|
|
// Assert we didn't delete the filtered evals
|
|
gotKeptEvals, err := store.EvalsByJob(nil, job.Namespace, "keepme")
|
|
must.NoError(t, err)
|
|
must.Len(t, 3, gotKeptEvals)
|
|
must.Eq(t, set.From(evalsToKeep), set.From(gotKeptEvals))
|
|
|
|
// Assert we didn't delete the evals that were not safe to delete
|
|
gotNotSafeEvals, err := store.EvalsByJob(nil, job.Namespace, "notsafetodelete")
|
|
must.NoError(t, err)
|
|
must.Len(t, 3, gotNotSafeEvals)
|
|
must.Eq(t, set.From(evalsNotSafeToDelete), set.From(gotNotSafeEvals))
|
|
|
|
})
|
|
|
|
}
|
|
|
|
func TestEvalEndpoint_List(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"
|
|
eval2 := mock.Eval()
|
|
eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9"
|
|
s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2})
|
|
|
|
// Lookup the eval
|
|
get := &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: structs.DefaultNamespace,
|
|
},
|
|
}
|
|
var resp structs.EvalListResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if resp.Index != 1000 {
|
|
t.Fatalf("Bad index: %d %d", resp.Index, 1000)
|
|
}
|
|
|
|
if len(resp.Evaluations) != 2 {
|
|
t.Fatalf("bad: %#v", resp.Evaluations)
|
|
}
|
|
|
|
// Lookup the eval by prefix
|
|
get = &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: structs.DefaultNamespace,
|
|
Prefix: "aaaabb",
|
|
},
|
|
}
|
|
var resp2 structs.EvalListResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp2); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if resp2.Index != 1000 {
|
|
t.Fatalf("Bad index: %d %d", resp2.Index, 1000)
|
|
}
|
|
|
|
if len(resp2.Evaluations) != 1 {
|
|
t.Fatalf("bad: %#v", resp2.Evaluations)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_List_order(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create register requests
|
|
uuid1 := uuid.Generate()
|
|
eval1 := mock.Eval()
|
|
eval1.ID = uuid1
|
|
|
|
uuid2 := uuid.Generate()
|
|
eval2 := mock.Eval()
|
|
eval2.ID = uuid2
|
|
|
|
uuid3 := uuid.Generate()
|
|
eval3 := mock.Eval()
|
|
eval3.ID = uuid3
|
|
|
|
err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1})
|
|
require.NoError(t, err)
|
|
|
|
err = s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval2})
|
|
require.NoError(t, err)
|
|
|
|
err = s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1002, []*structs.Evaluation{eval3})
|
|
require.NoError(t, err)
|
|
|
|
// update eval2 again so we can later assert create index order did not change
|
|
err = s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1003, []*structs.Evaluation{eval2})
|
|
require.NoError(t, err)
|
|
|
|
t.Run("default", func(t *testing.T) {
|
|
// Lookup the evaluations in the default order (oldest first)
|
|
get := &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: "*",
|
|
},
|
|
}
|
|
|
|
var resp structs.EvalListResponse
|
|
err = msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp)
|
|
require.NoError(t, err)
|
|
require.Equal(t, uint64(1003), resp.Index)
|
|
require.Len(t, resp.Evaluations, 3)
|
|
|
|
// Assert returned order is by CreateIndex (ascending)
|
|
require.Equal(t, uint64(1000), resp.Evaluations[0].CreateIndex)
|
|
require.Equal(t, uuid1, resp.Evaluations[0].ID)
|
|
|
|
require.Equal(t, uint64(1001), resp.Evaluations[1].CreateIndex)
|
|
require.Equal(t, uuid2, resp.Evaluations[1].ID)
|
|
|
|
require.Equal(t, uint64(1002), resp.Evaluations[2].CreateIndex)
|
|
require.Equal(t, uuid3, resp.Evaluations[2].ID)
|
|
})
|
|
|
|
t.Run("reverse", func(t *testing.T) {
|
|
// Lookup the evaluations in reverse order (newest first)
|
|
get := &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: "*",
|
|
Reverse: true,
|
|
},
|
|
}
|
|
|
|
var resp structs.EvalListResponse
|
|
err = msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp)
|
|
require.NoError(t, err)
|
|
require.Equal(t, uint64(1003), resp.Index)
|
|
require.Len(t, resp.Evaluations, 3)
|
|
|
|
// Assert returned order is by CreateIndex (descending)
|
|
require.Equal(t, uint64(1002), resp.Evaluations[0].CreateIndex)
|
|
require.Equal(t, uuid3, resp.Evaluations[0].ID)
|
|
|
|
require.Equal(t, uint64(1001), resp.Evaluations[1].CreateIndex)
|
|
require.Equal(t, uuid2, resp.Evaluations[1].ID)
|
|
|
|
require.Equal(t, uint64(1000), resp.Evaluations[2].CreateIndex)
|
|
require.Equal(t, uuid1, resp.Evaluations[2].ID)
|
|
})
|
|
}
|
|
|
|
func TestEvalEndpoint_ListAllNamespaces(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"
|
|
eval2 := mock.Eval()
|
|
eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9"
|
|
s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2})
|
|
|
|
// Lookup the eval
|
|
get := &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: "*",
|
|
},
|
|
}
|
|
var resp structs.EvalListResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if resp.Index != 1000 {
|
|
t.Fatalf("Bad index: %d %d", resp.Index, 1000)
|
|
}
|
|
|
|
if len(resp.Evaluations) != 2 {
|
|
t.Fatalf("bad: %#v", resp.Evaluations)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_List_ACL(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, root, cleanupS1 := TestACLServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create dev namespace
|
|
devNS := mock.Namespace()
|
|
devNS.Name = "dev"
|
|
err := s1.fsm.State().UpsertNamespaces(999, []*structs.Namespace{devNS})
|
|
require.NoError(t, err)
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"
|
|
eval2 := mock.Eval()
|
|
eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9"
|
|
eval3 := mock.Eval()
|
|
eval3.ID = "aaaacccc-3350-4b4b-d185-0e1992ed43e9"
|
|
eval3.Namespace = devNS.Name
|
|
state := s1.fsm.State()
|
|
err = state.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2, eval3})
|
|
require.NoError(t, err)
|
|
|
|
// Create ACL tokens
|
|
validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}))
|
|
invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs}))
|
|
devToken := mock.CreatePolicyAndToken(t, state, 1005, "test-dev",
|
|
mock.NamespacePolicy("dev", "", []string{acl.NamespaceCapabilityReadJob}))
|
|
|
|
testCases := []struct {
|
|
name string
|
|
namespace string
|
|
token string
|
|
expectedEvals []string
|
|
expectedError string
|
|
}{
|
|
{
|
|
name: "no token",
|
|
token: "",
|
|
namespace: structs.DefaultNamespace,
|
|
expectedError: structs.ErrPermissionDenied.Error(),
|
|
},
|
|
{
|
|
name: "invalid token",
|
|
token: invalidToken.SecretID,
|
|
namespace: structs.DefaultNamespace,
|
|
expectedError: structs.ErrPermissionDenied.Error(),
|
|
},
|
|
{
|
|
name: "valid token",
|
|
token: validToken.SecretID,
|
|
namespace: structs.DefaultNamespace,
|
|
expectedEvals: []string{eval1.ID, eval2.ID},
|
|
},
|
|
{
|
|
name: "root token default namespace",
|
|
token: root.SecretID,
|
|
namespace: structs.DefaultNamespace,
|
|
expectedEvals: []string{eval1.ID, eval2.ID},
|
|
},
|
|
{
|
|
name: "root token all namespaces",
|
|
token: root.SecretID,
|
|
namespace: structs.AllNamespacesSentinel,
|
|
expectedEvals: []string{eval1.ID, eval2.ID, eval3.ID},
|
|
},
|
|
{
|
|
name: "dev token all namespaces",
|
|
token: devToken.SecretID,
|
|
namespace: structs.AllNamespacesSentinel,
|
|
expectedEvals: []string{eval3.ID},
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
get := &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
AuthToken: tc.token,
|
|
Region: "global",
|
|
Namespace: tc.namespace,
|
|
},
|
|
}
|
|
|
|
var resp structs.EvalListResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp)
|
|
|
|
if tc.expectedError != "" {
|
|
require.Contains(t, err.Error(), tc.expectedError)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000)
|
|
|
|
got := make([]string, len(resp.Evaluations))
|
|
for i, eval := range resp.Evaluations {
|
|
got[i] = eval.ID
|
|
}
|
|
require.ElementsMatch(t, got, tc.expectedEvals)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_List_Blocking(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
state := s1.fsm.State()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the ieval
|
|
eval := mock.Eval()
|
|
|
|
// Upsert eval triggers watches
|
|
time.AfterFunc(100*time.Millisecond, func() {
|
|
if err := state.UpsertEvals(structs.MsgTypeTestSetup, 2, []*structs.Evaluation{eval}); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
req := &structs.EvalListRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: structs.DefaultNamespace,
|
|
MinQueryIndex: 1,
|
|
},
|
|
}
|
|
start := time.Now()
|
|
var resp structs.EvalListResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
|
|
t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
|
|
}
|
|
if resp.Index != 2 {
|
|
t.Fatalf("Bad index: %d %d", resp.Index, 2)
|
|
}
|
|
if len(resp.Evaluations) != 1 || resp.Evaluations[0].ID != eval.ID {
|
|
t.Fatalf("bad: %#v", resp.Evaluations)
|
|
}
|
|
|
|
// Eval deletion triggers watches
|
|
time.AfterFunc(100*time.Millisecond, func() {
|
|
if err := state.DeleteEval(3, []string{eval.ID}, nil, false); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
req.MinQueryIndex = 2
|
|
start = time.Now()
|
|
var resp2 structs.EvalListResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp2); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
|
|
t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
|
|
}
|
|
if resp2.Index != 3 {
|
|
t.Fatalf("Bad index: %d %d", resp2.Index, 3)
|
|
}
|
|
if len(resp2.Evaluations) != 0 {
|
|
t.Fatalf("bad: %#v", resp2.Evaluations)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
|
|
ci.Parallel(t)
|
|
s1, _, cleanupS1 := TestACLServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create non-default namespace
|
|
nondefaultNS := mock.Namespace()
|
|
nondefaultNS.Name = "non-default"
|
|
err := s1.fsm.State().UpsertNamespaces(999, []*structs.Namespace{nondefaultNS})
|
|
require.NoError(t, err)
|
|
|
|
// create a set of evals and field values to filter on. these are
|
|
// in the order that the state store will return them from the
|
|
// iterator (sorted by create index), for ease of writing tests
|
|
mocks := []struct {
|
|
ids []string
|
|
namespace string
|
|
jobID string
|
|
status string
|
|
}{
|
|
{ids: []string{"aaaa1111-3350-4b4b-d185-0e1992ed43e9"}, jobID: "example"}, // 0
|
|
{ids: []string{"aaaaaa22-3350-4b4b-d185-0e1992ed43e9"}, jobID: "example"}, // 1
|
|
{ids: []string{"aaaaaa33-3350-4b4b-d185-0e1992ed43e9"}, namespace: nondefaultNS.Name}, // 2
|
|
{ids: []string{"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"}, jobID: "example", status: "blocked"}, // 3
|
|
{ids: []string{"aaaaaabb-3350-4b4b-d185-0e1992ed43e9"}}, // 4
|
|
{ids: []string{"aaaaaacc-3350-4b4b-d185-0e1992ed43e9"}}, // 5
|
|
{ids: []string{"aaaaaadd-3350-4b4b-d185-0e1992ed43e9"}, jobID: "example"}, // 6
|
|
{ids: []string{"aaaaaaee-3350-4b4b-d185-0e1992ed43e9"}, jobID: "example"}, // 7
|
|
{ids: []string{"aaaaaaff-3350-4b4b-d185-0e1992ed43e9"}}, // 8
|
|
{ids: []string{"00000111-3350-4b4b-d185-0e1992ed43e9"}}, // 9
|
|
{ids: []string{ // 10
|
|
"00000222-3350-4b4b-d185-0e1992ed43e9",
|
|
"00000333-3350-4b4b-d185-0e1992ed43e9",
|
|
}},
|
|
{}, // 11, index missing
|
|
{ids: []string{"bbbb1111-3350-4b4b-d185-0e1992ed43e9"}}, // 12
|
|
}
|
|
|
|
state := s1.fsm.State()
|
|
|
|
var evals []*structs.Evaluation
|
|
for i, m := range mocks {
|
|
evalsInTx := []*structs.Evaluation{}
|
|
for _, id := range m.ids {
|
|
eval := mock.Eval()
|
|
eval.ID = id
|
|
if m.namespace != "" { // defaults to "default"
|
|
eval.Namespace = m.namespace
|
|
}
|
|
if m.jobID != "" { // defaults to some random UUID
|
|
eval.JobID = m.jobID
|
|
}
|
|
if m.status != "" { // defaults to "pending"
|
|
eval.Status = m.status
|
|
}
|
|
evals = append(evals, eval)
|
|
evalsInTx = append(evalsInTx, eval)
|
|
}
|
|
index := 1000 + uint64(i)
|
|
require.NoError(t, state.UpsertEvals(structs.MsgTypeTestSetup, index, evalsInTx))
|
|
}
|
|
|
|
aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read",
|
|
mock.NamespacePolicy("*", "read", nil)).
|
|
SecretID
|
|
|
|
cases := []struct {
|
|
name string
|
|
namespace string
|
|
prefix string
|
|
nextToken string
|
|
filterJobID string
|
|
filterStatus string
|
|
filter string
|
|
pageSize int32
|
|
expectedNextToken string
|
|
expectedIDs []string
|
|
expectedError string
|
|
}{
|
|
{
|
|
name: "test01 size-2 page-1 default NS",
|
|
pageSize: 2,
|
|
expectedIDs: []string{ // first two items
|
|
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // next one in default namespace
|
|
},
|
|
{
|
|
name: "test02 size-2 page-1 default NS with prefix",
|
|
prefix: "aaaa",
|
|
pageSize: 2,
|
|
expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // prefix results are not sorted by create index
|
|
expectedIDs: []string{
|
|
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test03 size-2 page-2 default NS",
|
|
pageSize: 2,
|
|
nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "1005.aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test04 size-2 page-2 default NS with prefix",
|
|
prefix: "aaaa",
|
|
pageSize: 2,
|
|
nextToken: "aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test05 size-2 page-1 with filters default NS",
|
|
pageSize: 2,
|
|
filterJobID: "example",
|
|
filterStatus: "pending",
|
|
// aaaaaaaa, bb, and cc are filtered by status
|
|
expectedNextToken: "1006.aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test06 size-2 page-1 with filters default NS with short prefix",
|
|
prefix: "aaaa",
|
|
pageSize: 2,
|
|
filterJobID: "example",
|
|
filterStatus: "pending",
|
|
// aaaaaaaa, bb, and cc are filtered by status
|
|
expectedNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test07 size-2 page-1 with filters default NS with longer prefix",
|
|
prefix: "aaaaaa",
|
|
pageSize: 2,
|
|
filterJobID: "example",
|
|
filterStatus: "pending",
|
|
expectedNextToken: "aaaaaaee-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test08 size-2 page-2 filter skip nextToken", //
|
|
pageSize: 3, // reads off the end
|
|
filterJobID: "example",
|
|
filterStatus: "pending",
|
|
nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "",
|
|
expectedIDs: []string{
|
|
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaaee-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test09 size-2 page-2 filters skip nextToken with prefix",
|
|
prefix: "aaaaaa",
|
|
pageSize: 3, // reads off the end
|
|
filterJobID: "example",
|
|
filterStatus: "pending",
|
|
nextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "",
|
|
expectedIDs: []string{
|
|
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaaee-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test10 size-2 page-2 all namespaces",
|
|
namespace: "*",
|
|
pageSize: 2,
|
|
nextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "1004.aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test11 no valid results with filters",
|
|
pageSize: 2,
|
|
filterJobID: "whatever",
|
|
nextToken: "",
|
|
expectedIDs: []string{},
|
|
},
|
|
{
|
|
name: "test12 no valid results with filters and prefix",
|
|
prefix: "aaaa",
|
|
pageSize: 2,
|
|
filterJobID: "whatever",
|
|
nextToken: "",
|
|
expectedIDs: []string{},
|
|
},
|
|
{
|
|
name: "test13 no valid results with filters page-2",
|
|
filterJobID: "whatever",
|
|
nextToken: "aaaaaa11-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{},
|
|
},
|
|
{
|
|
name: "test14 no valid results with filters page-2 with prefix",
|
|
prefix: "aaaa",
|
|
filterJobID: "whatever",
|
|
nextToken: "aaaaaa11-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{},
|
|
},
|
|
{
|
|
name: "test15 go-bexpr filter",
|
|
filter: `Status == "blocked"`,
|
|
nextToken: "",
|
|
expectedIDs: []string{"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"},
|
|
},
|
|
{
|
|
name: "test16 go-bexpr filter with pagination",
|
|
filter: `JobID == "example"`,
|
|
pageSize: 2,
|
|
expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
|
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test17 go-bexpr filter namespace",
|
|
namespace: "non-default",
|
|
filter: `ID contains "aaa"`,
|
|
expectedIDs: []string{
|
|
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test18 go-bexpr wrong namespace",
|
|
namespace: "default",
|
|
filter: `Namespace == "non-default"`,
|
|
expectedIDs: []string{},
|
|
},
|
|
{
|
|
name: "test19 incompatible filtering",
|
|
filter: `JobID == "example"`,
|
|
filterStatus: "complete",
|
|
expectedError: structs.ErrIncompatibleFiltering.Error(),
|
|
},
|
|
{
|
|
name: "test20 go-bexpr invalid expression",
|
|
filter: `NotValid`,
|
|
expectedError: "failed to read filter expression",
|
|
},
|
|
{
|
|
name: "test21 go-bexpr invalid field",
|
|
filter: `InvalidField == "value"`,
|
|
expectedError: "error finding value in datum",
|
|
},
|
|
{
|
|
name: "test22 non-lexicographic order",
|
|
pageSize: 1,
|
|
nextToken: "1009.00000111-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"00000111-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test23 same index",
|
|
pageSize: 1,
|
|
nextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedNextToken: "1010.00000333-3350-4b4b-d185-0e1992ed43e9",
|
|
expectedIDs: []string{
|
|
"00000222-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
{
|
|
name: "test24 missing index",
|
|
pageSize: 1,
|
|
nextToken: "1011.e9522802-0cd8-4b1d-9c9e-ab3d97938371",
|
|
expectedIDs: []string{
|
|
"bbbb1111-3350-4b4b-d185-0e1992ed43e9",
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := &structs.EvalListRequest{
|
|
FilterJobID: tc.filterJobID,
|
|
FilterEvalStatus: tc.filterStatus,
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: tc.namespace,
|
|
Prefix: tc.prefix,
|
|
PerPage: tc.pageSize,
|
|
NextToken: tc.nextToken,
|
|
Filter: tc.filter,
|
|
},
|
|
}
|
|
req.AuthToken = aclToken
|
|
var resp structs.EvalListResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp)
|
|
if tc.expectedError == "" {
|
|
require.NoError(t, err)
|
|
} else {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tc.expectedError)
|
|
return
|
|
}
|
|
|
|
gotIDs := []string{}
|
|
for _, eval := range resp.Evaluations {
|
|
gotIDs = append(gotIDs, eval.ID)
|
|
}
|
|
require.Equal(t, tc.expectedIDs, gotIDs, "unexpected page of evals")
|
|
require.Equal(t, tc.expectedNextToken, resp.QueryMeta.NextToken, "unexpected NextToken")
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
func TestEvalEndpoint_Count(t *testing.T) {
|
|
ci.Parallel(t)
|
|
s1, _, cleanupS1 := TestACLServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
index := uint64(100)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
store := s1.fsm.State()
|
|
|
|
// Create non-default namespace
|
|
nondefaultNS := mock.Namespace()
|
|
nondefaultNS.Name = "non-default"
|
|
err := store.UpsertNamespaces(index, []*structs.Namespace{nondefaultNS})
|
|
must.NoError(t, err)
|
|
|
|
// create a set of evals and field values to filter on.
|
|
mocks := []struct {
|
|
namespace string
|
|
status string
|
|
}{
|
|
{namespace: structs.DefaultNamespace, status: structs.EvalStatusPending},
|
|
{namespace: structs.DefaultNamespace, status: structs.EvalStatusPending},
|
|
{namespace: structs.DefaultNamespace, status: structs.EvalStatusPending},
|
|
{namespace: nondefaultNS.Name, status: structs.EvalStatusPending},
|
|
{namespace: structs.DefaultNamespace, status: structs.EvalStatusComplete},
|
|
{namespace: nondefaultNS.Name, status: structs.EvalStatusComplete},
|
|
}
|
|
|
|
evals := []*structs.Evaluation{}
|
|
for i, m := range mocks {
|
|
eval := mock.Eval()
|
|
eval.ID = fmt.Sprintf("%d", i) + uuid.Generate()[1:] // sorted for prefix count tests
|
|
eval.Namespace = m.namespace
|
|
eval.Status = m.status
|
|
evals = append(evals, eval)
|
|
}
|
|
|
|
index++
|
|
require.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, index, evals))
|
|
|
|
index++
|
|
aclToken := mock.CreatePolicyAndToken(t, store, index, "test-read-any",
|
|
mock.NamespacePolicy("*", "read", nil)).SecretID
|
|
|
|
limitedACLToken := mock.CreatePolicyAndToken(t, store, index, "test-read-limited",
|
|
mock.NamespacePolicy("default", "read", nil)).SecretID
|
|
|
|
cases := []struct {
|
|
name string
|
|
namespace string
|
|
prefix string
|
|
filter string
|
|
token string
|
|
expectedCount int
|
|
}{
|
|
{
|
|
name: "count wildcard namespace with read-any ACL",
|
|
namespace: "*",
|
|
token: aclToken,
|
|
expectedCount: 6,
|
|
},
|
|
{
|
|
name: "count wildcard namespace with limited-read ACL",
|
|
namespace: "*",
|
|
token: limitedACLToken,
|
|
expectedCount: 4,
|
|
},
|
|
{
|
|
name: "count wildcard namespace with prefix",
|
|
namespace: "*",
|
|
prefix: evals[2].ID[:2],
|
|
token: aclToken,
|
|
expectedCount: 1,
|
|
},
|
|
{
|
|
name: "count default namespace with filter",
|
|
namespace: structs.DefaultNamespace,
|
|
filter: "Status == \"pending\"",
|
|
token: aclToken,
|
|
expectedCount: 3,
|
|
},
|
|
{
|
|
name: "count nondefault namespace with filter",
|
|
namespace: "non-default",
|
|
filter: "Status == \"complete\"",
|
|
token: aclToken,
|
|
expectedCount: 1,
|
|
},
|
|
{
|
|
name: "count no results",
|
|
namespace: "non-default",
|
|
filter: "Status == \"never\"",
|
|
token: aclToken,
|
|
expectedCount: 0,
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := &structs.EvalCountRequest{
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
Namespace: tc.namespace,
|
|
Prefix: tc.prefix,
|
|
Filter: tc.filter,
|
|
},
|
|
}
|
|
req.AuthToken = tc.token
|
|
var resp structs.EvalCountResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.Count", req, &resp)
|
|
must.NoError(t, err)
|
|
must.Eq(t, tc.expectedCount, resp.Count)
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
func TestEvalEndpoint_Allocations(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the register request
|
|
alloc1 := mock.Alloc()
|
|
alloc2 := mock.Alloc()
|
|
alloc2.EvalID = alloc1.EvalID
|
|
state := s1.fsm.State()
|
|
state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))
|
|
state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))
|
|
err := state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc1, alloc2})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Lookup the eval
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: alloc1.EvalID,
|
|
QueryOptions: structs.QueryOptions{Region: "global"},
|
|
}
|
|
var resp structs.EvalAllocationsResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if resp.Index != 1000 {
|
|
t.Fatalf("Bad index: %d %d", resp.Index, 1000)
|
|
}
|
|
|
|
if len(resp.Allocations) != 2 {
|
|
t.Fatalf("bad: %#v", resp.Allocations)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Allocations_ACL(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, root, cleanupS1 := TestACLServer(t, nil)
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
assert := assert.New(t)
|
|
|
|
// Create the register request
|
|
alloc1 := mock.Alloc()
|
|
alloc2 := mock.Alloc()
|
|
alloc2.EvalID = alloc1.EvalID
|
|
state := s1.fsm.State()
|
|
assert.Nil(state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)))
|
|
assert.Nil(state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)))
|
|
assert.Nil(state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc1, alloc2}))
|
|
|
|
// Create ACL tokens
|
|
validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}))
|
|
invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid",
|
|
mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs}))
|
|
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: alloc1.EvalID,
|
|
QueryOptions: structs.QueryOptions{Region: "global"},
|
|
}
|
|
|
|
// Try with no token and expect permission denied
|
|
{
|
|
var resp structs.EvalAllocationsResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp)
|
|
assert.NotNil(err)
|
|
assert.Contains(err.Error(), structs.ErrPermissionDenied.Error())
|
|
}
|
|
|
|
// Try with an invalid token and expect permission denied
|
|
{
|
|
get.AuthToken = invalidToken.SecretID
|
|
var resp structs.EvalAllocationsResponse
|
|
err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp)
|
|
assert.NotNil(err)
|
|
assert.Contains(err.Error(), structs.ErrPermissionDenied.Error())
|
|
}
|
|
|
|
// Lookup the eval with a valid token
|
|
{
|
|
get.AuthToken = validToken.SecretID
|
|
var resp structs.EvalAllocationsResponse
|
|
assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp))
|
|
assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000)
|
|
assert.Lenf(resp.Allocations, 2, "bad: %#v", resp.Allocations)
|
|
}
|
|
|
|
// Lookup the eval with a root token
|
|
{
|
|
get.AuthToken = root.SecretID
|
|
var resp structs.EvalAllocationsResponse
|
|
assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp))
|
|
assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000)
|
|
assert.Lenf(resp.Allocations, 2, "bad: %#v", resp.Allocations)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Allocations_Blocking(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, nil)
|
|
defer cleanupS1()
|
|
state := s1.fsm.State()
|
|
codec := rpcClient(t, s1)
|
|
testutil.WaitForLeader(t, s1.RPC)
|
|
|
|
// Create the allocs
|
|
alloc1 := mock.Alloc()
|
|
alloc2 := mock.Alloc()
|
|
|
|
// Upsert an unrelated alloc first
|
|
time.AfterFunc(100*time.Millisecond, func() {
|
|
state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID))
|
|
err := state.UpsertAllocs(structs.MsgTypeTestSetup, 100, []*structs.Allocation{alloc1})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
// Upsert an alloc which will trigger the watch later
|
|
time.AfterFunc(200*time.Millisecond, func() {
|
|
state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID))
|
|
err := state.UpsertAllocs(structs.MsgTypeTestSetup, 200, []*structs.Allocation{alloc2})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
|
|
// Lookup the eval
|
|
get := &structs.EvalSpecificRequest{
|
|
EvalID: alloc2.EvalID,
|
|
QueryOptions: structs.QueryOptions{
|
|
Region: "global",
|
|
MinQueryIndex: 150,
|
|
},
|
|
}
|
|
var resp structs.EvalAllocationsResponse
|
|
start := time.Now()
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
|
|
t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
|
|
}
|
|
if resp.Index != 200 {
|
|
t.Fatalf("Bad index: %d %d", resp.Index, 200)
|
|
}
|
|
if len(resp.Allocations) != 1 || resp.Allocations[0].ID != alloc2.ID {
|
|
t.Fatalf("bad: %#v", resp.Allocations)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Reblock_Nonexistent(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the register request
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("missing eval")
|
|
}
|
|
|
|
get := &structs.EvalUpdateRequest{
|
|
Evals: []*structs.Evaluation{eval1},
|
|
EvalToken: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil {
|
|
t.Fatalf("expect error since eval does not exist")
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Reblock_NonBlocked(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the eval
|
|
eval1 := mock.Eval()
|
|
s1.evalBroker.Enqueue(eval1)
|
|
|
|
// Insert it into the state store
|
|
if err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
out, token, err := s1.evalBroker.Dequeue(defaultSched, 2*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("missing eval")
|
|
}
|
|
|
|
get := &structs.EvalUpdateRequest{
|
|
Evals: []*structs.Evaluation{eval1},
|
|
EvalToken: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil {
|
|
t.Fatalf("should error since eval was not in blocked state: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEvalEndpoint_Reblock(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
|
c.NumSchedulers = 0 // Prevent automatic dequeue
|
|
})
|
|
defer cleanupS1()
|
|
codec := rpcClient(t, s1)
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
return s1.evalBroker.Enabled(), nil
|
|
}, func(err error) {
|
|
t.Fatalf("should enable eval broker")
|
|
})
|
|
|
|
// Create the eval
|
|
eval1 := mock.Eval()
|
|
eval1.Status = structs.EvalStatusBlocked
|
|
s1.evalBroker.Enqueue(eval1)
|
|
|
|
// Insert it into the state store
|
|
if err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
out, token, err := s1.evalBroker.Dequeue(defaultSched, 7*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("bad: %v", out)
|
|
}
|
|
|
|
get := &structs.EvalUpdateRequest{
|
|
Evals: []*structs.Evaluation{eval1},
|
|
EvalToken: token,
|
|
WriteRequest: structs.WriteRequest{Region: "global"},
|
|
}
|
|
var resp structs.GenericResponse
|
|
if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Check that it is blocked
|
|
bStats := s1.blockedEvals.Stats()
|
|
if bStats.TotalBlocked+bStats.TotalEscaped == 0 {
|
|
t.Fatalf("ReblockEval didn't insert eval into the blocked eval tracker")
|
|
}
|
|
}
|