evaluations list pagination and filtering (#11648)
API queries can request pagination using the `NextToken` and `PerPage` fields of `QueryOptions`, when supported by the underlying API. Add a `NextToken` field to the `structs.QueryMeta` so that we have a common field across RPCs to tell the caller where to resume paging from on their next API call. Include this field on the `api.QueryMeta` as well so that it's available for future versions of List HTTP APIs that wrap the response with `QueryMeta` rather than returning a simple list of structs. In the meantime callers can get the `X-Nomad-NextToken`. Add pagination to the `Eval.List` RPC by checking for pagination token and page size in `QueryOptions`. This will allow resuming from the last ID seen so long as the query parameters and the state store itself are unchanged between requests. Add filtering by job ID or evaluation status over the results we get out of the state store. Parse the query parameters of the `Eval.List` API into the arguments expected for filtering in the RPC call.
This commit is contained in:
parent
3e6757f211
commit
624ecab901
|
@ -0,0 +1,3 @@
|
|||
```release-note:improvement
|
||||
api: Add pagination and filtering to Evaluations List API
|
||||
```
|
18
api/api.go
18
api/api.go
|
@ -69,8 +69,10 @@ type QueryOptions struct {
|
|||
// paginated lists.
|
||||
PerPage int32
|
||||
|
||||
// NextToken is the token used indicate where to start paging for queries
|
||||
// that support paginated lists.
|
||||
// NextToken is the token used to indicate where to start paging
|
||||
// for queries that support paginated lists. This token should be
|
||||
// the ID of the next object after the last one seen in the
|
||||
// previous response.
|
||||
NextToken string
|
||||
|
||||
// ctx is an optional context pass through to the underlying HTTP
|
||||
|
@ -113,6 +115,11 @@ type QueryMeta struct {
|
|||
|
||||
// How long did the request take
|
||||
RequestTime time.Duration
|
||||
|
||||
// NextToken is the token used to indicate where to start paging
|
||||
// for queries that support paginated lists. To resume paging from
|
||||
// this point, pass this token in the next request's QueryOptions
|
||||
NextToken string
|
||||
}
|
||||
|
||||
// WriteMeta is used to return meta data about a write
|
||||
|
@ -574,6 +581,12 @@ func (r *request) setQueryOptions(q *QueryOptions) {
|
|||
if q.Prefix != "" {
|
||||
r.params.Set("prefix", q.Prefix)
|
||||
}
|
||||
if q.PerPage != 0 {
|
||||
r.params.Set("per_page", fmt.Sprint(q.PerPage))
|
||||
}
|
||||
if q.NextToken != "" {
|
||||
r.params.Set("next_token", q.NextToken)
|
||||
}
|
||||
for k, v := range q.Params {
|
||||
r.params.Set(k, v)
|
||||
}
|
||||
|
@ -958,6 +971,7 @@ func parseQueryMeta(resp *http.Response, q *QueryMeta) error {
|
|||
return fmt.Errorf("Failed to parse X-Nomad-LastContact: %v", err)
|
||||
}
|
||||
q.LastContact = time.Duration(last) * time.Millisecond
|
||||
q.NextToken = header.Get("X-Nomad-NextToken")
|
||||
|
||||
// Parse the X-Nomad-KnownLeader
|
||||
switch header.Get("X-Nomad-KnownLeader") {
|
||||
|
|
|
@ -5,6 +5,8 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/nomad/api/internal/testutil"
|
||||
)
|
||||
|
||||
func TestEvaluations_List(t *testing.T) {
|
||||
|
@ -43,10 +45,38 @@ func TestEvaluations_List(t *testing.T) {
|
|||
|
||||
// if the eval fails fast there can be more than 1
|
||||
// but they are in order of most recent first, so look at the last one
|
||||
if len(result) == 0 {
|
||||
t.Fatalf("expected eval (%s), got none", resp.EvalID)
|
||||
}
|
||||
idx := len(result) - 1
|
||||
if len(result) == 0 || result[idx].ID != resp.EvalID {
|
||||
if result[idx].ID != resp.EvalID {
|
||||
t.Fatalf("expected eval (%s), got: %#v", resp.EvalID, result[idx])
|
||||
}
|
||||
|
||||
// wait until the 2nd eval shows up before we try paging
|
||||
results := []*Evaluation{}
|
||||
testutil.WaitForResult(func() (bool, error) {
|
||||
results, _, err = e.List(nil)
|
||||
if len(results) < 2 || err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}, func(err error) {
|
||||
t.Fatalf("err: %s", err)
|
||||
})
|
||||
|
||||
// Check the evaluations again with paging; note that while this
|
||||
// package sorts by timestamp, the actual HTTP API sorts by ID
|
||||
// so we need to use that for the NextToken
|
||||
ids := []string{results[0].ID, results[1].ID}
|
||||
sort.Strings(ids)
|
||||
result, qm, err = e.List(&QueryOptions{PerPage: int32(1), NextToken: ids[1]})
|
||||
if err != nil {
|
||||
t.Fatalf("err: %s", err)
|
||||
}
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected no evals after last one but got %v", result[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluations_PrefixList(t *testing.T) {
|
||||
|
|
|
@ -17,6 +17,10 @@ func (s *HTTPServer) EvalsRequest(resp http.ResponseWriter, req *http.Request) (
|
|||
return nil, nil
|
||||
}
|
||||
|
||||
query := req.URL.Query()
|
||||
args.FilterEvalStatus = query.Get("status")
|
||||
args.FilterJobID = query.Get("job")
|
||||
|
||||
var out structs.EvalListResponse
|
||||
if err := s.agent.RPC("Eval.List", &args, &out); err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -1,10 +1,13 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/hashicorp/nomad/nomad/mock"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
)
|
||||
|
@ -17,39 +20,42 @@ func TestHTTP_EvalList(t *testing.T) {
|
|||
eval1 := mock.Eval()
|
||||
eval2 := mock.Eval()
|
||||
err := state.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2})
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
// Make the HTTP request
|
||||
// simple list request
|
||||
req, err := http.NewRequest("GET", "/v1/evaluations", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
respW := httptest.NewRecorder()
|
||||
|
||||
// Make the request
|
||||
obj, err := s.Server.EvalsRequest(respW, req)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check for the index
|
||||
if respW.HeaderMap.Get("X-Nomad-Index") == "" {
|
||||
t.Fatalf("missing index")
|
||||
}
|
||||
if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
|
||||
t.Fatalf("missing known leader")
|
||||
}
|
||||
if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
|
||||
t.Fatalf("missing last contact")
|
||||
}
|
||||
// check headers and response body
|
||||
require.NotEqual(t, "", respW.HeaderMap.Get("X-Nomad-Index"), "missing index")
|
||||
require.Equal(t, "true", respW.HeaderMap.Get("X-Nomad-KnownLeader"), "missing known leader")
|
||||
require.NotEqual(t, "", respW.HeaderMap.Get("X-Nomad-LastContact"), "missing last contact")
|
||||
require.Len(t, obj.([]*structs.Evaluation), 2, "expected 2 evals")
|
||||
|
||||
// paginated list request
|
||||
req, err = http.NewRequest("GET", "/v1/evaluations?per_page=1", nil)
|
||||
require.NoError(t, err)
|
||||
respW = httptest.NewRecorder()
|
||||
obj, err = s.Server.EvalsRequest(respW, req)
|
||||
require.NoError(t, err)
|
||||
|
||||
// check response body
|
||||
require.Len(t, obj.([]*structs.Evaluation), 1, "expected 1 eval")
|
||||
|
||||
// filtered list request
|
||||
req, err = http.NewRequest("GET",
|
||||
fmt.Sprintf("/v1/evaluations?per_page=10&job=%s", eval2.JobID), nil)
|
||||
require.NoError(t, err)
|
||||
respW = httptest.NewRecorder()
|
||||
obj, err = s.Server.EvalsRequest(respW, req)
|
||||
require.NoError(t, err)
|
||||
|
||||
// check response body
|
||||
require.Len(t, obj.([]*structs.Evaluation), 1, "expected 1 eval")
|
||||
|
||||
// Check the eval
|
||||
e := obj.([]*structs.Evaluation)
|
||||
if len(e) != 2 {
|
||||
t.Fatalf("bad: %#v", e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -608,11 +608,19 @@ func setLastContact(resp http.ResponseWriter, last time.Duration) {
|
|||
resp.Header().Set("X-Nomad-LastContact", strconv.FormatUint(lastMsec, 10))
|
||||
}
|
||||
|
||||
// setNextToken is used to set the next token header for pagination
|
||||
func setNextToken(resp http.ResponseWriter, nextToken string) {
|
||||
if nextToken != "" {
|
||||
resp.Header().Set("X-Nomad-NextToken", nextToken)
|
||||
}
|
||||
}
|
||||
|
||||
// setMeta is used to set the query response meta data
|
||||
func setMeta(resp http.ResponseWriter, m *structs.QueryMeta) {
|
||||
setIndex(resp, m.Index)
|
||||
setLastContact(resp, m.LastContact)
|
||||
setKnownLeader(resp, m.KnownLeader)
|
||||
setNextToken(resp, m.NextToken)
|
||||
}
|
||||
|
||||
// setHeaders is used to set canonical response header fields
|
||||
|
@ -746,8 +754,7 @@ func parsePagination(req *http.Request, b *structs.QueryOptions) {
|
|||
}
|
||||
}
|
||||
|
||||
nextToken := query.Get("next_token")
|
||||
b.NextToken = nextToken
|
||||
b.NextToken = query.Get("next_token")
|
||||
}
|
||||
|
||||
// parseWriteRequest is a convenience method for endpoints that need to parse a
|
||||
|
|
|
@ -349,32 +349,39 @@ func (e *Eval) List(args *structs.EvalListRequest,
|
|||
opts := blockingOptions{
|
||||
queryOpts: &args.QueryOptions,
|
||||
queryMeta: &reply.QueryMeta,
|
||||
run: func(ws memdb.WatchSet, state *state.StateStore) error {
|
||||
run: func(ws memdb.WatchSet, store *state.StateStore) error {
|
||||
// Scan all the evaluations
|
||||
var err error
|
||||
var iter memdb.ResultIterator
|
||||
if prefix := args.QueryOptions.Prefix; prefix != "" {
|
||||
iter, err = state.EvalsByIDPrefix(ws, args.RequestNamespace(), prefix)
|
||||
iter, err = store.EvalsByIDPrefix(ws, args.RequestNamespace(), prefix)
|
||||
} else {
|
||||
iter, err = state.EvalsByNamespace(ws, args.RequestNamespace())
|
||||
iter, err = store.EvalsByNamespace(ws, args.RequestNamespace())
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var evals []*structs.Evaluation
|
||||
for {
|
||||
raw := iter.Next()
|
||||
if raw == nil {
|
||||
break
|
||||
iter = memdb.NewFilterIterator(iter, func(raw interface{}) bool {
|
||||
if eval := raw.(*structs.Evaluation); eval != nil {
|
||||
return args.ShouldBeFiltered(eval)
|
||||
}
|
||||
eval := raw.(*structs.Evaluation)
|
||||
evals = append(evals, eval)
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
var evals []*structs.Evaluation
|
||||
paginator := state.NewPaginator(iter, args.QueryOptions,
|
||||
func(raw interface{}) {
|
||||
eval := raw.(*structs.Evaluation)
|
||||
evals = append(evals, eval)
|
||||
})
|
||||
|
||||
nextToken := paginator.Page()
|
||||
reply.QueryMeta.NextToken = nextToken
|
||||
reply.Evaluations = evals
|
||||
|
||||
// Use the last index that affected the jobs table
|
||||
index, err := state.Index("evals")
|
||||
index, err := store.Index("evals")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -851,6 +851,225 @@ func TestEvalEndpoint_List_Blocking(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
|
||||
t.Parallel()
|
||||
s1, _, cleanupS1 := TestACLServer(t, nil)
|
||||
defer cleanupS1()
|
||||
codec := rpcClient(t, s1)
|
||||
testutil.WaitForLeader(t, s1.RPC)
|
||||
|
||||
// create a set of evals and field values to filter on. these are
|
||||
// in the order that the state store will return them from the
|
||||
// iterator (sorted by key), for ease of writing tests
|
||||
mocks := []struct {
|
||||
id string
|
||||
namespace string
|
||||
jobID string
|
||||
status string
|
||||
}{
|
||||
{id: "aaaa1111-3350-4b4b-d185-0e1992ed43e9", jobID: "example"},
|
||||
{id: "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", jobID: "example"},
|
||||
{id: "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", namespace: "non-default"},
|
||||
{id: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", jobID: "example", status: "blocked"},
|
||||
{id: "aaaaaabb-3350-4b4b-d185-0e1992ed43e9"},
|
||||
{id: "aaaaaacc-3350-4b4b-d185-0e1992ed43e9"},
|
||||
{id: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9", jobID: "example"},
|
||||
{id: "aaaaaaee-3350-4b4b-d185-0e1992ed43e9", jobID: "example"},
|
||||
{id: "aaaaaaff-3350-4b4b-d185-0e1992ed43e9"},
|
||||
}
|
||||
|
||||
mockEvals := []*structs.Evaluation{}
|
||||
for _, m := range mocks {
|
||||
eval := mock.Eval()
|
||||
eval.ID = m.id
|
||||
if m.namespace != "" { // defaults to "default"
|
||||
eval.Namespace = m.namespace
|
||||
}
|
||||
if m.jobID != "" { // defaults to some random UUID
|
||||
eval.JobID = m.jobID
|
||||
}
|
||||
if m.status != "" { // defaults to "pending"
|
||||
eval.Status = m.status
|
||||
}
|
||||
mockEvals = append(mockEvals, eval)
|
||||
}
|
||||
|
||||
state := s1.fsm.State()
|
||||
require.NoError(t, state.UpsertEvals(structs.MsgTypeTestSetup, 1000, mockEvals))
|
||||
|
||||
aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read",
|
||||
mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).
|
||||
SecretID
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
namespace string
|
||||
prefix string
|
||||
nextToken string
|
||||
filterJobID string
|
||||
filterStatus string
|
||||
pageSize int32
|
||||
expectedNextToken string
|
||||
expectedIDs []string
|
||||
}{
|
||||
{
|
||||
name: "test01 size-2 page-1 default NS",
|
||||
pageSize: 2,
|
||||
expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test02 size-2 page-1 default NS with prefix",
|
||||
prefix: "aaaa",
|
||||
pageSize: 2,
|
||||
expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test03 size-2 page-2 default NS",
|
||||
pageSize: 2,
|
||||
nextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedNextToken: "aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test04 size-2 page-2 default NS with prefix",
|
||||
prefix: "aaaa",
|
||||
pageSize: 2,
|
||||
nextToken: "aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test05 size-2 page-1 with filters default NS",
|
||||
pageSize: 2,
|
||||
filterJobID: "example",
|
||||
filterStatus: "pending",
|
||||
// aaaaaaaa, bb, and cc are filtered by status
|
||||
expectedNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test06 size-2 page-1 with filters default NS with short prefix",
|
||||
prefix: "aaaa",
|
||||
pageSize: 2,
|
||||
filterJobID: "example",
|
||||
filterStatus: "pending",
|
||||
// aaaaaaaa, bb, and cc are filtered by status
|
||||
expectedNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test07 size-2 page-1 with filters default NS with longer prefix",
|
||||
prefix: "aaaaaa",
|
||||
pageSize: 2,
|
||||
filterJobID: "example",
|
||||
filterStatus: "pending",
|
||||
expectedNextToken: "aaaaaaee-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{
|
||||
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test08 size-2 page-2 filter skip nextToken",
|
||||
pageSize: 3, // reads off the end
|
||||
filterJobID: "example",
|
||||
filterStatus: "pending",
|
||||
nextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedNextToken: "",
|
||||
expectedIDs: []string{
|
||||
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaaee-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test09 size-2 page-2 filters skip nextToken with prefix",
|
||||
prefix: "aaaaaa",
|
||||
pageSize: 3, // reads off the end
|
||||
filterJobID: "example",
|
||||
filterStatus: "pending",
|
||||
nextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedNextToken: "",
|
||||
expectedIDs: []string{
|
||||
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
|
||||
"aaaaaaee-3350-4b4b-d185-0e1992ed43e9",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test10 no valid results with filters",
|
||||
pageSize: 2,
|
||||
filterJobID: "whatever",
|
||||
nextToken: "",
|
||||
expectedIDs: []string{},
|
||||
},
|
||||
{
|
||||
name: "test11 no valid results with filters and prefix",
|
||||
prefix: "aaaa",
|
||||
pageSize: 2,
|
||||
filterJobID: "whatever",
|
||||
nextToken: "",
|
||||
expectedIDs: []string{},
|
||||
},
|
||||
{
|
||||
name: "test12 no valid results with filters page-2",
|
||||
filterJobID: "whatever",
|
||||
nextToken: "aaaaaa11-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{},
|
||||
},
|
||||
{
|
||||
name: "test13 no valid results with filters page-2 with prefix",
|
||||
prefix: "aaaa",
|
||||
filterJobID: "whatever",
|
||||
nextToken: "aaaaaa11-3350-4b4b-d185-0e1992ed43e9",
|
||||
expectedIDs: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := &structs.EvalListRequest{
|
||||
FilterJobID: tc.filterJobID,
|
||||
FilterEvalStatus: tc.filterStatus,
|
||||
QueryOptions: structs.QueryOptions{
|
||||
Region: "global",
|
||||
Namespace: tc.namespace,
|
||||
Prefix: tc.prefix,
|
||||
PerPage: tc.pageSize,
|
||||
NextToken: tc.nextToken,
|
||||
},
|
||||
}
|
||||
req.AuthToken = aclToken
|
||||
var resp structs.EvalListResponse
|
||||
require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp))
|
||||
gotIDs := []string{}
|
||||
for _, eval := range resp.Evaluations {
|
||||
gotIDs = append(gotIDs, eval.ID)
|
||||
}
|
||||
require.Equal(t, tc.expectedIDs, gotIDs, "unexpected page of evals")
|
||||
require.Equal(t, tc.expectedNextToken, resp.QueryMeta.NextToken, "unexpected NextToken")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvalEndpoint_Allocations(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
|
|
@ -0,0 +1,88 @@
|
|||
package state
|
||||
|
||||
import (
|
||||
memdb "github.com/hashicorp/go-memdb"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
)
|
||||
|
||||
// Paginator is an iterator over a memdb.ResultIterator that returns
|
||||
// only the expected number of pages.
|
||||
type Paginator struct {
|
||||
iter memdb.ResultIterator
|
||||
perPage int32
|
||||
itemCount int32
|
||||
seekingToken string
|
||||
nextToken string
|
||||
nextTokenFound bool
|
||||
|
||||
// appendFunc is the function the caller should use to append raw
|
||||
// entries to the results set. The object is guaranteed to be
|
||||
// non-nil.
|
||||
appendFunc func(interface{})
|
||||
}
|
||||
|
||||
func NewPaginator(iter memdb.ResultIterator, opts structs.QueryOptions, appendFunc func(interface{})) *Paginator {
|
||||
return &Paginator{
|
||||
iter: iter,
|
||||
perPage: opts.PerPage,
|
||||
seekingToken: opts.NextToken,
|
||||
nextTokenFound: opts.NextToken == "",
|
||||
appendFunc: appendFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// Page populates a page by running the append function
|
||||
// over all results. Returns the next token
|
||||
func (p *Paginator) Page() string {
|
||||
DONE:
|
||||
for {
|
||||
raw, andThen := p.next()
|
||||
switch andThen {
|
||||
case paginatorInclude:
|
||||
p.appendFunc(raw)
|
||||
case paginatorSkip:
|
||||
continue
|
||||
case paginatorComplete:
|
||||
break DONE
|
||||
}
|
||||
}
|
||||
return p.nextToken
|
||||
}
|
||||
|
||||
func (p *Paginator) next() (interface{}, paginatorState) {
|
||||
raw := p.iter.Next()
|
||||
if raw == nil {
|
||||
p.nextToken = ""
|
||||
return nil, paginatorComplete
|
||||
}
|
||||
|
||||
// have we found the token we're seeking (if any)?
|
||||
id := raw.(IDGetter).GetID()
|
||||
p.nextToken = id
|
||||
if !p.nextTokenFound && id < p.seekingToken {
|
||||
return nil, paginatorSkip
|
||||
}
|
||||
p.nextTokenFound = true
|
||||
|
||||
// have we produced enough results for this page?
|
||||
p.itemCount++
|
||||
if p.perPage != 0 && p.itemCount > p.perPage {
|
||||
return raw, paginatorComplete
|
||||
}
|
||||
|
||||
return raw, paginatorInclude
|
||||
}
|
||||
|
||||
// IDGetter must be implemented for the results of any iterator we
|
||||
// want to paginate
|
||||
type IDGetter interface {
|
||||
GetID() string
|
||||
}
|
||||
|
||||
type paginatorState int
|
||||
|
||||
const (
|
||||
paginatorInclude paginatorState = iota
|
||||
paginatorSkip
|
||||
paginatorComplete
|
||||
)
|
|
@ -0,0 +1,112 @@
|
|||
package state
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
memdb "github.com/hashicorp/go-memdb"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
)
|
||||
|
||||
func TestPaginator(t *testing.T) {
|
||||
t.Parallel()
|
||||
ids := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
perPage int32
|
||||
nextToken string
|
||||
expected []string
|
||||
expectedNextToken string
|
||||
}{
|
||||
{
|
||||
name: "size-3 page-1",
|
||||
perPage: 3,
|
||||
expected: []string{"0", "1", "2"},
|
||||
expectedNextToken: "3",
|
||||
},
|
||||
{
|
||||
name: "size-5 page-2 stop before end",
|
||||
perPage: 5,
|
||||
nextToken: "3",
|
||||
expected: []string{"3", "4", "5", "6", "7"},
|
||||
expectedNextToken: "8",
|
||||
},
|
||||
{
|
||||
name: "page-2 reading off the end",
|
||||
perPage: 10,
|
||||
nextToken: "5",
|
||||
expected: []string{"5", "6", "7", "8", "9"},
|
||||
expectedNextToken: "",
|
||||
},
|
||||
{
|
||||
name: "starting off the end",
|
||||
perPage: 5,
|
||||
nextToken: "a",
|
||||
expected: []string{},
|
||||
expectedNextToken: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
iter := newTestIterator(ids)
|
||||
results := []string{}
|
||||
|
||||
paginator := NewPaginator(iter,
|
||||
structs.QueryOptions{
|
||||
PerPage: tc.perPage, NextToken: tc.nextToken,
|
||||
},
|
||||
func(raw interface{}) {
|
||||
result := raw.(*mockObject)
|
||||
results = append(results, result.GetID())
|
||||
},
|
||||
)
|
||||
|
||||
nextToken := paginator.Page()
|
||||
require.Equal(t, tc.expected, results)
|
||||
require.Equal(t, tc.expectedNextToken, nextToken)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// helpers for pagination tests
|
||||
|
||||
// implements memdb.ResultIterator interface
|
||||
type testResultIterator struct {
|
||||
results chan interface{}
|
||||
idx int
|
||||
}
|
||||
|
||||
func (i testResultIterator) Next() interface{} {
|
||||
select {
|
||||
case result := <-i.results:
|
||||
return result
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// not used, but required to implement memdb.ResultIterator
|
||||
func (i testResultIterator) WatchCh() <-chan struct{} {
|
||||
return make(<-chan struct{})
|
||||
}
|
||||
|
||||
type mockObject struct {
|
||||
id string
|
||||
}
|
||||
|
||||
func (m *mockObject) GetID() string {
|
||||
return m.id
|
||||
}
|
||||
|
||||
func newTestIterator(ids []string) memdb.ResultIterator {
|
||||
iter := testResultIterator{results: make(chan interface{}, 20)}
|
||||
for _, id := range ids {
|
||||
iter.results <- &mockObject{id: id}
|
||||
}
|
||||
return iter
|
||||
}
|
|
@ -277,8 +277,10 @@ type QueryOptions struct {
|
|||
// paginated lists.
|
||||
PerPage int32
|
||||
|
||||
// NextToken is the token used indicate where to start paging for queries
|
||||
// that support paginated lists.
|
||||
// NextToken is the token used to indicate where to start paging
|
||||
// for queries that support paginated lists. This token should be
|
||||
// the ID of the next object after the last one seen in the
|
||||
// previous response.
|
||||
NextToken string
|
||||
|
||||
InternalRpcInfo
|
||||
|
@ -436,6 +438,11 @@ type QueryMeta struct {
|
|||
|
||||
// Used to indicate if there is a known leader node
|
||||
KnownLeader bool
|
||||
|
||||
// NextToken is the token returned with queries that support
|
||||
// paginated lists. To resume paging from this point, pass
|
||||
// this token in the next request's QueryOptions.
|
||||
NextToken string
|
||||
}
|
||||
|
||||
// WriteMeta allows a write response to include potentially
|
||||
|
@ -844,9 +851,23 @@ type EvalDequeueRequest struct {
|
|||
|
||||
// EvalListRequest is used to list the evaluations
|
||||
type EvalListRequest struct {
|
||||
FilterJobID string
|
||||
FilterEvalStatus string
|
||||
QueryOptions
|
||||
}
|
||||
|
||||
// ShouldBeFiltered indicates that the eval should be filtered (that
|
||||
// is, removed) from the results
|
||||
func (req *EvalListRequest) ShouldBeFiltered(e *Evaluation) bool {
|
||||
if req.FilterJobID != "" && req.FilterJobID != e.JobID {
|
||||
return true
|
||||
}
|
||||
if req.FilterEvalStatus != "" && req.FilterEvalStatus != e.Status {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// PlanRequest is used to submit an allocation plan to the leader
|
||||
type PlanRequest struct {
|
||||
Plan *Plan
|
||||
|
@ -10391,6 +10412,14 @@ type Evaluation struct {
|
|||
ModifyTime int64
|
||||
}
|
||||
|
||||
// GetID implements the IDGetter interface, required for pagination
|
||||
func (e *Evaluation) GetID() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.ID
|
||||
}
|
||||
|
||||
// TerminalStatus returns if the current status is terminal and
|
||||
// will no longer transition.
|
||||
func (e *Evaluation) TerminalStatus() bool {
|
||||
|
|
|
@ -31,6 +31,24 @@ The table below shows this endpoint's support for
|
|||
even number of hexadecimal characters (0-9a-f). This is specified as a query
|
||||
string parameter.
|
||||
|
||||
- `next_token` `(string: "")` - This endpoint supports paging. The
|
||||
`next_token` parameter accepts a string which is the `ID` field of
|
||||
the next expected evaluation. This value can be obtained from the
|
||||
`X-Nomad-NextToken` header from the previous response.
|
||||
|
||||
- `per_page` `(int: 0)` - Specifies a maximum number of evaluations to
|
||||
return for this request. If omitted, the response is not
|
||||
paginated. The `ID` of the last evaluation in the response can be
|
||||
used as the `last_token` of the next request to fetch additional
|
||||
pages.
|
||||
|
||||
- `job` `(string: "")` - Filter the list of evaluations to a specific
|
||||
job ID.
|
||||
|
||||
- `status` `(string: "")` - Filter the list of evaluations to a
|
||||
specific evaluation status (one of `blocked`, `pending`, `complete`,
|
||||
`failed`, or `canceled`).
|
||||
|
||||
### Sample Request
|
||||
|
||||
```shell-session
|
||||
|
|
Loading…
Reference in New Issue