luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Add `go-bexpr` filters to evals and deployment list endpoints (#12034)

This commit is contained in:
Luiz Aoqui 2022-02-16 11:40:30 -05:00 committed by GitHub
parent c30b4617aa
commit 110dbeeb9d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 509 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/12034.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
api: filter values of evaluation and deployment list api endpoints
```

7
api/api.go
View File

@ -65,6 +65,10 @@ type QueryOptions struct {
// AuthToken is the secret ID of an ACL token
AuthToken string
// Filter specifies the go-bexpr filter expression to be used for
// filtering the data prior to returning a response
Filter string
// PerPage is the number of entries to be returned in queries that support
// paginated lists.
PerPage int32
@ -586,6 +590,9 @@ func (r *request) setQueryOptions(q *QueryOptions) {
if q.Prefix != "" {
r.params.Set("prefix", q.Prefix)
}
if q.Filter != "" {
r.params.Set("filter", q.Filter)
}
if q.PerPage != 0 {
r.params.Set("per_page", fmt.Sprint(q.PerPage))
}

8
api/evaluations_test.go
View File

@ -77,6 +77,14 @@ func TestEvaluations_List(t *testing.T) {
if len(result) != 1 {
t.Fatalf("expected no evals after last one but got %v", result[0])
}
// Query evaluations using a filter.
results, _, err = e.List(&QueryOptions{
Filter: `TriggeredBy == "job-register"`,
})
if len(result) != 1 {
t.Fatalf("expected 1 eval, got %d", len(result))
}
}
func TestEvaluations_PrefixList(t *testing.T) {

12
command/agent/http.go
View File

@ -537,6 +537,9 @@ func (s *HTTPServer) wrap(handler func(resp http.ResponseWriter, req *http.Reque
} else if strings.HasSuffix(errMsg, structs.ErrJobRegistrationDisabled.Error()) {
errMsg = structs.ErrJobRegistrationDisabled.Error()
code = 403
} else if strings.HasSuffix(errMsg, structs.ErrIncompatibleFiltering.Error()) {
errMsg = structs.ErrIncompatibleFiltering.Error()
code = 400
}
}
@ -784,6 +787,7 @@ func (s *HTTPServer) parse(resp http.ResponseWriter, req *http.Request, r *strin
parsePrefix(req, b)
parseNamespace(req, &b.Namespace)
parsePagination(req, b)
parseFilter(req, b)
return parseWait(resp, req, b)
}
@ -801,6 +805,14 @@ func parsePagination(req *http.Request, b *structs.QueryOptions) {
b.NextToken = query.Get("next_token")
}
// parseFilter parses the filter query parameter for QueryOptions
func parseFilter(req *http.Request, b *structs.QueryOptions) {
query := req.URL.Query()
if filter := query.Get("filter"); filter != "" {
b.Filter = filter
}
}
// parseWriteRequest is a convenience method for endpoints that need to parse a
// write request.
func (s *HTTPServer) parseWriteRequest(req *http.Request, w *structs.WriteRequest) {

12
command/deployment_list.go
View File

@ -30,6 +30,9 @@ List Options:
-json
Output the deployments in a JSON format.
-filter
Specifies an expression used to filter query results.
-t
Format and display the deployments using a Go template.
@ -43,6 +46,7 @@ func (c *DeploymentListCommand) AutocompleteFlags() complete.Flags {
return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
complete.Flags{
"-json": complete.PredictNothing,
"-filter": complete.PredictAnything,
"-t": complete.PredictAnything,
"-verbose": complete.PredictNothing,
})
@ -60,12 +64,13 @@ func (c *DeploymentListCommand) Name() string { return "deployment list" }
func (c *DeploymentListCommand) Run(args []string) int {
var json, verbose bool
var tmpl string
var filter, tmpl string
flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
flags.Usage = func() { c.Ui.Output(c.Help()) }
flags.BoolVar(&verbose, "verbose", false, "")
flags.BoolVar(&json, "json", false, "")
flags.StringVar(&filter, "filter", "", "")
flags.StringVar(&tmpl, "t", "", "")
if err := flags.Parse(args); err != nil {
@ -93,7 +98,10 @@ func (c *DeploymentListCommand) Run(args []string) int {
return 1
}
deploys, _, err := client.Deployments().List(nil)
opts := &api.QueryOptions{
Filter: filter,
}
deploys, _, err := client.Deployments().List(opts)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error retrieving deployments: %s", err))
return 1

8
command/eval_list.go
View File

@ -35,6 +35,9 @@ Eval List Options:
-page-token
Where to start pagination.
-filter
Specifies an expression used to filter query results.
-job
Only show evaluations for this job ID.
@ -61,6 +64,7 @@ func (c *EvalListCommand) AutocompleteFlags() complete.Flags {
"-json": complete.PredictNothing,
"-t": complete.PredictAnything,
"-verbose": complete.PredictNothing,
"-filter": complete.PredictAnything,
"-job": complete.PredictAnything,
"-status": complete.PredictAnything,
"-per-page": complete.PredictAnything,
@ -88,7 +92,7 @@ func (c *EvalListCommand) Name() string { return "eval list" }
func (c *EvalListCommand) Run(args []string) int {
var monitor, verbose, json bool
var perPage int
var tmpl, pageToken, filterJobID, filterStatus string
var tmpl, pageToken, filter, filterJobID, filterStatus string
flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
flags.Usage = func() { c.Ui.Output(c.Help()) }
@ -98,6 +102,7 @@ func (c *EvalListCommand) Run(args []string) int {
flags.StringVar(&tmpl, "t", "", "")
flags.IntVar(&perPage, "per-page", 0, "")
flags.StringVar(&pageToken, "page-token", "", "")
flags.StringVar(&filter, "filter", "", "")
flags.StringVar(&filterJobID, "job", "", "")
flags.StringVar(&filterStatus, "status", "", "")
@ -120,6 +125,7 @@ func (c *EvalListCommand) Run(args []string) int {
}
opts := &api.QueryOptions{
Filter: filter,
PerPage: int32(perPage),
NextToken: pageToken,
Params: map[string]string{},

2
go.mod
View File

@ -48,6 +48,7 @@ require (
github.com/hashicorp/consul/api v1.9.1
github.com/hashicorp/consul/sdk v0.8.0
github.com/hashicorp/cronexpr v1.1.1
github.com/hashicorp/go-bexpr v0.1.11
github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-connlimit v0.3.0
@ -209,6 +210,7 @@ require (
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/pointerstructure v1.2.1 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/mrunalp/fileutils v0.5.0 // indirect

5
go.sum
View File

@ -667,6 +667,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU=
github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY=
github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE=
github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de h1:XDCSythtg8aWSRSO29uwhgh7b127fWr+m5SemqjSUL8=
github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de/go.mod h1:xIwEieBHERyEvaeKF/TcHh1Hu+lxPM+n2vT1+g9I4m4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@ -938,9 +940,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw=
github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=

14
nomad/deployment_endpoint.go
View File

@ -2,6 +2,7 @@ package nomad
import (
"fmt"
"net/http"
"time"
metrics "github.com/armon/go-metrics"
@ -421,13 +422,22 @@ func (d *Deployment) List(args *structs.DeploymentListRequest, reply *structs.De
}
var deploys []*structs.Deployment
paginator := state.NewPaginator(iter, args.QueryOptions,
paginator, err := state.NewPaginator(iter, args.QueryOptions,
func(raw interface{}) {
deploy := raw.(*structs.Deployment)
deploys = append(deploys, deploy)
})
if err != nil {
return structs.NewErrRPCCodedf(
http.StatusBadRequest, "failed to create result paginator: %v", err)
}
nextToken, err := paginator.Page()
if err != nil {
return structs.NewErrRPCCodedf(
http.StatusBadRequest, "failed to read result page: %v", err)
}
nextToken := paginator.Page()
reply.QueryMeta.NextToken = nextToken
reply.Deployments = deploys

62
nomad/deployment_endpoint_test.go
View File

@ -1288,17 +1288,19 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
}
aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read",
mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).
mock.NamespacePolicy("*", "read", nil)).
SecretID
cases := []struct {
name string
namespace string
prefix string
filter string
nextToken string
pageSize int32
expectedNextToken string
expectedIDs []string
expectedError string
}{
{
name: "test01 size-2 page-1 default NS",
@ -1341,12 +1343,57 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
},
},
{
name: "test5 no valid results with filters and prefix",
name: "test05 no valid results with filters and prefix",
prefix: "cccc",
pageSize: 2,
nextToken: "",
expectedIDs: []string{},
},
{
name: "test06 go-bexpr filter",
namespace: "*",
filter: `ID matches "^a+[123]"`,
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test07 go-bexpr filter with pagination",
namespace: "*",
filter: `ID matches "^a+[123]"`,
pageSize: 2,
expectedNextToken: "aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test08 go-bexpr filter in namespace",
namespace: "non-default",
filter: `Status == "cancelled"`,
expectedIDs: []string{
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test09 go-bexpr wrong namespace",
namespace: "default",
filter: `Namespace == "non-default"`,
expectedIDs: []string{},
},
{
name: "test10 go-bexpr invalid expression",
filter: `NotValid`,
expectedError: "failed to read filter expression",
},
{
name: "test11 go-bexpr invalid field",
filter: `InvalidField == "value"`,
expectedError: "error finding value in datum",
},
}
for _, tc := range cases {
@ -1357,13 +1404,22 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
Region: "global",
Namespace: tc.namespace,
Prefix: tc.prefix,
Filter: tc.filter,
PerPage: tc.pageSize,
NextToken: tc.nextToken,
},
}
req.AuthToken = aclToken
var resp structs.DeploymentListResponse
require.NoError(t, msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp))
err := msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp)
if tc.expectedError == "" {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Contains(t, err.Error(), tc.expectedError)
return
}
gotIDs := []string{}
for _, deployment := range resp.Deployments {
gotIDs = append(gotIDs, deployment.ID)

22
nomad/eval_endpoint.go
View File

@ -2,6 +2,7 @@ package nomad
import (
"fmt"
"net/http"
"time"
metrics "github.com/armon/go-metrics"
@ -397,6 +398,14 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon
return structs.ErrPermissionDenied
}
if args.Filter != "" {
// Check for incompatible filtering.
hasLegacyFilter := args.FilterJobID != "" || args.FilterEvalStatus != ""
if hasLegacyFilter {
return structs.ErrIncompatibleFiltering
}
}
// Setup the blocking query
opts := blockingOptions{
queryOpts: &args.QueryOptions,
@ -425,13 +434,22 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon
})
var evals []*structs.Evaluation
paginator := state.NewPaginator(iter, args.QueryOptions,
paginator, err := state.NewPaginator(iter, args.QueryOptions,
func(raw interface{}) {
eval := raw.(*structs.Evaluation)
evals = append(evals, eval)
})
if err != nil {
return structs.NewErrRPCCodedf(
http.StatusBadRequest, "failed to create result paginator: %v", err)
}
nextToken, err := paginator.Page()
if err != nil {
return structs.NewErrRPCCodedf(
http.StatusBadRequest, "failed to read result page: %v", err)
}
nextToken := paginator.Page()
reply.QueryMeta.NextToken = nextToken
reply.Evaluations = evals

61
nomad/eval_endpoint_test.go
View File

@ -1050,7 +1050,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
}
aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read",
mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).
mock.NamespacePolicy("*", "read", nil)).
SecretID
cases := []struct {
@ -1060,9 +1060,11 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
nextToken string
filterJobID string
filterStatus string
filter string
pageSize int32
expectedNextToken string
expectedIDs []string
expectedError string
}{
{
name: "test01 size-2 page-1 default NS",
@ -1194,6 +1196,52 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
nextToken: "aaaaaa11-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{},
},
{
name: "test14 go-bexpr filter",
filter: `Status == "blocked"`,
nextToken: "",
expectedIDs: []string{"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"},
},
{
name: "test15 go-bexpr filter with pagination",
filter: `JobID == "example"`,
pageSize: 2,
expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test16 go-bexpr filter namespace",
namespace: "non-default",
filter: `ID contains "aaa"`,
expectedIDs: []string{
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test17 go-bexpr wrong namespace",
namespace: "default",
filter: `Namespace == "non-default"`,
expectedIDs: []string{},
},
{
name: "test18 incompatible filtering",
filter: `JobID == "example"`,
filterStatus: "complete",
expectedError: structs.ErrIncompatibleFiltering.Error(),
},
{
name: "test19 go-bexpr invalid expression",
filter: `NotValid`,
expectedError: "failed to read filter expression",
},
{
name: "test20 go-bexpr invalid field",
filter: `InvalidField == "value"`,
expectedError: "error finding value in datum",
},
}
for _, tc := range cases {
@ -1208,11 +1256,20 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
Prefix: tc.prefix,
PerPage: tc.pageSize,
NextToken: tc.nextToken,
Filter: tc.filter,
},
}
req.AuthToken = aclToken
var resp structs.EvalListResponse
require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp))
err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp)
if tc.expectedError == "" {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Contains(t, err.Error(), tc.expectedError)
return
}
gotIDs := []string{}
for _, eval := range resp.Evaluations {
gotIDs = append(gotIDs, eval.ID)

228
nomad/state/filter_test.go Normal file
View File

@ -0,0 +1,228 @@
package state
import (
"testing"
"time"
"github.com/hashicorp/go-bexpr"
"github.com/hashicorp/nomad/helper/uuid"
"github.com/hashicorp/nomad/nomad/structs"
)
func BenchmarkEvalListFilter(b *testing.B) {
const evalCount = 100_000
b.Run("filter with index", func(b *testing.B) {
state := setupPopulatedState(b, evalCount)
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace)
var lastSeen string
var countSeen int
for {
raw := iter.Next()
if raw == nil {
break
}
eval := raw.(*structs.Evaluation)
lastSeen = eval.ID
countSeen++
}
if countSeen < evalCount/2 {
b.Fatalf("failed: %d evals seen, lastSeen=%s", countSeen, lastSeen)
}
}
})
b.Run("filter with go-bexpr", func(b *testing.B) {
state := setupPopulatedState(b, evalCount)
evaluator, err := bexpr.CreateEvaluator(`Namespace == "default"`)
if err != nil {
b.Fatalf("failed: %v", err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter, _ := state.Evals(nil, false)
var lastSeen string
var countSeen int
for {
raw := iter.Next()
if raw == nil {
break
}
match, err := evaluator.Evaluate(raw)
if !match || err != nil {
continue
}
eval := raw.(*structs.Evaluation)
lastSeen = eval.ID
countSeen++
}
if countSeen < evalCount/2 {
b.Fatalf("failed: %d evals seen, lastSeen=%s", countSeen, lastSeen)
}
}
})
b.Run("paginated filter with index", func(b *testing.B) {
state := setupPopulatedState(b, evalCount)
opts := structs.QueryOptions{
PerPage: 100,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace)
var evals []*structs.Evaluation
paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
eval := raw.(*structs.Evaluation)
evals = append(evals, eval)
})
if err != nil {
b.Fatalf("failed: %v", err)
}
paginator.Page()
}
})
b.Run("paginated filter with go-bexpr", func(b *testing.B) {
state := setupPopulatedState(b, evalCount)
opts := structs.QueryOptions{
PerPage: 100,
Filter: `Namespace == "default"`,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter, _ := state.Evals(nil, false)
var evals []*structs.Evaluation
paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
eval := raw.(*structs.Evaluation)
evals = append(evals, eval)
})
if err != nil {
b.Fatalf("failed: %v", err)
}
paginator.Page()
}
})
b.Run("paginated filter with index last page", func(b *testing.B) {
state := setupPopulatedState(b, evalCount)
// Find the last eval ID.
iter, _ := state.Evals(nil, false)
var lastSeen string
for {
raw := iter.Next()
if raw == nil {
break
}
eval := raw.(*structs.Evaluation)
lastSeen = eval.ID
}
opts := structs.QueryOptions{
PerPage: 100,
NextToken: lastSeen,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace)
var evals []*structs.Evaluation
paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
eval := raw.(*structs.Evaluation)
evals = append(evals, eval)
})
if err != nil {
b.Fatalf("failed: %v", err)
}
paginator.Page()
}
})
b.Run("paginated filter with go-bexpr last page", func(b *testing.B) {
state := setupPopulatedState(b, evalCount)
// Find the last eval ID.
iter, _ := state.Evals(nil, false)
var lastSeen string
for {
raw := iter.Next()
if raw == nil {
break
}
eval := raw.(*structs.Evaluation)
lastSeen = eval.ID
}
opts := structs.QueryOptions{
PerPage: 100,
NextToken: lastSeen,
Filter: `Namespace == "default"`,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter, _ := state.Evals(nil, false)
var evals []*structs.Evaluation
paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
eval := raw.(*structs.Evaluation)
evals = append(evals, eval)
})
if err != nil {
b.Fatalf("failed: %v", err)
}
paginator.Page()
}
})
}
// -----------------
// BENCHMARK HELPER FUNCTIONS
func setupPopulatedState(b *testing.B, evalCount int) *StateStore {
evals := generateEvals(evalCount)
index := uint64(0)
var err error
state := TestStateStore(b)
for _, eval := range evals {
index++
err = state.UpsertEvals(
structs.MsgTypeTestSetup, index, []*structs.Evaluation{eval})
}
if err != nil {
b.Fatalf("failed: %v", err)
}
return state
}
func generateEvals(count int) []*structs.Evaluation {
evals := []*structs.Evaluation{}
ns := structs.DefaultNamespace
for i := 0; i < count; i++ {
if i > count/2 {
ns = "other"
}
evals = append(evals, generateEval(i, ns))
}
return evals
}
func generateEval(i int, ns string) *structs.Evaluation {
now := time.Now().UTC().UnixNano()
return &structs.Evaluation{
ID: uuid.Generate(),
Namespace: ns,
Priority: 50,
Type: structs.JobTypeService,
JobID: uuid.Generate(),
Status: structs.EvalStatusPending,
CreateTime: now,
ModifyTime: now,
}
}

60
nomad/state/paginator.go
View File

@ -1,19 +1,33 @@
package state
import (
memdb "github.com/hashicorp/go-memdb"
"fmt"
"github.com/hashicorp/go-bexpr"
"github.com/hashicorp/nomad/nomad/structs"
)
// Iterator is the interface that must be implemented to use the Paginator.
type Iterator interface {
// Next returns the next element to be considered for pagination.
// The page will end if nil is returned.
Next() interface{}
}
// Paginator is an iterator over a memdb.ResultIterator that returns
// only the expected number of pages.
type Paginator struct {
iter memdb.ResultIterator
iter Iterator
perPage int32
itemCount int32
seekingToken string
nextToken string
nextTokenFound bool
pageErr error
// filterEvaluator is used to filter results using go-bexpr. It's nil if
// no filter expression is defined.
filterEvaluator *bexpr.Evaluator
// appendFunc is the function the caller should use to append raw
// entries to the results set. The object is guaranteed to be
@ -21,19 +35,30 @@ type Paginator struct {
appendFunc func(interface{})
}
func NewPaginator(iter memdb.ResultIterator, opts structs.QueryOptions, appendFunc func(interface{})) *Paginator {
return &Paginator{
iter: iter,
perPage: opts.PerPage,
seekingToken: opts.NextToken,
nextTokenFound: opts.NextToken == "",
appendFunc: appendFunc,
func NewPaginator(iter Iterator, opts structs.QueryOptions, appendFunc func(interface{})) (*Paginator, error) {
var evaluator *bexpr.Evaluator
var err error
if opts.Filter != "" {
evaluator, err = bexpr.CreateEvaluator(opts.Filter)
if err != nil {
return nil, fmt.Errorf("failed to read filter expression: %v", err)
}
}
return &Paginator{
iter: iter,
perPage: opts.PerPage,
seekingToken: opts.NextToken,
nextTokenFound: opts.NextToken == "",
filterEvaluator: evaluator,
appendFunc: appendFunc,
}, nil
}
// Page populates a page by running the append function
// over all results. Returns the next token
func (p *Paginator) Page() string {
func (p *Paginator) Page() (string, error) {
DONE:
for {
raw, andThen := p.next()
@ -46,7 +71,7 @@ DONE:
break DONE
}
}
return p.nextToken
return p.nextToken, p.pageErr
}
func (p *Paginator) next() (interface{}, paginatorState) {
@ -62,6 +87,19 @@ func (p *Paginator) next() (interface{}, paginatorState) {
if !p.nextTokenFound && id < p.seekingToken {
return nil, paginatorSkip
}
// apply filter if defined
if p.filterEvaluator != nil {
match, err := p.filterEvaluator.Evaluate(raw)
if err != nil {
p.pageErr = err
return nil, paginatorComplete
}
if !match {
return nil, paginatorSkip
}
}
p.nextTokenFound = true
// have we produced enough results for this page?

6
nomad/state/paginator_test.go
View File

@ -55,7 +55,7 @@ func TestPaginator(t *testing.T) {
iter := newTestIterator(ids)
results := []string{}
paginator := NewPaginator(iter,
paginator, err := NewPaginator(iter,
structs.QueryOptions{
PerPage: tc.perPage, NextToken: tc.nextToken,
},
@ -64,8 +64,10 @@ func TestPaginator(t *testing.T) {
results = append(results, result.GetID())
},
)
require.NoError(t, err)
nextToken := paginator.Page()
nextToken, err := paginator.Page()
require.NoError(t, err)
require.Equal(t, tc.expected, results)
require.Equal(t, tc.expectedNextToken, nextToken)
})

2
nomad/structs/errors.go
View File

@ -19,6 +19,7 @@ const (
errUnknownNomadVersion = "Unable to determine Nomad version"
errNodeLacksRpc = "Node does not support RPC; requires 0.8 or later"
errMissingAllocID = "Missing allocation ID"
errIncompatibleFiltering = "Filter expression cannot be used with other filter parameters"
// Prefix based errors that are used to check if the error is of a given
// type. These errors should be created with the associated constructor.
@ -53,6 +54,7 @@ var (
ErrUnknownNomadVersion = errors.New(errUnknownNomadVersion)
ErrNodeLacksRpc = errors.New(errNodeLacksRpc)
ErrMissingAllocID = errors.New(errMissingAllocID)
ErrIncompatibleFiltering = errors.New(errIncompatibleFiltering)
ErrUnknownNode = errors.New(ErrUnknownNodePrefix)

4
nomad/structs/structs.go
View File

@ -274,6 +274,10 @@ type QueryOptions struct {
// AuthToken is secret portion of the ACL token used for the request
AuthToken string
// Filter specifies the go-bexpr filter expression to be used for
// filtering the data prior to returning a response
Filter string
// PerPage is the number of entries to be returned in queries that support
// paginated lists.
PerPage int32

12
website/content/api-docs/deployments.mdx
View File

@ -29,11 +29,11 @@ The table below shows this endpoint's support for
- `prefix` `(string: "")`- Specifies a string to filter deployments based on
an ID prefix. Because the value is decoded to bytes, the prefix must have an
even number of hexadecimal characters (0-9a-f) .This is specified as a query
string parameter.
string parameter and is used before any `filter` expression is applied.
- `namespace` `(string: "default")` - Specifies the target
namespace. Specifying `*` will return all evaluations across all
authorized namespaces.
- `namespace` `(string: "default")` - Specifies the target namespace.
Specifying `*` will return all evaluations across all authorized namespaces.
This parameter is used before any `filter` expression is applied.
- `next_token` `(string: "")` - This endpoint supports paging. The
`next_token` parameter accepts a string which is the `ID` field of
@ -46,6 +46,10 @@ The table below shows this endpoint's support for
used as the `last_token` of the next request to fetch additional
pages.
- `filter` `(string: "")` - Specifies the expression used to filter the query
results. Consider using pagination or a query parameter to reduce resource
used to serve the request.
- `ascending` `(bool: false)` - Specifies the list of returned deployments should
be sorted in chronological order (oldest evaluations first). By default deployments
are returned sorted in reverse chronological order (newest deployments first).

12
website/content/api-docs/evaluations.mdx
View File

@ -29,7 +29,7 @@ The table below shows this endpoint's support for
- `prefix` `(string: "")`- Specifies a string to filter evaluations based on an
ID prefix. Because the value is decoded to bytes, the prefix must have an
even number of hexadecimal characters (0-9a-f). This is specified as a query
string parameter.
string parameter and and is used before any `filter` expression is applied.
- `next_token` `(string: "")` - This endpoint supports paging. The
`next_token` parameter accepts a string which is the `ID` field of
@ -42,6 +42,10 @@ The table below shows this endpoint's support for
used as the `last_token` of the next request to fetch additional
pages.
- `filter` `(string: "")` - Specifies the expression used to filter the query
results. Consider using pagination or a query parameter to reduce resource
used to serve the request.
- `job` `(string: "")` - Filter the list of evaluations to a specific
job ID.
@ -49,9 +53,9 @@ The table below shows this endpoint's support for
specific evaluation status (one of `blocked`, `pending`, `complete`,
`failed`, or `canceled`).
- `namespace` `(string: "default")` - Specifies the target
namespace. Specifying `*` will return all evaluations across all
authorized namespaces.
- `namespace` `(string: "default")` - Specifies the target namespace.
Specifying `*` will return all evaluations across all authorized namespaces.
This parameter is used before any `filter` expression is applied.
- `ascending` `(bool: false)` - Specifies the list of returned evaluations should
be sorted in chronological order (oldest evaluations first). By default evaluations

1
website/content/docs/commands/deployment/list.mdx
View File

@ -27,6 +27,7 @@ capability for the deployment's namespace.
## List Options
- `-json` : Output the deployments in their JSON format.
- `-filter`: Specifies an expression used to filter query results.
- `-t` : Format and display the deployments using a Go template.
- `-verbose`: Show full information.

1
website/content/docs/commands/eval/list.mdx
View File

@ -29,6 +29,7 @@ capability for the requested namespace.
- `-verbose`: Show full information.
- `-per-page`: How many results to show per page.
- `-page-token`: Where to start pagination.
- `-filter`: Specifies an expression used to filter query results.
- `-job`: Only show evaluations for this job ID.
- `-status`: Only show evaluations with this status.
- `-json`: Output the evaluation in its JSON format.