Add `go-bexpr` filters to evals and deployment list endpoints (#12034)

.changelog/12034.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+api: filter values of evaluation and deployment list api endpoints
+```

api/api.go

@@ -65,6 +65,10 @@ type QueryOptions struct {
 	// AuthToken is the secret ID of an ACL token
 	AuthToken string
 
+	// Filter specifies the go-bexpr filter expression to be used for
+	// filtering the data prior to returning a response
+	Filter string
+
 	// PerPage is the number of entries to be returned in queries that support
 	// paginated lists.
 	PerPage int32
@@ -586,6 +590,9 @@ func (r *request) setQueryOptions(q *QueryOptions) {
 	if q.Prefix != "" {
 		r.params.Set("prefix", q.Prefix)
 	}
+	if q.Filter != "" {
+		r.params.Set("filter", q.Filter)
+	}
 	if q.PerPage != 0 {
 		r.params.Set("per_page", fmt.Sprint(q.PerPage))
 	}

api/evaluations_test.go

@@ -77,6 +77,14 @@ func TestEvaluations_List(t *testing.T) {
 	if len(result) != 1 {
 		t.Fatalf("expected no evals after last one but got %v", result[0])
 	}
+
+	// Query evaluations using a filter.
+	results, _, err = e.List(&QueryOptions{
+		Filter: `TriggeredBy == "job-register"`,
+	})
+	if len(result) != 1 {
+		t.Fatalf("expected 1 eval, got %d", len(result))
+	}
 }
 
 func TestEvaluations_PrefixList(t *testing.T) {

command/agent/http.go

@@ -537,6 +537,9 @@ func (s *HTTPServer) wrap(handler func(resp http.ResponseWriter, req *http.Reque
 				} else if strings.HasSuffix(errMsg, structs.ErrJobRegistrationDisabled.Error()) {
 					errMsg = structs.ErrJobRegistrationDisabled.Error()
 					code = 403
+				} else if strings.HasSuffix(errMsg, structs.ErrIncompatibleFiltering.Error()) {
+					errMsg = structs.ErrIncompatibleFiltering.Error()
+					code = 400
 				}
 			}
 
@@ -784,6 +787,7 @@ func (s *HTTPServer) parse(resp http.ResponseWriter, req *http.Request, r *strin
 	parsePrefix(req, b)
 	parseNamespace(req, &b.Namespace)
 	parsePagination(req, b)
+	parseFilter(req, b)
 	return parseWait(resp, req, b)
 }
 
@@ -801,6 +805,14 @@ func parsePagination(req *http.Request, b *structs.QueryOptions) {
 	b.NextToken = query.Get("next_token")
 }
 
+// parseFilter parses the filter query parameter for QueryOptions
+func parseFilter(req *http.Request, b *structs.QueryOptions) {
+	query := req.URL.Query()
+	if filter := query.Get("filter"); filter != "" {
+		b.Filter = filter
+	}
+}
+
 // parseWriteRequest is a convenience method for endpoints that need to parse a
 // write request.
 func (s *HTTPServer) parseWriteRequest(req *http.Request, w *structs.WriteRequest) {

command/deployment_list.go

@@ -30,6 +30,9 @@ List Options:
   -json
     Output the deployments in a JSON format.
 
+  -filter
+    Specifies an expression used to filter query results.
+
   -t
     Format and display the deployments using a Go template.
 
@@ -43,6 +46,7 @@ func (c *DeploymentListCommand) AutocompleteFlags() complete.Flags {
 	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
 		complete.Flags{
 			"-json":    complete.PredictNothing,
+			"-filter":  complete.PredictAnything,
 			"-t":       complete.PredictAnything,
 			"-verbose": complete.PredictNothing,
 		})
@@ -60,12 +64,13 @@ func (c *DeploymentListCommand) Name() string { return "deployment list" }
 
 func (c *DeploymentListCommand) Run(args []string) int {
 	var json, verbose bool
-	var tmpl string
+	var filter, tmpl string
 
 	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
 	flags.Usage = func() { c.Ui.Output(c.Help()) }
 	flags.BoolVar(&verbose, "verbose", false, "")
 	flags.BoolVar(&json, "json", false, "")
+	flags.StringVar(&filter, "filter", "", "")
 	flags.StringVar(&tmpl, "t", "", "")
 
 	if err := flags.Parse(args); err != nil {
@@ -93,7 +98,10 @@ func (c *DeploymentListCommand) Run(args []string) int {
 		return 1
 	}
 
-	deploys, _, err := client.Deployments().List(nil)
+	opts := &api.QueryOptions{
+		Filter: filter,
+	}
+	deploys, _, err := client.Deployments().List(opts)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error retrieving deployments: %s", err))
 		return 1

command/eval_list.go

@@ -35,6 +35,9 @@ Eval List Options:
   -page-token
     Where to start pagination.
 
+  -filter
+    Specifies an expression used to filter query results.
+
   -job
     Only show evaluations for this job ID.
 
@@ -61,6 +64,7 @@ func (c *EvalListCommand) AutocompleteFlags() complete.Flags {
 			"-json":       complete.PredictNothing,
 			"-t":          complete.PredictAnything,
 			"-verbose":    complete.PredictNothing,
+			"-filter":     complete.PredictAnything,
 			"-job":        complete.PredictAnything,
 			"-status":     complete.PredictAnything,
 			"-per-page":   complete.PredictAnything,
@@ -88,7 +92,7 @@ func (c *EvalListCommand) Name() string { return "eval list" }
 func (c *EvalListCommand) Run(args []string) int {
 	var monitor, verbose, json bool
 	var perPage int
-	var tmpl, pageToken, filterJobID, filterStatus string
+	var tmpl, pageToken, filter, filterJobID, filterStatus string
 
 	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
 	flags.Usage = func() { c.Ui.Output(c.Help()) }
@@ -98,6 +102,7 @@ func (c *EvalListCommand) Run(args []string) int {
 	flags.StringVar(&tmpl, "t", "", "")
 	flags.IntVar(&perPage, "per-page", 0, "")
 	flags.StringVar(&pageToken, "page-token", "", "")
+	flags.StringVar(&filter, "filter", "", "")
 	flags.StringVar(&filterJobID, "job", "", "")
 	flags.StringVar(&filterStatus, "status", "", "")
 
@@ -120,6 +125,7 @@ func (c *EvalListCommand) Run(args []string) int {
 	}
 
 	opts := &api.QueryOptions{
+		Filter:    filter,
 		PerPage:   int32(perPage),
 		NextToken: pageToken,
 		Params:    map[string]string{},

go.mod

@@ -48,6 +48,7 @@ require (
 	github.com/hashicorp/consul/api v1.9.1
 	github.com/hashicorp/consul/sdk v0.8.0
 	github.com/hashicorp/cronexpr v1.1.1
+	github.com/hashicorp/go-bexpr v0.1.11
 	github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-connlimit v0.3.0
@@ -209,6 +210,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mrunalp/fileutils v0.5.0 // indirect

go.sum

@@ -667,6 +667,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU=
+github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY=
+github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE=
 github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de h1:XDCSythtg8aWSRSO29uwhgh7b127fWr+m5SemqjSUL8=
 github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de/go.mod h1:xIwEieBHERyEvaeKF/TcHh1Hu+lxPM+n2vT1+g9I4m4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -938,9 +940,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
+github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw=
+github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=

nomad/deployment_endpoint.go

@@ -2,6 +2,7 @@ package nomad
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	metrics "github.com/armon/go-metrics"
@@ -421,13 +422,22 @@ func (d *Deployment) List(args *structs.DeploymentListRequest, reply *structs.De
 			}
 
 			var deploys []*structs.Deployment
-			paginator := state.NewPaginator(iter, args.QueryOptions,
+			paginator, err := state.NewPaginator(iter, args.QueryOptions,
 				func(raw interface{}) {
 					deploy := raw.(*structs.Deployment)
 					deploys = append(deploys, deploy)
 				})
+			if err != nil {
+				return structs.NewErrRPCCodedf(
+					http.StatusBadRequest, "failed to create result paginator: %v", err)
+			}
+
+			nextToken, err := paginator.Page()
+			if err != nil {
+				return structs.NewErrRPCCodedf(
+					http.StatusBadRequest, "failed to read result page: %v", err)
+			}
 
-			nextToken := paginator.Page()
 			reply.QueryMeta.NextToken = nextToken
 			reply.Deployments = deploys
 

nomad/deployment_endpoint_test.go

@@ -1288,17 +1288,19 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
 	}
 
 	aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read",
-		mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).
+		mock.NamespacePolicy("*", "read", nil)).
 		SecretID
 
 	cases := []struct {
 		name              string
 		namespace         string
 		prefix            string
+		filter            string
 		nextToken         string
 		pageSize          int32
 		expectedNextToken string
 		expectedIDs       []string
+		expectedError     string
 	}{
 		{
 			name:              "test01 size-2 page-1 default NS",
@@ -1341,12 +1343,57 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
 			},
 		},
 		{
-			name:        "test5 no valid results with filters and prefix",
+			name:        "test05 no valid results with filters and prefix",
 			prefix:      "cccc",
 			pageSize:    2,
 			nextToken:   "",
 			expectedIDs: []string{},
 		},
+		{
+			name:      "test06 go-bexpr filter",
+			namespace: "*",
+			filter:    `ID matches "^a+[123]"`,
+			expectedIDs: []string{
+				"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
+				"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
+				"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
+			},
+		},
+		{
+			name:              "test07 go-bexpr filter with pagination",
+			namespace:         "*",
+			filter:            `ID matches "^a+[123]"`,
+			pageSize:          2,
+			expectedNextToken: "aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
+			expectedIDs: []string{
+				"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
+				"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
+			},
+		},
+		{
+			name:      "test08 go-bexpr filter in namespace",
+			namespace: "non-default",
+			filter:    `Status == "cancelled"`,
+			expectedIDs: []string{
+				"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
+			},
+		},
+		{
+			name:        "test09 go-bexpr wrong namespace",
+			namespace:   "default",
+			filter:      `Namespace == "non-default"`,
+			expectedIDs: []string{},
+		},
+		{
+			name:          "test10 go-bexpr invalid expression",
+			filter:        `NotValid`,
+			expectedError: "failed to read filter expression",
+		},
+		{
+			name:          "test11 go-bexpr invalid field",
+			filter:        `InvalidField == "value"`,
+			expectedError: "error finding value in datum",
+		},
 	}
 
 	for _, tc := range cases {
@@ -1357,13 +1404,22 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
 					Region:    "global",
 					Namespace: tc.namespace,
 					Prefix:    tc.prefix,
+					Filter:    tc.filter,
 					PerPage:   tc.pageSize,
 					NextToken: tc.nextToken,
 				},
 			}
 			req.AuthToken = aclToken
 			var resp structs.DeploymentListResponse
-			require.NoError(t, msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp))
+			err := msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp)
+			if tc.expectedError == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedError)
+				return
+			}
+
 			gotIDs := []string{}
 			for _, deployment := range resp.Deployments {
 				gotIDs = append(gotIDs, deployment.ID)

nomad/eval_endpoint.go

@@ -2,6 +2,7 @@ package nomad
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	metrics "github.com/armon/go-metrics"
@@ -397,6 +398,14 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon
 		return structs.ErrPermissionDenied
 	}
 
+	if args.Filter != "" {
+		// Check for incompatible filtering.
+		hasLegacyFilter := args.FilterJobID != "" || args.FilterEvalStatus != ""
+		if hasLegacyFilter {
+			return structs.ErrIncompatibleFiltering
+		}
+	}
+
 	// Setup the blocking query
 	opts := blockingOptions{
 		queryOpts: &args.QueryOptions,
@@ -425,13 +434,22 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon
 			})
 
 			var evals []*structs.Evaluation
-			paginator := state.NewPaginator(iter, args.QueryOptions,
+			paginator, err := state.NewPaginator(iter, args.QueryOptions,
 				func(raw interface{}) {
 					eval := raw.(*structs.Evaluation)
 					evals = append(evals, eval)
 				})
+			if err != nil {
+				return structs.NewErrRPCCodedf(
+					http.StatusBadRequest, "failed to create result paginator: %v", err)
+			}
+
+			nextToken, err := paginator.Page()
+			if err != nil {
+				return structs.NewErrRPCCodedf(
+					http.StatusBadRequest, "failed to read result page: %v", err)
+			}
 
-			nextToken := paginator.Page()
 			reply.QueryMeta.NextToken = nextToken
 			reply.Evaluations = evals
 

nomad/eval_endpoint_test.go

@@ -1050,7 +1050,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
 	}
 
 	aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read",
-		mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).
+		mock.NamespacePolicy("*", "read", nil)).
 		SecretID
 
 	cases := []struct {
@@ -1060,9 +1060,11 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
 		nextToken         string
 		filterJobID       string
 		filterStatus      string
+		filter            string
 		pageSize          int32
 		expectedNextToken string
 		expectedIDs       []string
+		expectedError     string
 	}{
 		{
 			name:     "test01 size-2 page-1 default NS",
@@ -1194,6 +1196,52 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
 			nextToken:   "aaaaaa11-3350-4b4b-d185-0e1992ed43e9",
 			expectedIDs: []string{},
 		},
+		{
+			name:        "test14 go-bexpr filter",
+			filter:      `Status == "blocked"`,
+			nextToken:   "",
+			expectedIDs: []string{"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"},
+		},
+		{
+			name:              "test15 go-bexpr filter with pagination",
+			filter:            `JobID == "example"`,
+			pageSize:          2,
+			expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
+			expectedIDs: []string{
+				"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
+				"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
+			},
+		},
+		{
+			name:      "test16 go-bexpr filter namespace",
+			namespace: "non-default",
+			filter:    `ID contains "aaa"`,
+			expectedIDs: []string{
+				"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
+			},
+		},
+		{
+			name:        "test17 go-bexpr wrong namespace",
+			namespace:   "default",
+			filter:      `Namespace == "non-default"`,
+			expectedIDs: []string{},
+		},
+		{
+			name:          "test18 incompatible filtering",
+			filter:        `JobID == "example"`,
+			filterStatus:  "complete",
+			expectedError: structs.ErrIncompatibleFiltering.Error(),
+		},
+		{
+			name:          "test19 go-bexpr invalid expression",
+			filter:        `NotValid`,
+			expectedError: "failed to read filter expression",
+		},
+		{
+			name:          "test20 go-bexpr invalid field",
+			filter:        `InvalidField == "value"`,
+			expectedError: "error finding value in datum",
+		},
 	}
 
 	for _, tc := range cases {
@@ -1208,11 +1256,20 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
 					Prefix:    tc.prefix,
 					PerPage:   tc.pageSize,
 					NextToken: tc.nextToken,
+					Filter:    tc.filter,
 				},
 			}
 			req.AuthToken = aclToken
 			var resp structs.EvalListResponse
-			require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp))
+			err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp)
+			if tc.expectedError == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedError)
+				return
+			}
+
 			gotIDs := []string{}
 			for _, eval := range resp.Evaluations {
 				gotIDs = append(gotIDs, eval.ID)

nomad/state/filter_test.go

@@ -0,0 +1,228 @@
+package state
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hashicorp/go-bexpr"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+func BenchmarkEvalListFilter(b *testing.B) {
+	const evalCount = 100_000
+
+	b.Run("filter with index", func(b *testing.B) {
+		state := setupPopulatedState(b, evalCount)
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace)
+			var lastSeen string
+			var countSeen int
+			for {
+				raw := iter.Next()
+				if raw == nil {
+					break
+				}
+				eval := raw.(*structs.Evaluation)
+				lastSeen = eval.ID
+				countSeen++
+			}
+			if countSeen < evalCount/2 {
+				b.Fatalf("failed: %d evals seen, lastSeen=%s", countSeen, lastSeen)
+			}
+		}
+	})
+
+	b.Run("filter with go-bexpr", func(b *testing.B) {
+		state := setupPopulatedState(b, evalCount)
+		evaluator, err := bexpr.CreateEvaluator(`Namespace == "default"`)
+		if err != nil {
+			b.Fatalf("failed: %v", err)
+		}
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			iter, _ := state.Evals(nil, false)
+			var lastSeen string
+			var countSeen int
+			for {
+				raw := iter.Next()
+				if raw == nil {
+					break
+				}
+				match, err := evaluator.Evaluate(raw)
+				if !match || err != nil {
+					continue
+				}
+				eval := raw.(*structs.Evaluation)
+				lastSeen = eval.ID
+				countSeen++
+			}
+			if countSeen < evalCount/2 {
+				b.Fatalf("failed: %d evals seen, lastSeen=%s", countSeen, lastSeen)
+			}
+		}
+	})
+
+	b.Run("paginated filter with index", func(b *testing.B) {
+		state := setupPopulatedState(b, evalCount)
+		opts := structs.QueryOptions{
+			PerPage: 100,
+		}
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace)
+			var evals []*structs.Evaluation
+			paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
+				eval := raw.(*structs.Evaluation)
+				evals = append(evals, eval)
+			})
+			if err != nil {
+				b.Fatalf("failed: %v", err)
+			}
+			paginator.Page()
+		}
+	})
+
+	b.Run("paginated filter with go-bexpr", func(b *testing.B) {
+		state := setupPopulatedState(b, evalCount)
+		opts := structs.QueryOptions{
+			PerPage: 100,
+			Filter:  `Namespace == "default"`,
+		}
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			iter, _ := state.Evals(nil, false)
+			var evals []*structs.Evaluation
+			paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
+				eval := raw.(*structs.Evaluation)
+				evals = append(evals, eval)
+			})
+			if err != nil {
+				b.Fatalf("failed: %v", err)
+			}
+			paginator.Page()
+		}
+	})
+
+	b.Run("paginated filter with index last page", func(b *testing.B) {
+		state := setupPopulatedState(b, evalCount)
+
+		// Find the last eval ID.
+		iter, _ := state.Evals(nil, false)
+		var lastSeen string
+		for {
+			raw := iter.Next()
+			if raw == nil {
+				break
+			}
+			eval := raw.(*structs.Evaluation)
+			lastSeen = eval.ID
+		}
+
+		opts := structs.QueryOptions{
+			PerPage:   100,
+			NextToken: lastSeen,
+		}
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace)
+			var evals []*structs.Evaluation
+			paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
+				eval := raw.(*structs.Evaluation)
+				evals = append(evals, eval)
+			})
+			if err != nil {
+				b.Fatalf("failed: %v", err)
+			}
+			paginator.Page()
+		}
+	})
+
+	b.Run("paginated filter with go-bexpr last page", func(b *testing.B) {
+		state := setupPopulatedState(b, evalCount)
+
+		// Find the last eval ID.
+		iter, _ := state.Evals(nil, false)
+		var lastSeen string
+		for {
+			raw := iter.Next()
+			if raw == nil {
+				break
+			}
+			eval := raw.(*structs.Evaluation)
+			lastSeen = eval.ID
+		}
+
+		opts := structs.QueryOptions{
+			PerPage:   100,
+			NextToken: lastSeen,
+			Filter:    `Namespace == "default"`,
+		}
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			iter, _ := state.Evals(nil, false)
+			var evals []*structs.Evaluation
+			paginator, err := NewPaginator(iter, opts, func(raw interface{}) {
+				eval := raw.(*structs.Evaluation)
+				evals = append(evals, eval)
+			})
+			if err != nil {
+				b.Fatalf("failed: %v", err)
+			}
+			paginator.Page()
+		}
+	})
+}
+
+// -----------------
+// BENCHMARK HELPER FUNCTIONS
+
+func setupPopulatedState(b *testing.B, evalCount int) *StateStore {
+	evals := generateEvals(evalCount)
+
+	index := uint64(0)
+	var err error
+	state := TestStateStore(b)
+	for _, eval := range evals {
+		index++
+		err = state.UpsertEvals(
+			structs.MsgTypeTestSetup, index, []*structs.Evaluation{eval})
+	}
+	if err != nil {
+		b.Fatalf("failed: %v", err)
+	}
+	return state
+}
+
+func generateEvals(count int) []*structs.Evaluation {
+	evals := []*structs.Evaluation{}
+	ns := structs.DefaultNamespace
+	for i := 0; i < count; i++ {
+		if i > count/2 {
+			ns = "other"
+		}
+		evals = append(evals, generateEval(i, ns))
+	}
+	return evals
+}
+
+func generateEval(i int, ns string) *structs.Evaluation {
+	now := time.Now().UTC().UnixNano()
+	return &structs.Evaluation{
+		ID:         uuid.Generate(),
+		Namespace:  ns,
+		Priority:   50,
+		Type:       structs.JobTypeService,
+		JobID:      uuid.Generate(),
+		Status:     structs.EvalStatusPending,
+		CreateTime: now,
+		ModifyTime: now,
+	}
+}

nomad/state/paginator.go

@@ -1,19 +1,33 @@
 package state
 
 import (
-	memdb "github.com/hashicorp/go-memdb"
+	"fmt"
+
+	"github.com/hashicorp/go-bexpr"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
+// Iterator is the interface that must be implemented to use the Paginator.
+type Iterator interface {
+	// Next returns the next element to be considered for pagination.
+	// The page will end if nil is returned.
+	Next() interface{}
+}
+
 // Paginator is an iterator over a memdb.ResultIterator that returns
 // only the expected number of pages.
 type Paginator struct {
-	iter           memdb.ResultIterator
+	iter           Iterator
 	perPage        int32
 	itemCount      int32
 	seekingToken   string
 	nextToken      string
 	nextTokenFound bool
+	pageErr        error
+
+	// filterEvaluator is used to filter results using go-bexpr. It's nil if
+	// no filter expression is defined.
+	filterEvaluator *bexpr.Evaluator
 
 	// appendFunc is the function the caller should use to append raw
 	// entries to the results set. The object is guaranteed to be
@@ -21,19 +35,30 @@ type Paginator struct {
 	appendFunc func(interface{})
 }
 
-func NewPaginator(iter memdb.ResultIterator, opts structs.QueryOptions, appendFunc func(interface{})) *Paginator {
+func NewPaginator(iter Iterator, opts structs.QueryOptions, appendFunc func(interface{})) (*Paginator, error) {
+	var evaluator *bexpr.Evaluator
+	var err error
+
+	if opts.Filter != "" {
+		evaluator, err = bexpr.CreateEvaluator(opts.Filter)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read filter expression: %v", err)
+		}
+	}
+
 	return &Paginator{
 		iter:            iter,
 		perPage:         opts.PerPage,
 		seekingToken:    opts.NextToken,
 		nextTokenFound:  opts.NextToken == "",
+		filterEvaluator: evaluator,
 		appendFunc:      appendFunc,
-	}
+	}, nil
 }
 
 // Page populates a page by running the append function
 // over all results. Returns the next token
-func (p *Paginator) Page() string {
+func (p *Paginator) Page() (string, error) {
 DONE:
 	for {
 		raw, andThen := p.next()
@@ -46,7 +71,7 @@ DONE:
 			break DONE
 		}
 	}
-	return p.nextToken
+	return p.nextToken, p.pageErr
 }
 
 func (p *Paginator) next() (interface{}, paginatorState) {
@@ -62,6 +87,19 @@ func (p *Paginator) next() (interface{}, paginatorState) {
 	if !p.nextTokenFound && id < p.seekingToken {
 		return nil, paginatorSkip
 	}
+
+	// apply filter if defined
+	if p.filterEvaluator != nil {
+		match, err := p.filterEvaluator.Evaluate(raw)
+		if err != nil {
+			p.pageErr = err
+			return nil, paginatorComplete
+		}
+		if !match {
+			return nil, paginatorSkip
+		}
+	}
+
 	p.nextTokenFound = true
 
 	// have we produced enough results for this page?

nomad/state/paginator_test.go

@@ -55,7 +55,7 @@ func TestPaginator(t *testing.T) {
 			iter := newTestIterator(ids)
 			results := []string{}
 
-			paginator := NewPaginator(iter,
+			paginator, err := NewPaginator(iter,
 				structs.QueryOptions{
 					PerPage: tc.perPage, NextToken: tc.nextToken,
 				},
@@ -64,8 +64,10 @@ func TestPaginator(t *testing.T) {
 					results = append(results, result.GetID())
 				},
 			)
+			require.NoError(t, err)
 
-			nextToken := paginator.Page()
+			nextToken, err := paginator.Page()
+			require.NoError(t, err)
 			require.Equal(t, tc.expected, results)
 			require.Equal(t, tc.expectedNextToken, nextToken)
 		})

nomad/structs/errors.go

@@ -19,6 +19,7 @@ const (
 	errUnknownNomadVersion        = "Unable to determine Nomad version"
 	errNodeLacksRpc               = "Node does not support RPC; requires 0.8 or later"
 	errMissingAllocID             = "Missing allocation ID"
+	errIncompatibleFiltering      = "Filter expression cannot be used with other filter parameters"
 
 	// Prefix based errors that are used to check if the error is of a given
 	// type. These errors should be created with the associated constructor.
@@ -53,6 +54,7 @@ var (
 	ErrUnknownNomadVersion        = errors.New(errUnknownNomadVersion)
 	ErrNodeLacksRpc               = errors.New(errNodeLacksRpc)
 	ErrMissingAllocID             = errors.New(errMissingAllocID)
+	ErrIncompatibleFiltering      = errors.New(errIncompatibleFiltering)
 
 	ErrUnknownNode = errors.New(ErrUnknownNodePrefix)
 

nomad/structs/structs.go

@@ -274,6 +274,10 @@ type QueryOptions struct {
 	// AuthToken is secret portion of the ACL token used for the request
 	AuthToken string
 
+	// Filter specifies the go-bexpr filter expression to be used for
+	// filtering the data prior to returning a response
+	Filter string
+
 	// PerPage is the number of entries to be returned in queries that support
 	// paginated lists.
 	PerPage int32

website/content/api-docs/deployments.mdx

@@ -29,11 +29,11 @@ The table below shows this endpoint's support for
 - `prefix` `(string: "")`- Specifies a string to filter deployments based on
   an ID prefix. Because the value is decoded to bytes, the prefix must have an
   even number of hexadecimal characters (0-9a-f) .This is specified as a query
-  string parameter.
+  string parameter and is used before any `filter` expression is applied.
 
-- `namespace` `(string: "default")` - Specifies the target
+- `namespace` `(string: "default")` - Specifies the target namespace.
-  namespace. Specifying `*` will return all evaluations across all
+  Specifying `*` will return all evaluations across all authorized namespaces.
-  authorized namespaces.
+  This parameter is used before any `filter` expression is applied.
 
 - `next_token` `(string: "")` - This endpoint supports paging. The
   `next_token` parameter accepts a string which is the `ID` field of
@@ -46,6 +46,10 @@ The table below shows this endpoint's support for
   used as the `last_token` of the next request to fetch additional
   pages.
 
+- `filter` `(string: "")` - Specifies the expression used to filter the query
+  results. Consider using pagination or a query parameter to reduce resource
+  used to serve the request.
+
 - `ascending` `(bool: false)` - Specifies the list of returned deployments should
   be sorted in chronological order (oldest evaluations first). By default deployments
   are returned sorted in reverse chronological order (newest deployments first).

website/content/api-docs/evaluations.mdx

@@ -29,7 +29,7 @@ The table below shows this endpoint's support for
 - `prefix` `(string: "")`- Specifies a string to filter evaluations based on an
   ID prefix. Because the value is decoded to bytes, the prefix must have an
   even number of hexadecimal characters (0-9a-f). This is specified as a query
-  string parameter.
+  string parameter and and is used before any `filter` expression is applied.
 
 - `next_token` `(string: "")` - This endpoint supports paging. The
   `next_token` parameter accepts a string which is the `ID` field of
@@ -42,6 +42,10 @@ The table below shows this endpoint's support for
   used as the `last_token` of the next request to fetch additional
   pages.
 
+- `filter` `(string: "")` - Specifies the expression used to filter the query
+  results. Consider using pagination or a query parameter to reduce resource
+  used to serve the request.
+
 - `job` `(string: "")` - Filter the list of evaluations to a specific
   job ID.
 
@@ -49,9 +53,9 @@ The table below shows this endpoint's support for
   specific evaluation status (one of `blocked`, `pending`, `complete`,
   `failed`, or `canceled`).
 
-- `namespace` `(string: "default")` - Specifies the target
+- `namespace` `(string: "default")` - Specifies the target namespace.
-  namespace. Specifying `*` will return all evaluations across all
+  Specifying `*` will return all evaluations across all authorized namespaces.
-  authorized namespaces.
+  This parameter is used before any `filter` expression is applied.
 
 - `ascending` `(bool: false)` - Specifies the list of returned evaluations should
   be sorted in chronological order (oldest evaluations first). By default evaluations

website/content/docs/commands/deployment/list.mdx

@@ -27,6 +27,7 @@ capability for the deployment's namespace.
 ## List Options
 
 - `-json` : Output the deployments in their JSON format.
+- `-filter`: Specifies an expression used to filter query results.
 - `-t` : Format and display the deployments using a Go template.
 - `-verbose`: Show full information.
 

website/content/docs/commands/eval/list.mdx

@@ -29,6 +29,7 @@ capability for the requested namespace.
 - `-verbose`: Show full information.
 - `-per-page`: How many results to show per page.
 - `-page-token`: Where to start pagination.
+- `-filter`: Specifies an expression used to filter query results.
 - `-job`: Only show evaluations for this job ID.
 - `-status`: Only show evaluations with this status.
 - `-json`: Output the evaluation in its JSON format.