luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

Implement HTTP search API for Variables (#13257)

Browse source 
* Add Path only index for SecureVariables
* Add GetSecureVariablesByPrefix; refactor tests
* Add search for SecureVariables
* Add prefix search for secure variables
This commit is contained in:
Charlie Voiselle 2022-06-19 20:39:00 -07:00 committed by Tim Gross
parent 48679fdf99
commit 1fe080c6de
8 changed files with 497 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/contexts/contexts.go
View file

@ -17,6 +17,7 @@ const (
Recommendations Context = "recommendations"
ScalingPolicies Context = "scaling_policy"
Plugins Context = "plugins"
SecureVariables Context = "vars"
Volumes Context = "volumes"
// These Context types are used to associate a search result from a lower

269
command/agent/search_endpoint_test.go
View file

@ -622,3 +622,272 @@ func TestHTTP_FuzzySearch_AllContext(t *testing.T) {
require.Equal(t, "8000", header(respW, "X-Nomad-Index"))
})
}
func TestHTTP_PrefixSearch_SecureVariables(t *testing.T) {
ci.Parallel(t)
testPath := "alpha/beta/charlie"
testPathPrefix := "alpha/beta"
httpTest(t, nil, func(s *TestAgent) {
sv := mock.SecureVariableEncrypted()
state := s.Agent.server.State()
sv.Path = testPath
err := state.UpsertSecureVariables(structs.MsgTypeTestSetup, 8000, []*structs.SecureVariableEncrypted{sv})
require.NoError(t, err)
data := structs.SearchRequest{Prefix: testPathPrefix, Context: structs.SecureVariables}
req, err := http.NewRequest("POST", "/v1/search", encodeReq(data))
require.NoError(t, err)
respW := httptest.NewRecorder()
resp, err := s.Server.SearchRequest(respW, req)
require.NoError(t, err)
res := resp.(structs.SearchResponse)
matchedVars := res.Matches[structs.SecureVariables]
require.Len(t, matchedVars, 1)
require.Equal(t, testPath, matchedVars[0])
require.Equal(t, "8000", header(respW, "X-Nomad-Index"))
})
}
func TestHTTP_FuzzySearch_SecureVariables(t *testing.T) {
ci.Parallel(t)
testPath := "alpha/beta/charlie"
testPathText := "beta"
httpTest(t, nil, func(s *TestAgent) {
state := s.Agent.server.State()
sv := mock.SecureVariableEncrypted()
sv.Path = testPath
err := state.UpsertSecureVariables(structs.MsgTypeTestSetup, 8000, []*structs.SecureVariableEncrypted{sv})
require.NoError(t, err)
data := structs.FuzzySearchRequest{Text: testPathText, Context: structs.SecureVariables}
req, err := http.NewRequest("POST", "/v1/search/", encodeReq(data))
require.NoError(t, err)
respW := httptest.NewRecorder()
resp, err := s.Server.FuzzySearchRequest(respW, req)
require.NoError(t, err)
res := resp.(structs.FuzzySearchResponse)
matchedVars := res.Matches[structs.SecureVariables]
require.Len(t, matchedVars, 1)
require.Equal(t, testPath, matchedVars[0].ID)
require.Equal(t, []string{
"default", testPath,
}, matchedVars[0].Scope)
require.Equal(t, "8000", header(respW, "X-Nomad-Index"))
})
}
func TestHTTP_PrefixSearch_SecureVariables_ACL(t *testing.T) {
ci.Parallel(t)
testPath := "alpha/beta/charlie"
testPathPrefix := "alpha/beta"
httpACLTest(t, nil, func(s *TestAgent) {
state := s.Agent.server.State()
ns := mock.Namespace()
sv1 := mock.SecureVariableEncrypted()
sv1.Path = testPath
sv2 := sv1.Copy()
sv2.Namespace = ns.Name
_ = state.UpsertNamespaces(7000, []*structs.Namespace{mock.Namespace()})
_ = state.UpsertSecureVariables(structs.MsgTypeTestSetup, 8000, []*structs.SecureVariableEncrypted{sv1})
_ = state.UpsertSecureVariables(structs.MsgTypeTestSetup, 8001, []*structs.SecureVariableEncrypted{&sv2})
rootToken := s.RootToken
defNSToken := mock.CreatePolicyAndToken(t, state, 8002, "default", mock.NamespacePolicy("default", "read", nil))
ns1NSToken := mock.CreatePolicyAndToken(t, state, 8004, "ns-"+ns.Name, mock.NamespacePolicy(ns.Name, "read", nil))
denyToken := mock.CreatePolicyAndToken(t, state, 8006, "none", mock.NamespacePolicy("default", "deny", nil))
testCases := []struct {
desc string
token *structs.ACLToken
namespace string
expectedCount int
expectedNamespaces []string
expectedErr string
}{
{
desc: "management token",
token: rootToken,
namespace: "*",
expectedCount: 2,
expectedNamespaces: []string{"default", ns.Name},
},
{
desc: "default ns token",
token: defNSToken,
namespace: "default",
expectedCount: 1,
expectedNamespaces: []string{"default"},
},
{
desc: "ns specific token",
token: ns1NSToken,
namespace: ns.Name,
expectedCount: 1,
expectedNamespaces: []string{ns.Name},
},
{
desc: "denied token",
token: denyToken,
namespace: "default",
expectedCount: 0,
expectedErr: structs.ErrPermissionDenied.Error(),
},
}
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
tC := tC
data := structs.SearchRequest{
Prefix: testPathPrefix,
Context: structs.SecureVariables,
QueryOptions: structs.QueryOptions{
AuthToken: tC.token.SecretID,
Namespace: tC.namespace,
},
}
req, err := http.NewRequest("POST", "/v1/search", encodeReq(data))
require.NoError(t, err)
respW := httptest.NewRecorder()
resp, err := s.Server.SearchRequest(respW, req)
if tC.expectedErr != "" {
require.Error(t, err)
require.Equal(t, tC.expectedErr, err.Error())
return
}
require.NoError(t, err)
res := resp.(structs.SearchResponse)
matchedVars := res.Matches[structs.SecureVariables]
require.Len(t, matchedVars, tC.expectedCount)
for _, mv := range matchedVars {
require.Equal(t, testPath, mv)
}
require.Equal(t, "8001", header(respW, "X-Nomad-Index"))
})
}
})
}
func TestHTTP_FuzzySearch_SecureVariables_ACL(t *testing.T) {
ci.Parallel(t)
testPath := "alpha/beta/charlie"
testPathText := "beta"
httpACLTest(t, nil, func(s *TestAgent) {
state := s.Agent.server.State()
ns := mock.Namespace()
sv1 := mock.SecureVariableEncrypted()
sv1.Path = testPath
sv2 := sv1.Copy()
sv2.Namespace = ns.Name
_ = state.UpsertNamespaces(7000, []*structs.Namespace{mock.Namespace()})
_ = state.UpsertSecureVariables(structs.MsgTypeTestSetup, 8000, []*structs.SecureVariableEncrypted{sv1})
_ = state.UpsertSecureVariables(structs.MsgTypeTestSetup, 8001, []*structs.SecureVariableEncrypted{&sv2})
rootToken := s.RootToken
defNSToken := mock.CreatePolicyAndToken(t, state, 8002, "default", mock.NamespacePolicy("default", "read", nil))
ns1NSToken := mock.CreatePolicyAndToken(t, state, 8004, "ns-"+ns.Name, mock.NamespacePolicy(ns.Name, "read", nil))
denyToken := mock.CreatePolicyAndToken(t, state, 8006, "none", mock.NamespacePolicy("default", "deny", nil))
type testCase struct {
desc string
token *structs.ACLToken
namespace string
expectedCount int
expectedNamespaces []string
expectedErr string
}
testCases := []testCase{
{
desc: "management token",
token: rootToken,
expectedCount: 2,
expectedNamespaces: []string{"default", ns.Name},
},
{
desc: "default ns token",
token: defNSToken,
expectedCount: 1,
expectedNamespaces: []string{"default"},
},
{
desc: "ns specific token",
token: ns1NSToken,
expectedCount: 1,
expectedNamespaces: []string{ns.Name},
},
{
desc: "denied token",
token: denyToken,
expectedCount: 0,
// You would think that this should error out, but when it is
// the wildcard namespace, objects that fail the access check
// are filtered out rather than throwing a permissions error.
},
{
desc: "denied token",
token: denyToken,
namespace: "default",
expectedCount: 0,
expectedErr: structs.ErrPermissionDenied.Error(),
},
}
tcNS := func(tC testCase) string {
if tC.namespace == "" {
return "*"
}
return tC.namespace
}
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
data := structs.FuzzySearchRequest{
Text: testPathText,
Context: structs.SecureVariables,
QueryOptions: structs.QueryOptions{
AuthToken: tC.token.SecretID,
Namespace: tcNS(tC),
},
}
req, err := http.NewRequest("POST", "/v1/search/fuzzy", encodeReq(data))
require.NoError(t, err)
setToken(req, tC.token)
respW := httptest.NewRecorder()
resp, err := s.Server.FuzzySearchRequest(respW, req)
if tC.expectedErr != "" {
require.Error(t, err)
require.Equal(t, tC.expectedErr, err.Error())
return
}
res := resp.(structs.FuzzySearchResponse)
matchedVars := res.Matches[structs.SecureVariables]
require.Len(t, matchedVars, tC.expectedCount)
for _, mv := range matchedVars {
require.Equal(t, testPath, mv.ID)
require.Len(t, mv.Scope, 2)
require.Contains(t, tC.expectedNamespaces, mv.Scope[0])
require.Equal(t, "8001", header(respW, "X-Nomad-Index"))
}
})
}
})
}

27
nomad/search_endpoint.go
View file

@ -35,6 +35,7 @@ var (
structs.Plugins,
structs.Volumes,
structs.ScalingPolicies,
structs.SecureVariables,
structs.Namespaces,
}
)
@ -76,6 +77,8 @@ func (s *Search) getPrefixMatches(iter memdb.ResultIterator, prefix string) ([]s
id = t.ID
case *structs.Namespace:
id = t.Name
case *structs.SecureVariableEncrypted:
id = t.Path
default:
matchID, ok := getEnterpriseMatch(raw)
if !ok {
@ -214,6 +217,10 @@ func (s *Search) fuzzyMatchSingle(raw interface{}, text string) (structs.Context
case *structs.CSIPlugin:
name = t.ID
ctx = structs.Plugins
case *structs.SecureVariableEncrypted:
name = t.Path
scope = []string{t.Namespace, t.Path}
ctx = structs.SecureVariables
}
if idx := fuzzyIndex(name, text); idx >= 0 {
@ -382,6 +389,15 @@ func getResourceIter(context structs.Context, aclObj *acl.ACL, namespace, prefix
return iter, nil
}
return memdb.NewFilterIterator(iter, nsCapFilter(aclObj)), nil
case structs.SecureVariables:
iter, err := store.GetSecureVariablesByPrefix(ws, prefix)
if err != nil {
return nil, err
}
if aclObj == nil {
return iter, nil
}
return memdb.NewFilterIterator(iter, nsCapFilter(aclObj)), nil
default:
return getEnterpriseResourceIter(context, aclObj, namespace, prefix, ws, store)
}
@ -410,6 +426,13 @@ func getFuzzyResourceIterator(context structs.Context, aclObj *acl.ACL, namespac
}
return store.AllocsByNamespace(ws, namespace)
case structs.SecureVariables:
if wildcard(namespace) {
iter, err := store.SecureVariables(ws)
return nsCapIterFilter(iter, err, aclObj)
}
return store.GetSecureVariablesByNamespace(ws, namespace)
case structs.Nodes:
if wildcard(namespace) {
iter, err := store.Nodes(ws)
@ -457,6 +480,10 @@ func nsCapFilter(aclObj *acl.ACL) memdb.FilterFunc {
case *structs.Allocation:
return !aclObj.AllowNsOp(t.Namespace, acl.NamespaceCapabilityReadJob)
case *structs.SecureVariableEncrypted:
// FIXME: Update to final implementation.
return !aclObj.AllowNsOp(t.Namespace, acl.NamespaceCapabilityReadJob)
case *structs.Namespace:
return !aclObj.AllowNamespace(t.Name)

26
nomad/search_endpoint_oss.go
View file

@ -20,8 +20,14 @@ var (
// contextToIndex returns the index name to lookup in the state store.
func contextToIndex(ctx structs.Context) string {
switch ctx {
// Handle cases where context name and state store table name do not match
case structs.SecureVariables:
return state.TableSecureVariables
default:
return string(ctx)
}
}
// getEnterpriseMatch is a no-op in oss since there are no enterprise objects.
func getEnterpriseMatch(match interface{}) (id string, ok bool) {
@ -50,7 +56,9 @@ func sufficientSearchPerms(aclObj *acl.ACL, namespace string, context structs.Co
if aclObj == nil {
return true
}
if aclObj.IsManagement() {
return true
}
nodeRead := aclObj.AllowNodeRead()
allowNS := aclObj.AllowNamespace(namespace)
jobRead := aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityReadJob)
@ -59,8 +67,10 @@ func sufficientSearchPerms(aclObj *acl.ACL, namespace string, context structs.Co
acl.NamespaceCapabilityListJobs,
acl.NamespaceCapabilityReadJob)
volRead := allowVolume(aclObj, namespace)
// FIXME: Replace with real variables capability
allowVariables := aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityReadJob)
if !nodeRead && !jobRead && !volRead && !allowNS {
if !nodeRead && !jobRead && !volRead && !allowNS && !allowVariables {
return false
}
@ -80,6 +90,10 @@ func sufficientSearchPerms(aclObj *acl.ACL, namespace string, context structs.Co
return false
}
}
if !allowVariables && context == structs.SecureVariables {
return false
}
if !volRead && context == structs.Volumes {
return false
}
@ -98,7 +112,9 @@ func filteredSearchContexts(aclObj *acl.ACL, namespace string, context structs.C
if aclObj == nil {
return desired
}
if aclObj.IsManagement() {
return desired
}
jobRead := aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityReadJob)
allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIListVolume,
acl.NamespaceCapabilityCSIReadVolume,
@ -123,6 +139,10 @@ func filteredSearchContexts(aclObj *acl.ACL, namespace string, context structs.C
if aclObj.AllowNamespace(namespace) {
available = append(available, c)
}
case structs.SecureVariables:
if jobRead {
available = append(available, c)
}
case structs.Nodes:
if aclObj.AllowNodeRead() {
available = append(available, c)

9
nomad/state/schema.go
View file

@ -26,6 +26,7 @@ const (
indexAllocID = "alloc_id"
indexServiceName = "service_name"
indexKeyID = "key_id"
indexPath = "path"
)
var (
@ -1236,6 +1237,14 @@ func secureVariablesTableSchema() *memdb.TableSchema {
AllowMissing: false,
Indexer: &secureVariableKeyIDFieldIndexer{},
},
indexPath: {
Name: indexPath,
AllowMissing: false,
Unique: false,
Indexer: &memdb.StringFieldIndex{
Field: "Path",
},
},
},
}
}

22
nomad/state/state_store_secure_variables.go
View file

@ -54,6 +54,23 @@ func (s *StateStore) GetSecureVariablesByNamespaceAndPrefix(
return iter, nil
}
// GetSecureVariablesByPrefix returns an iterator that contains all variables that
// match the prefix in any namespace. Namespace filtering is the responsibility
// of the caller.
func (s *StateStore) GetSecureVariablesByPrefix(
ws memdb.WatchSet, prefix string) (memdb.ResultIterator, error) {
txn := s.db.ReadTxn()
// Walk the entire table.
iter, err := txn.Get(TableSecureVariables, indexPath+"_prefix", prefix)
if err != nil {
return nil, fmt.Errorf("secure variable lookup failed: %v", err)
}
ws.Add(iter.WatchCh())
return iter, nil
}
// GetSecureVariablesByKeyID returns an iterator that contains all
// variables that were encrypted with a particular key
func (s *StateStore) GetSecureVariablesByKeyID(
@ -69,7 +86,7 @@ func (s *StateStore) GetSecureVariablesByKeyID(
return iter, nil
}
// GetSecureVariable returns an single secure variable at a given namespace and
// GetSecureVariable returns a single secure variable at a given namespace and
// path.
func (s *StateStore) GetSecureVariable(
ws memdb.WatchSet, namespace, path string) (*structs.SecureVariableEncrypted, error) {
@ -154,6 +171,7 @@ func (s *StateStore) upsertSecureVariableImpl(index uint64, txn *txn, sv *struct
// shouldWrite can be used to determine if a write needs to happen.
func shouldWrite(sv, existing *structs.SecureVariableEncrypted) bool {
// FIXME: Move this to the RPC layer eventually.
if existing == nil {
return true
}
@ -208,7 +226,7 @@ func (s *StateStore) DeleteSecureVariableTxn(index uint64, namespace, path strin
return fmt.Errorf("secure variable not found")
}
// Delete the launch
// Delete the variable
if err := txn.Delete(TableSecureVariables, existing); err != nil {
return fmt.Errorf("secure variable delete failed: %v", err)
}

170
nomad/state/state_store_secure_variables_test.go
View file

@ -263,8 +263,7 @@ func TestStateStore_DeleteSecureVariable(t *testing.T) {
}
require.Equal(t, 0, delete2Count, "unexpected number of variables in table")
}
func TestStateStore_ListSecureVariablesByNamespace(t *testing.T) {
func TestStateStore_GetSecureVariables(t *testing.T) {
ci.Parallel(t)
testState := testStateStore(t)
@ -325,80 +324,165 @@ func TestStateStore_ListSecureVariablesByNamespaceAndPrefix(t *testing.T) {
testState := testStateStore(t)
// Generate some test secure variables and upsert them.
svs, _ := mockSecureVariables(4, 4)
svs[0].Namespace = "~*magical*~"
svs[0].Path = "a/b/c"
svs, _ := mockSecureVariables(6, 6)
svs[0].Path = "a/b"
svs[1].Path = "a/b/c"
svs[2].Path = "a/b"
svs[3].Path = "unrelated/b/c"
svs[2].Path = "unrelated/b/c"
svs[3].Namespace = "other"
svs[3].Path = "a/b/c"
svs[4].Namespace = "other"
svs[4].Path = "a/q/z"
svs[5].Namespace = "other"
svs[5].Path = "a/z/z"
initialIndex := uint64(10)
require.NoError(t, testState.UpsertSecureVariables(structs.MsgTypeTestSetup, initialIndex, svs))
// Look up secure variables using the namespace of the first mock variable
// and a path prefix.
t.Run("ByNamespace", func(t *testing.T) {
testCases := []struct {
desc string
namespace string
expectedCount int
}{
{
desc: "default",
namespace: "default",
expectedCount: 2,
},
{
desc: "other",
namespace: "other",
expectedCount: 3,
},
{
desc: "nonexistent",
namespace: "BAD",
expectedCount: 0,
},
}
ws := memdb.NewWatchSet()
iter, err := testState.GetSecureVariablesByNamespaceAndPrefix(ws, svs[0].Namespace, "a")
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
iter, err := testState.GetSecureVariablesByNamespace(ws, tC.namespace)
require.NoError(t, err)
var count1 int
var count int = 0
for raw := iter.Next(); raw != nil; raw = iter.Next() {
count1++
count++
sv := raw.(*structs.SecureVariableEncrypted)
t.Logf("- sv: n=%q p=%q ci=%v mi=%v ed.ki=%q", sv.Namespace, sv.Path, sv.CreateIndex, sv.ModifyIndex, sv.KeyID)
require.Equal(t, initialIndex, sv.CreateIndex, "incorrect create index", sv.Path)
require.Equal(t, initialIndex, sv.ModifyIndex, "incorrect modify index", sv.Path)
require.Equal(t, svs[0].Namespace, sv.Namespace)
require.Equal(t, tC.namespace, sv.Namespace)
}
require.Equal(t, 1, count1)
})
}
})
// Look up secure variables using the namespace of the first mock variable
// and a path prefix that doesn't exist.
iter, err = testState.GetSecureVariablesByNamespaceAndPrefix(ws, svs[0].Namespace, "b")
t.Run("ByNamespaceAndPrefix", func(t *testing.T) {
testCases := []struct {
desc string
namespace string
prefix string
expectedCount int
}{
{
desc: "ns1 with good path",
namespace: "default",
prefix: "a",
expectedCount: 2,
},
{
desc: "ns2 with good path",
namespace: "other",
prefix: "a",
expectedCount: 3,
},
{
desc: "ns1 path valid for ns2",
namespace: "default",
prefix: "a/b/c",
expectedCount: 1,
},
{
desc: "ns2 empty prefix",
namespace: "other",
prefix: "",
expectedCount: 3,
},
{
desc: "nonexistent ns",
namespace: "BAD",
prefix: "",
expectedCount: 0,
},
}
ws := memdb.NewWatchSet()
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
iter, err := testState.GetSecureVariablesByNamespaceAndPrefix(ws, tC.namespace, tC.prefix)
require.NoError(t, err)
count1 = 0
var count int = 0
for raw := iter.Next(); raw != nil; raw = iter.Next() {
count1++
count++
sv := raw.(*structs.SecureVariableEncrypted)
t.Logf("- sv: n=%q p=%q ci=%v mi=%v ed.ki=%q", sv.Namespace, sv.Path, sv.CreateIndex, sv.ModifyIndex, sv.KeyID)
require.Equal(t, initialIndex, sv.CreateIndex, "incorrect create index", sv.Path)
require.Equal(t, initialIndex, sv.ModifyIndex, "incorrect modify index", sv.Path)
require.Equal(t, svs[0].Namespace, sv.Namespace)
require.Equal(t, tC.namespace, sv.Namespace)
require.True(t, strings.HasPrefix(sv.Path, tC.prefix))
}
require.Equal(t, 0, count1)
require.Equal(t, tC.expectedCount, count)
})
}
})
// Look up variables using the namespace of the second mock variable and a
// good prefix.
iter, err = testState.GetSecureVariablesByNamespaceAndPrefix(ws, svs[1].Namespace, "a")
t.Run("ByPrefix", func(t *testing.T) {
testCases := []struct {
desc string
prefix string
expectedCount int
}{
{
desc: "bad prefix",
prefix: "bad",
expectedCount: 0,
},
{
desc: "multiple ns",
prefix: "a/b/c",
expectedCount: 2,
},
{
desc: "all",
prefix: "",
expectedCount: 6,
},
}
ws := memdb.NewWatchSet()
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
iter, err := testState.GetSecureVariablesByPrefix(ws, tC.prefix)
require.NoError(t, err)
var count2 int
var count int = 0
for raw := iter.Next(); raw != nil; raw = iter.Next() {
count2++
count++
sv := raw.(*structs.SecureVariableEncrypted)
t.Logf("- sv: n=%q p=%q ci=%v mi=%v ed.ki=%q", sv.Namespace, sv.Path, sv.CreateIndex, sv.ModifyIndex, sv.KeyID)
require.Equal(t, initialIndex, sv.CreateIndex, "incorrect create index", sv.Path)
require.Equal(t, initialIndex, sv.ModifyIndex, "incorrect modify index", sv.Path)
require.Equal(t, svs[1].Namespace, sv.Namespace)
require.True(t, strings.HasPrefix(sv.Path, tC.prefix))
}
require.Equal(t, 2, count2)
// Look up variables using a namespace that shouldn't contain any
// secure variables.
iter, err = testState.GetSecureVariablesByNamespace(ws, "pony-club")
require.NoError(t, err)
var count3 int
for raw := iter.Next(); raw != nil; raw = iter.Next() {
count3++
require.Equal(t, tC.expectedCount, count)
})
}
require.Equal(t, 0, count3)
})
}
func TestStateStore_ListSecureVariablesByKeyID(t *testing.T) {
ci.Parallel(t)
testState := testStateStore(t)

1
nomad/structs/search.go
View file

@ -16,6 +16,7 @@ const (
Recommendations Context = "recommendations"
ScalingPolicies Context = "scaling_policy"
Plugins Context = "plugins"
SecureVariables Context = "vars"
Volumes Context = "volumes"
// Subtypes used in fuzzy matching.