luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge pull request #9911 from hashicorp/dnephin/state-index-acl-roles 

state: convert ACLRoles policies index to new functional indexer pattern
This commit is contained in:
Daniel Nephin 2021-03-24 18:28:19 -04:00 committed by GitHub
parent 6731cf6061 1d3fe64bba
commit 4a3b462c28
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 197 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
agent/consul/state/acl.go
View File

@ -113,57 +113,6 @@ func (s *TokenRolesIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) {
return val, nil
}
type RolePoliciesIndex struct {
}
func (s *RolePoliciesIndex) FromObject(obj interface{}) (bool, [][]byte, error) {
role, ok := obj.(*structs.ACLRole)
if !ok {
return false, nil, fmt.Errorf("object is not an ACLRole")
}
links := role.Policies
numLinks := len(links)
if numLinks == 0 {
return false, nil, nil
}
vals := make([][]byte, 0, numLinks)
for _, link := range links {
vals = append(vals, []byte(link.ID+"\x00"))
}
return true, vals, nil
}
func (s *RolePoliciesIndex) FromArgs(args ...interface{}) ([]byte, error) {
if len(args) != 1 {
return nil, fmt.Errorf("must provide only a single argument")
}
arg, ok := args[0].(string)
if !ok {
return nil, fmt.Errorf("argument must be a string: %#v", args[0])
}
// Add the null character as a terminator
arg += "\x00"
return []byte(arg), nil
}
func (s *RolePoliciesIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) {
val, err := s.FromArgs(args...)
if err != nil {
return nil, err
}
// Strip the null terminator, the rest is a prefix
n := len(val)
if n > 0 {
return val[:n-1], nil
}
return val, nil
}
type TokenExpirationIndex struct {
LocalFilter bool
}
@ -237,7 +186,7 @@ func (s *Restore) ACLPolicy(policy *structs.ACLPolicy) error {
// ACLRoles is used when saving a snapshot
func (s *Snapshot) ACLRoles() (memdb.ResultIterator, error) {
iter, err := s.tx.Get("acl-roles", "id")
iter, err := s.tx.Get(tableACLRoles, indexID)
if err != nil {
return nil, err
}
@ -544,22 +493,21 @@ func fixupTokenRoleLinks(tx ReadTxn, original *structs.ACLToken) (*structs.ACLTo
func resolveRolePolicyLinks(tx *txn, role *structs.ACLRole, allowMissing bool) error {
for linkIndex, link := range role.Policies {
if link.ID != "" {
policy, err := getPolicyWithTxn(tx, nil, link.ID, aclPolicyGetByID, &role.EnterpriseMeta)
if err != nil {
return err
}
if policy != nil {
// the name doesn't matter here
role.Policies[linkIndex].Name = policy.Name
} else if !allowMissing {
return fmt.Errorf("No such policy with ID: %s", link.ID)
}
} else {
if link.ID == "" {
return fmt.Errorf("Encountered a Role with policies linked by Name in the state store")
}
policy, err := getPolicyWithTxn(tx, nil, link.ID, aclPolicyGetByID, &role.EnterpriseMeta)
if err != nil {
return err
}
if policy != nil {
// the name doesn't matter here
role.Policies[linkIndex].Name = policy.Name
} else if !allowMissing {
return fmt.Errorf("No such policy with ID: %s", link.ID)
}
}
return nil
}
@ -1371,7 +1319,8 @@ func aclRoleSetTxn(tx *txn, idx uint64, role *structs.ACLRole, allowMissing bool
}
// ensure the name is unique (cannot conflict with another role with a different ID)
_, nameMatch, err := aclRoleGetByName(tx, role.Name, &role.EnterpriseMeta)
q := Query{EnterpriseMeta: role.EnterpriseMeta, Value: role.Name}
nameMatch, err := tx.First(tableACLRoles, indexName, q)
if err != nil {
return fmt.Errorf("failed acl role lookup: %v", err)
}
@ -1424,6 +1373,15 @@ func (s *Store) ACLRoleGetByName(ws memdb.WatchSet, name string, entMeta *struct
return s.aclRoleGet(ws, name, aclRoleGetByName, entMeta)
}
func aclRoleGetByName(tx ReadTxn, name string, entMeta *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) {
// TODO: accept non-pointer value
if entMeta == nil {
entMeta = structs.DefaultEnterpriseMeta()
}
q := Query{EnterpriseMeta: *entMeta, Value: name}
return tx.FirstWatch(tableACLRoles, indexName, q)
}
func (s *Store) ACLRoleBatchGet(ws memdb.WatchSet, ids []string) (uint64, structs.ACLRoles, error) {
tx := s.db.Txn(false)
defer tx.Abort()
@ -1440,7 +1398,7 @@ func (s *Store) ACLRoleBatchGet(ws memdb.WatchSet, ids []string) (uint64, struct
}
}
idx := maxIndexTxn(tx, "acl-roles")
idx := maxIndexTxn(tx, tableACLRoles)
return idx, roles, nil
}
@ -1485,10 +1443,16 @@ func (s *Store) ACLRoleList(ws memdb.WatchSet, policy string, entMeta *structs.E
var iter memdb.ResultIterator
var err error
// TODO: accept non-pointer value
if entMeta == nil {
entMeta = structs.DefaultEnterpriseMeta()
}
if policy != "" {
iter, err = aclRoleListByPolicy(tx, policy, entMeta)
q := Query{Value: policy, EnterpriseMeta: *entMeta}
iter, err = tx.Get(tableACLRoles, indexPolicies, q)
} else {
iter, err = aclRoleList(tx, entMeta)
iter, err = tx.Get(tableACLRoles, indexName+"_prefix", entMeta)
}
if err != nil {

5
agent/consul/state/acl_events.go
View File

@ -20,7 +20,7 @@ func aclChangeUnsubscribeEvent(tx ReadTxn, changes Changes) ([]stream.Event, err
token := changeObject(change).(*structs.ACLToken)
secretIDs = append(secretIDs, token.SecretID)
case "acl-roles":
case tableACLRoles:
role := changeObject(change).(*structs.ACLRole)
tokens, err := aclTokenListByRole(tx, role.ID, &role.EnterpriseMeta)
if err != nil {
@ -36,7 +36,8 @@ func aclChangeUnsubscribeEvent(tx ReadTxn, changes Changes) ([]stream.Event, err
}
secretIDs = appendSecretIDsFromTokenIterator(secretIDs, tokens)
roles, err := aclRoleListByPolicy(tx, policy.ID, &policy.EnterpriseMeta)
q := Query{Value: policy.ID, EnterpriseMeta: policy.EnterpriseMeta}
roles, err := tx.Get(tableACLRoles, indexPolicies, q)
if err != nil {
return nil, err
}

62
agent/consul/state/acl_oss.go
View File

@ -38,6 +38,44 @@ func indexNameFromACLPolicy(raw interface{}) ([]byte, error) {
return b.Bytes(), nil
}
func indexNameFromACLRole(raw interface{}) ([]byte, error) {
p, ok := raw.(*structs.ACLRole)
if !ok {
return nil, fmt.Errorf("unexpected type %T for structs.ACLRole index", raw)
}
if p.Name == "" {
return nil, errMissingValueForIndex
}
var b indexBuilder
b.String(strings.ToLower(p.Name))
return b.Bytes(), nil
}
func multiIndexPolicyFromACLRole(raw interface{}) ([][]byte, error) {
role, ok := raw.(*structs.ACLRole)
if !ok {
return nil, fmt.Errorf("unexpected type %T for structs.ACLRole index", raw)
}
count := len(role.Policies)
if count == 0 {
return nil, errMissingValueForIndex
}
vals := make([][]byte, 0, count)
for _, link := range role.Policies {
v, err := uuidStringToBytes(link.ID)
if err != nil {
return nil, err
}
vals = append(vals, v)
}
return vals, nil
}
func aclPolicyGetByID(tx ReadTxn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) {
return tx.FirstWatch(tableACLPolicies, indexID, id)
}
@ -144,48 +182,36 @@ func (s *Store) ACLTokenUpsertValidateEnterprise(token *structs.ACLToken, existi
func aclRoleInsert(tx *txn, role *structs.ACLRole) error {
// insert the role into memdb
if err := tx.Insert("acl-roles", role); err != nil {
if err := tx.Insert(tableACLRoles, role); err != nil {
return fmt.Errorf("failed inserting acl role: %v", err)
}
// update the overall acl-roles index
if err := indexUpdateMaxTxn(tx, role.ModifyIndex, "acl-roles"); err != nil {
if err := indexUpdateMaxTxn(tx, role.ModifyIndex, tableACLRoles); err != nil {
return fmt.Errorf("failed updating acl roles index: %v", err)
}
return nil
}
func aclRoleGetByID(tx ReadTxn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) {
return tx.FirstWatch("acl-roles", "id", id)
}
func aclRoleGetByName(tx ReadTxn, name string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) {
return tx.FirstWatch("acl-roles", "name", name)
}
func aclRoleList(tx ReadTxn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
return tx.Get("acl-roles", "id")
}
func aclRoleListByPolicy(tx ReadTxn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
return tx.Get("acl-roles", "policies", policy)
return tx.FirstWatch(tableACLRoles, indexID, id)
}
func aclRoleDeleteWithRole(tx *txn, role *structs.ACLRole, idx uint64) error {
// remove the role
if err := tx.Delete("acl-roles", role); err != nil {
if err := tx.Delete(tableACLRoles, role); err != nil {
return fmt.Errorf("failed deleting acl role: %v", err)
}
// update the overall acl-roles index
if err := indexUpdateMaxTxn(tx, idx, "acl-roles"); err != nil {
if err := indexUpdateMaxTxn(tx, idx, tableACLRoles); err != nil {
return fmt.Errorf("failed updating acl policies index: %v", err)
}
return nil
}
func aclRoleMaxIndex(tx ReadTxn, _ *structs.ACLRole, _ *structs.EnterpriseMeta) uint64 {
return maxIndexTxn(tx, "acl-roles")
return maxIndexTxn(tx, tableACLRoles)
}
func aclRoleUpsertValidateEnterprise(tx *txn, role *structs.ACLRole, existing *structs.ACLRole) error {

47
agent/consul/state/acl_oss_test.go
View File

@ -33,3 +33,50 @@ func testIndexerTableACLPolicies() map[string]indexerTestCase {
},
}
}
func testIndexerTableACLRoles() map[string]indexerTestCase {
policyID1 := "123e4567-e89a-12d7-a456-426614174001"
policyID2 := "123e4567-e89a-12d7-a456-426614174002"
obj := &structs.ACLRole{
ID: "123e4567-e89a-12d7-a456-426614174abc",
Name: "RoLe",
Policies: []structs.ACLRolePolicyLink{
{ID: policyID1}, {ID: policyID2},
},
}
encodedID := []byte{0x12, 0x3e, 0x45, 0x67, 0xe8, 0x9a, 0x12, 0xd7, 0xa4, 0x56, 0x42, 0x66, 0x14, 0x17, 0x4a, 0xbc}
encodedPID1 := []byte{0x12, 0x3e, 0x45, 0x67, 0xe8, 0x9a, 0x12, 0xd7, 0xa4, 0x56, 0x42, 0x66, 0x14, 0x17, 0x40, 0x01}
encodedPID2 := []byte{0x12, 0x3e, 0x45, 0x67, 0xe8, 0x9a, 0x12, 0xd7, 0xa4, 0x56, 0x42, 0x66, 0x14, 0x17, 0x40, 0x02}
return map[string]indexerTestCase{
indexID: {
read: indexValue{
source: obj.ID,
expected: encodedID,
},
write: indexValue{
source: obj,
expected: encodedID,
},
},
indexName: {
read: indexValue{
source: Query{Value: "RoLe"},
expected: []byte("role\x00"),
},
write: indexValue{
source: obj,
expected: []byte("role\x00"),
},
},
indexPolicies: {
read: indexValue{
source: Query{Value: policyID1},
expected: encodedPID1,
},
writeMulti: indexValueMulti{
source: obj,
expected: [][]byte{encodedPID1, encodedPID2},
},
},
}
}

12
agent/consul/state/acl_schema.go
View File

@ -151,9 +151,10 @@ func rolesTableSchema() *memdb.TableSchema {
Name: indexName,
AllowMissing: false,
Unique: true,
Indexer: &memdb.StringFieldIndex{
Field: "Name",
Lowercase: true,
Indexer: indexerSingleWithPrefix{
readIndex: readIndex(indexFromQuery),
writeIndex: writeIndex(indexNameFromACLRole),
prefixIndex: prefixIndex(prefixIndexFromQuery),
},
},
indexPolicies: {
@ -161,7 +162,10 @@ func rolesTableSchema() *memdb.TableSchema {
// Need to allow missing for the anonymous token
AllowMissing: true,
Unique: false,
Indexer: &RolePoliciesIndex{},
Indexer: indexerMulti{
readIndex: readIndex(indexFromUUIDQuery),
writeIndexMulti: writeIndexMulti(multiIndexPolicyFromACLRole),
},
},
},
}

2
agent/consul/state/acl_test.go
View File

@ -4082,7 +4082,7 @@ func TestStateStore_ACLRoles_Snapshot_Restore(t *testing.T) {
require.NoError(t, err)
require.Equal(t, uint64(2), idx)
require.ElementsMatch(t, roles, res)
require.Equal(t, uint64(2), s.maxIndex("acl-roles"))
require.Equal(t, uint64(2), s.maxIndex(tableACLRoles))
}()
}

7
agent/consul/state/catalog.go
View File

@ -28,13 +28,6 @@ const (
minUUIDLookupLen = 2
)
// Query is a type used to query any single value index that may include an
// enterprise identifier.
type Query struct {
Value string
structs.EnterpriseMeta
}
func resizeNodeLookupKey(s string) string {
l := len(s)

7
agent/consul/state/indexer.go
View File

@ -107,6 +107,13 @@ func (b *indexBuilder) String(v string) {
(*bytes.Buffer)(b).WriteString(null)
}
// Raw appends the bytes without a null terminator to the buffer. Raw should
// only be used when v has a fixed length, or when building the last segment of
// a prefix index.
func (b *indexBuilder) Raw(v []byte) {
(*bytes.Buffer)(b).Write(v)
}
func (b *indexBuilder) Bytes() []byte {
return (*bytes.Buffer)(b).Bytes()
}

42
agent/consul/state/query.go Normal file
View File

@ -0,0 +1,42 @@
package state
import (
"encoding/hex"
"fmt"
"strings"
"github.com/hashicorp/consul/agent/structs"
)
// Query is a type used to query any single value index that may include an
// enterprise identifier.
type Query struct {
Value string
structs.EnterpriseMeta
}
// uuidStringToBytes is a modified version of memdb.UUIDFieldIndex.parseString
func uuidStringToBytes(uuid string) ([]byte, error) {
l := len(uuid)
if l != 36 {
return nil, fmt.Errorf("UUID must be 36 characters")
}
hyphens := strings.Count(uuid, "-")
if hyphens > 4 {
return nil, fmt.Errorf(`UUID should have maximum of 4 "-"; got %d`, hyphens)
}
// The sanitized length is the length of the original string without the "-".
sanitized := strings.Replace(uuid, "-", "", -1)
sanitizedLength := len(sanitized)
if sanitizedLength%2 != 0 {
return nil, fmt.Errorf("UUID (without hyphens) must be even length")
}
dec, err := hex.DecodeString(sanitized)
if err != nil {
return nil, fmt.Errorf("invalid UUID: %w", err)
}
return dec, nil
}

8
agent/consul/state/query_oss.go
View File

@ -36,3 +36,11 @@ func prefixIndexFromQuery(arg interface{}) ([]byte, error) {
return nil, fmt.Errorf("unexpected type %T for Query prefix index", arg)
}
func indexFromUUIDQuery(raw interface{}) ([]byte, error) {
q, ok := raw.(Query)
if !ok {
return nil, fmt.Errorf("unexpected type %T for UUIDQuery index", raw)
}
return uuidStringToBytes(q.Value)
}

1
agent/consul/state/schema_test.go
View File

@ -129,6 +129,7 @@ func TestNewDBSchema_Indexers(t *testing.T) {
var testcases = map[string]func() map[string]indexerTestCase{
tableACLPolicies: testIndexerTableACLPolicies,
tableACLRoles: testIndexerTableACLRoles,
tableChecks: testIndexerTableChecks,
tableServices: testIndexerTableServices,
tableNodes: testIndexerTableNodes,

4
agent/consul/state/testdata/TestStateStoreSchema.golden vendored
View File

@ -18,9 +18,9 @@ table=acl-roles
index=id unique
indexer=github.com/hashicorp/go-memdb.UUIDFieldIndex Field=ID
index=name unique
indexer=github.com/hashicorp/go-memdb.StringFieldIndex Field=Name Lowercase=true
indexer=github.com/hashicorp/consul/agent/consul/state.indexerSingleWithPrefix readIndex=github.com/hashicorp/consul/agent/consul/state.indexFromQuery writeIndex=github.com/hashicorp/consul/agent/consul/state.indexNameFromACLRole prefixIndex=github.com/hashicorp/consul/agent/consul/state.prefixIndexFromQuery
index=policies allow-missing
indexer=github.com/hashicorp/consul/agent/consul/state.RolePoliciesIndex
indexer=github.com/hashicorp/consul/agent/consul/state.indexerMulti readIndex=github.com/hashicorp/consul/agent/consul/state.indexFromUUIDQuery writeIndexMulti=github.com/hashicorp/consul/agent/consul/state.multiIndexPolicyFromACLRole
table=acl-tokens
index=accessor unique allow-missing