ProcessChanges: use stream.Event

Also remove secretHash, which was used to hash tokens. We don't expose
these tokens anywhere, so we can use the string itself instead of a
Hash.

Fix acl_events_test.go for storing a structs type.
This commit is contained in:
Daniel Nephin 2020-06-15 18:49:00 -04:00
parent 2e45bbbb3e
commit 2020e9c7c7
10 changed files with 261 additions and 406 deletions

View File

@ -869,11 +869,11 @@ func (s *Store) ACLTokenList(ws memdb.WatchSet, local, global bool, policy, role
}
} else if policy != "" && role == "" && methodName == "" {
iter, err = s.aclTokenListByPolicy(tx, policy, entMeta)
iter, err = aclTokenListByPolicy(tx, policy, entMeta)
needLocalityFilter = true
} else if policy == "" && role != "" && methodName == "" {
iter, err = s.aclTokenListByRole(tx, role, entMeta)
iter, err = aclTokenListByRole(tx, role, entMeta)
needLocalityFilter = true
} else if policy == "" && role == "" && methodName != "" {
@ -1464,7 +1464,7 @@ func (s *Store) ACLRoleList(ws memdb.WatchSet, policy string, entMeta *structs.E
var err error
if policy != "" {
iter, err = s.aclRoleListByPolicy(tx, policy, entMeta)
iter, err = aclRoleListByPolicy(tx, policy, entMeta)
} else {
iter, err = s.aclRoleList(tx, entMeta)
}

View File

@ -1,81 +1,54 @@
package state
import (
"github.com/hashicorp/consul/agent/agentpb"
"github.com/hashicorp/consul/agent/consul/stream"
"github.com/hashicorp/consul/agent/structs"
memdb "github.com/hashicorp/go-memdb"
)
// ACLEventsFromChanges returns all the ACL token, policy or role events that
// should be emitted given a set of changes to the state store.
func (s *Store) ACLEventsFromChanges(tx *txn, changes memdb.Changes) ([]agentpb.Event, error) {
// Don't allocate yet since in majority of update transactions no ACL token
// will be changed.
var events []agentpb.Event
getObj := func(change memdb.Change) interface{} {
if change.Deleted() {
return change.Before
}
return change.After
}
getOp := func(change memdb.Change) agentpb.ACLOp {
if change.Deleted() {
return agentpb.ACLOp_Delete
}
return agentpb.ACLOp_Update
}
// TODO: Add OpDelete/OpUpdate to the event or payload?
func aclEventsFromChanges(tx *txn, changes memdb.Changes) ([]stream.Event, error) {
var events []stream.Event
// TODO: mapping of table->topic?
for _, change := range changes {
switch change.Table {
case "acl-tokens":
token := getObj(change).(*structs.ACLToken)
e := agentpb.Event{
Topic: agentpb.Topic_ACLTokens,
Index: tx.Index,
Payload: &agentpb.Event_ACLToken{
ACLToken: &agentpb.ACLTokenUpdate{
Op: getOp(change),
Token: &agentpb.ACLTokenIdentifier{
AccessorID: token.AccessorID,
SecretID: token.SecretID,
},
},
},
token := changeObject(change).(*structs.ACLToken)
e := stream.Event{
Topic: stream.Topic_ACLTokens,
Index: tx.Index,
Payload: token,
}
events = append(events, e)
case "acl-policies":
policy := getObj(change).(*structs.ACLPolicy)
e := agentpb.Event{
Topic: agentpb.Topic_ACLPolicies,
Index: tx.Index,
Payload: &agentpb.Event_ACLPolicy{
ACLPolicy: &agentpb.ACLPolicyUpdate{
Op: getOp(change),
PolicyID: policy.ID,
},
},
policy := changeObject(change).(*structs.ACLPolicy)
e := stream.Event{
Topic: stream.Topic_ACLPolicies,
Index: tx.Index,
Payload: policy,
}
events = append(events, e)
case "acl-roles":
role := getObj(change).(*structs.ACLRole)
e := agentpb.Event{
Topic: agentpb.Topic_ACLRoles,
Index: tx.Index,
Payload: &agentpb.Event_ACLRole{
ACLRole: &agentpb.ACLRoleUpdate{
Op: getOp(change),
RoleID: role.ID,
},
},
role := changeObject(change).(*structs.ACLRole)
e := stream.Event{
Topic: stream.Topic_ACLRoles,
Index: tx.Index,
Payload: role,
}
events = append(events, e)
default:
continue
}
}
return events, nil
}
// changeObject returns the object before it was deleted if the change was a delete,
// otherwise returns the object after the change.
func changeObject(change memdb.Change) interface{} {
if change.Deleted() {
return change.Before
}
return change.After
}

View File

@ -5,291 +5,121 @@ import (
"strings"
"testing"
"github.com/hashicorp/consul/agent/agentpb"
"github.com/hashicorp/consul/agent/consul/stream"
"github.com/hashicorp/consul/agent/structs"
"github.com/stretchr/testify/require"
)
func testACLTokenEvent(t *testing.T, idx uint64, n int, delete bool) agentpb.Event {
t.Helper()
uuid := strings.ReplaceAll("11111111-????-????-????-????????????", "?",
strconv.Itoa(n))
op := agentpb.ACLOp_Update
if delete {
op = agentpb.ACLOp_Delete
}
return agentpb.Event{
Topic: agentpb.Topic_ACLTokens,
Index: idx,
Payload: &agentpb.Event_ACLToken{
ACLToken: &agentpb.ACLTokenUpdate{
Op: op,
Token: &agentpb.ACLTokenIdentifier{
AccessorID: uuid,
SecretID: uuid,
},
},
},
}
}
func testACLPolicyEvent(t *testing.T, idx uint64, n int, delete bool) agentpb.Event {
t.Helper()
uuid := strings.ReplaceAll("22222222-????-????-????-????????????", "?",
strconv.Itoa(n))
op := agentpb.ACLOp_Update
if delete {
op = agentpb.ACLOp_Delete
}
return agentpb.Event{
Topic: agentpb.Topic_ACLPolicies,
Index: idx,
Payload: &agentpb.Event_ACLPolicy{
ACLPolicy: &agentpb.ACLPolicyUpdate{
Op: op,
PolicyID: uuid,
},
},
}
}
func testACLRoleEvent(t *testing.T, idx uint64, n int, delete bool) agentpb.Event {
t.Helper()
uuid := strings.ReplaceAll("33333333-????-????-????-????????????", "?",
strconv.Itoa(n))
op := agentpb.ACLOp_Update
if delete {
op = agentpb.ACLOp_Delete
}
return agentpb.Event{
Topic: agentpb.Topic_ACLRoles,
Index: idx,
Payload: &agentpb.Event_ACLRole{
ACLRole: &agentpb.ACLRoleUpdate{
Op: op,
RoleID: uuid,
},
},
}
}
func testToken(t *testing.T, n int) *structs.ACLToken {
uuid := strings.ReplaceAll("11111111-????-????-????-????????????", "?",
strconv.Itoa(n))
return &structs.ACLToken{
AccessorID: uuid,
SecretID: uuid,
}
}
func testPolicy(t *testing.T, n int) *structs.ACLPolicy {
numStr := strconv.Itoa(n)
uuid := strings.ReplaceAll("22222222-????-????-????-????????????", "?", numStr)
return &structs.ACLPolicy{
ID: uuid,
Name: "test_policy_" + numStr,
Rules: `operator = "read"`,
}
}
func testRole(t *testing.T, n, p int) *structs.ACLRole {
numStr := strconv.Itoa(n)
uuid := strings.ReplaceAll("33333333-????-????-????-????????????", "?", numStr)
policy := testPolicy(t, p)
return &structs.ACLRole{
ID: uuid,
Name: "test_role_" + numStr,
Policies: []structs.ACLRolePolicyLink{{
ID: policy.ID,
Name: policy.Name,
}},
}
}
func TestACLEventsFromChanges(t *testing.T) {
cases := []struct {
Name string
Setup func(s *Store, tx *txn) error
Mutate func(s *Store, tx *txn) error
WantEvents []agentpb.Event
WantErr bool
Name string
Setup func(s *Store, tx *txn) error
Mutate func(s *Store, tx *txn) error
expected stream.Event
}{
{
Name: "token create",
Mutate: func(s *Store, tx *txn) error {
if err := s.aclTokenSetTxn(tx, tx.Index, testToken(t, 1), false, false, false, false); err != nil {
return err
}
return nil
return s.aclTokenSetTxn(tx, tx.Index, newACLToken(1), false, false, false, false)
},
WantEvents: []agentpb.Event{
testACLTokenEvent(t, 100, 1, false),
},
WantErr: false,
expected: newACLTokenEvent(100, 1),
},
{
Name: "token update",
Setup: func(s *Store, tx *txn) error {
if err := s.aclTokenSetTxn(tx, tx.Index, testToken(t, 1), false, false, false, false); err != nil {
return err
}
return nil
return s.aclTokenSetTxn(tx, tx.Index, newACLToken(1), false, false, false, false)
},
Mutate: func(s *Store, tx *txn) error {
// Add a policy to the token (never mind it doesn't exist for now) we
// allow it in the set command below.
token := testToken(t, 1)
token := newACLToken(1)
token.Policies = []structs.ACLTokenPolicyLink{{ID: "33333333-1111-1111-1111-111111111111"}}
if err := s.aclTokenSetTxn(tx, tx.Index, token, false, true, false, false); err != nil {
return err
}
return nil
return s.aclTokenSetTxn(tx, tx.Index, token, false, true, false, false)
},
WantEvents: []agentpb.Event{
// Should see an event from the update
testACLTokenEvent(t, 100, 1, false),
},
WantErr: false,
expected: newACLTokenEvent(100, 1, structs.ACLTokenPolicyLink{ID: "33333333-1111-1111-1111-111111111111"}),
},
{
Name: "token delete",
Setup: func(s *Store, tx *txn) error {
if err := s.aclTokenSetTxn(tx, tx.Index, testToken(t, 1), false, false, false, false); err != nil {
return err
}
return nil
return s.aclTokenSetTxn(tx, tx.Index, newACLToken(1), false, false, false, false)
},
Mutate: func(s *Store, tx *txn) error {
// Delete it
token := testToken(t, 1)
if err := s.aclTokenDeleteTxn(tx, tx.Index, token.AccessorID, "id", nil); err != nil {
return err
}
return nil
token := newACLToken(1)
return s.aclTokenDeleteTxn(tx, tx.Index, token.AccessorID, "id", nil)
},
WantEvents: []agentpb.Event{
// Should see a delete event
testACLTokenEvent(t, 100, 1, true),
},
WantErr: false,
expected: newACLTokenEvent(100, 1),
},
{
Name: "policy create",
Mutate: func(s *Store, tx *txn) error {
if err := s.aclPolicySetTxn(tx, tx.Index, testPolicy(t, 1)); err != nil {
return err
}
return nil
return s.aclPolicySetTxn(tx, tx.Index, newACLPolicy(1))
},
WantEvents: []agentpb.Event{
testACLPolicyEvent(t, 100, 1, false),
},
WantErr: false,
expected: newACLPolicyEvent(100, 1),
},
{
Name: "policy update",
Setup: func(s *Store, tx *txn) error {
if err := s.aclPolicySetTxn(tx, tx.Index, testPolicy(t, 1)); err != nil {
return err
}
return nil
return s.aclPolicySetTxn(tx, tx.Index, newACLPolicy(1))
},
Mutate: func(s *Store, tx *txn) error {
policy := testPolicy(t, 1)
policy := newACLPolicy(1)
policy.Rules = `operator = "write"`
if err := s.aclPolicySetTxn(tx, tx.Index, policy); err != nil {
return err
}
return nil
return s.aclPolicySetTxn(tx, tx.Index, policy)
},
WantEvents: []agentpb.Event{
// Should see an event from the update
testACLPolicyEvent(t, 100, 1, false),
expected: stream.Event{
Topic: stream.Topic_ACLPolicies,
Index: 100,
Payload: &structs.ACLPolicy{
ID: "22222222-1111-1111-1111-111111111111",
Name: "test_policy_1",
Rules: `operator = "write"`,
},
},
WantErr: false,
},
{
Name: "policy delete",
Setup: func(s *Store, tx *txn) error {
if err := s.aclPolicySetTxn(tx, tx.Index, testPolicy(t, 1)); err != nil {
return err
}
return nil
return s.aclPolicySetTxn(tx, tx.Index, newACLPolicy(1))
},
Mutate: func(s *Store, tx *txn) error {
// Delete it
policy := testPolicy(t, 1)
if err := s.aclPolicyDeleteTxn(tx, tx.Index, policy.ID, s.aclPolicyGetByID, nil); err != nil {
return err
}
return nil
policy := newACLPolicy(1)
return s.aclPolicyDeleteTxn(tx, tx.Index, policy.ID, s.aclPolicyGetByID, nil)
},
WantEvents: []agentpb.Event{
// Should see a delete event
testACLPolicyEvent(t, 100, 1, true),
},
WantErr: false,
expected: newACLPolicyEvent(100, 1),
},
{
Name: "role create",
Mutate: func(s *Store, tx *txn) error {
if err := s.aclRoleSetTxn(tx, tx.Index, testRole(t, 1, 1), true); err != nil {
return err
}
return nil
return s.aclRoleSetTxn(tx, tx.Index, newACLRole(1, newACLRolePolicyLink(1)), true)
},
WantEvents: []agentpb.Event{
testACLRoleEvent(t, 100, 1, false),
},
WantErr: false,
expected: newACLRoleEvent(100, 1, newACLRolePolicyLink(1)),
},
{
Name: "role update",
Setup: func(s *Store, tx *txn) error {
if err := s.aclRoleSetTxn(tx, tx.Index, testRole(t, 1, 1), true); err != nil {
return err
}
return nil
return s.aclRoleSetTxn(tx, tx.Index, newACLRole(1, newACLRolePolicyLink(1)), true)
},
Mutate: func(s *Store, tx *txn) error {
role := testRole(t, 1, 1)
policy2 := testPolicy(t, 2)
role := newACLRole(1, newACLRolePolicyLink(1))
policy2 := newACLPolicy(2)
role.Policies = append(role.Policies, structs.ACLRolePolicyLink{
ID: policy2.ID,
Name: policy2.Name,
})
if err := s.aclRoleSetTxn(tx, tx.Index, role, true); err != nil {
return err
}
return nil
return s.aclRoleSetTxn(tx, tx.Index, role, true)
},
WantEvents: []agentpb.Event{
// Should see an event from the update
testACLRoleEvent(t, 100, 1, false),
},
WantErr: false,
expected: newACLRoleEvent(100, 1, newACLRolePolicyLink(1), newACLRolePolicyLink(2)),
},
{
Name: "role delete",
Setup: func(s *Store, tx *txn) error {
if err := s.aclRoleSetTxn(tx, tx.Index, testRole(t, 1, 1), true); err != nil {
return err
}
return nil
return s.aclRoleSetTxn(tx, tx.Index, newACLRole(1, newACLRolePolicyLink(1)), true)
},
Mutate: func(s *Store, tx *txn) error {
// Delete it
role := testRole(t, 1, 1)
if err := s.aclRoleDeleteTxn(tx, tx.Index, role.ID, s.aclRoleGetByID, nil); err != nil {
return err
}
return nil
role := newACLRole(1, newACLRolePolicyLink(1))
return s.aclRoleDeleteTxn(tx, tx.Index, role.ID, s.aclRoleGetByID, nil)
},
WantEvents: []agentpb.Event{
// Should see a delete event
testACLRoleEvent(t, 100, 1, true),
},
WantErr: false,
expected: newACLRoleEvent(100, 1, newACLRolePolicyLink(1)),
},
}
@ -315,28 +145,95 @@ func TestACLEventsFromChanges(t *testing.T) {
// Note we call the func under test directly rather than publishChanges so
// we can test this in isolation.
got, err := s.ACLEventsFromChanges(tx, tx.Changes())
if tc.WantErr {
require.Error(t, err)
return
}
events, err := aclEventsFromChanges(tx, tx.Changes())
require.NoError(t, err)
// Make sure we have the right events, only taking ordering into account
// where it matters to account for non-determinism.
requireEventsInCorrectPartialOrder(t, tc.WantEvents, got, func(e agentpb.Event) string {
// We only care that events affecting the same actual token are ordered
// with respect ot each other so use it's ID as the key.
switch v := e.Payload.(type) {
case *agentpb.Event_ACLToken:
return "token:" + v.ACLToken.Token.AccessorID
case *agentpb.Event_ACLPolicy:
return "policy:" + v.ACLPolicy.PolicyID
case *agentpb.Event_ACLRole:
return "role:" + v.ACLRole.RoleID
}
return ""
})
require.Len(t, events, 1)
actual := events[0]
// ignore modified and created index because we don't set them in our expected values
// TODO: gotest.tools/assert would make this easier
normalizePayload(&actual)
require.Equal(t, tc.expected, actual)
})
}
}
func normalizePayload(s *stream.Event) {
switch s := s.Payload.(type) {
case *structs.ACLToken:
s.ModifyIndex = 0
s.CreateIndex = 0
s.Hash = nil
case *structs.ACLPolicy:
s.ModifyIndex = 0
s.CreateIndex = 0
case *structs.ACLRole:
s.ModifyIndex = 0
s.CreateIndex = 0
}
}
func newACLTokenEvent(idx uint64, n int, policies ...structs.ACLTokenPolicyLink) stream.Event {
uuid := strings.ReplaceAll("11111111-????-????-????-????????????", "?", strconv.Itoa(n))
return stream.Event{
Topic: stream.Topic_ACLTokens,
Index: idx,
Payload: &structs.ACLToken{
AccessorID: uuid,
SecretID: uuid,
Policies: policies,
},
}
}
func newACLPolicyEvent(idx uint64, n int) stream.Event {
return stream.Event{
Topic: stream.Topic_ACLPolicies,
Index: idx,
Payload: newACLPolicy(n),
}
}
func newACLRoleEvent(idx uint64, n int, policies ...structs.ACLRolePolicyLink) stream.Event {
return stream.Event{
Topic: stream.Topic_ACLRoles,
Index: idx,
Payload: newACLRole(n, policies...),
}
}
func newACLToken(n int) *structs.ACLToken {
uuid := strings.ReplaceAll("11111111-????-????-????-????????????", "?", strconv.Itoa(n))
return &structs.ACLToken{
AccessorID: uuid,
SecretID: uuid,
}
}
func newACLPolicy(n int) *structs.ACLPolicy {
numStr := strconv.Itoa(n)
uuid := strings.ReplaceAll("22222222-????-????-????-????????????", "?", numStr)
return &structs.ACLPolicy{
ID: uuid,
Name: "test_policy_" + numStr,
Rules: `operator = "read"`,
}
}
func newACLRole(n int, policies ...structs.ACLRolePolicyLink) *structs.ACLRole {
numStr := strconv.Itoa(n)
uuid := strings.ReplaceAll("33333333-????-????-????-????????????", "?", numStr)
return &structs.ACLRole{
ID: uuid,
Name: "test_role_" + numStr,
Policies: policies,
}
}
func newACLRolePolicyLink(n int) structs.ACLRolePolicyLink {
policy := newACLPolicy(n)
return structs.ACLRolePolicyLink{
ID: policy.ID,
Name: policy.Name,
}
}

View File

@ -289,11 +289,11 @@ func (s *Store) aclTokenListGlobal(tx *txn, _ *structs.EnterpriseMeta) (memdb.Re
return tx.Get("acl-tokens", "local", false)
}
func (s *Store) aclTokenListByPolicy(tx *txn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
func aclTokenListByPolicy(tx *txn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
return tx.Get("acl-tokens", "policies", policy)
}
func (s *Store) aclTokenListByRole(tx *txn, role string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
func aclTokenListByRole(tx *txn, role string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
return tx.Get("acl-tokens", "roles", role)
}
@ -355,7 +355,7 @@ func (s *Store) aclRoleList(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIte