luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Merge pull request #9865 from hashicorp/dnephin/state-index-config-entries 

state: convert config-entries table to the new pattern of functional indexers
This commit is contained in:
Daniel Nephin 2021-03-17 17:40:59 -04:00 committed by GitHub
parent 98c32599e4 d2591312f8
commit af17ab54e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 149 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
agent/consul/state/config_entry.go
View File

@ -106,7 +106,7 @@ func configEntryTxn(tx ReadTxn, ws memdb.WatchSet, kind, name string, entMeta *s
idx := maxIndexTxn(tx, tableConfigEntries)
// Get the existing config entry.
watchCh, existing, err := firstWatchConfigEntryWithTxn(tx, kind, name, entMeta)
watchCh, existing, err := tx.FirstWatch(tableConfigEntries, "id", NewConfigEntryKindName(kind, name, entMeta))
if err != nil {
return 0, nil, fmt.Errorf("failed config entry lookup: %s", err)
}
@ -175,7 +175,7 @@ func (s *Store) EnsureConfigEntry(idx uint64, conf structs.ConfigEntry) error {
// ensureConfigEntryTxn upserts a config entry inside of a transaction.
func ensureConfigEntryTxn(tx WriteTxn, idx uint64, conf structs.ConfigEntry) error {
// Check for existing configuration.
existing, err := firstConfigEntryWithTxn(tx, conf.GetKind(), conf.GetName(), conf.GetEnterpriseMeta())
existing, err := tx.First(tableConfigEntries, indexID, newConfigEntryQuery(conf))
if err != nil {
return fmt.Errorf("failed configuration lookup: %s", err)
}
@ -214,7 +214,7 @@ func (s *Store) EnsureConfigEntryCAS(idx, cidx uint64, conf structs.ConfigEntry)
defer tx.Abort()
// Check for existing configuration.
existing, err := firstConfigEntryWithTxn(tx, conf.GetKind(), conf.GetName(), conf.GetEnterpriseMeta())
existing, err := tx.First(tableConfigEntries, indexID, newConfigEntryQuery(conf))
if err != nil {
return false, fmt.Errorf("failed configuration lookup: %s", err)
}
@ -254,9 +254,9 @@ func (s *Store) DeleteConfigEntry(idx uint64, kind, name string, entMeta *struct
return tx.Commit()
}
// TODO: accept structs.ConfigEntry instead of individual fields
func deleteConfigEntryTxn(tx WriteTxn, idx uint64, kind, name string, entMeta *structs.EnterpriseMeta) error {
// Try to retrieve the existing config entry.
existing, err := firstConfigEntryWithTxn(tx, kind, name, entMeta)
existing, err := tx.First(tableConfigEntries, indexID, NewConfigEntryKindName(kind, name, entMeta))
if err != nil {
return fmt.Errorf("failed config entry lookup: %s", err)
}
@ -1242,3 +1242,13 @@ func NewConfigEntryKindName(kind, name string, entMeta *structs.EnterpriseMeta)
ret.EnterpriseMeta.Normalize()
return ret
}
func newConfigEntryQuery(c structs.ConfigEntry) ConfigEntryKindName {
return NewConfigEntryKindName(c.GetKind(), c.GetName(), c.GetEnterpriseMeta())
}
// ConfigEntryKindQuery is used to lookup config entries by their kind.
type ConfigEntryKindQuery struct {
Kind string
structs.EnterpriseMeta
}

4
agent/consul/state/config_entry_intention.go
View File

@ -123,7 +123,7 @@ func (s *ServiceIntentionSourceIndex) FromArgs(args ...interface{}) ([]byte, err
return []byte(arg.String() + "\x00"), nil
}
func (s *Store) configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
func configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
// unrolled part of configEntriesByKindTxn
idx := maxIndexTxn(tx, tableConfigEntries)
@ -144,7 +144,7 @@ func (s *Store) configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *
return idx, results, true, nil
}
func (s *Store) configIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.ServiceIntentionsConfigEntry, *structs.Intention, error) {
func configIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.ServiceIntentionsConfigEntry, *structs.Intention, error) {
idx := maxIndexTxn(tx, tableConfigEntries)
if idx < 1 {
idx = 1

67
agent/consul/state/config_entry_oss.go
View File

@ -3,22 +3,67 @@
package state
import (
"fmt"
"strings"
memdb "github.com/hashicorp/go-memdb"
"github.com/hashicorp/consul/agent/structs"
)
func firstConfigEntryWithTxn(tx ReadTxn, kind, name string, _ *structs.EnterpriseMeta) (interface{}, error) {
return tx.First(tableConfigEntries, "id", kind, name)
func indexFromConfigEntryKindName(arg interface{}) ([]byte, error) {
n, ok := arg.(ConfigEntryKindName)
if !ok {
return nil, fmt.Errorf("invalid type for ConfigEntryKindName query: %T", arg)
}
var b indexBuilder
b.String(strings.ToLower(n.Kind))
b.String(strings.ToLower(n.Name))
return b.Bytes(), nil
}
func firstWatchConfigEntryWithTxn(
tx ReadTxn,
kind string,
name string,
_ *structs.EnterpriseMeta,
) (<-chan struct{}, interface{}, error) {
return tx.FirstWatch(tableConfigEntries, "id", kind, name)
func indexFromConfigEntry(raw interface{}) ([]byte, error) {
c, ok := raw.(structs.ConfigEntry)
if !ok {
return nil, fmt.Errorf("type must be structs.ConfigEntry: %T", raw)
}
if c.GetName() == "" || c.GetKind() == "" {
return nil, errMissingValueForIndex
}
var b indexBuilder
b.String(strings.ToLower(c.GetKind()))
b.String(strings.ToLower(c.GetName()))
return b.Bytes(), nil
}
// indexKindFromConfigEntry indexes kinds, it is a shim for enterprise.
func indexKindFromConfigEntry(raw interface{}) ([]byte, error) {
c, ok := raw.(structs.ConfigEntry)
if !ok {
return nil, fmt.Errorf("type must be structs.ConfigEntry: %T", raw)
}
if c.GetKind() == "" {
return nil, errMissingValueForIndex
}
var b indexBuilder
b.String(strings.ToLower(c.GetKind()))
return b.Bytes(), nil
}
func indexFromConfigEntryKindQuery(raw interface{}) ([]byte, error) {
q, ok := raw.(ConfigEntryKindQuery)
if !ok {
return nil, fmt.Errorf("type must be structs.ConfigEntry: %T", raw)
}
var b indexBuilder
b.String(strings.ToLower(q.Kind))
return b.Bytes(), nil
}
func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error {
@ -26,11 +71,11 @@ func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error {
}
func getAllConfigEntriesWithTxn(tx ReadTxn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
return tx.Get(tableConfigEntries, "id")
return tx.Get(tableConfigEntries, indexID)
}
func getConfigEntryKindsWithTxn(tx ReadTxn, kind string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
return tx.Get(tableConfigEntries, "kind", kind)
return tx.Get(tableConfigEntries, indexKind, ConfigEntryKindQuery{Kind: kind})
}
func configIntentionsConvertToList(iter memdb.ResultIterator, _ *structs.EnterpriseMeta) structs.Intentions {

35
agent/consul/state/config_entry_oss_test.go Normal file
View File

@ -0,0 +1,35 @@
// +build !consulent
package state
import "github.com/hashicorp/consul/agent/structs"
func testIndexerTableConfigEntries() map[string]indexerTestCase {
return map[string]indexerTestCase{
indexID: {
read: indexValue{
source: ConfigEntryKindName{
Kind: "Proxy-Defaults",
Name: "NaMe",
},
expected: []byte("proxy-defaults\x00name\x00"),
},
write: indexValue{
source: &structs.ProxyConfigEntry{Name: "NaMe"},
expected: []byte("proxy-defaults\x00name\x00"),
},
},
indexKind: {
read: indexValue{
source: ConfigEntryKindQuery{
Kind: "Service-Defaults",
},
expected: []byte("service-defaults\x00"),
},
write: indexValue{
source: &structs.ServiceConfigEntry{},
expected: []byte("service-defaults\x00"),
},
},
}
}

25
agent/consul/state/config_entry_schema.go
View File

@ -1,6 +1,8 @@
package state
import "github.com/hashicorp/go-memdb"
import (
"github.com/hashicorp/go-memdb"
)
const (
tableConfigEntries = "config-entries"
@ -20,26 +22,19 @@ func configTableSchema() *memdb.TableSchema {
Name: indexID,
AllowMissing: false,
Unique: true,
Indexer: &memdb.CompoundIndex{
Indexes: []memdb.Indexer{
&memdb.StringFieldIndex{
Field: "Kind",
Lowercase: true,
},
&memdb.StringFieldIndex{
Field: "Name",
Lowercase: true,
},
},
Indexer: indexerSingleWithPrefix{
readIndex: readIndex(indexFromConfigEntryKindName),
writeIndex: writeIndex(indexFromConfigEntry),
prefixIndex: prefixIndex(indexFromConfigEntryKindName),
},
},
indexKind: {
Name: indexKind,
AllowMissing: false,
Unique: false,
Indexer: &memdb.StringFieldIndex{
Field: "Kind",
Lowercase: true,
Indexer: indexerSingle{
readIndex: readIndex(indexFromConfigEntryKindQuery),
writeIndex: writeIndex(indexKindFromConfigEntry),
},
},
indexLink: {

18
agent/consul/state/indexer.go
View File

@ -30,6 +30,13 @@ type indexerMulti struct {
writeIndexMulti
}
// indexerSingleWithPrefix is a indexerSingle which also supports prefix queries.
type indexerSingleWithPrefix struct {
readIndex
writeIndex
prefixIndex
}
// readIndex implements memdb.Indexer. It exists so that a function can be used
// to provide the interface.
//
@ -78,6 +85,17 @@ func (f writeIndexMulti) FromObject(raw interface{}) (bool, [][]byte, error) {
return err == nil, v, err
}
// prefixIndex implements memdb.PrefixIndexer. It exists so that a function
// can be used to provide this interface.
type prefixIndex func(args interface{}) ([]byte, error)
func (f prefixIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) {
if len(args) != 1 {
return nil, fmt.Errorf("index supports only a single arg")
}
return f(args[0])
}
const null = "\x00"
// indexBuilder is a buffer used to construct memdb index values.

14
agent/consul/state/intention.go
View File

@ -154,7 +154,7 @@ func (s *Store) LegacyIntentions(ws memdb.WatchSet, entMeta *structs.EnterpriseM
tx := s.db.Txn(false)
defer tx.Abort()
idx, results, _, err := s.legacyIntentionsListTxn(tx, ws, entMeta)
idx, results, _, err := legacyIntentionsListTxn(tx, ws, entMeta)
return idx, results, err
}
@ -168,12 +168,12 @@ func (s *Store) Intentions(ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (
return 0, nil, false, err
}
if !usingConfigEntries {
return s.legacyIntentionsListTxn(tx, ws, entMeta)
return legacyIntentionsListTxn(tx, ws, entMeta)
}
return s.configIntentionsListTxn(tx, ws, entMeta)
return configIntentionsListTxn(tx, ws, entMeta)
}
func (s *Store) legacyIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
func legacyIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
// Get the index
idx := maxIndexTxn(tx, tableConnectIntentions)
if idx < 1 {
@ -578,13 +578,13 @@ func (s *Store) IntentionGet(ws memdb.WatchSet, id string) (uint64, *structs.Ser
return 0, nil, nil, err
}
if !usingConfigEntries {
idx, ixn, err := s.legacyIntentionGetTxn(tx, ws, id)
idx, ixn, err := legacyIntentionGetTxn(tx, ws, id)
return idx, nil, ixn, err
}
return s.configIntentionGetTxn(tx, ws, id)
return configIntentionGetTxn(tx, ws, id)
}
func (s *Store) legacyIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.Intention, error) {
func legacyIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.Intention, error) {
// Get the table index.
idx := maxIndexTxn(tx, tableConnectIntentions)
if idx < 1 {

1
agent/consul/state/schema_test.go
View File

@ -131,6 +131,7 @@ func TestNewDBSchema_Indexers(t *testing.T) {
tableChecks: testIndexerTableChecks,
tableServices: testIndexerTableServices,
tableNodes: testIndexerTableNodes,
tableConfigEntries: testIndexerTableConfigEntries,
}
for _, table := range schema.Tables {

4
agent/consul/state/testdata/TestStateStoreSchema.golden vendored
View File

@ -60,13 +60,13 @@ table=checks
table=config-entries
index=id unique
indexer=github.com/hashicorp/go-memdb.CompoundIndex Indexes=[github.com/hashicorp/go-memdb.StringFieldIndex Field=Kind Lowercase=true, github.com/hashicorp/go-memdb.StringFieldIndex Field=Name Lowercase=true] AllowMissing=false
indexer=github.com/hashicorp/consul/agent/consul/state.indexerSingleWithPrefix readIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntryKindName writeIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntry prefixIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntryKindName
index=intention-legacy-id unique allow-missing
indexer=github.com/hashicorp/consul/agent/consul/state.ServiceIntentionLegacyIDIndex uuidFieldIndex={}
index=intention-source allow-missing
indexer=github.com/hashicorp/consul/agent/consul/state.ServiceIntentionSourceIndex
index=kind
indexer=github.com/hashicorp/go-memdb.StringFieldIndex Field=Kind Lowercase=true
indexer=github.com/hashicorp/consul/agent/consul/state.indexerSingle readIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntryKindQuery writeIndex=github.com/hashicorp/consul/agent/consul/state.indexKindFromConfigEntry
index=link allow-missing
indexer=github.com/hashicorp/consul/agent/consul/state.ConfigEntryLinkIndex