Merge pull request #9865 from hashicorp/dnephin/state-index-config-entries

state: convert config-entries table to the new pattern of functional indexers

agent/consul/state/config_entry.go

@@ -106,7 +106,7 @@ func configEntryTxn(tx ReadTxn, ws memdb.WatchSet, kind, name string, entMeta *s
 	idx := maxIndexTxn(tx, tableConfigEntries)
 
 	// Get the existing config entry.
-	watchCh, existing, err := firstWatchConfigEntryWithTxn(tx, kind, name, entMeta)
+	watchCh, existing, err := tx.FirstWatch(tableConfigEntries, "id", NewConfigEntryKindName(kind, name, entMeta))
 	if err != nil {
 		return 0, nil, fmt.Errorf("failed config entry lookup: %s", err)
 	}
@@ -175,7 +175,7 @@ func (s *Store) EnsureConfigEntry(idx uint64, conf structs.ConfigEntry) error {
 // ensureConfigEntryTxn upserts a config entry inside of a transaction.
 func ensureConfigEntryTxn(tx WriteTxn, idx uint64, conf structs.ConfigEntry) error {
 	// Check for existing configuration.
-	existing, err := firstConfigEntryWithTxn(tx, conf.GetKind(), conf.GetName(), conf.GetEnterpriseMeta())
+	existing, err := tx.First(tableConfigEntries, indexID, newConfigEntryQuery(conf))
 	if err != nil {
 		return fmt.Errorf("failed configuration lookup: %s", err)
 	}
@@ -214,7 +214,7 @@ func (s *Store) EnsureConfigEntryCAS(idx, cidx uint64, conf structs.ConfigEntry)
 	defer tx.Abort()
 
 	// Check for existing configuration.
-	existing, err := firstConfigEntryWithTxn(tx, conf.GetKind(), conf.GetName(), conf.GetEnterpriseMeta())
+	existing, err := tx.First(tableConfigEntries, indexID, newConfigEntryQuery(conf))
 	if err != nil {
 		return false, fmt.Errorf("failed configuration lookup: %s", err)
 	}
@@ -254,9 +254,9 @@ func (s *Store) DeleteConfigEntry(idx uint64, kind, name string, entMeta *struct
 	return tx.Commit()
 }
 
+// TODO: accept structs.ConfigEntry instead of individual fields
 func deleteConfigEntryTxn(tx WriteTxn, idx uint64, kind, name string, entMeta *structs.EnterpriseMeta) error {
-	// Try to retrieve the existing config entry.
+	existing, err := tx.First(tableConfigEntries, indexID, NewConfigEntryKindName(kind, name, entMeta))
-	existing, err := firstConfigEntryWithTxn(tx, kind, name, entMeta)
 	if err != nil {
 		return fmt.Errorf("failed config entry lookup: %s", err)
 	}
@@ -1242,3 +1242,13 @@ func NewConfigEntryKindName(kind, name string, entMeta *structs.EnterpriseMeta)
 	ret.EnterpriseMeta.Normalize()
 	return ret
 }
+
+func newConfigEntryQuery(c structs.ConfigEntry) ConfigEntryKindName {
+	return NewConfigEntryKindName(c.GetKind(), c.GetName(), c.GetEnterpriseMeta())
+}
+
+// ConfigEntryKindQuery is used to lookup config entries by their kind.
+type ConfigEntryKindQuery struct {
+	Kind string
+	structs.EnterpriseMeta
+}

agent/consul/state/config_entry_intention.go

@@ -123,7 +123,7 @@ func (s *ServiceIntentionSourceIndex) FromArgs(args ...interface{}) ([]byte, err
 	return []byte(arg.String() + "\x00"), nil
 }
 
-func (s *Store) configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
+func configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
 	// unrolled part of configEntriesByKindTxn
 
 	idx := maxIndexTxn(tx, tableConfigEntries)
@@ -144,7 +144,7 @@ func (s *Store) configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *
 	return idx, results, true, nil
 }
 
-func (s *Store) configIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.ServiceIntentionsConfigEntry, *structs.Intention, error) {
+func configIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.ServiceIntentionsConfigEntry, *structs.Intention, error) {
 	idx := maxIndexTxn(tx, tableConfigEntries)
 	if idx < 1 {
 		idx = 1

agent/consul/state/config_entry_oss.go

@@ -3,22 +3,67 @@
 package state
 
 import (
+	"fmt"
+	"strings"
+
 	memdb "github.com/hashicorp/go-memdb"
 
 	"github.com/hashicorp/consul/agent/structs"
 )
 
-func firstConfigEntryWithTxn(tx ReadTxn, kind, name string, _ *structs.EnterpriseMeta) (interface{}, error) {
+func indexFromConfigEntryKindName(arg interface{}) ([]byte, error) {
-	return tx.First(tableConfigEntries, "id", kind, name)
+	n, ok := arg.(ConfigEntryKindName)
+	if !ok {
+		return nil, fmt.Errorf("invalid type for ConfigEntryKindName query: %T", arg)
+	}
+
+	var b indexBuilder
+	b.String(strings.ToLower(n.Kind))
+	b.String(strings.ToLower(n.Name))
+	return b.Bytes(), nil
 }
 
-func firstWatchConfigEntryWithTxn(
+func indexFromConfigEntry(raw interface{}) ([]byte, error) {
-	tx ReadTxn,
+	c, ok := raw.(structs.ConfigEntry)
-	kind string,
+	if !ok {
-	name string,
+		return nil, fmt.Errorf("type must be structs.ConfigEntry: %T", raw)
-	_ *structs.EnterpriseMeta,
+	}
-) (<-chan struct{}, interface{}, error) {
+
-	return tx.FirstWatch(tableConfigEntries, "id", kind, name)
+	if c.GetName() == "" || c.GetKind() == "" {
+		return nil, errMissingValueForIndex
+	}
+
+	var b indexBuilder
+	b.String(strings.ToLower(c.GetKind()))
+	b.String(strings.ToLower(c.GetName()))
+	return b.Bytes(), nil
+}
+
+// indexKindFromConfigEntry indexes kinds, it is a shim for enterprise.
+func indexKindFromConfigEntry(raw interface{}) ([]byte, error) {
+	c, ok := raw.(structs.ConfigEntry)
+	if !ok {
+		return nil, fmt.Errorf("type must be structs.ConfigEntry: %T", raw)
+	}
+
+	if c.GetKind() == "" {
+		return nil, errMissingValueForIndex
+	}
+
+	var b indexBuilder
+	b.String(strings.ToLower(c.GetKind()))
+	return b.Bytes(), nil
+}
+
+func indexFromConfigEntryKindQuery(raw interface{}) ([]byte, error) {
+	q, ok := raw.(ConfigEntryKindQuery)
+	if !ok {
+		return nil, fmt.Errorf("type must be structs.ConfigEntry: %T", raw)
+	}
+
+	var b indexBuilder
+	b.String(strings.ToLower(q.Kind))
+	return b.Bytes(), nil
 }
 
 func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error {
@@ -26,11 +71,11 @@ func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error {
 }
 
 func getAllConfigEntriesWithTxn(tx ReadTxn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
-	return tx.Get(tableConfigEntries, "id")
+	return tx.Get(tableConfigEntries, indexID)
 }
 
 func getConfigEntryKindsWithTxn(tx ReadTxn, kind string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) {
-	return tx.Get(tableConfigEntries, "kind", kind)
+	return tx.Get(tableConfigEntries, indexKind, ConfigEntryKindQuery{Kind: kind})
 }
 
 func configIntentionsConvertToList(iter memdb.ResultIterator, _ *structs.EnterpriseMeta) structs.Intentions {

agent/consul/state/config_entry_oss_test.go

@@ -0,0 +1,35 @@
+// +build !consulent
+
+package state
+
+import "github.com/hashicorp/consul/agent/structs"
+
+func testIndexerTableConfigEntries() map[string]indexerTestCase {
+	return map[string]indexerTestCase{
+		indexID: {
+			read: indexValue{
+				source: ConfigEntryKindName{
+					Kind: "Proxy-Defaults",
+					Name: "NaMe",
+				},
+				expected: []byte("proxy-defaults\x00name\x00"),
+			},
+			write: indexValue{
+				source:   &structs.ProxyConfigEntry{Name: "NaMe"},
+				expected: []byte("proxy-defaults\x00name\x00"),
+			},
+		},
+		indexKind: {
+			read: indexValue{
+				source: ConfigEntryKindQuery{
+					Kind: "Service-Defaults",
+				},
+				expected: []byte("service-defaults\x00"),
+			},
+			write: indexValue{
+				source:   &structs.ServiceConfigEntry{},
+				expected: []byte("service-defaults\x00"),
+			},
+		},
+	}
+}

agent/consul/state/config_entry_schema.go

@@ -1,6 +1,8 @@
 package state
 
-import "github.com/hashicorp/go-memdb"
+import (
+	"github.com/hashicorp/go-memdb"
+)
 
 const (
 	tableConfigEntries = "config-entries"
@@ -20,26 +22,19 @@ func configTableSchema() *memdb.TableSchema {
 				Name:         indexID,
 				AllowMissing: false,
 				Unique:       true,
-				Indexer: &memdb.CompoundIndex{
+				Indexer: indexerSingleWithPrefix{
-					Indexes: []memdb.Indexer{
+					readIndex:   readIndex(indexFromConfigEntryKindName),
-						&memdb.StringFieldIndex{
+					writeIndex:  writeIndex(indexFromConfigEntry),
-							Field:     "Kind",
+					prefixIndex: prefixIndex(indexFromConfigEntryKindName),
-							Lowercase: true,
-						},
-						&memdb.StringFieldIndex{
-							Field:     "Name",
-							Lowercase: true,
-						},
-					},
 				},
 			},
 			indexKind: {
 				Name:         indexKind,
 				AllowMissing: false,
 				Unique:       false,
-				Indexer: &memdb.StringFieldIndex{
+				Indexer: indexerSingle{
-					Field:     "Kind",
+					readIndex:  readIndex(indexFromConfigEntryKindQuery),
-					Lowercase: true,
+					writeIndex: writeIndex(indexKindFromConfigEntry),
 				},
 			},
 			indexLink: {

agent/consul/state/indexer.go

@@ -30,6 +30,13 @@ type indexerMulti struct {
 	writeIndexMulti
 }
 
+// indexerSingleWithPrefix is a indexerSingle which also supports prefix queries.
+type indexerSingleWithPrefix struct {
+	readIndex
+	writeIndex
+	prefixIndex
+}
+
 // readIndex implements memdb.Indexer. It exists so that a function can be used
 // to provide the interface.
 //
@@ -78,6 +85,17 @@ func (f writeIndexMulti) FromObject(raw interface{}) (bool, [][]byte, error) {
 	return err == nil, v, err
 }
 
+// prefixIndex implements memdb.PrefixIndexer. It exists so that a function
+// can be used to provide this interface.
+type prefixIndex func(args interface{}) ([]byte, error)
+
+func (f prefixIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) {
+	if len(args) != 1 {
+		return nil, fmt.Errorf("index supports only a single arg")
+	}
+	return f(args[0])
+}
+
 const null = "\x00"
 
 // indexBuilder is a buffer used to construct memdb index values.

agent/consul/state/intention.go

@@ -154,7 +154,7 @@ func (s *Store) LegacyIntentions(ws memdb.WatchSet, entMeta *structs.EnterpriseM
 	tx := s.db.Txn(false)
 	defer tx.Abort()
 
-	idx, results, _, err := s.legacyIntentionsListTxn(tx, ws, entMeta)
+	idx, results, _, err := legacyIntentionsListTxn(tx, ws, entMeta)
 	return idx, results, err
 }
 
@@ -168,12 +168,12 @@ func (s *Store) Intentions(ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (
 		return 0, nil, false, err
 	}
 	if !usingConfigEntries {
-		return s.legacyIntentionsListTxn(tx, ws, entMeta)
+		return legacyIntentionsListTxn(tx, ws, entMeta)
 	}
-	return s.configIntentionsListTxn(tx, ws, entMeta)
+	return configIntentionsListTxn(tx, ws, entMeta)
 }
 
-func (s *Store) legacyIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
+func legacyIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) {
 	// Get the index
 	idx := maxIndexTxn(tx, tableConnectIntentions)
 	if idx < 1 {
@@ -578,13 +578,13 @@ func (s *Store) IntentionGet(ws memdb.WatchSet, id string) (uint64, *structs.Ser
 		return 0, nil, nil, err
 	}
 	if !usingConfigEntries {
-		idx, ixn, err := s.legacyIntentionGetTxn(tx, ws, id)
+		idx, ixn, err := legacyIntentionGetTxn(tx, ws, id)
 		return idx, nil, ixn, err
 	}
-	return s.configIntentionGetTxn(tx, ws, id)
+	return configIntentionGetTxn(tx, ws, id)
 }
 
-func (s *Store) legacyIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.Intention, error) {
+func legacyIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.Intention, error) {
 	// Get the table index.
 	idx := maxIndexTxn(tx, tableConnectIntentions)
 	if idx < 1 {

agent/consul/state/schema_test.go

@@ -128,9 +128,10 @@ func TestNewDBSchema_Indexers(t *testing.T) {
 	require.NoError(t, schema.Validate())
 
 	var testcases = map[string]func() map[string]indexerTestCase{
-		tableChecks:   testIndexerTableChecks,
+		tableChecks:        testIndexerTableChecks,
-		tableServices: testIndexerTableServices,
+		tableServices:      testIndexerTableServices,
-		tableNodes:    testIndexerTableNodes,
+		tableNodes:         testIndexerTableNodes,
+		tableConfigEntries: testIndexerTableConfigEntries,
 	}
 
 	for _, table := range schema.Tables {

agent/consul/state/testdata/TestStateStoreSchema.golden

@@ -60,13 +60,13 @@ table=checks
 
 table=config-entries
   index=id unique
-    indexer=github.com/hashicorp/go-memdb.CompoundIndex Indexes=[github.com/hashicorp/go-memdb.StringFieldIndex Field=Kind Lowercase=true, github.com/hashicorp/go-memdb.StringFieldIndex Field=Name Lowercase=true] AllowMissing=false
+    indexer=github.com/hashicorp/consul/agent/consul/state.indexerSingleWithPrefix readIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntryKindName writeIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntry prefixIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntryKindName
   index=intention-legacy-id unique allow-missing
     indexer=github.com/hashicorp/consul/agent/consul/state.ServiceIntentionLegacyIDIndex uuidFieldIndex={}
   index=intention-source allow-missing
     indexer=github.com/hashicorp/consul/agent/consul/state.ServiceIntentionSourceIndex
   index=kind
-    indexer=github.com/hashicorp/go-memdb.StringFieldIndex Field=Kind Lowercase=true
+    indexer=github.com/hashicorp/consul/agent/consul/state.indexerSingle readIndex=github.com/hashicorp/consul/agent/consul/state.indexFromConfigEntryKindQuery writeIndex=github.com/hashicorp/consul/agent/consul/state.indexKindFromConfigEntry
   index=link allow-missing
     indexer=github.com/hashicorp/consul/agent/consul/state.ConfigEntryLinkIndex