From 7d17e2027061db544f1be188f17da6bc3a8d6f0e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 28 Jan 2021 20:34:15 -0500 Subject: [PATCH 1/2] state: move config-entries table to new pattern --- agent/consul/state/config_entry.go | 4 -- agent/consul/state/config_entry_oss.go | 57 +------------------- agent/consul/state/config_entry_schema.go | 65 +++++++++++++++++++++++ 3 files changed, 67 insertions(+), 59 deletions(-) create mode 100644 agent/consul/state/config_entry_schema.go diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 68cbbd73c..cb70373b4 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -12,10 +12,6 @@ import ( "github.com/hashicorp/consul/lib" ) -const ( - configTableName = "config-entries" -) - type ConfigEntryLinkIndex struct { } diff --git a/agent/consul/state/config_entry_oss.go b/agent/consul/state/config_entry_oss.go index bea2c01bc..eefe83272 100644 --- a/agent/consul/state/config_entry_oss.go +++ b/agent/consul/state/config_entry_oss.go @@ -3,63 +3,10 @@ package state import ( - "github.com/hashicorp/consul/agent/structs" memdb "github.com/hashicorp/go-memdb" -) -// configTableSchema returns a new table schema used to store global -// config entries. -func configTableSchema() *memdb.TableSchema { - return &memdb.TableSchema{ - Name: configTableName, - Indexes: map[string]*memdb.IndexSchema{ - "id": { - Name: "id", - AllowMissing: false, - Unique: true, - Indexer: &memdb.CompoundIndex{ - Indexes: []memdb.Indexer{ - &memdb.StringFieldIndex{ - Field: "Kind", - Lowercase: true, - }, - &memdb.StringFieldIndex{ - Field: "Name", - Lowercase: true, - }, - }, - }, - }, - "kind": { - Name: "kind", - AllowMissing: false, - Unique: false, - Indexer: &memdb.StringFieldIndex{ - Field: "Kind", - Lowercase: true, - }, - }, - "link": { - Name: "link", - AllowMissing: true, - Unique: false, - Indexer: &ConfigEntryLinkIndex{}, - }, - "intention-legacy-id": { - Name: "intention-legacy-id", - AllowMissing: true, - Unique: true, - Indexer: &ServiceIntentionLegacyIDIndex{}, - }, - "intention-source": { - Name: "intention-source", - AllowMissing: true, - Unique: false, - Indexer: &ServiceIntentionSourceIndex{}, - }, - }, - } -} + "github.com/hashicorp/consul/agent/structs" +) func firstConfigEntryWithTxn(tx ReadTxn, kind, name string, _ *structs.EnterpriseMeta) (interface{}, error) { return tx.First(configTableName, "id", kind, name) diff --git a/agent/consul/state/config_entry_schema.go b/agent/consul/state/config_entry_schema.go new file mode 100644 index 000000000..72a6320b0 --- /dev/null +++ b/agent/consul/state/config_entry_schema.go @@ -0,0 +1,65 @@ +package state + +import "github.com/hashicorp/go-memdb" + +const ( + configTableName = "config-entries" + + indexLink = "link" + indexIntentionLegacyID = "intention-legacy-id" + indexSource = "intention-source" +) + +// configTableSchema returns a new table schema used to store global +// config entries. +func configTableSchema() *memdb.TableSchema { + return &memdb.TableSchema{ + Name: configTableName, + Indexes: map[string]*memdb.IndexSchema{ + indexID: { + Name: indexID, + AllowMissing: false, + Unique: true, + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{ + Field: "Kind", + Lowercase: true, + }, + &memdb.StringFieldIndex{ + Field: "Name", + Lowercase: true, + }, + }, + }, + }, + indexKind: { + Name: indexKind, + AllowMissing: false, + Unique: false, + Indexer: &memdb.StringFieldIndex{ + Field: "Kind", + Lowercase: true, + }, + }, + indexLink: { + Name: indexLink, + AllowMissing: true, + Unique: false, + Indexer: &ConfigEntryLinkIndex{}, + }, + indexIntentionLegacyID: { + Name: indexIntentionLegacyID, + AllowMissing: true, + Unique: true, + Indexer: &ServiceIntentionLegacyIDIndex{}, + }, + indexSource: { + Name: indexSource, + AllowMissing: true, + Unique: false, + Indexer: &ServiceIntentionSourceIndex{}, + }, + }, + } +} From 09425b22a114845f5c5af6f99f3f1e6173921e19 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 28 Jan 2021 20:34:34 -0500 Subject: [PATCH 2/2] state: rename config-entries table const to match new pattern --- agent/consul/state/config_entry.go | 18 ++++++------- agent/consul/state/config_entry_intention.go | 12 ++++----- agent/consul/state/config_entry_oss.go | 8 +++--- agent/consul/state/config_entry_schema.go | 4 +-- agent/consul/state/intention.go | 2 +- agent/consul/state/intention_test.go | 27 ++++++++++---------- 6 files changed, 36 insertions(+), 35 deletions(-) diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index cb70373b4..0b825b6d4 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -80,7 +80,7 @@ func init() { // ConfigEntries is used to pull all the config entries for the snapshot. func (s *Snapshot) ConfigEntries() ([]structs.ConfigEntry, error) { - entries, err := s.tx.Get(configTableName, "id") + entries, err := s.tx.Get(tableConfigEntries, "id") if err != nil { return nil, err } @@ -107,7 +107,7 @@ func (s *Store) ConfigEntry(ws memdb.WatchSet, kind, name string, entMeta *struc func configEntryTxn(tx ReadTxn, ws memdb.WatchSet, kind, name string, entMeta *structs.EnterpriseMeta) (uint64, structs.ConfigEntry, error) { // Get the index - idx := maxIndexTxn(tx, configTableName) + idx := maxIndexTxn(tx, tableConfigEntries) // Get the existing config entry. watchCh, existing, err := firstWatchConfigEntryWithTxn(tx, kind, name, entMeta) @@ -142,7 +142,7 @@ func (s *Store) ConfigEntriesByKind(ws memdb.WatchSet, kind string, entMeta *str func configEntriesByKindTxn(tx ReadTxn, ws memdb.WatchSet, kind string, entMeta *structs.EnterpriseMeta) (uint64, []structs.ConfigEntry, error) { // Get the index and watch for updates - idx := maxIndexWatchTxn(tx, ws, configTableName) + idx := maxIndexWatchTxn(tx, ws, tableConfigEntries) // Lookup by kind, or all if kind is empty var iter memdb.ResultIterator @@ -296,10 +296,10 @@ func deleteConfigEntryTxn(tx WriteTxn, idx uint64, kind, name string, entMeta *s } // Delete the config entry from the DB and update the index. - if err := tx.Delete(configTableName, existing); err != nil { + if err := tx.Delete(tableConfigEntries, existing); err != nil { return fmt.Errorf("failed removing config entry: %s", err) } - if err := tx.Insert("index", &IndexEntry{configTableName, idx}); err != nil { + if err := tx.Insert("index", &IndexEntry{tableConfigEntries, idx}); err != nil { return fmt.Errorf("failed updating index: %s", err) } @@ -320,10 +320,10 @@ func insertConfigEntryWithTxn(tx WriteTxn, idx uint64, conf structs.ConfigEntry) } // Insert the config entry and update the index - if err := tx.Insert(configTableName, conf); err != nil { + if err := tx.Insert(tableConfigEntries, conf); err != nil { return fmt.Errorf("failed inserting config entry: %s", err) } - if err := indexUpdateMaxTxn(tx, idx, configTableName); err != nil { + if err := indexUpdateMaxTxn(tx, idx, tableConfigEntries); err != nil { return fmt.Errorf("failed updating index: %v", err) } @@ -429,7 +429,7 @@ func (s *Store) discoveryChainSourcesTxn(tx ReadTxn, ws memdb.WatchSet, dc strin queue := []structs.ServiceName{destination} for len(queue) > 0 { // The "link" index returns config entries that reference a service - iter, err := tx.Get(configTableName, "link", queue[0].ToServiceID()) + iter, err := tx.Get(tableConfigEntries, "link", queue[0].ToServiceID()) if err != nil { return 0, nil, err } @@ -601,7 +601,7 @@ func validateProposedConfigEntryInServiceGraph( sid := structs.NewServiceID(name, entMeta) checkChains[sid] = struct{}{} - iter, err := tx.Get(configTableName, "link", sid) + iter, err := tx.Get(tableConfigEntries, "link", sid) if err != nil { return err } diff --git a/agent/consul/state/config_entry_intention.go b/agent/consul/state/config_entry_intention.go index c22e48427..fa5625c91 100644 --- a/agent/consul/state/config_entry_intention.go +++ b/agent/consul/state/config_entry_intention.go @@ -126,7 +126,7 @@ func (s *ServiceIntentionSourceIndex) FromArgs(args ...interface{}) ([]byte, err func (s *Store) configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, bool, error) { // unrolled part of configEntriesByKindTxn - idx := maxIndexTxn(tx, configTableName) + idx := maxIndexTxn(tx, tableConfigEntries) iter, err := getConfigEntryKindsWithTxn(tx, structs.ServiceIntentions, structs.WildcardEnterpriseMeta()) if err != nil { @@ -145,12 +145,12 @@ func (s *Store) configIntentionsListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta * } func (s *Store) configIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *structs.ServiceIntentionsConfigEntry, *structs.Intention, error) { - idx := maxIndexTxn(tx, configTableName) + idx := maxIndexTxn(tx, tableConfigEntries) if idx < 1 { idx = 1 } - watchCh, existing, err := tx.FirstWatch(configTableName, "intention-legacy-id", id) + watchCh, existing, err := tx.FirstWatch(tableConfigEntries, "intention-legacy-id", id) if err != nil { return 0, nil, nil, fmt.Errorf("failed config entry lookup: %s", err) } @@ -239,7 +239,7 @@ func configIntentionMatchOneTxn( } func readSourceIntentionsFromConfigEntriesTxn(tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, error) { - idx := maxIndexTxn(tx, configTableName) + idx := maxIndexTxn(tx, tableConfigEntries) var ( results structs.Intentions @@ -265,7 +265,7 @@ func readSourceIntentionsFromConfigEntriesTxn(tx ReadTxn, ws memdb.WatchSet, ser func readSourceIntentionsFromConfigEntriesForServiceTxn(tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta *structs.EnterpriseMeta, results structs.Intentions) (structs.Intentions, error) { sn := structs.NewServiceName(serviceName, entMeta) - iter, err := tx.Get(configTableName, "intention-source", sn) + iter, err := tx.Get(tableConfigEntries, "intention-source", sn) if err != nil { return nil, fmt.Errorf("failed config entry lookup: %s", err) } @@ -284,7 +284,7 @@ func readSourceIntentionsFromConfigEntriesForServiceTxn(tx ReadTxn, ws memdb.Wat } func readDestinationIntentionsFromConfigEntriesTxn(tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta *structs.EnterpriseMeta) (uint64, structs.Intentions, error) { - idx := maxIndexTxn(tx, configTableName) + idx := maxIndexTxn(tx, tableConfigEntries) var results structs.Intentions diff --git a/agent/consul/state/config_entry_oss.go b/agent/consul/state/config_entry_oss.go index eefe83272..f58913b76 100644 --- a/agent/consul/state/config_entry_oss.go +++ b/agent/consul/state/config_entry_oss.go @@ -9,7 +9,7 @@ import ( ) func firstConfigEntryWithTxn(tx ReadTxn, kind, name string, _ *structs.EnterpriseMeta) (interface{}, error) { - return tx.First(configTableName, "id", kind, name) + return tx.First(tableConfigEntries, "id", kind, name) } func firstWatchConfigEntryWithTxn( @@ -18,7 +18,7 @@ func firstWatchConfigEntryWithTxn( name string, _ *structs.EnterpriseMeta, ) (<-chan struct{}, interface{}, error) { - return tx.FirstWatch(configTableName, "id", kind, name) + return tx.FirstWatch(tableConfigEntries, "id", kind, name) } func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error { @@ -26,11 +26,11 @@ func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error { } func getAllConfigEntriesWithTxn(tx ReadTxn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { - return tx.Get(configTableName, "id") + return tx.Get(tableConfigEntries, "id") } func getConfigEntryKindsWithTxn(tx ReadTxn, kind string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { - return tx.Get(configTableName, "kind", kind) + return tx.Get(tableConfigEntries, "kind", kind) } func configIntentionsConvertToList(iter memdb.ResultIterator, _ *structs.EnterpriseMeta) structs.Intentions { diff --git a/agent/consul/state/config_entry_schema.go b/agent/consul/state/config_entry_schema.go index 72a6320b0..6e349a615 100644 --- a/agent/consul/state/config_entry_schema.go +++ b/agent/consul/state/config_entry_schema.go @@ -3,7 +3,7 @@ package state import "github.com/hashicorp/go-memdb" const ( - configTableName = "config-entries" + tableConfigEntries = "config-entries" indexLink = "link" indexIntentionLegacyID = "intention-legacy-id" @@ -14,7 +14,7 @@ const ( // config entries. func configTableSchema() *memdb.TableSchema { return &memdb.TableSchema{ - Name: configTableName, + Name: tableConfigEntries, Indexes: map[string]*memdb.IndexSchema{ indexID: { Name: indexID, diff --git a/agent/consul/state/intention.go b/agent/consul/state/intention.go index f1f67866c..ca7a948d2 100644 --- a/agent/consul/state/intention.go +++ b/agent/consul/state/intention.go @@ -718,7 +718,7 @@ func (s *Store) LegacyIntentionDeleteAll(idx uint64) error { // Also bump the index for the config entry table so that // secondaries can correctly know when they've replicated all of the service-intentions // config entries that USED to exist in the old intentions table. - if err := tx.Insert("index", &IndexEntry{configTableName, idx}); err != nil { + if err := tx.Insert("index", &IndexEntry{tableConfigEntries, idx}); err != nil { return fmt.Errorf("failed updating index: %s", err) } diff --git a/agent/consul/state/intention_test.go b/agent/consul/state/intention_test.go index 23b9caa75..e7c9fd0f3 100644 --- a/agent/consul/state/intention_test.go +++ b/agent/consul/state/intention_test.go @@ -4,13 +4,14 @@ import ( "testing" "time" + "github.com/hashicorp/go-memdb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/sdk/testutil" - "github.com/hashicorp/go-memdb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var ( @@ -90,7 +91,7 @@ func TestStore_IntentionSetGet_basic(t *testing.T) { // Make sure the right index got updated. require.Equal(t, lastIndex, s.maxIndex(intentionsTableName)) - require.Equal(t, uint64(0), s.maxIndex(configTableName)) + require.Equal(t, uint64(0), s.maxIndex(tableConfigEntries)) expected = &structs.Intention{ ID: legacyIxn.ID, @@ -131,7 +132,7 @@ func TestStore_IntentionSetGet_basic(t *testing.T) { require.NoError(t, s.EnsureConfigEntry(lastIndex, configEntry.Clone(), nil)) // Make sure the config entry index got updated instead of the old intentions one - require.Equal(t, lastIndex, s.maxIndex(configTableName)) + require.Equal(t, lastIndex, s.maxIndex(tableConfigEntries)) require.Equal(t, uint64(0), s.maxIndex(intentionsTableName)) expected = &structs.Intention{ @@ -178,7 +179,7 @@ func TestStore_IntentionSetGet_basic(t *testing.T) { // Make sure the index got updated. require.Equal(t, lastIndex, s.maxIndex(intentionsTableName)) - require.Equal(t, uint64(0), s.maxIndex(configTableName)) + require.Equal(t, uint64(0), s.maxIndex(tableConfigEntries)) expected.SourceNS = legacyIxn.SourceNS expected.Action = structs.IntentionActionDeny @@ -201,7 +202,7 @@ func TestStore_IntentionSetGet_basic(t *testing.T) { require.NoError(t, s.EnsureConfigEntry(lastIndex, configEntry.Clone(), nil)) // Make sure the config entry index got updated instead of the old intentions one - require.Equal(t, lastIndex, s.maxIndex(configTableName)) + require.Equal(t, lastIndex, s.maxIndex(tableConfigEntries)) require.Equal(t, uint64(0), s.maxIndex(intentionsTableName)) expected.Description = configEntry.Sources[0].Description @@ -240,7 +241,7 @@ func TestStore_IntentionSetGet_basic(t *testing.T) { // Make sure the index did NOT get updated. require.Equal(t, lastIndex-1, s.maxIndex(intentionsTableName)) - require.Equal(t, uint64(0), s.maxIndex(configTableName)) + require.Equal(t, uint64(0), s.maxIndex(tableConfigEntries)) require.False(t, watchFired(ws), "watch not fired") } }) @@ -815,7 +816,7 @@ func TestStore_LegacyIntentionSet_emptyId(t *testing.T) { // Index is not updated if nothing is saved. require.Equal(t, s.maxIndex(intentionsTableName), uint64(0)) - require.Equal(t, uint64(0), s.maxIndex(configTableName)) + require.Equal(t, uint64(0), s.maxIndex(tableConfigEntries)) require.False(t, watchFired(ws), "watch fired") } @@ -1005,7 +1006,7 @@ func TestStore_IntentionDelete(t *testing.T) { // Make sure the index got updated. require.Equal(t, s.maxIndex(intentionsTableName), lastIndex) - require.Equal(t, uint64(0), s.maxIndex(configTableName)) + require.Equal(t, uint64(0), s.maxIndex(tableConfigEntries)) } else { conf := &structs.ServiceIntentionsConfigEntry{ Kind: structs.ServiceIntentions, @@ -1027,7 +1028,7 @@ func TestStore_IntentionDelete(t *testing.T) { require.NoError(t, s.EnsureConfigEntry(1, conf.Clone(), nil)) // Make sure the index got updated. - require.Equal(t, s.maxIndex(configTableName), lastIndex) + require.Equal(t, s.maxIndex(tableConfigEntries), lastIndex) require.Equal(t, uint64(0), s.maxIndex(intentionsTableName)) } require.True(t, watchFired(ws), "watch fired") @@ -1045,13 +1046,13 @@ func TestStore_IntentionDelete(t *testing.T) { // Make sure the index got updated. require.Equal(t, s.maxIndex(intentionsTableName), lastIndex) - require.Equal(t, uint64(0), s.maxIndex(configTableName)) + require.Equal(t, uint64(0), s.maxIndex(tableConfigEntries)) } else { lastIndex++ require.NoError(t, s.DeleteConfigEntry(lastIndex, structs.ServiceIntentions, "web", nil)) // Make sure the index got updated. - require.Equal(t, s.maxIndex(configTableName), lastIndex) + require.Equal(t, s.maxIndex(tableConfigEntries), lastIndex) require.Equal(t, uint64(0), s.maxIndex(intentionsTableName)) } require.True(t, watchFired(ws), "watch fired")