From 698fc291a9eea090d1004b42279f7b66553e35bf Mon Sep 17 00:00:00 2001 From: Connor Kelly Date: Fri, 17 Sep 2021 14:36:34 -0500 Subject: [PATCH 1/6] Add KVUsage to consul state usage metrics This change will add the number of entries in the consul KV store to the already existing usage metrics. --- agent/consul/state/usage.go | 29 +++ agent/consul/state/usage_oss.go | 8 + agent/consul/state/usage_test.go | 38 ++++ agent/consul/usagemetrics/usagemetrics.go | 13 ++ agent/consul/usagemetrics/usagemetrics_oss.go | 9 + .../usagemetrics/usagemetrics_oss_test.go | 174 ++++++++++++++++++ 6 files changed, 271 insertions(+) diff --git a/agent/consul/state/usage.go b/agent/consul/state/usage.go index 5e1509766..9d5277fa0 100644 --- a/agent/consul/state/usage.go +++ b/agent/consul/state/usage.go @@ -10,6 +10,7 @@ import ( const ( serviceNamesUsageTable = "service-names" + kvUsageTable = "kv-entries" tableUsage = "usage" ) @@ -54,6 +55,11 @@ type NodeUsage struct { EnterpriseNodeUsage } +type KVUsage struct { + KVCount int + EnterpriseKVUsage +} + type uniqueServiceState int const ( @@ -95,6 +101,9 @@ func updateUsage(tx WriteTxn, changes Changes) error { } else { serviceNameChanges[svc.CompoundServiceName()] += delta } + case "kvs": + usageDeltas[change.Table] += delta + addEnterpriseKVUsage(usageDeltas, change) } } @@ -269,6 +278,26 @@ func (s *Store) ServiceUsage() (uint64, ServiceUsage, error) { return serviceInstances.Index, results, nil } +func (s *Store) KVUsage() (uint64, KVUsage, error) { + tx := s.db.ReadTxn() + defer tx.Abort() + + kvs, err := firstUsageEntry(tx, "kvs") + if err != nil { + return 0, KVUsage{}, fmt.Errorf("failed kvs lookup: %s", err) + } + + usage := KVUsage{ + KVCount: kvs.Count, + } + results, err := compileEnterpriseKVUsage(tx, usage) + if err != nil { + return 0, KVUsage{}, fmt.Errorf("failed kvs lookup: %s", err) + } + + return kvs.Index, results, nil +} + func firstUsageEntry(tx ReadTxn, id string) (*UsageEntry, error) { usage, err := tx.First(tableUsage, indexID, id) if err != nil { diff --git a/agent/consul/state/usage_oss.go b/agent/consul/state/usage_oss.go index c45bf74d5..cb8233963 100644 --- a/agent/consul/state/usage_oss.go +++ b/agent/consul/state/usage_oss.go @@ -1,3 +1,4 @@ +//go:build !consulent // +build !consulent package state @@ -10,6 +11,7 @@ import ( type EnterpriseServiceUsage struct{} type EnterpriseNodeUsage struct{} +type EnterpriseKVUsage struct{} func addEnterpriseNodeUsage(map[string]int, memdb.Change) {} @@ -17,6 +19,8 @@ func addEnterpriseServiceInstanceUsage(map[string]int, memdb.Change) {} func addEnterpriseServiceUsage(map[string]int, map[structs.ServiceName]uniqueServiceState) {} +func addEnterpriseKVUsage(map[string]int, memdb.Change) {} + func compileEnterpriseServiceUsage(tx ReadTxn, usage ServiceUsage) (ServiceUsage, error) { return usage, nil } @@ -24,3 +28,7 @@ func compileEnterpriseServiceUsage(tx ReadTxn, usage ServiceUsage) (ServiceUsage func compileEnterpriseNodeUsage(tx ReadTxn, usage NodeUsage) (NodeUsage, error) { return usage, nil } + +func compileEnterpriseKVUsage(tx ReadTxn, usage KVUsage) (KVUsage, error) { + return usage, nil +} diff --git a/agent/consul/state/usage_test.go b/agent/consul/state/usage_test.go index 2d650a591..5b231d568 100644 --- a/agent/consul/state/usage_test.go +++ b/agent/consul/state/usage_test.go @@ -45,6 +45,44 @@ func TestStateStore_Usage_NodeUsage_Delete(t *testing.T) { require.Equal(t, usage.Nodes, 1) } +func TestStateStore_Usage_KVUsage(t *testing.T) { + s := testStateStore(t) + + // No nodes have been registered, and thus no usage entry exists + idx, usage, err := s.KVUsage() + require.NoError(t, err) + require.Equal(t, idx, uint64(0)) + require.Equal(t, usage.KVCount, 0) + + testSetKey(t, s, 0, "key-1", "0", nil) + testSetKey(t, s, 1, "key-2", "0", nil) + testSetKey(t, s, 2, "key-2", "1", nil) + + idx, usage, err = s.KVUsage() + require.NoError(t, err) + require.Equal(t, idx, uint64(2)) + require.Equal(t, usage.KVCount, 2) +} + +func TestStateStore_Usage_KVUsage_Delete(t *testing.T) { + s := testStateStore(t) + + testSetKey(t, s, 0, "key-1", "0", nil) + testSetKey(t, s, 1, "key-2", "0", nil) + testSetKey(t, s, 2, "key-2", "1", nil) + + idx, usage, err := s.KVUsage() + require.NoError(t, err) + require.Equal(t, idx, uint64(2)) + require.Equal(t, usage.KVCount, 2) + + require.NoError(t, s.KVSDelete(3, "key-2", nil)) + idx, usage, err = s.KVUsage() + require.NoError(t, err) + require.Equal(t, idx, uint64(3)) + require.Equal(t, usage.KVCount, 1) +} + func TestStateStore_Usage_ServiceUsageEmpty(t *testing.T) { s := testStateStore(t) diff --git a/agent/consul/usagemetrics/usagemetrics.go b/agent/consul/usagemetrics/usagemetrics.go index 9669125c6..78fa7ffc3 100644 --- a/agent/consul/usagemetrics/usagemetrics.go +++ b/agent/consul/usagemetrics/usagemetrics.go @@ -36,6 +36,10 @@ var Gauges = []prometheus.GaugeDefinition{ Name: []string{"consul", "members", "servers"}, Help: "Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6.", }, + { + Name: []string{"consul", "kv", "entries"}, + Help: "Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.10.3.", + }, } type getMembersFunc func() []serf.Member @@ -145,6 +149,7 @@ func (u *UsageMetricsReporter) Run(ctx context.Context) { } func (u *UsageMetricsReporter) runOnce() { + u.logger.Debug("Starting usage run") state := u.stateProvider.State() _, nodeUsage, err := state.NodeUsage() @@ -163,6 +168,14 @@ func (u *UsageMetricsReporter) runOnce() { members := u.memberUsage() u.emitMemberUsage(members) + + _, kvUsage, err := state.KVUsage() + if err != nil { + u.logger.Warn("failed to retrieve kv entries from state store", "error", err) + } + + u.emitKVUsage(kvUsage) + } func (u *UsageMetricsReporter) memberUsage() []serf.Member { diff --git a/agent/consul/usagemetrics/usagemetrics_oss.go b/agent/consul/usagemetrics/usagemetrics_oss.go index 68c885211..d936a1632 100644 --- a/agent/consul/usagemetrics/usagemetrics_oss.go +++ b/agent/consul/usagemetrics/usagemetrics_oss.go @@ -1,3 +1,4 @@ +//go:build !consulent // +build !consulent package usagemetrics @@ -58,3 +59,11 @@ func (u *UsageMetricsReporter) emitServiceUsage(serviceUsage state.ServiceUsage) u.metricLabels, ) } + +func (u *UsageMetricsReporter) emitKVUsage(kvUsage state.KVUsage) { + metrics.SetGaugeWithLabels( + []string{"consul", "state", "kv_entries"}, + float32(kvUsage.KVCount), + u.metricLabels, + ) +} diff --git a/agent/consul/usagemetrics/usagemetrics_oss_test.go b/agent/consul/usagemetrics/usagemetrics_oss_test.go index 810c05f3c..d36441ff0 100644 --- a/agent/consul/usagemetrics/usagemetrics_oss_test.go +++ b/agent/consul/usagemetrics/usagemetrics_oss_test.go @@ -1,3 +1,4 @@ +//go:build !consulent // +build !consulent package usagemetrics @@ -57,6 +58,11 @@ func TestUsageReporter_emitNodeUsage_OSS(t *testing.T) { Value: 0, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, }, + "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.consul.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, }, getMembersFunc: func() []serf.Member { return []serf.Member{} }, }, @@ -114,6 +120,11 @@ func TestUsageReporter_emitNodeUsage_OSS(t *testing.T) { Value: 0, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, }, + "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.consul.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, }, }, } @@ -199,6 +210,11 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { {Name: "datacenter", Value: "dc1"}, }, }, + "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.consul.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, }, getMembersFunc: func() []serf.Member { return []serf.Member{} }, }, @@ -276,6 +292,11 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { {Name: "datacenter", Value: "dc1"}, }, }, + "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.consul.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, }, }, } @@ -314,3 +335,156 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { }) } } + +func TestUsageReporter_emitKVUsage_OSS(t *testing.T) { + type testCase struct { + modfiyStateStore func(t *testing.T, s *state.Store) + getMembersFunc getMembersFunc + expectedGauges map[string]metrics.GaugeValue + } + cases := map[string]testCase{ + "empty-state": { + expectedGauges: map[string]metrics.GaugeValue{ + // --- node --- + "consul.usage.test.consul.state.nodes;datacenter=dc1": { + Name: "consul.usage.test.consul.state.nodes", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- member --- + "consul.usage.test.consul.members.clients;datacenter=dc1": { + Name: "consul.usage.test.consul.members.clients", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.consul.members.servers;datacenter=dc1": { + Name: "consul.usage.test.consul.members.servers", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- service --- + "consul.usage.test.consul.state.services;datacenter=dc1": { + Name: "consul.usage.test.consul.state.services", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.consul.state.service_instances;datacenter=dc1": { + Name: "consul.usage.test.consul.state.service_instances", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.consul.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + }, + getMembersFunc: func() []serf.Member { return []serf.Member{} }, + }, + "nodes": { + modfiyStateStore: func(t *testing.T, s *state.Store) { + require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"})) + require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"})) + require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"})) + + require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}})) + require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}})) + require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}})) + require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}})) + require.NoError(t, s.KVSDelete(8, "d", &structs.EnterpriseMeta{})) + require.NoError(t, s.KVSDelete(9, "c", &structs.EnterpriseMeta{})) + require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}})) + require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}})) + }, + getMembersFunc: func() []serf.Member { + return []serf.Member{ + { + Name: "foo", + Tags: map[string]string{"role": "consul"}, + Status: serf.StatusAlive, + }, + { + Name: "bar", + Tags: map[string]string{"role": "consul"}, + Status: serf.StatusAlive, + }, + { + Name: "baz", + Tags: map[string]string{"role": "node"}, + Status: serf.StatusAlive, + }, + } + }, + expectedGauges: map[string]metrics.GaugeValue{ + // --- node --- + "consul.usage.test.consul.state.nodes;datacenter=dc1": { + Name: "consul.usage.test.consul.state.nodes", + Value: 3, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- member --- + "consul.usage.test.consul.members.servers;datacenter=dc1": { + Name: "consul.usage.test.consul.members.servers", + Value: 2, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.consul.members.clients;datacenter=dc1": { + Name: "consul.usage.test.consul.members.clients", + Value: 1, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- service --- + "consul.usage.test.consul.state.services;datacenter=dc1": { + Name: "consul.usage.test.consul.state.services", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.consul.state.service_instances;datacenter=dc1": { + Name: "consul.usage.test.consul.state.service_instances", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.consul.state.kv_entries", + Value: 5, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + }, + }, + } + + for name, tcase := range cases { + t.Run(name, func(t *testing.T) { + // Only have a single interval for the test + sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) + cfg := metrics.DefaultConfig("consul.usage.test") + cfg.EnableHostname = false + metrics.NewGlobal(cfg, sink) + + mockStateProvider := &mockStateProvider{} + s, err := newStateStore() + require.NoError(t, err) + if tcase.modfiyStateStore != nil { + tcase.modfiyStateStore(t, s) + } + mockStateProvider.On("State").Return(s) + + reporter, err := NewUsageMetricsReporter( + new(Config). + WithStateProvider(mockStateProvider). + WithLogger(testutil.Logger(t)). + WithDatacenter("dc1"). + WithGetMembersFunc(tcase.getMembersFunc), + ) + require.NoError(t, err) + + reporter.runOnce() + + intervals := sink.Data() + require.Len(t, intervals, 1) + intv := intervals[0] + + assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) + }) + } +} From 5e6828a1173c21f6d5d8887cc2c77bc84eae1132 Mon Sep 17 00:00:00 2001 From: Connor Kelly Date: Mon, 20 Sep 2021 12:43:40 -0500 Subject: [PATCH 2/6] Add new telemetry to website This will add information about the new kv_usage metric to the website --- website/content/docs/agent/telemetry.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index 9d4e0c4b4..8cce2ed43 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -313,6 +313,7 @@ This is a full list of metrics emitted by Consul. | `consul.state.nodes` | Measures the current number of nodes registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | | `consul.state.services` | Measures the current number of unique services registered with Consul, based on service name. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | | `consul.state.service_instances` | Measures the current number of unique service instances registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | +| `consul.state.kv_entries` | Measures the current number of unique KV entries written in Consul. It is only emitted by Consul servers. Added in v1.10.3. | number of objects | gauge | | `consul.members.clients` | Measures the current number of client agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of clients | gauge | | `consul.members.servers` | Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of servers | gauge | | `consul.dns.stale_queries` | Increments when an agent serves a query within the allowed stale threshold. | queries | counter | From 9fb8656915a3779feff482ad0d734604b14d0e21 Mon Sep 17 00:00:00 2001 From: Connor Kelly Date: Mon, 20 Sep 2021 12:52:19 -0500 Subject: [PATCH 3/6] Add changelog entry --- .changelog/11090.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/11090.txt diff --git a/.changelog/11090.txt b/.changelog/11090.txt new file mode 100644 index 000000000..e6d8ee20c --- /dev/null +++ b/.changelog/11090.txt @@ -0,0 +1,3 @@ +```release-note:improvement +telemetry: Add new metrics for the count of KV entries in the Consul store. +``` From 973b7b5c78298a1ea0642775e63b397b8da4fbc5 Mon Sep 17 00:00:00 2001 From: Connor Kelly Date: Mon, 20 Sep 2021 13:44:43 -0500 Subject: [PATCH 4/6] Fix test --- agent/consul/usagemetrics/usagemetrics_oss_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/consul/usagemetrics/usagemetrics_oss_test.go b/agent/consul/usagemetrics/usagemetrics_oss_test.go index d36441ff0..ed2bf7b16 100644 --- a/agent/consul/usagemetrics/usagemetrics_oss_test.go +++ b/agent/consul/usagemetrics/usagemetrics_oss_test.go @@ -446,7 +446,7 @@ func TestUsageReporter_emitKVUsage_OSS(t *testing.T) { }, "consul.usage.test.consul.state.kv_entries;datacenter=dc1": { Name: "consul.usage.test.consul.state.kv_entries", - Value: 5, + Value: 4, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, }, }, From 64852cd3e56e51532b5b670b3932838fdcb90dad Mon Sep 17 00:00:00 2001 From: Connor Date: Tue, 21 Sep 2021 10:52:46 -0500 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: Matt Keeler --- agent/consul/state/usage_test.go | 2 +- agent/consul/usagemetrics/usagemetrics.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/consul/state/usage_test.go b/agent/consul/state/usage_test.go index 5b231d568..28d492e20 100644 --- a/agent/consul/state/usage_test.go +++ b/agent/consul/state/usage_test.go @@ -48,7 +48,7 @@ func TestStateStore_Usage_NodeUsage_Delete(t *testing.T) { func TestStateStore_Usage_KVUsage(t *testing.T) { s := testStateStore(t) - // No nodes have been registered, and thus no usage entry exists + // No keys have been registered, and thus no usage entry exists idx, usage, err := s.KVUsage() require.NoError(t, err) require.Equal(t, idx, uint64(0)) diff --git a/agent/consul/usagemetrics/usagemetrics.go b/agent/consul/usagemetrics/usagemetrics.go index 78fa7ffc3..53c011199 100644 --- a/agent/consul/usagemetrics/usagemetrics.go +++ b/agent/consul/usagemetrics/usagemetrics.go @@ -149,7 +149,7 @@ func (u *UsageMetricsReporter) Run(ctx context.Context) { } func (u *UsageMetricsReporter) runOnce() { - u.logger.Debug("Starting usage run") + u.logger.Trace("Starting usage run") state := u.stateProvider.State() _, nodeUsage, err := state.NodeUsage() @@ -171,7 +171,7 @@ func (u *UsageMetricsReporter) runOnce() { _, kvUsage, err := state.KVUsage() if err != nil { - u.logger.Warn("failed to retrieve kv entries from state store", "error", err) + u.logger.Warn("failed to retrieve kv entry usage from state store", "error", err) } u.emitKVUsage(kvUsage) From bfe6b64ca725f2196a53a81c9dde764b8dc71fe8 Mon Sep 17 00:00:00 2001 From: Connor Kelly Date: Wed, 22 Sep 2021 11:04:48 -0500 Subject: [PATCH 6/6] Strip out go 1.17 bits --- agent/consul/state/usage_oss.go | 1 - agent/consul/usagemetrics/usagemetrics_oss.go | 1 - agent/consul/usagemetrics/usagemetrics_oss_test.go | 1 - 3 files changed, 3 deletions(-) diff --git a/agent/consul/state/usage_oss.go b/agent/consul/state/usage_oss.go index cb8233963..4fc35b7d7 100644 --- a/agent/consul/state/usage_oss.go +++ b/agent/consul/state/usage_oss.go @@ -1,4 +1,3 @@ -//go:build !consulent // +build !consulent package state diff --git a/agent/consul/usagemetrics/usagemetrics_oss.go b/agent/consul/usagemetrics/usagemetrics_oss.go index d936a1632..427bde74a 100644 --- a/agent/consul/usagemetrics/usagemetrics_oss.go +++ b/agent/consul/usagemetrics/usagemetrics_oss.go @@ -1,4 +1,3 @@ -//go:build !consulent // +build !consulent package usagemetrics diff --git a/agent/consul/usagemetrics/usagemetrics_oss_test.go b/agent/consul/usagemetrics/usagemetrics_oss_test.go index ed2bf7b16..708985ecf 100644 --- a/agent/consul/usagemetrics/usagemetrics_oss_test.go +++ b/agent/consul/usagemetrics/usagemetrics_oss_test.go @@ -1,4 +1,3 @@ -//go:build !consulent // +build !consulent package usagemetrics