[VAULT-2776] Add prefix_filter option to Vault (#12025)

* [VAULT-2776] Add prefix_filter support to vault

* [VAULT-2776] Add filter_default config, update docs

* [VAULT-2776] Add changelog file

* [VAULT-2776] Update telemetry tests and error handling

* [VAULT-2776] Add test fixtures, update test

* [VAULT-2776] Update gitignore hcl filter

.gitignore

@@ -48,7 +48,7 @@ Vagrantfile
 # Configs
 *.hcl
 !command/agent/config/test-fixtures/*.hcl
-!command/server/test-fixtures/*.hcl
+!command/server/test-fixtures/**/*.hcl
 
 
 .DS_Store

changelog/12025.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: Add `prefix_filter` to telemetry config
+```

command/server/config_telemetry_test.go

@@ -0,0 +1,41 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMetricFilterConfigs(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		configFile            string
+		expectedFilterDefault *bool
+		expectedPrefixFilter  []string
+	}{
+		{
+			"./test-fixtures/telemetry/valid_prefix_filter.hcl",
+			nil,
+			[]string{"-vault.expire", "-vault.audit", "+vault.expire.num_irrevocable_leases"},
+		},
+		{
+			"./test-fixtures/telemetry/filter_default_override.hcl",
+			boolPointer(false),
+			[]string(nil),
+		},
+	}
+	t.Run("validate metric filter configs", func(t *testing.T) {
+		t.Parallel()
+
+		for _, tc := range cases {
+			config, err := LoadConfigFile(tc.configFile)
+
+			if err != nil {
+				t.Fatalf("Error encountered when loading config %+v", err)
+			}
+
+			assert.Equal(t, tc.expectedFilterDefault, config.SharedConfig.Telemetry.FilterDefault)
+			assert.Equal(t, tc.expectedPrefixFilter, config.SharedConfig.Telemetry.PrefixFilter)
+		}
+	})
+}

command/server/config_test_helpers.go

@@ -16,6 +16,10 @@ import (
 	"github.com/hashicorp/vault/internalshared/configutil"
 )
 
+func boolPointer(x bool) *bool {
+	return &x
+}
+
 func testConfigRaftRetryJoin(t *testing.T) {
 	config, err := LoadConfigFile("./test-fixtures/raft_retry_join.hcl")
 	if err != nil {

command/server/test-fixtures/telemetry/filter_default_override.hcl

@@ -0,0 +1,7 @@
+disable_mlock = true
+ui            = true
+
+telemetry {
+  statsd_address = "foo"
+  filter_default = false
+}

command/server/test-fixtures/telemetry/valid_prefix_filter.hcl

@@ -0,0 +1,7 @@
+disable_mlock = true
+ui            = true
+
+telemetry {
+  statsd_address = "foo"
+  prefix_filter = ["-vault.expire", "-vault.audit", "+vault.expire.num_irrevocable_leases"]
+}

internalshared/configutil/telemetry.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/hashicorp/vault/sdk/helper/parseutil"
 	"time"
 
+	"github.com/hashicorp/vault/sdk/helper/parseutil"
+
 	monitoring "cloud.google.com/go/monitoring/apiv3"
 	"github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/circonus"
@@ -150,6 +151,14 @@ type Telemetry struct {
 
 	// Whether or not telemetry should add labels for namespaces
 	LeaseMetricsNameSpaceLabels bool `hcl:"add_lease_metrics_namespace_labels"`
+
+	// FilterDefault is the default for whether to allow a metric that's not
+	// covered by the prefix filter.
+	FilterDefault *bool `hcl:"filter_default"`
+
+	// PrefixFilter is a list of filter rules to apply for allowing
+	// or blocking metrics by prefix.
+	PrefixFilter []string `hcl:"prefix_filter"`
 }
 
 func (t *Telemetry) Validate(source string) []ConfigError {
@@ -259,6 +268,9 @@ func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil.
 	metricsConf := metrics.DefaultConfig(opts.ServiceName)
 	metricsConf.EnableHostname = !opts.Config.DisableHostname
 	metricsConf.EnableHostnameLabel = opts.Config.EnableHostnameLabel
+	if opts.Config.FilterDefault != nil {
+		metricsConf.FilterDefault = *opts.Config.FilterDefault
+	}
 
 	// Configure the statsite sink
 	var fanout metrics.FanoutSink
@@ -388,5 +400,32 @@ func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil.
 	wrapper.TelemetryConsts.LeaseMetricsNameSpaceLabels = opts.Config.LeaseMetricsNameSpaceLabels
 	wrapper.TelemetryConsts.NumLeaseMetricsTimeBuckets = opts.Config.NumLeaseMetricsTimeBuckets
 
+	// Parse the metric filters
+	telemetryAllowedPrefixes, telemetryBlockedPrefixes, err := parsePrefixFilter(opts.Config.PrefixFilter)
+
+	if err != nil {
+		return nil, nil, false, err
+	}
+
+	metrics.UpdateFilter(telemetryAllowedPrefixes, telemetryBlockedPrefixes)
 	return inm, wrapper, prometheusEnabled, nil
 }
+
+func parsePrefixFilter(prefixFilters []string) ([]string, []string, error) {
+	var telemetryAllowedPrefixes, telemetryBlockedPrefixes []string
+
+	for _, rule := range prefixFilters {
+		if rule == "" {
+			return nil, nil, fmt.Errorf("Cannot have empty filter rule in prefix_filter")
+		}
+		switch rule[0] {
+		case '+':
+			telemetryAllowedPrefixes = append(telemetryAllowedPrefixes, rule[1:])
+		case '-':
+			telemetryBlockedPrefixes = append(telemetryBlockedPrefixes, rule[1:])
+		default:
+			return nil, nil, fmt.Errorf("Filter rule must begin with either '+' or '-': %q", rule)
+		}
+	}
+	return telemetryAllowedPrefixes, telemetryBlockedPrefixes, nil
+}

internalshared/configutil/telemetry_test.go

@@ -0,0 +1,53 @@
+package configutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParsePrefixFilters(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		inputFilters            []string
+		expectedErrStr          string
+		expectedAllowedPrefixes []string
+		expectedBlockedPrefixes []string
+	}{
+		{
+			[]string{""},
+			"Cannot have empty filter rule in prefix_filter",
+			[]string(nil),
+			[]string(nil),
+		},
+		{
+			[]string{"vault.abc"},
+			"Filter rule must begin with either '+' or '-': \"vault.abc\"",
+			[]string(nil),
+			[]string(nil),
+		},
+		{
+			[]string{"+vault.abc", "-vault.bcd"},
+			"",
+			[]string{"vault.abc"},
+			[]string{"vault.bcd"},
+		},
+	}
+	t.Run("validate metric filter configs", func(t *testing.T) {
+		t.Parallel()
+
+		for _, tc := range cases {
+
+			allowedPrefixes, blockedPrefixes, err := parsePrefixFilter(tc.inputFilters)
+
+			if err != nil {
+				assert.EqualError(t, err, tc.expectedErrStr)
+			} else {
+				assert.Equal(t, "", tc.expectedErrStr)
+				assert.Equal(t, tc.expectedAllowedPrefixes, allowedPrefixes)
+
+				assert.Equal(t, tc.expectedBlockedPrefixes, blockedPrefixes)
+			}
+		}
+	})
+}

website/content/docs/configuration/telemetry.mdx

@@ -51,6 +51,17 @@ The following options are available on all telemetry configurations.
 - `add_lease_metrics_namespace_labels` `(bool: false)` - If this value is set to true, then `vault.expire.leases.by_expiration`
   will break down expiring leases by both time and namespace. This parameter is disabled by default because enabling it can lead
   to a large-cardinality metric.
+- `filter_default` `(bool: true)` - This controls whether to allow metrics that have not been specified by the filter.
+  Defaults to `true`, which will allow all metrics when no filters are provided.
+  When set to `false` with no filters, no metrics will be sent.
+- `prefix_filter` `(string array: [])` - This is a list of filter rules to apply for allowing/blocking metrics by
+  prefix in the following format:
+  ```json
+  ["+vault.token", "-vault.expire", "+vault.expire.num_leases"]
+  ```
+  A leading "**+**" will enable any metrics with the given prefix, and a leading "**-**" will block them. 
+  If there is overlap between two rules, the more specific rule will take precedence. Blocking will take priority if the same prefix is listed multiple times.
+
 
 ### `statsite`