Add experiment system + events experiment (#18682)

changelog/18682.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+core: Add experiments system and `events.beta1` experiment.
+```
+

command/commands.go

@@ -87,6 +87,12 @@ const (
 	// EnvVaultLogLevel is used to specify the log level applied to logging
 	// Supported log levels: Trace, Debug, Error, Warn, Info
 	EnvVaultLogLevel = "VAULT_LOG_LEVEL"
+	// EnvVaultExperiments defines the experiments to enable for a server as a
+	// comma separated list. See experiments.ValidExperiments() for the list of
+	// valid experiments. Not mutable or persisted in storage, only read and
+	// logged at startup _per node_. This was initially introduced for the events
+	// system being developed over multiple release cycles.
+	EnvVaultExperiments = "VAULT_EXPERIMENTS"
 
 	// DisableSSCTokens is an env var used to disable index bearing
 	// token functionality

command/server.go

@@ -36,6 +36,7 @@ import (
 	"github.com/hashicorp/vault/command/server"
 	"github.com/hashicorp/vault/helper/builtinplugins"
 	"github.com/hashicorp/vault/helper/constants"
+	"github.com/hashicorp/vault/helper/experiments"
 	loghelper "github.com/hashicorp/vault/helper/logging"
 	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
@@ -116,6 +117,7 @@ type ServerCommand struct {
 
 	flagConfigs            []string
 	flagRecovery           bool
+	flagExperiments        []string
 	flagDev                bool
 	flagDevTLS             bool
 	flagDevTLSCertDir      string
@@ -204,6 +206,17 @@ func (c *ServerCommand) Flags() *FlagSets {
 			"Using a recovery operation token, \"sys/raw\" API can be used to manipulate the storage.",
 	})
 
+	f.StringSliceVar(&StringSliceVar{
+		Name:       "experiment",
+		Target:     &c.flagExperiments,
+		Completion: complete.PredictSet(experiments.ValidExperiments()...),
+		Usage: "Name of an experiment to enable. Experiments should NOT be used in production, and " +
+			"the associated APIs may have backwards incompatible changes between releases. This " +
+			"flag can be specified multiple times to specify multiple experiments. This can also be " +
+			fmt.Sprintf("specified via the %s environment variable as a comma-separated list. ", EnvVaultExperiments) +
+			"Valid experiments are: " + strings.Join(experiments.ValidExperiments(), ", "),
+	})
+
 	f = set.NewFlagSet("Dev Options")
 
 	f.BoolVar(&BoolVar{
@@ -1105,6 +1118,11 @@ func (c *ServerCommand) Run(args []string) int {
 		}
 	}
 
+	if err := server.ExperimentsFromEnvAndCLI(config, EnvVaultExperiments, c.flagExperiments); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
 	// If mlockall(2) isn't supported, show a warning. We disable this in dev
 	// because it is quite scary to see when first using Vault. We also disable
 	// this if the user has explicitly disabled mlock in configuration.
@@ -1173,6 +1191,12 @@ func (c *ServerCommand) Run(args []string) int {
 	info[key] = strings.Join(envVarKeys, ", ")
 	infoKeys = append(infoKeys, key)
 
+	if len(config.Experiments) != 0 {
+		expKey := "experiments"
+		info[expKey] = strings.Join(config.Experiments, ", ")
+		infoKeys = append(infoKeys, expKey)
+	}
+
 	barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(c, config, infoKeys, info)
 	// Check error here
 	if err != nil {
@@ -2637,6 +2661,7 @@ func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.
 		License:                        config.License,
 		LicensePath:                    config.LicensePath,
 		DisableSSCTokens:               config.DisableSSCTokens,
+		Experiments:                    config.Experiments,
 	}
 
 	if c.flagDev {

command/server/config.go

@@ -17,9 +17,11 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/hcl/hcl/ast"
+	"github.com/hashicorp/vault/helper/experiments"
 	"github.com/hashicorp/vault/helper/osutil"
 	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/sdk/helper/consts"
+	"github.com/hashicorp/vault/sdk/helper/strutil"
 )
 
 const (
@@ -28,9 +30,14 @@ const (
 	VaultDevKeyFilename  = "vault-key.pem"
 )
 
-var entConfigValidate = func(_ *Config, _ string) []configutil.ConfigError {
+var (
+	entConfigValidate = func(_ *Config, _ string) []configutil.ConfigError {
 		return nil
-}
+	}
+
+	// Modified internally for testing.
+	validExperiments = experiments.ValidExperiments()
+)
 
 // Config is the configuration for the vault server.
 type Config struct {
@@ -45,6 +52,8 @@ type Config struct {
 
 	ServiceRegistration *ServiceRegistration `hcl:"-"`
 
+	Experiments []string `hcl:"experiments"`
+
 	CacheSize                int         `hcl:"cache_size"`
 	DisableCache             bool        `hcl:"-"`
 	DisableCacheRaw          interface{} `hcl:"disable_cache"`
@@ -433,6 +442,8 @@ func (c *Config) Merge(c2 *Config) *Config {
 
 	result.entConfig = c.entConfig.Merge(c2.entConfig)
 
+	result.Experiments = mergeExperiments(c.Experiments, c2.Experiments)
+
 	return result
 }
 
@@ -699,6 +710,10 @@ func ParseConfig(d, source string) (*Config, error) {
 		}
 	}
 
+	if err := validateExperiments(result.Experiments); err != nil {
+		return nil, fmt.Errorf("error validating experiment(s) from config: %w", err)
+	}
+
 	if err := result.parseConfig(list); err != nil {
 		return nil, fmt.Errorf("error parsing enterprise config: %w", err)
 	}
@@ -715,6 +730,69 @@ func ParseConfig(d, source string) (*Config, error) {
 	return result, nil
 }
 
+func ExperimentsFromEnvAndCLI(config *Config, envKey string, flagExperiments []string) error {
+	if envExperimentsRaw := os.Getenv(envKey); envExperimentsRaw != "" {
+		envExperiments := strings.Split(envExperimentsRaw, ",")
+		err := validateExperiments(envExperiments)
+		if err != nil {
+			return fmt.Errorf("error validating experiment(s) from environment variable %q: %w", envKey, err)
+		}
+
+		config.Experiments = mergeExperiments(config.Experiments, envExperiments)
+	}
+
+	if len(flagExperiments) != 0 {
+		err := validateExperiments(flagExperiments)
+		if err != nil {
+			return fmt.Errorf("error validating experiment(s) from command line flag: %w", err)
+		}
+
+		config.Experiments = mergeExperiments(config.Experiments, flagExperiments)
+	}
+
+	return nil
+}
+
+// Validate checks each experiment is a known experiment.
+func validateExperiments(experiments []string) error {
+	var invalid []string
+
+	for _, experiment := range experiments {
+		if !strutil.StrListContains(validExperiments, experiment) {
+			invalid = append(invalid, experiment)
+		}
+	}
+
+	if len(invalid) != 0 {
+		return fmt.Errorf("valid experiment(s) are %s, but received the following invalid experiment(s): %s",
+			strings.Join(validExperiments, ", "),
+			strings.Join(invalid, ", "))
+	}
+
+	return nil
+}
+
+// mergeExperiments returns the logical OR of the two sets.
+func mergeExperiments(left, right []string) []string {
+	processed := map[string]struct{}{}
+	var result []string
+	for _, l := range left {
+		if _, seen := processed[l]; !seen {
+			result = append(result, l)
+		}
+		processed[l] = struct{}{}
+	}
+
+	for _, r := range right {
+		if _, seen := processed[r]; !seen {
+			result = append(result, r)
+			processed[r] = struct{}{}
+		}
+	}
+
+	return result
+}
+
 // LoadConfigDir loads all the configurations in the given directory
 // in alphabetical order.
 func LoadConfigDir(dir string) (*Config, error) {
@@ -1032,6 +1110,7 @@ func (c *Config) Sanitized() map[string]interface{} {
 		"enable_response_header_raft_node_id": c.EnableResponseHeaderRaftNodeID,
 
 		"log_requests_level": c.LogRequestsLevel,
+		"experiments":        c.Experiments,
 
 		"detect_deadlocks": c.DetectDeadlocks,
 	}

command/server/config_test.go

@@ -1,6 +1,9 @@
 package server
 
 import (
+	"fmt"
+	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -71,3 +74,112 @@ func TestUnknownFieldValidationHcl(t *testing.T) {
 func TestUnknownFieldValidationListenerAndStorage(t *testing.T) {
 	testUnknownFieldValidationStorageAndListener(t)
 }
+
+func TestExperimentsConfigParsing(t *testing.T) {
+	const envKey = "VAULT_EXPERIMENTS"
+	originalValue := validExperiments
+	validExperiments = []string{"foo", "bar", "baz"}
+	t.Cleanup(func() {
+		validExperiments = originalValue
+	})
+
+	for name, tc := range map[string]struct {
+		fromConfig    []string
+		fromEnv       []string
+		fromCLI       []string
+		expected      []string
+		expectedError string
+	}{
+		// Multiple sources.
+		"duplication":  {[]string{"foo"}, []string{"foo"}, []string{"foo"}, []string{"foo"}, ""},
+		"disjoint set": {[]string{"foo"}, []string{"bar"}, []string{"baz"}, []string{"foo", "bar", "baz"}, ""},
+
+		// Single source.
+		"config only": {[]string{"foo"}, nil, nil, []string{"foo"}, ""},
+		"env only":    {nil, []string{"foo"}, nil, []string{"foo"}, ""},
+		"CLI only":    {nil, nil, []string{"foo"}, []string{"foo"}, ""},
+
+		// Validation errors.
+		"config invalid": {[]string{"invalid"}, nil, nil, nil, "from config"},
+		"env invalid":    {nil, []string{"invalid"}, nil, nil, "from environment variable"},
+		"CLI invalid":    {nil, nil, []string{"invalid"}, nil, "from command line flag"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			var configString string
+			t.Setenv(envKey, strings.Join(tc.fromEnv, ","))
+			if len(tc.fromConfig) != 0 {
+				configString = fmt.Sprintf("experiments = [\"%s\"]", strings.Join(tc.fromConfig, "\", \""))
+			}
+			config, err := ParseConfig(configString, "")
+			if err == nil {
+				err = ExperimentsFromEnvAndCLI(config, envKey, tc.fromCLI)
+			}
+
+			switch tc.expectedError {
+			case "":
+				if err != nil {
+					t.Fatal(err)
+				}
+
+			default:
+				if err == nil || !strings.Contains(err.Error(), tc.expectedError) {
+					t.Fatalf("Expected error to contain %q, but got: %s", tc.expectedError, err)
+				}
+			}
+		})
+	}
+}
+
+func TestValidate(t *testing.T) {
+	originalValue := validExperiments
+	for name, tc := range map[string]struct {
+		validSet    []string
+		input       []string
+		expectError bool
+	}{
+		// Valid cases
+		"minimal valid": {[]string{"foo"}, []string{"foo"}, false},
+		"valid subset":  {[]string{"foo", "bar"}, []string{"bar"}, false},
+		"repeated":      {[]string{"foo"}, []string{"foo", "foo"}, false},
+
+		// Error cases
+		"partially valid":      {[]string{"foo", "bar"}, []string{"foo", "baz"}, true},
+		"empty":                {[]string{"foo"}, []string{""}, true},
+		"no valid experiments": {[]string{}, []string{"foo"}, true},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Cleanup(func() {
+				validExperiments = originalValue
+			})
+
+			validExperiments = tc.validSet
+			err := validateExperiments(tc.input)
+			if tc.expectError && err == nil {
+				t.Fatal("Expected error but got none")
+			}
+			if !tc.expectError && err != nil {
+				t.Fatal("Did not expect error but got", err)
+			}
+		})
+	}
+}
+
+func TestMerge(t *testing.T) {
+	for name, tc := range map[string]struct {
+		left     []string
+		right    []string
+		expected []string
+	}{
+		"disjoint":    {[]string{"foo"}, []string{"bar"}, []string{"foo", "bar"}},
+		"empty left":  {[]string{}, []string{"foo"}, []string{"foo"}},
+		"empty right": {[]string{"foo"}, []string{}, []string{"foo"}},
+		"overlapping": {[]string{"foo", "bar"}, []string{"foo", "baz"}, []string{"foo", "bar", "baz"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			result := mergeExperiments(tc.left, tc.right)
+			if !reflect.DeepEqual(tc.expected, result) {
+				t.Fatalf("Expected %v but got %v", tc.expected, result)
+			}
+		})
+	}
+}

command/server/config_test_helpers.go

@@ -738,6 +738,7 @@ func testConfig_Sanitized(t *testing.T) {
 		"disable_indexing":                    false,
 		"disable_mlock":                       true,
 		"disable_performance_standby":         false,
+		"experiments":                         []string(nil),
 		"plugin_file_uid":                     0,
 		"plugin_file_permissions":             0,
 		"disable_printable_check":             false,

helper/experiments/experiments.go

@@ -0,0 +1,16 @@
+package experiments
+
+const VaultExperimentEventsBeta1 = "events.beta1"
+
+var validExperiments = []string{
+	VaultExperimentEventsBeta1,
+}
+
+// ValidExperiments exposes the list without exposing a mutable global variable.
+// Experiments can only be enabled when starting a server, and will typically
+// enable pre-GA API functionality.
+func ValidExperiments() []string {
+	result := make([]string, len(validExperiments))
+	copy(result, validExperiments)
+	return result
+}

http/sys_config_state_test.go

@@ -38,6 +38,7 @@ func TestSysConfigState_Sanitized(t *testing.T) {
 		"disable_performance_standby":         false,
 		"disable_printable_check":             false,
 		"disable_sealwrap":                    false,
+		"experiments":                         nil,
 		"raw_storage_endpoint":                false,
 		"detect_deadlocks":                    "",
 		"introspection_endpoint":              false,

vault/core.go

@@ -673,6 +673,8 @@ type Core struct {
 
 	rollbackPeriod time.Duration
 
+	experiments []string
+
 	pendingRemovalMountsAllowed bool
 	expirationRevokeRetryBase   time.Duration
 }
@@ -823,6 +825,8 @@ type CoreConfig struct {
 
 	RollbackPeriod time.Duration
 
+	Experiments []string
+
 	PendingRemovalMountsAllowed bool
 
 	ExpirationRevokeRetryBase time.Duration
@@ -989,6 +993,7 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
 		disableSSCTokens:               conf.DisableSSCTokens,
 		effectiveSDKVersion:            effectiveSDKVersion,
 		userFailedLoginInfo:            make(map[FailedLoginUser]*FailedLoginInfo),
+		experiments:                    conf.Experiments,
 		pendingRemovalMountsAllowed:    conf.PendingRemovalMountsAllowed,
 		expirationRevokeRetryBase:      conf.ExpirationRevokeRetryBase,
 	}
@@ -3871,6 +3876,10 @@ func (c *Core) GetHCPLinkStatus() (string, string) {
 	return status, resourceID
 }
 
+func (c *Core) isExperimentEnabled(experiment string) bool {
+	return strutil.StrListContains(c.experiments, experiment)
+}
+
 // ListenerAddresses provides a slice of configured listener addresses
 func (c *Core) ListenerAddresses() ([]string, error) {
 	addresses := make([]string, 0)

vault/logical_system.go

@@ -27,6 +27,7 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 	semver "github.com/hashicorp/go-version"
+	"github.com/hashicorp/vault/helper/experiments"
 	"github.com/hashicorp/vault/helper/hostutil"
 	"github.com/hashicorp/vault/helper/identity"
 	"github.com/hashicorp/vault/helper/logging"
@@ -143,6 +144,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend {
 				"unseal",
 				"leader",
 				"health",
+				"experiments",
 				"generate-root/attempt",
 				"generate-root/update",
 				"rekey/init",
@@ -192,6 +194,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend {
 	b.Backend.Paths = append(b.Backend.Paths, b.quotasPaths()...)
 	b.Backend.Paths = append(b.Backend.Paths, b.rootActivityPaths()...)
 	b.Backend.Paths = append(b.Backend.Paths, b.loginMFAPaths()...)
+	b.Backend.Paths = append(b.Backend.Paths, b.experimentPaths()...)
 	b.Backend.Paths = append(b.Backend.Paths, b.introspectionPaths()...)
 
 	if core.rawEnabled {
@@ -5086,6 +5089,24 @@ func (b *SystemBackend) handleLoggersByNameDelete(ctx context.Context, req *logi
 	return nil, nil
 }
 
+// handleReadExperiments returns the available and enabled experiments on this node.
+// Each node within a cluster could have different values for each, but it's not
+// recommended.
+func (b *SystemBackend) handleReadExperiments(ctx context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) {
+	enabled := b.Core.experiments
+	if len(enabled) == 0 {
+		// Return empty slice instead of nil, so the JSON shows [] instead of null
+		enabled = []string{}
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"available": experiments.ValidExperiments(),
+			"enabled":   enabled,
+		},
+	}, nil
+}
+
 func sanitizePath(path string) string {
 	if !strings.HasSuffix(path, "/") {
 		path += "/"
@@ -5947,4 +5968,12 @@ This path responds to the following HTTP methods.
         Returns a list historical version changes sorted by installation time in ascending order.
 		`,
 	},
+	"experiments": {
+		"Returns information about Vault's experimental features. Should NOT be used in production.",
+		`
+This path responds to the following HTTP methods.
+		GET /
+			Returns the available and enabled experiments.
+		`,
+	},
 }

vault/logical_system_paths.go

@@ -2050,6 +2050,22 @@ func (b *SystemBackend) mountPaths() []*framework.Path {
 	}
 }
 
+func (b *SystemBackend) experimentPaths() []*framework.Path {
+	return []*framework.Path{
+		{
+			Pattern: "experiments$",
+			Operations: map[logical.Operation]framework.OperationHandler{
+				logical.ReadOperation: &framework.PathOperation{
+					Callback: b.handleReadExperiments,
+					Summary:  "Returns the available and enabled experiments",
+				},
+			},
+			HelpSynopsis:    strings.TrimSpace(sysHelp["experiments"][0]),
+			HelpDescription: strings.TrimSpace(sysHelp["experiments"][1]),
+		},
+	}
+}
+
 func (b *SystemBackend) lockedUserPaths() []*framework.Path {
 	return []*framework.Path{
 		{

vault/logical_system_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hashicorp/vault/audit"
 	credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
 	"github.com/hashicorp/vault/helper/builtinplugins"
+	"github.com/hashicorp/vault/helper/experiments"
 	"github.com/hashicorp/vault/helper/identity"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/random"
@@ -5444,3 +5445,32 @@ func TestCanUnseal_WithNonExistentBuiltinPluginVersion_InMountStorage(t *testing
 		}
 	}
 }
+
+func TestSystemBackend_ReadExperiments(t *testing.T) {
+	c, _, _ := TestCoreUnsealed(t)
+
+	for name, tc := range map[string][]string{
+		"no experiments enabled": {},
+		"one experiment enabled": {experiments.VaultExperimentEventsBeta1},
+	} {
+		t.Run(name, func(t *testing.T) {
+			// Set the enabled experiments.
+			c.experiments = tc
+
+			req := logical.TestRequest(t, logical.ReadOperation, "experiments")
+			resp, err := c.systemBackend.HandleRequest(namespace.RootContext(nil), req)
+			if err != nil {
+				t.Fatalf("err: %v", err)
+			}
+			if resp == nil {
+				t.Fatal("Expected a response")
+			}
+			if !reflect.DeepEqual(experiments.ValidExperiments(), resp.Data["available"]) {
+				t.Fatalf("Expected %v but got %v", experiments.ValidExperiments(), resp.Data["available"])
+			}
+			if !reflect.DeepEqual(tc, resp.Data["enabled"]) {
+				t.Fatal("No experiments should be enabled by default")
+			}
+		})
+	}
+}

website/content/api-docs/system/experiments.mdx

@@ -0,0 +1,46 @@
+---
+layout: api
+page_title: /sys/experiments - HTTP API
+description: The `/sys/experiments` endpoint returns information about experiments on the Vault node.
+---
+
+# `/sys/experiments`
+
+The `/sys/experiments` endpoint returns information about experiments on the Vault node.
+
+## Read Experiments
+
+This endpoint returns the experiments available and enabled on the Vault node.
+Experiments are per-node and cannot be changed while the node is running. See
+the [`-experiment`](/docs/commands/server#experiment) flag and the
+[`experiments`](/docs/configuration#experiments) config key documentation for
+details on enabling experiments.
+
+| Method | Path               |
+| :----- | :----------------- |
+| `GET`  | `/sys/experiments` |
+
+### Sample Request
+
+```shell-session
+$ curl \
+    http://127.0.0.1:8200/v1/sys/experiments
+```
+
+### Sample Response
+
+```json
+{
+  "request_id": "cb48b1e2-635c-52e9-db79-ad9a54ed3e88",
+  "lease_id": "",
+  "lease_duration": 0,
+  "renewable": false,
+  "data": {
+    "available": [
+      "events.beta1"
+    ],
+    "enabled": []
+  },
+  "warnings": null
+}
+```

website/content/docs/commands/server.mdx

@@ -81,6 +81,13 @@ flags](/docs/commands) included on all commands.
   number of older log file archives to keep. Defaults to 0 (no files are ever deleted).
   Set to -1 to discard old log files when a new one is created.
 
+- `-experiment` `(string array: [])` - The name of an experiment to enable for this node.
+  This flag can be specified multiple times to enable multiple experiments. Experiments
+  should NOT be used in production, and the associated APIs may have backwards incompatible
+  changes between releases. Additional experiments can also be specified via the
+  `VAULT_EXPERIMENTS` environment variable as a comma-separated list, or via the
+  [`experiments`](/docs/configuration#experiments) config key.
+
 - `VAULT_ALLOW_PENDING_REMOVAL_MOUNTS` `(bool: false)` - (environment variable)
   Allow Vault to be started with builtin engines which have the `Pending Removal`
   deprecation state. This is a temporary stopgap in place in order to perform an

website/content/docs/configuration/index.mdx

@@ -205,6 +205,12 @@ a negative effect on performance due to the tracking of each lock attempt.
 
 - `log_rotate_max_files` - Equivalent to the [`-log-rotate-max-files` command-line flag](/docs/commands/server#_log_rotate_max_files).
 
+- `experiments` `(string array: [])` - The list of experiments to enable for this node.
+  Experiments should NOT be used in production, and the associated APIs may have backwards
+  incompatible changes between releases. Additional experiments can also be specified via
+  the `VAULT_EXPERIMENTS` environment variable as a comma-separated list, or via the
+  [`-experiment`](/docs/commands/server#experiment) flag.
+
 ### High Availability Parameters
 
 The following parameters are used on backends that support [high availability][high-availability].

website/data/api-docs-nav-data.json

@@ -446,6 +446,10 @@
         "title": "<code>/sys/control-group</code>",
         "path": "system/control-group"
       },
+      {
+        "title": "<code>/sys/experiments</code>",
+        "path": "system/experiments"
+      },
       {
         "title": "<code>/sys/generate-recovery-token</code>",
         "path": "system/generate-recovery-token"