From d5c35f39c34e7fa1bb9e44a760d3637070814413 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 16 Jan 2023 16:07:18 +0000 Subject: [PATCH] Add experiment system + events experiment (#18682) --- changelog/18682.txt | 4 + command/commands.go | 6 + command/server.go | 25 ++++ command/server/config.go | 85 ++++++++++++- command/server/config_test.go | 112 ++++++++++++++++++ command/server/config_test_helpers.go | 1 + helper/experiments/experiments.go | 16 +++ http/sys_config_state_test.go | 1 + vault/core.go | 9 ++ vault/logical_system.go | 29 +++++ vault/logical_system_paths.go | 16 +++ vault/logical_system_test.go | 30 +++++ .../content/api-docs/system/experiments.mdx | 46 +++++++ website/content/docs/commands/server.mdx | 7 ++ website/content/docs/configuration/index.mdx | 6 + website/data/api-docs-nav-data.json | 4 + 16 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 changelog/18682.txt create mode 100644 helper/experiments/experiments.go create mode 100644 website/content/api-docs/system/experiments.mdx diff --git a/changelog/18682.txt b/changelog/18682.txt new file mode 100644 index 000000000..d1a7987d6 --- /dev/null +++ b/changelog/18682.txt @@ -0,0 +1,4 @@ +```release-note:improvement +core: Add experiments system and `events.beta1` experiment. +``` + diff --git a/command/commands.go b/command/commands.go index d11388e98..5b309c6cd 100644 --- a/command/commands.go +++ b/command/commands.go @@ -87,6 +87,12 @@ const ( // EnvVaultLogLevel is used to specify the log level applied to logging // Supported log levels: Trace, Debug, Error, Warn, Info EnvVaultLogLevel = "VAULT_LOG_LEVEL" + // EnvVaultExperiments defines the experiments to enable for a server as a + // comma separated list. See experiments.ValidExperiments() for the list of + // valid experiments. Not mutable or persisted in storage, only read and + // logged at startup _per node_. This was initially introduced for the events + // system being developed over multiple release cycles. + EnvVaultExperiments = "VAULT_EXPERIMENTS" // DisableSSCTokens is an env var used to disable index bearing // token functionality diff --git a/command/server.go b/command/server.go index eff271e7e..e3da4072a 100644 --- a/command/server.go +++ b/command/server.go @@ -36,6 +36,7 @@ import ( "github.com/hashicorp/vault/command/server" "github.com/hashicorp/vault/helper/builtinplugins" "github.com/hashicorp/vault/helper/constants" + "github.com/hashicorp/vault/helper/experiments" loghelper "github.com/hashicorp/vault/helper/logging" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" @@ -116,6 +117,7 @@ type ServerCommand struct { flagConfigs []string flagRecovery bool + flagExperiments []string flagDev bool flagDevTLS bool flagDevTLSCertDir string @@ -204,6 +206,17 @@ func (c *ServerCommand) Flags() *FlagSets { "Using a recovery operation token, \"sys/raw\" API can be used to manipulate the storage.", }) + f.StringSliceVar(&StringSliceVar{ + Name: "experiment", + Target: &c.flagExperiments, + Completion: complete.PredictSet(experiments.ValidExperiments()...), + Usage: "Name of an experiment to enable. Experiments should NOT be used in production, and " + + "the associated APIs may have backwards incompatible changes between releases. This " + + "flag can be specified multiple times to specify multiple experiments. This can also be " + + fmt.Sprintf("specified via the %s environment variable as a comma-separated list. ", EnvVaultExperiments) + + "Valid experiments are: " + strings.Join(experiments.ValidExperiments(), ", "), + }) + f = set.NewFlagSet("Dev Options") f.BoolVar(&BoolVar{ @@ -1105,6 +1118,11 @@ func (c *ServerCommand) Run(args []string) int { } } + if err := server.ExperimentsFromEnvAndCLI(config, EnvVaultExperiments, c.flagExperiments); err != nil { + c.UI.Error(err.Error()) + return 1 + } + // If mlockall(2) isn't supported, show a warning. We disable this in dev // because it is quite scary to see when first using Vault. We also disable // this if the user has explicitly disabled mlock in configuration. @@ -1173,6 +1191,12 @@ func (c *ServerCommand) Run(args []string) int { info[key] = strings.Join(envVarKeys, ", ") infoKeys = append(infoKeys, key) + if len(config.Experiments) != 0 { + expKey := "experiments" + info[expKey] = strings.Join(config.Experiments, ", ") + infoKeys = append(infoKeys, expKey) + } + barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(c, config, infoKeys, info) // Check error here if err != nil { @@ -2637,6 +2661,7 @@ func createCoreConfig(c *ServerCommand, config *server.Config, backend physical. License: config.License, LicensePath: config.LicensePath, DisableSSCTokens: config.DisableSSCTokens, + Experiments: config.Experiments, } if c.flagDev { diff --git a/command/server/config.go b/command/server/config.go index 63b43def4..d2d30b40e 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -17,9 +17,11 @@ import ( "github.com/hashicorp/go-secure-stdlib/parseutil" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/vault/helper/experiments" "github.com/hashicorp/vault/helper/osutil" "github.com/hashicorp/vault/internalshared/configutil" "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/hashicorp/vault/sdk/helper/strutil" ) const ( @@ -28,9 +30,14 @@ const ( VaultDevKeyFilename = "vault-key.pem" ) -var entConfigValidate = func(_ *Config, _ string) []configutil.ConfigError { - return nil -} +var ( + entConfigValidate = func(_ *Config, _ string) []configutil.ConfigError { + return nil + } + + // Modified internally for testing. + validExperiments = experiments.ValidExperiments() +) // Config is the configuration for the vault server. type Config struct { @@ -45,6 +52,8 @@ type Config struct { ServiceRegistration *ServiceRegistration `hcl:"-"` + Experiments []string `hcl:"experiments"` + CacheSize int `hcl:"cache_size"` DisableCache bool `hcl:"-"` DisableCacheRaw interface{} `hcl:"disable_cache"` @@ -433,6 +442,8 @@ func (c *Config) Merge(c2 *Config) *Config { result.entConfig = c.entConfig.Merge(c2.entConfig) + result.Experiments = mergeExperiments(c.Experiments, c2.Experiments) + return result } @@ -699,6 +710,10 @@ func ParseConfig(d, source string) (*Config, error) { } } + if err := validateExperiments(result.Experiments); err != nil { + return nil, fmt.Errorf("error validating experiment(s) from config: %w", err) + } + if err := result.parseConfig(list); err != nil { return nil, fmt.Errorf("error parsing enterprise config: %w", err) } @@ -715,6 +730,69 @@ func ParseConfig(d, source string) (*Config, error) { return result, nil } +func ExperimentsFromEnvAndCLI(config *Config, envKey string, flagExperiments []string) error { + if envExperimentsRaw := os.Getenv(envKey); envExperimentsRaw != "" { + envExperiments := strings.Split(envExperimentsRaw, ",") + err := validateExperiments(envExperiments) + if err != nil { + return fmt.Errorf("error validating experiment(s) from environment variable %q: %w", envKey, err) + } + + config.Experiments = mergeExperiments(config.Experiments, envExperiments) + } + + if len(flagExperiments) != 0 { + err := validateExperiments(flagExperiments) + if err != nil { + return fmt.Errorf("error validating experiment(s) from command line flag: %w", err) + } + + config.Experiments = mergeExperiments(config.Experiments, flagExperiments) + } + + return nil +} + +// Validate checks each experiment is a known experiment. +func validateExperiments(experiments []string) error { + var invalid []string + + for _, experiment := range experiments { + if !strutil.StrListContains(validExperiments, experiment) { + invalid = append(invalid, experiment) + } + } + + if len(invalid) != 0 { + return fmt.Errorf("valid experiment(s) are %s, but received the following invalid experiment(s): %s", + strings.Join(validExperiments, ", "), + strings.Join(invalid, ", ")) + } + + return nil +} + +// mergeExperiments returns the logical OR of the two sets. +func mergeExperiments(left, right []string) []string { + processed := map[string]struct{}{} + var result []string + for _, l := range left { + if _, seen := processed[l]; !seen { + result = append(result, l) + } + processed[l] = struct{}{} + } + + for _, r := range right { + if _, seen := processed[r]; !seen { + result = append(result, r) + processed[r] = struct{}{} + } + } + + return result +} + // LoadConfigDir loads all the configurations in the given directory // in alphabetical order. func LoadConfigDir(dir string) (*Config, error) { @@ -1032,6 +1110,7 @@ func (c *Config) Sanitized() map[string]interface{} { "enable_response_header_raft_node_id": c.EnableResponseHeaderRaftNodeID, "log_requests_level": c.LogRequestsLevel, + "experiments": c.Experiments, "detect_deadlocks": c.DetectDeadlocks, } diff --git a/command/server/config_test.go b/command/server/config_test.go index 21ebd38b6..5b3aeb54b 100644 --- a/command/server/config_test.go +++ b/command/server/config_test.go @@ -1,6 +1,9 @@ package server import ( + "fmt" + "reflect" + "strings" "testing" ) @@ -71,3 +74,112 @@ func TestUnknownFieldValidationHcl(t *testing.T) { func TestUnknownFieldValidationListenerAndStorage(t *testing.T) { testUnknownFieldValidationStorageAndListener(t) } + +func TestExperimentsConfigParsing(t *testing.T) { + const envKey = "VAULT_EXPERIMENTS" + originalValue := validExperiments + validExperiments = []string{"foo", "bar", "baz"} + t.Cleanup(func() { + validExperiments = originalValue + }) + + for name, tc := range map[string]struct { + fromConfig []string + fromEnv []string + fromCLI []string + expected []string + expectedError string + }{ + // Multiple sources. + "duplication": {[]string{"foo"}, []string{"foo"}, []string{"foo"}, []string{"foo"}, ""}, + "disjoint set": {[]string{"foo"}, []string{"bar"}, []string{"baz"}, []string{"foo", "bar", "baz"}, ""}, + + // Single source. + "config only": {[]string{"foo"}, nil, nil, []string{"foo"}, ""}, + "env only": {nil, []string{"foo"}, nil, []string{"foo"}, ""}, + "CLI only": {nil, nil, []string{"foo"}, []string{"foo"}, ""}, + + // Validation errors. + "config invalid": {[]string{"invalid"}, nil, nil, nil, "from config"}, + "env invalid": {nil, []string{"invalid"}, nil, nil, "from environment variable"}, + "CLI invalid": {nil, nil, []string{"invalid"}, nil, "from command line flag"}, + } { + t.Run(name, func(t *testing.T) { + var configString string + t.Setenv(envKey, strings.Join(tc.fromEnv, ",")) + if len(tc.fromConfig) != 0 { + configString = fmt.Sprintf("experiments = [\"%s\"]", strings.Join(tc.fromConfig, "\", \"")) + } + config, err := ParseConfig(configString, "") + if err == nil { + err = ExperimentsFromEnvAndCLI(config, envKey, tc.fromCLI) + } + + switch tc.expectedError { + case "": + if err != nil { + t.Fatal(err) + } + + default: + if err == nil || !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("Expected error to contain %q, but got: %s", tc.expectedError, err) + } + } + }) + } +} + +func TestValidate(t *testing.T) { + originalValue := validExperiments + for name, tc := range map[string]struct { + validSet []string + input []string + expectError bool + }{ + // Valid cases + "minimal valid": {[]string{"foo"}, []string{"foo"}, false}, + "valid subset": {[]string{"foo", "bar"}, []string{"bar"}, false}, + "repeated": {[]string{"foo"}, []string{"foo", "foo"}, false}, + + // Error cases + "partially valid": {[]string{"foo", "bar"}, []string{"foo", "baz"}, true}, + "empty": {[]string{"foo"}, []string{""}, true}, + "no valid experiments": {[]string{}, []string{"foo"}, true}, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(func() { + validExperiments = originalValue + }) + + validExperiments = tc.validSet + err := validateExperiments(tc.input) + if tc.expectError && err == nil { + t.Fatal("Expected error but got none") + } + if !tc.expectError && err != nil { + t.Fatal("Did not expect error but got", err) + } + }) + } +} + +func TestMerge(t *testing.T) { + for name, tc := range map[string]struct { + left []string + right []string + expected []string + }{ + "disjoint": {[]string{"foo"}, []string{"bar"}, []string{"foo", "bar"}}, + "empty left": {[]string{}, []string{"foo"}, []string{"foo"}}, + "empty right": {[]string{"foo"}, []string{}, []string{"foo"}}, + "overlapping": {[]string{"foo", "bar"}, []string{"foo", "baz"}, []string{"foo", "bar", "baz"}}, + } { + t.Run(name, func(t *testing.T) { + result := mergeExperiments(tc.left, tc.right) + if !reflect.DeepEqual(tc.expected, result) { + t.Fatalf("Expected %v but got %v", tc.expected, result) + } + }) + } +} diff --git a/command/server/config_test_helpers.go b/command/server/config_test_helpers.go index bb06dda93..94535b438 100644 --- a/command/server/config_test_helpers.go +++ b/command/server/config_test_helpers.go @@ -738,6 +738,7 @@ func testConfig_Sanitized(t *testing.T) { "disable_indexing": false, "disable_mlock": true, "disable_performance_standby": false, + "experiments": []string(nil), "plugin_file_uid": 0, "plugin_file_permissions": 0, "disable_printable_check": false, diff --git a/helper/experiments/experiments.go b/helper/experiments/experiments.go new file mode 100644 index 000000000..4b7ded689 --- /dev/null +++ b/helper/experiments/experiments.go @@ -0,0 +1,16 @@ +package experiments + +const VaultExperimentEventsBeta1 = "events.beta1" + +var validExperiments = []string{ + VaultExperimentEventsBeta1, +} + +// ValidExperiments exposes the list without exposing a mutable global variable. +// Experiments can only be enabled when starting a server, and will typically +// enable pre-GA API functionality. +func ValidExperiments() []string { + result := make([]string, len(validExperiments)) + copy(result, validExperiments) + return result +} diff --git a/http/sys_config_state_test.go b/http/sys_config_state_test.go index d55897854..08361af49 100644 --- a/http/sys_config_state_test.go +++ b/http/sys_config_state_test.go @@ -38,6 +38,7 @@ func TestSysConfigState_Sanitized(t *testing.T) { "disable_performance_standby": false, "disable_printable_check": false, "disable_sealwrap": false, + "experiments": nil, "raw_storage_endpoint": false, "detect_deadlocks": "", "introspection_endpoint": false, diff --git a/vault/core.go b/vault/core.go index f84672b5b..e837ee98a 100644 --- a/vault/core.go +++ b/vault/core.go @@ -673,6 +673,8 @@ type Core struct { rollbackPeriod time.Duration + experiments []string + pendingRemovalMountsAllowed bool expirationRevokeRetryBase time.Duration } @@ -823,6 +825,8 @@ type CoreConfig struct { RollbackPeriod time.Duration + Experiments []string + PendingRemovalMountsAllowed bool ExpirationRevokeRetryBase time.Duration @@ -989,6 +993,7 @@ func CreateCore(conf *CoreConfig) (*Core, error) { disableSSCTokens: conf.DisableSSCTokens, effectiveSDKVersion: effectiveSDKVersion, userFailedLoginInfo: make(map[FailedLoginUser]*FailedLoginInfo), + experiments: conf.Experiments, pendingRemovalMountsAllowed: conf.PendingRemovalMountsAllowed, expirationRevokeRetryBase: conf.ExpirationRevokeRetryBase, } @@ -3871,6 +3876,10 @@ func (c *Core) GetHCPLinkStatus() (string, string) { return status, resourceID } +func (c *Core) isExperimentEnabled(experiment string) bool { + return strutil.StrListContains(c.experiments, experiment) +} + // ListenerAddresses provides a slice of configured listener addresses func (c *Core) ListenerAddresses() ([]string, error) { addresses := make([]string, 0) diff --git a/vault/logical_system.go b/vault/logical_system.go index 95dc965ec..886a06663 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/go-secure-stdlib/parseutil" "github.com/hashicorp/go-secure-stdlib/strutil" semver "github.com/hashicorp/go-version" + "github.com/hashicorp/vault/helper/experiments" "github.com/hashicorp/vault/helper/hostutil" "github.com/hashicorp/vault/helper/identity" "github.com/hashicorp/vault/helper/logging" @@ -143,6 +144,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend { "unseal", "leader", "health", + "experiments", "generate-root/attempt", "generate-root/update", "rekey/init", @@ -192,6 +194,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend { b.Backend.Paths = append(b.Backend.Paths, b.quotasPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.rootActivityPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.loginMFAPaths()...) + b.Backend.Paths = append(b.Backend.Paths, b.experimentPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.introspectionPaths()...) if core.rawEnabled { @@ -5086,6 +5089,24 @@ func (b *SystemBackend) handleLoggersByNameDelete(ctx context.Context, req *logi return nil, nil } +// handleReadExperiments returns the available and enabled experiments on this node. +// Each node within a cluster could have different values for each, but it's not +// recommended. +func (b *SystemBackend) handleReadExperiments(ctx context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) { + enabled := b.Core.experiments + if len(enabled) == 0 { + // Return empty slice instead of nil, so the JSON shows [] instead of null + enabled = []string{} + } + + return &logical.Response{ + Data: map[string]interface{}{ + "available": experiments.ValidExperiments(), + "enabled": enabled, + }, + }, nil +} + func sanitizePath(path string) string { if !strings.HasSuffix(path, "/") { path += "/" @@ -5947,4 +5968,12 @@ This path responds to the following HTTP methods. Returns a list historical version changes sorted by installation time in ascending order. `, }, + "experiments": { + "Returns information about Vault's experimental features. Should NOT be used in production.", + ` +This path responds to the following HTTP methods. + GET / + Returns the available and enabled experiments. + `, + }, } diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index 5b424c329..f83dd9acc 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -2050,6 +2050,22 @@ func (b *SystemBackend) mountPaths() []*framework.Path { } } +func (b *SystemBackend) experimentPaths() []*framework.Path { + return []*framework.Path{ + { + Pattern: "experiments$", + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handleReadExperiments, + Summary: "Returns the available and enabled experiments", + }, + }, + HelpSynopsis: strings.TrimSpace(sysHelp["experiments"][0]), + HelpDescription: strings.TrimSpace(sysHelp["experiments"][1]), + }, + } +} + func (b *SystemBackend) lockedUserPaths() []*framework.Path { return []*framework.Path{ { diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index e0e4dd50f..75a6b67f9 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -22,6 +22,7 @@ import ( "github.com/hashicorp/vault/audit" credUserpass "github.com/hashicorp/vault/builtin/credential/userpass" "github.com/hashicorp/vault/helper/builtinplugins" + "github.com/hashicorp/vault/helper/experiments" "github.com/hashicorp/vault/helper/identity" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/random" @@ -5444,3 +5445,32 @@ func TestCanUnseal_WithNonExistentBuiltinPluginVersion_InMountStorage(t *testing } } } + +func TestSystemBackend_ReadExperiments(t *testing.T) { + c, _, _ := TestCoreUnsealed(t) + + for name, tc := range map[string][]string{ + "no experiments enabled": {}, + "one experiment enabled": {experiments.VaultExperimentEventsBeta1}, + } { + t.Run(name, func(t *testing.T) { + // Set the enabled experiments. + c.experiments = tc + + req := logical.TestRequest(t, logical.ReadOperation, "experiments") + resp, err := c.systemBackend.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp == nil { + t.Fatal("Expected a response") + } + if !reflect.DeepEqual(experiments.ValidExperiments(), resp.Data["available"]) { + t.Fatalf("Expected %v but got %v", experiments.ValidExperiments(), resp.Data["available"]) + } + if !reflect.DeepEqual(tc, resp.Data["enabled"]) { + t.Fatal("No experiments should be enabled by default") + } + }) + } +} diff --git a/website/content/api-docs/system/experiments.mdx b/website/content/api-docs/system/experiments.mdx new file mode 100644 index 000000000..5a72b1303 --- /dev/null +++ b/website/content/api-docs/system/experiments.mdx @@ -0,0 +1,46 @@ +--- +layout: api +page_title: /sys/experiments - HTTP API +description: The `/sys/experiments` endpoint returns information about experiments on the Vault node. +--- + +# `/sys/experiments` + +The `/sys/experiments` endpoint returns information about experiments on the Vault node. + +## Read Experiments + +This endpoint returns the experiments available and enabled on the Vault node. +Experiments are per-node and cannot be changed while the node is running. See +the [`-experiment`](/docs/commands/server#experiment) flag and the +[`experiments`](/docs/configuration#experiments) config key documentation for +details on enabling experiments. + +| Method | Path | +| :----- | :----------------- | +| `GET` | `/sys/experiments` | + +### Sample Request + +```shell-session +$ curl \ + http://127.0.0.1:8200/v1/sys/experiments +``` + +### Sample Response + +```json +{ + "request_id": "cb48b1e2-635c-52e9-db79-ad9a54ed3e88", + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "data": { + "available": [ + "events.beta1" + ], + "enabled": [] + }, + "warnings": null +} +``` diff --git a/website/content/docs/commands/server.mdx b/website/content/docs/commands/server.mdx index b1dd24bf4..053afbac8 100644 --- a/website/content/docs/commands/server.mdx +++ b/website/content/docs/commands/server.mdx @@ -81,6 +81,13 @@ flags](/docs/commands) included on all commands. number of older log file archives to keep. Defaults to 0 (no files are ever deleted). Set to -1 to discard old log files when a new one is created. +- `-experiment` `(string array: [])` - The name of an experiment to enable for this node. + This flag can be specified multiple times to enable multiple experiments. Experiments + should NOT be used in production, and the associated APIs may have backwards incompatible + changes between releases. Additional experiments can also be specified via the + `VAULT_EXPERIMENTS` environment variable as a comma-separated list, or via the + [`experiments`](/docs/configuration#experiments) config key. + - `VAULT_ALLOW_PENDING_REMOVAL_MOUNTS` `(bool: false)` - (environment variable) Allow Vault to be started with builtin engines which have the `Pending Removal` deprecation state. This is a temporary stopgap in place in order to perform an diff --git a/website/content/docs/configuration/index.mdx b/website/content/docs/configuration/index.mdx index 287d64d1f..291b353f3 100644 --- a/website/content/docs/configuration/index.mdx +++ b/website/content/docs/configuration/index.mdx @@ -205,6 +205,12 @@ a negative effect on performance due to the tracking of each lock attempt. - `log_rotate_max_files` - Equivalent to the [`-log-rotate-max-files` command-line flag](/docs/commands/server#_log_rotate_max_files). +- `experiments` `(string array: [])` - The list of experiments to enable for this node. + Experiments should NOT be used in production, and the associated APIs may have backwards + incompatible changes between releases. Additional experiments can also be specified via + the `VAULT_EXPERIMENTS` environment variable as a comma-separated list, or via the + [`-experiment`](/docs/commands/server#experiment) flag. + ### High Availability Parameters The following parameters are used on backends that support [high availability][high-availability]. diff --git a/website/data/api-docs-nav-data.json b/website/data/api-docs-nav-data.json index 224c75963..d302f4ab9 100644 --- a/website/data/api-docs-nav-data.json +++ b/website/data/api-docs-nav-data.json @@ -446,6 +446,10 @@ "title": "/sys/control-group", "path": "system/control-group" }, + { + "title": "/sys/experiments", + "path": "system/experiments" + }, { "title": "/sys/generate-recovery-token", "path": "system/generate-recovery-token"