From dc85e37cf4edad8b7fb64142d3ee36e8586263fa Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 18 Nov 2022 17:58:16 +0000 Subject: [PATCH] storage/raft: Add retry_join_as_non_voter config option (#18030) --- changelog/18030.txt | 3 + physical/raft/raft.go | 32 +++++++++ physical/raft/raft_test.go | 66 +++++++++++++++++++ vault/raft.go | 2 +- .../content/docs/commands/operator/raft.mdx | 2 + .../docs/configuration/storage/raft.mdx | 10 +++ 6 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 changelog/18030.txt diff --git a/changelog/18030.txt b/changelog/18030.txt new file mode 100644 index 000000000..30a49c5d1 --- /dev/null +++ b/changelog/18030.txt @@ -0,0 +1,3 @@ +```release-note:improvement +storage/raft: Add `retry_join_as_non_voter` config option. +``` diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 7400794f1..78f01834e 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -43,6 +43,10 @@ const ( // EnvVaultRaftPath is used to fetch the path where Raft data is stored from the environment. EnvVaultRaftPath = "VAULT_RAFT_PATH" + + // EnvVaultRaftNonVoter is used to override the non_voter config option, telling Vault to join as a non-voter (i.e. read replica). + EnvVaultRaftNonVoter = "VAULT_RAFT_RETRY_JOIN_AS_NON_VOTER" + raftNonVoterConfigKey = "retry_join_as_non_voter" ) var getMmapFlags = func(string) int { return 0 } @@ -172,6 +176,10 @@ type RaftBackend struct { // redundancyZone specifies a redundancy zone for autopilot. redundancyZone string + // nonVoter specifies whether the node should join the cluster as a non-voter. Non-voters get + // replicated to and can serve reads, but do not take part in leader elections. + nonVoter bool + effectiveSDKVersion string } @@ -473,6 +481,22 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend } } + var nonVoter bool + if v := os.Getenv(EnvVaultRaftNonVoter); v != "" { + // Consistent with handling of other raft boolean env vars + // VAULT_RAFT_AUTOPILOT_DISABLE and VAULT_RAFT_FREELIST_SYNC + nonVoter = true + } else if v, ok := conf[raftNonVoterConfigKey]; ok { + nonVoter, err = strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("failed to parse %s config value %q as a boolean: %w", raftNonVoterConfigKey, v, err) + } + } + + if nonVoter && conf["retry_join"] == "" { + return nil, fmt.Errorf("setting %s to true is only valid if at least one retry_join stanza is specified", raftNonVoterConfigKey) + } + return &RaftBackend{ logger: logger, fsm: fsm, @@ -489,6 +513,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend autopilotReconcileInterval: reconcileInterval, autopilotUpdateInterval: updateInterval, redundancyZone: conf["autopilot_redundancy_zone"], + nonVoter: nonVoter, upgradeVersion: upgradeVersion, }, nil } @@ -554,6 +579,13 @@ func (b *RaftBackend) RedundancyZone() string { return b.redundancyZone } +func (b *RaftBackend) NonVoter() bool { + b.l.RLock() + defer b.l.RUnlock() + + return b.nonVoter +} + func (b *RaftBackend) EffectiveVersion() string { b.l.RLock() defer b.l.RUnlock() diff --git a/physical/raft/raft_test.go b/physical/raft/raft_test.go index b3fff3851..50171fd68 100644 --- a/physical/raft/raft_test.go +++ b/physical/raft/raft_test.go @@ -249,6 +249,72 @@ func TestRaft_ParseAutopilotUpgradeVersion(t *testing.T) { } } +func TestRaft_ParseNonVoter(t *testing.T) { + p := func(s string) *string { + return &s + } + + for _, retryJoinConf := range []string{"", "not-empty"} { + t.Run(retryJoinConf, func(t *testing.T) { + for name, tc := range map[string]struct { + envValue *string + configValue *string + expectNonVoter bool + invalidNonVoterValue bool + }{ + "valid false": {nil, p("false"), false, false}, + "valid true": {nil, p("true"), true, false}, + "invalid empty": {nil, p(""), false, true}, + "invalid truthy": {nil, p("no"), false, true}, + "invalid": {nil, p("totallywrong"), false, true}, + "valid env false": {p("false"), nil, true, false}, + "valid env true": {p("true"), nil, true, false}, + "valid env not boolean": {p("anything"), nil, true, false}, + "valid env empty": {p(""), nil, false, false}, + "neither set, default false": {nil, nil, false, false}, + "both set, env preferred": {p("true"), p("false"), true, false}, + } { + t.Run(name, func(t *testing.T) { + if tc.envValue != nil { + t.Setenv(EnvVaultRaftNonVoter, *tc.envValue) + } + raftDir, err := ioutil.TempDir("", "vault-raft-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(raftDir) + + conf := map[string]string{ + "path": raftDir, + "node_id": "abc123", + "retry_join": retryJoinConf, + } + if tc.configValue != nil { + conf[raftNonVoterConfigKey] = *tc.configValue + } + + backend, err := NewRaftBackend(conf, hclog.NewNullLogger()) + switch { + case tc.invalidNonVoterValue || (retryJoinConf == "" && tc.expectNonVoter): + if err == nil { + t.Fatal("expected an error but got none") + } + default: + if err != nil { + t.Fatalf("expected no error but got: %s", err) + } + + raftBackend := backend.(*RaftBackend) + if tc.expectNonVoter != raftBackend.NonVoter() { + t.Fatalf("expected %s %v but got %v", raftNonVoterConfigKey, tc.expectNonVoter, raftBackend.NonVoter()) + } + } + }) + } + }) + } +} + func TestRaft_Backend_LargeKey(t *testing.T) { b, dir := getRaft(t, true, true) defer os.RemoveAll(dir) diff --git a/vault/raft.go b/vault/raft.go index d32d99a25..98987e8ea 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -853,7 +853,7 @@ func (c *Core) InitiateRetryJoin(ctx context.Context) error { c.logger.Info("raft retry join initiated") - if _, err = c.JoinRaftCluster(ctx, leaderInfos, false); err != nil { + if _, err = c.JoinRaftCluster(ctx, leaderInfos, raftBackend.NonVoter()); err != nil { return err } diff --git a/website/content/docs/commands/operator/raft.mdx b/website/content/docs/commands/operator/raft.mdx index 529feed93..9156beb59 100644 --- a/website/content/docs/commands/operator/raft.mdx +++ b/website/content/docs/commands/operator/raft.mdx @@ -79,6 +79,8 @@ The following flags are available for the `operator raft join` command. server not participate in the Raft quorum, and have it only receive the data replication stream. This can be used to add read scalability to a cluster in cases where a high volume of reads to servers are needed. The default is false. + See [`retry_join_as_non_voter`](/docs/configuration/storage/raft#retry_join_as_non_voter) + for the equivalent config option when using `retry_join` stanzas instead. - `-retry` `(bool: false)` - Continuously retry joining the Raft cluster upon failures. The default is false. diff --git a/website/content/docs/configuration/storage/raft.mdx b/website/content/docs/configuration/storage/raft.mdx index 7dc63fbb7..28476736e 100644 --- a/website/content/docs/configuration/storage/raft.mdx +++ b/website/content/docs/configuration/storage/raft.mdx @@ -95,6 +95,16 @@ set [`disable_mlock`](/docs/configuration#disable_mlock) to `true`, and to disab See [the section below](#retry_join-stanza) that describes the parameters accepted by the [`retry_join`](#retry_join-stanza) stanza. +- `retry_join_as_non_voter` `(boolean: false)` - If set, causes any `retry_join` + config to join the Raft cluster as a non-voter. The node will not participate + in the Raft quorum but will still receive the data replication stream, adding + read scalability to a cluster. This option has the same effect as the + [`-non-voter`](/docs/commands/operator/raft#non-voter) flag for the + `vault operator raft join` command, but only affects voting status when joining + via `retry_join` config. This setting can be overridden to true by setting the + `VAULT_RAFT_RETRY_JOIN_AS_NON_VOTER` environment variable to any non-empty value. + Only valid if there is at least one `retry_join` stanza. + - `max_entry_size` `(integer: 1048576)` - This configures the maximum number of bytes for a Raft entry. It applies to both Put operations and transactions. Any put or transaction operation exceeding this configuration value will cause