storage/raft: Add retry_join_as_non_voter config option (#18030)

changelog/18030.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+storage/raft: Add `retry_join_as_non_voter` config option.
+```

physical/raft/raft.go

@@ -43,6 +43,10 @@ const (
 
 	// EnvVaultRaftPath is used to fetch the path where Raft data is stored from the environment.
 	EnvVaultRaftPath = "VAULT_RAFT_PATH"
+
+	// EnvVaultRaftNonVoter is used to override the non_voter config option, telling Vault to join as a non-voter (i.e. read replica).
+	EnvVaultRaftNonVoter  = "VAULT_RAFT_RETRY_JOIN_AS_NON_VOTER"
+	raftNonVoterConfigKey = "retry_join_as_non_voter"
 )
 
 var getMmapFlags = func(string) int { return 0 }
@@ -172,6 +176,10 @@ type RaftBackend struct {
 	// redundancyZone specifies a redundancy zone for autopilot.
 	redundancyZone string
 
+	// nonVoter specifies whether the node should join the cluster as a non-voter. Non-voters get
+	// replicated to and can serve reads, but do not take part in leader elections.
+	nonVoter bool
+
 	effectiveSDKVersion string
 }
 
@@ -473,6 +481,22 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		}
 	}
 
+	var nonVoter bool
+	if v := os.Getenv(EnvVaultRaftNonVoter); v != "" {
+		// Consistent with handling of other raft boolean env vars
+		// VAULT_RAFT_AUTOPILOT_DISABLE and VAULT_RAFT_FREELIST_SYNC
+		nonVoter = true
+	} else if v, ok := conf[raftNonVoterConfigKey]; ok {
+		nonVoter, err = strconv.ParseBool(v)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %s config value %q as a boolean: %w", raftNonVoterConfigKey, v, err)
+		}
+	}
+
+	if nonVoter && conf["retry_join"] == "" {
+		return nil, fmt.Errorf("setting %s to true is only valid if at least one retry_join stanza is specified", raftNonVoterConfigKey)
+	}
+
 	return &RaftBackend{
 		logger:                     logger,
 		fsm:                        fsm,
@@ -489,6 +513,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		autopilotReconcileInterval: reconcileInterval,
 		autopilotUpdateInterval:    updateInterval,
 		redundancyZone:             conf["autopilot_redundancy_zone"],
+		nonVoter:                   nonVoter,
 		upgradeVersion:             upgradeVersion,
 	}, nil
 }
@@ -554,6 +579,13 @@ func (b *RaftBackend) RedundancyZone() string {
 	return b.redundancyZone
 }
 
+func (b *RaftBackend) NonVoter() bool {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	return b.nonVoter
+}
+
 func (b *RaftBackend) EffectiveVersion() string {
 	b.l.RLock()
 	defer b.l.RUnlock()

physical/raft/raft_test.go

@@ -249,6 +249,72 @@ func TestRaft_ParseAutopilotUpgradeVersion(t *testing.T) {
 	}
 }
 
+func TestRaft_ParseNonVoter(t *testing.T) {
+	p := func(s string) *string {
+		return &s
+	}
+
+	for _, retryJoinConf := range []string{"", "not-empty"} {
+		t.Run(retryJoinConf, func(t *testing.T) {
+			for name, tc := range map[string]struct {
+				envValue             *string
+				configValue          *string
+				expectNonVoter       bool
+				invalidNonVoterValue bool
+			}{
+				"valid false":                {nil, p("false"), false, false},
+				"valid true":                 {nil, p("true"), true, false},
+				"invalid empty":              {nil, p(""), false, true},
+				"invalid truthy":             {nil, p("no"), false, true},
+				"invalid":                    {nil, p("totallywrong"), false, true},
+				"valid env false":            {p("false"), nil, true, false},
+				"valid env true":             {p("true"), nil, true, false},
+				"valid env not boolean":      {p("anything"), nil, true, false},
+				"valid env empty":            {p(""), nil, false, false},
+				"neither set, default false": {nil, nil, false, false},
+				"both set, env preferred":    {p("true"), p("false"), true, false},
+			} {
+				t.Run(name, func(t *testing.T) {
+					if tc.envValue != nil {
+						t.Setenv(EnvVaultRaftNonVoter, *tc.envValue)
+					}
+					raftDir, err := ioutil.TempDir("", "vault-raft-")
+					if err != nil {
+						t.Fatal(err)
+					}
+					defer os.RemoveAll(raftDir)
+
+					conf := map[string]string{
+						"path":       raftDir,
+						"node_id":    "abc123",
+						"retry_join": retryJoinConf,
+					}
+					if tc.configValue != nil {
+						conf[raftNonVoterConfigKey] = *tc.configValue
+					}
+
+					backend, err := NewRaftBackend(conf, hclog.NewNullLogger())
+					switch {
+					case tc.invalidNonVoterValue || (retryJoinConf == "" && tc.expectNonVoter):
+						if err == nil {
+							t.Fatal("expected an error but got none")
+						}
+					default:
+						if err != nil {
+							t.Fatalf("expected no error but got: %s", err)
+						}
+
+						raftBackend := backend.(*RaftBackend)
+						if tc.expectNonVoter != raftBackend.NonVoter() {
+							t.Fatalf("expected %s %v but got %v", raftNonVoterConfigKey, tc.expectNonVoter, raftBackend.NonVoter())
+						}
+					}
+				})
+			}
+		})
+	}
+}
+
 func TestRaft_Backend_LargeKey(t *testing.T) {
 	b, dir := getRaft(t, true, true)
 	defer os.RemoveAll(dir)

vault/raft.go

@@ -853,7 +853,7 @@ func (c *Core) InitiateRetryJoin(ctx context.Context) error {
 
 	c.logger.Info("raft retry join initiated")
 
-	if _, err = c.JoinRaftCluster(ctx, leaderInfos, false); err != nil {
+	if _, err = c.JoinRaftCluster(ctx, leaderInfos, raftBackend.NonVoter()); err != nil {
 		return err
 	}
 

website/content/docs/commands/operator/raft.mdx

@@ -79,6 +79,8 @@ The following flags are available for the `operator raft join` command.
   server not participate in the Raft quorum, and have it only receive the data
   replication stream. This can be used to add read scalability to a cluster in
   cases where a high volume of reads to servers are needed. The default is false.
+  See [`retry_join_as_non_voter`](/docs/configuration/storage/raft#retry_join_as_non_voter)
+  for the equivalent config option when using `retry_join` stanzas instead.
 
 - `-retry` `(bool: false)` - Continuously retry joining the Raft cluster upon
   failures. The default is false.

website/content/docs/configuration/storage/raft.mdx

@@ -95,6 +95,16 @@ set [`disable_mlock`](/docs/configuration#disable_mlock) to `true`, and to disab
   See [the section below](#retry_join-stanza) that describes the parameters
   accepted by the [`retry_join`](#retry_join-stanza) stanza.
 
+- `retry_join_as_non_voter` `(boolean: false)` - If set, causes any `retry_join`
+  config to join the Raft cluster as a non-voter. The node will not participate
+  in the Raft quorum but will still receive the data replication stream, adding
+  read scalability to a cluster. This option has the same effect as the
+  [`-non-voter`](/docs/commands/operator/raft#non-voter) flag for the
+  `vault operator raft join` command, but only affects voting status when joining
+  via `retry_join` config. This setting can be overridden to true by setting the
+  `VAULT_RAFT_RETRY_JOIN_AS_NON_VOTER` environment variable to any non-empty value.
+  Only valid if there is at least one `retry_join` stanza.
+
 - `max_entry_size` `(integer: 1048576)` - This configures the maximum number of
   bytes for a Raft entry. It applies to both Put operations and transactions.
   Any put or transaction operation exceeding this configuration value will cause