From a2394e7353b6c3d6e180feedf36b1c0784ffebae Mon Sep 17 00:00:00 2001 From: Vishal Nayak Date: Wed, 10 Feb 2021 09:58:18 -0500 Subject: [PATCH] Read-replica instead of non-voter (#10875) --- api/sys_raft.go | 2 +- command/operator_raft_join.go | 23 +++++++++++++++++-- http/sys_raft.go | 8 +++---- http/util.go | 2 +- physical/raft/raft.go | 2 +- physical/raft/raft_util.go | 7 +++--- vault/core.go | 2 +- vault/logical_system_raft.go | 16 ++++++++++--- vault/raft.go | 6 ++--- .../hashicorp/vault/api/sys_raft.go | 2 +- .../content/api-docs/system/storage/raft.mdx | 13 +++++++++++ 11 files changed, 63 insertions(+), 20 deletions(-) diff --git a/api/sys_raft.go b/api/sys_raft.go index fb7be34d0..17f0efdec 100644 --- a/api/sys_raft.go +++ b/api/sys_raft.go @@ -24,7 +24,7 @@ type RaftJoinRequest struct { LeaderClientCert string `json:"leader_client_cert"` LeaderClientKey string `json:"leader_client_key"` Retry bool `json:"retry"` - NonVoter bool `json:"non_voter"` + ReadReplica bool `json:"read_replica"` } // RaftJoin adds the node from which this call is invoked from to the raft diff --git a/command/operator_raft_join.go b/command/operator_raft_join.go index 52f0c7d8e..0ebef36ed 100644 --- a/command/operator_raft_join.go +++ b/command/operator_raft_join.go @@ -14,13 +14,16 @@ var _ cli.CommandAutocomplete = (*OperatorRaftJoinCommand)(nil) type OperatorRaftJoinCommand struct { flagRetry bool - flagNonVoter bool flagLeaderCACert string flagLeaderClientCert string flagLeaderClientKey string flagAutoJoinScheme string flagAutoJoinPort uint + flagReadReplica bool *BaseCommand + + // Deprecated flags + flagNonVoter bool } func (c *OperatorRaftJoinCommand) Synopsis() string { @@ -113,6 +116,13 @@ func (c *OperatorRaftJoinCommand) Flags() *FlagSets { Name: "non-voter", Target: &c.flagNonVoter, Default: false, + Usage: "DEPRECATED: Use -read-replica instead.", + }) + + f.BoolVar(&BoolVar{ + Name: "read-replica", + Target: &c.flagReadReplica, + Default: false, Usage: "(Enterprise-only) This flag is used to make the server not participate in the Raft quorum, and have it only receive the data replication stream. This can be used to add read scalability to a cluster in cases where a high volume of reads to servers are needed.", }) @@ -148,6 +158,15 @@ func (c *OperatorRaftJoinCommand) Run(args []string) int { return 1 } + switch { + case c.flagReadReplica: + // Prioritize -read-replica flag. + c.flagNonVoter = true + case c.flagNonVoter: + // If the deprecated -non-voter is used, update the -read-replica flag value. + c.flagReadReplica = true + } + leaderCACert, err := parseFlagFile(c.flagLeaderCACert) if err != nil { c.UI.Error(fmt.Sprintf("Failed to parse leader CA certificate: %s", err)) @@ -182,7 +201,7 @@ func (c *OperatorRaftJoinCommand) Run(args []string) int { LeaderClientCert: leaderClientCert, LeaderClientKey: leaderClientKey, Retry: c.flagRetry, - NonVoter: c.flagNonVoter, + ReadReplica: c.flagReadReplica, } if strings.Contains(leaderInfo, "provider=") { diff --git a/http/sys_raft.go b/http/sys_raft.go index 3411cbf03..ca2c225c6 100644 --- a/http/sys_raft.go +++ b/http/sys_raft.go @@ -51,8 +51,8 @@ func handleSysRaftJoinPost(core *vault.Core, w http.ResponseWriter, r *http.Requ return } - if req.NonVoter && !nonVotersAllowed { - respondError(w, http.StatusBadRequest, errors.New("non-voting nodes not allowed")) + if req.ReadReplica && !readReplicasAllowed { + respondError(w, http.StatusBadRequest, errors.New("read-replica nodes not allowed")) return } @@ -83,7 +83,7 @@ func handleSysRaftJoinPost(core *vault.Core, w http.ResponseWriter, r *http.Requ }, } - joined, err := core.JoinRaftCluster(context.Background(), leaderInfos, req.NonVoter) + joined, err := core.JoinRaftCluster(context.Background(), leaderInfos, req.ReadReplica) if err != nil { respondError(w, http.StatusInternalServerError, err) return @@ -109,5 +109,5 @@ type JoinRequest struct { LeaderClientKey string `json:"leader_client_key"` LeaderTLSServerName string `json:"leader_tls_servername"` Retry bool `json:"retry"` - NonVoter bool `json:"non_voter"` + ReadReplica bool `json:"read_replica"` } diff --git a/http/util.go b/http/util.go index f16a16236..d25a201bd 100644 --- a/http/util.go +++ b/http/util.go @@ -27,7 +27,7 @@ var ( additionalRoutes = func(mux *http.ServeMux, core *vault.Core) {} - nonVotersAllowed = false + readReplicasAllowed = false ) func rateLimitQuotaWrapping(handler http.Handler, core *vault.Core) http.Handler { diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 5481552e3..607b218f8 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -713,7 +713,7 @@ func (b *RaftBackend) SetupCluster(ctx context.Context, opts SetupOpts) error { // Non-voting servers are only allowed in enterprise. If Suffrage is disabled, // error out to indicate that it isn't allowed. for idx := range recoveryConfig.Servers { - if !nonVotersAllowed && recoveryConfig.Servers[idx].Suffrage == raft.Nonvoter { + if !readReplicasAllowed && recoveryConfig.Servers[idx].Suffrage == raft.Nonvoter { return fmt.Errorf("raft recovery failed to parse configuration for node %q: setting `non_voter` is only supported in enterprise", recoveryConfig.Servers[idx].ID) } } diff --git a/physical/raft/raft_util.go b/physical/raft/raft_util.go index 514bdf0d0..f792601da 100644 --- a/physical/raft/raft_util.go +++ b/physical/raft/raft_util.go @@ -7,9 +7,10 @@ import ( "errors" ) -const nonVotersAllowed = false +const readReplicasAllowed = false -// AddPeer adds a new server to the raft cluster -func (b *RaftBackend) AddNonVotingPeer(ctx context.Context, peerID, clusterAddr string) error { +// AddReadReplicaPeer adds a new server to the raft cluster which does not have +// voting rights but gets all the data replicated to it. +func (b *RaftBackend) AddReadReplicaPeer(ctx context.Context, peerID, clusterAddr string) error { return errors.New("not implemented") } diff --git a/vault/core.go b/vault/core.go index 21c85b789..c0df802af 100644 --- a/vault/core.go +++ b/vault/core.go @@ -163,7 +163,7 @@ type raftInformation struct { challenge *wrapping.EncryptedBlobInfo leaderClient *api.Client leaderBarrierConfig *SealConfig - nonVoter bool + readReplica bool joinInProgress bool } diff --git a/vault/logical_system_raft.go b/vault/logical_system_raft.go index f705002d5..bae810775 100644 --- a/vault/logical_system_raft.go +++ b/vault/logical_system_raft.go @@ -34,6 +34,10 @@ func (b *SystemBackend) raftStoragePaths() []*framework.Path { Type: framework.TypeString, }, "non_voter": { + Type: framework.TypeBool, + Deprecated: true, + }, + "read_replica": { Type: framework.TypeBool, }, }, @@ -256,7 +260,13 @@ func (b *SystemBackend) handleRaftBootstrapAnswerWrite() framework.OperationFunc return logical.ErrorResponse("no cluster_addr provided"), logical.ErrInvalidRequest } - nonVoter := d.Get("non_voter").(bool) + // Prioritize read_replica parameter + readReplica := d.Get("read_replica").(bool) + + // If the deprecated non_voter is used, consider that as well + if !readReplica && d.Get("non_voter").(bool) { + readReplica = true + } answer, err := base64.StdEncoding.DecodeString(answerRaw) if err != nil { @@ -286,9 +296,9 @@ func (b *SystemBackend) handleRaftBootstrapAnswerWrite() framework.OperationFunc return nil, errors.New("could not decode raft TLS configuration") } - switch nonVoter { + switch readReplica { case true: - err = raftBackend.AddNonVotingPeer(ctx, serverID, clusterAddr) + err = raftBackend.AddReadReplicaPeer(ctx, serverID, clusterAddr) default: err = raftBackend.AddPeer(ctx, serverID, clusterAddr) } diff --git a/vault/raft.go b/vault/raft.go index a4d0f4223..1ec64c919 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -710,7 +710,7 @@ func (c *Core) InitiateRetryJoin(ctx context.Context) error { return nil } -func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJoinInfo, nonVoter bool) (bool, error) { +func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJoinInfo, readReplica bool) (bool, error) { raftBackend := c.getRaftBackend() if raftBackend == nil { return false, errors.New("raft backend not in use") @@ -881,7 +881,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo challenge: eBlob, leaderClient: apiClient, leaderBarrierConfig: &sealConfig, - nonVoter: nonVoter, + readReplica: readReplica, } // If we're using Shamir and using raft for both physical and HA, we @@ -1077,7 +1077,7 @@ func (c *Core) joinRaftSendAnswer(ctx context.Context, sealAccess *seal.Access, "answer": base64.StdEncoding.EncodeToString(plaintext), "cluster_addr": clusterAddr, "server_id": raftBackend.NodeID(), - "non_voter": raftInfo.nonVoter, + "read_replica": raftInfo.readReplica, }); err != nil { return err } diff --git a/vendor/github.com/hashicorp/vault/api/sys_raft.go b/vendor/github.com/hashicorp/vault/api/sys_raft.go index fb7be34d0..17f0efdec 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_raft.go +++ b/vendor/github.com/hashicorp/vault/api/sys_raft.go @@ -24,7 +24,7 @@ type RaftJoinRequest struct { LeaderClientCert string `json:"leader_client_cert"` LeaderClientKey string `json:"leader_client_key"` Retry bool `json:"retry"` - NonVoter bool `json:"non_voter"` + ReadReplica bool `json:"read_replica"` } // RaftJoin adds the node from which this call is invoked from to the raft diff --git a/website/content/api-docs/system/storage/raft.mdx b/website/content/api-docs/system/storage/raft.mdx index beda167ca..5dd228154 100644 --- a/website/content/api-docs/system/storage/raft.mdx +++ b/website/content/api-docs/system/storage/raft.mdx @@ -39,6 +39,19 @@ leader node. - `leader_client_key` `(string: "")` - Client key used to communicate with Raft's leader node. +- `auto_join` `(string: "")` - Defines any cloud auto-join metadata. If + supplied, Vault will attempt to automatically discover peers in addition to what + can be provided via 'leader_api_addr'. + +- `auto_join_scheme` `(string: "https")` - URI scheme to be used for `auto_join`. + +- `auto_join_port` `(int: 8200)` - Port to be used for `auto_join`. + +- `-read-replica` `(bool: false) (enterprise)` - This flag is used to make the + server not participate in the Raft quorum, and have it only receive the data + replication stream. This can be used to add read scalability to a cluster in + cases where a high volume of reads to servers are needed. The default is false. + ### Sample Payload ```json