luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

Prevent autopilot from demoting voters when they join a 2nd time (#18263)

This commit is contained in:
Nick Cabatoff 2022-12-07 14:17:45 -05:00 committed by GitHub
parent 342b61984a
commit 1b745aef58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 25 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
changelog/18263.txt Normal file
View File

@ -0,0 +1,5 @@
```release-note:bug
storage/raft (enterprise): An already joined node can rejoin by wiping storage
and re-issueing a join request, but in doing so could transiently become a
non-voter. In some scenarios this resulted in loss of quorum.
```

10
vault/external_tests/raft/raft_autopilot_test.go
View File

@ -24,7 +24,7 @@ import (
) )
func TestRaft_Autopilot_Disable(t *testing.T) { func TestRaft_Autopilot_Disable(t *testing.T) {
cluster := raftCluster(t, &RaftClusterOpts{ cluster, _ := raftCluster(t, &RaftClusterOpts{
DisableFollowerJoins: true, DisableFollowerJoins: true,
InmemCluster: true, InmemCluster: true,
// Not setting EnableAutopilot here. // Not setting EnableAutopilot here.
@ -38,7 +38,7 @@ func TestRaft_Autopilot_Disable(t *testing.T) {
} }
func TestRaft_Autopilot_Stabilization_And_State(t *testing.T) { func TestRaft_Autopilot_Stabilization_And_State(t *testing.T) {
cluster := raftCluster(t, &RaftClusterOpts{ cluster, _ := raftCluster(t, &RaftClusterOpts{
DisableFollowerJoins: true, DisableFollowerJoins: true,
InmemCluster: true, InmemCluster: true,
EnableAutopilot: true, EnableAutopilot: true,
@ -109,7 +109,7 @@ func TestRaft_Autopilot_Stabilization_And_State(t *testing.T) {
} }
func TestRaft_Autopilot_Configuration(t *testing.T) { func TestRaft_Autopilot_Configuration(t *testing.T) {
cluster := raftCluster(t, &RaftClusterOpts{ cluster, _ := raftCluster(t, &RaftClusterOpts{
DisableFollowerJoins: true, DisableFollowerJoins: true,
InmemCluster: true, InmemCluster: true,
EnableAutopilot: true, EnableAutopilot: true,
@ -301,7 +301,7 @@ func TestRaft_Autopilot_Stabilization_Delay(t *testing.T) {
} }
func TestRaft_AutoPilot_Peersets_Equivalent(t *testing.T) { func TestRaft_AutoPilot_Peersets_Equivalent(t *testing.T) {
cluster := raftCluster(t, &RaftClusterOpts{ cluster, _ := raftCluster(t, &RaftClusterOpts{
InmemCluster: true, InmemCluster: true,
EnableAutopilot: true, EnableAutopilot: true,
DisableFollowerJoins: true, DisableFollowerJoins: true,
@ -417,7 +417,7 @@ func join(t *testing.T, core *vault.TestClusterCore, client *api.Client, cluster
// TestRaft_VotersStayVoters ensures that autopilot doesn't demote a node just // TestRaft_VotersStayVoters ensures that autopilot doesn't demote a node just
// because it hasn't been heard from in some time. // because it hasn't been heard from in some time.
func TestRaft_VotersStayVoters(t *testing.T) { func TestRaft_VotersStayVoters(t *testing.T) {
cluster := raftCluster(t, &RaftClusterOpts{ cluster, _ := raftCluster(t, &RaftClusterOpts{
DisableFollowerJoins: true, DisableFollowerJoins: true,
InmemCluster: true, InmemCluster: true,
EnableAutopilot: true, EnableAutopilot: true,

26
vault/external_tests/raft/raft_test.go
View File

@ -48,7 +48,7 @@ type RaftClusterOpts struct {
EffectiveSDKVersionMap map[int]string EffectiveSDKVersionMap map[int]string
} }
func raftCluster(t testing.TB, ropts *RaftClusterOpts) *vault.TestCluster { func raftCluster(t testing.TB, ropts *RaftClusterOpts) (*vault.TestCluster, *vault.TestClusterOptions) {
if ropts == nil { if ropts == nil {
ropts = &RaftClusterOpts{} ropts = &RaftClusterOpts{}
} }
@ -82,7 +82,7 @@ func raftCluster(t testing.TB, ropts *RaftClusterOpts) *vault.TestCluster {
cluster := vault.NewTestCluster(benchhelpers.TBtoT(t), conf, &opts) cluster := vault.NewTestCluster(benchhelpers.TBtoT(t), conf, &opts)
cluster.Start() cluster.Start()
vault.TestWaitActive(benchhelpers.TBtoT(t), cluster.Cores[0].Core) vault.TestWaitActive(benchhelpers.TBtoT(t), cluster.Cores[0].Core)
return cluster return cluster, &opts
} }
func TestRaft_BoltDBMetrics(t *testing.T) { func TestRaft_BoltDBMetrics(t *testing.T) {
@ -322,7 +322,7 @@ func TestRaft_Join(t *testing.T) {
func TestRaft_RemovePeer(t *testing.T) { func TestRaft_RemovePeer(t *testing.T) {
t.Parallel() t.Parallel()
cluster := raftCluster(t, nil) cluster, _ := raftCluster(t, nil)
defer cluster.Cleanup() defer cluster.Cleanup()
for i, c := range cluster.Cores { for i, c := range cluster.Cores {
@ -395,7 +395,7 @@ func TestRaft_NodeIDHeader(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) { t.Run(tc.description, func(t *testing.T) {
cluster := raftCluster(t, tc.ropts) cluster, _ := raftCluster(t, tc.ropts)
defer cluster.Cleanup() defer cluster.Cleanup()
for i, c := range cluster.Cores { for i, c := range cluster.Cores {
@ -432,7 +432,7 @@ func TestRaft_NodeIDHeader(t *testing.T) {
func TestRaft_Configuration(t *testing.T) { func TestRaft_Configuration(t *testing.T) {
t.Parallel() t.Parallel()
cluster := raftCluster(t, nil) cluster, _ := raftCluster(t, nil)
defer cluster.Cleanup() defer cluster.Cleanup()
for i, c := range cluster.Cores { for i, c := range cluster.Cores {
@ -479,7 +479,7 @@ func TestRaft_Configuration(t *testing.T) {
func TestRaft_ShamirUnseal(t *testing.T) { func TestRaft_ShamirUnseal(t *testing.T) {
t.Parallel() t.Parallel()
cluster := raftCluster(t, nil) cluster, _ := raftCluster(t, nil)
defer cluster.Cleanup() defer cluster.Cleanup()
for i, c := range cluster.Cores { for i, c := range cluster.Cores {
@ -491,7 +491,7 @@ func TestRaft_ShamirUnseal(t *testing.T) {
func TestRaft_SnapshotAPI(t *testing.T) { func TestRaft_SnapshotAPI(t *testing.T) {
t.Parallel() t.Parallel()
cluster := raftCluster(t, nil) cluster, _ := raftCluster(t, nil)
defer cluster.Cleanup() defer cluster.Cleanup()
leaderClient := cluster.Cores[0].Client leaderClient := cluster.Cores[0].Client
@ -555,7 +555,7 @@ func TestRaft_SnapshotAPI_MidstreamFailure(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
cluster := raftCluster(t, &RaftClusterOpts{ cluster, _ := raftCluster(t, &RaftClusterOpts{
NumCores: 1, NumCores: 1,
Seal: autoSeal, Seal: autoSeal,
}) })
@ -660,7 +660,7 @@ func TestRaft_SnapshotAPI_RekeyRotate_Backward(t *testing.T) {
tCaseLocal := tCase tCaseLocal := tCase
t.Parallel() t.Parallel()
cluster := raftCluster(t, &RaftClusterOpts{DisablePerfStandby: tCaseLocal.DisablePerfStandby}) cluster, _ := raftCluster(t, &RaftClusterOpts{DisablePerfStandby: tCaseLocal.DisablePerfStandby})
defer cluster.Cleanup() defer cluster.Cleanup()
leaderClient := cluster.Cores[0].Client leaderClient := cluster.Cores[0].Client
@ -861,7 +861,7 @@ func TestRaft_SnapshotAPI_RekeyRotate_Forward(t *testing.T) {
tCaseLocal := tCase tCaseLocal := tCase
t.Parallel() t.Parallel()
cluster := raftCluster(t, &RaftClusterOpts{DisablePerfStandby: tCaseLocal.DisablePerfStandby}) cluster, _ := raftCluster(t, &RaftClusterOpts{DisablePerfStandby: tCaseLocal.DisablePerfStandby})
defer cluster.Cleanup() defer cluster.Cleanup()
leaderClient := cluster.Cores[0].Client leaderClient := cluster.Cores[0].Client
@ -1048,7 +1048,7 @@ func TestRaft_SnapshotAPI_RekeyRotate_Forward(t *testing.T) {
func TestRaft_SnapshotAPI_DifferentCluster(t *testing.T) { func TestRaft_SnapshotAPI_DifferentCluster(t *testing.T) {
t.Parallel() t.Parallel()
cluster := raftCluster(t, nil) cluster, _ := raftCluster(t, nil)
defer cluster.Cleanup() defer cluster.Cleanup()
leaderClient := cluster.Cores[0].Client leaderClient := cluster.Cores[0].Client
@ -1094,7 +1094,7 @@ func TestRaft_SnapshotAPI_DifferentCluster(t *testing.T) {
// Cluster 2 // Cluster 2
{ {
cluster2 := raftCluster(t, nil) cluster2, _ := raftCluster(t, nil)
defer cluster2.Cleanup() defer cluster2.Cleanup()
leaderClient := cluster2.Cores[0].Client leaderClient := cluster2.Cores[0].Client
@ -1141,7 +1141,7 @@ func TestRaft_SnapshotAPI_DifferentCluster(t *testing.T) {
} }
func BenchmarkRaft_SingleNode(b *testing.B) { func BenchmarkRaft_SingleNode(b *testing.B) {
cluster := raftCluster(b, nil) cluster, _ := raftCluster(b, nil)
defer cluster.Cleanup() defer cluster.Cleanup()
leaderClient := cluster.Cores[0].Client leaderClient := cluster.Cores[0].Client

4
vault/logical_system_raft.go
View File

@ -361,9 +361,9 @@ func (b *SystemBackend) handleRaftBootstrapAnswerWrite() framework.OperationFunc
var desiredSuffrage string var desiredSuffrage string
switch nonVoter { switch nonVoter {
case true: case true:
desiredSuffrage = "voter"
default:
desiredSuffrage = "non-voter" desiredSuffrage = "non-voter"
default:
desiredSuffrage = "voter"
} }
if b.Core.raftFollowerStates != nil { if b.Core.raftFollowerStates != nil {