From dac3382e1589fcf609bf95e050bcfe14296c7a53 Mon Sep 17 00:00:00 2001 From: Calvin Leung Huang Date: Thu, 13 Feb 2020 13:18:53 -0800 Subject: [PATCH] external_tests: ensure derived cores are stable before proceeding on tests (#8342) * external_tests: ensure derived cores are stable before proceeding on tests * testhelpers: add min duration tolerance when checking stability on derived core --- helper/testhelpers/testhelpers.go | 23 ++++++++++++++++++++++- vault/external_tests/raft/raft_test.go | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/helper/testhelpers/testhelpers.go b/helper/testhelpers/testhelpers.go index 1a657d58e..ad67f1fe9 100644 --- a/helper/testhelpers/testhelpers.go +++ b/helper/testhelpers/testhelpers.go @@ -186,7 +186,16 @@ func AttemptUnsealCore(c *vault.TestCluster, core *vault.TestClusterCore) error } func EnsureStableActiveNode(t testing.T, cluster *vault.TestCluster) { + deriveStableActiveCore(t, cluster) +} + +func DeriveStableActiveCore(t testing.T, cluster *vault.TestCluster) *vault.TestClusterCore { + return deriveStableActiveCore(t, cluster) +} + +func deriveStableActiveCore(t testing.T, cluster *vault.TestCluster) *vault.TestClusterCore { activeCore := DeriveActiveCore(t, cluster) + minDuration := time.NewTimer(3 * time.Second) for i := 0; i < 30; i++ { leaderResp, err := activeCore.Client.Sys().Leader() @@ -194,10 +203,22 @@ func EnsureStableActiveNode(t testing.T, cluster *vault.TestCluster) { t.Fatal(err) } if !leaderResp.IsSelf { - t.Fatal("unstable active node") + minDuration.Reset(3 * time.Second) } time.Sleep(200 * time.Millisecond) } + + select { + case <-minDuration.C: + default: + if stopped := minDuration.Stop(); stopped { + t.Fatal("unstable active node") + } + // Drain the value + <-minDuration.C + } + + return activeCore } func DeriveActiveCore(t testing.T, cluster *vault.TestCluster) *vault.TestClusterCore { diff --git a/vault/external_tests/raft/raft_test.go b/vault/external_tests/raft/raft_test.go index 8ac863812..cd0ecc7d4 100644 --- a/vault/external_tests/raft/raft_test.go +++ b/vault/external_tests/raft/raft_test.go @@ -504,7 +504,7 @@ func TestRaft_SnapshotAPI_RekeyRotate_Backward(t *testing.T) { cluster.BarrierKeys = barrierKeys testhelpers.EnsureCoresUnsealed(t, cluster) testhelpers.WaitForActiveNode(t, cluster) - activeCore := testhelpers.DeriveActiveCore(t, cluster) + activeCore := testhelpers.DeriveStableActiveCore(t, cluster) // Read the value. data, err := activeCore.Client.Logical().Read("secret/foo") @@ -707,7 +707,7 @@ func TestRaft_SnapshotAPI_RekeyRotate_Forward(t *testing.T) { cluster.BarrierKeys = newBarrierKeys testhelpers.EnsureCoresUnsealed(t, cluster) testhelpers.WaitForActiveNode(t, cluster) - activeCore := testhelpers.DeriveActiveCore(t, cluster) + activeCore := testhelpers.DeriveStableActiveCore(t, cluster) // Read the value. data, err := activeCore.Client.Logical().Read("secret/foo")