unflake TestLeader_SecondaryCA_Initialize (#6631)

agent/consul/leader_connect_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/agent/connect"
+	ca "github.com/hashicorp/consul/agent/connect/ca"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/agent/token"
 	tokenStore "github.com/hashicorp/consul/agent/token"
@@ -23,14 +24,12 @@ import (
 func TestLeader_SecondaryCA_Initialize(t *testing.T) {
 	t.Parallel()
 
-	require := require.New(t)
-
 	masterToken := "8a85f086-dd95-4178-b128-e10902767c5c"
 
 	// Initialize primary as the primary DC
 	dir1, s1 := testServerWithConfig(t, func(c *Config) {
 		c.Datacenter = "primary"
-		c.PrimaryDatacenter = "primary"
+		c.ACLDatacenter = "primary"
 		c.Build = "1.6.0"
 		c.ACLsEnabled = true
 		c.ACLMasterToken = masterToken
@@ -46,10 +45,11 @@ func TestLeader_SecondaryCA_Initialize(t *testing.T) {
 	// secondary as a secondary DC
 	dir2, s2 := testServerWithConfig(t, func(c *Config) {
 		c.Datacenter = "secondary"
-		c.PrimaryDatacenter = "primary"
+		c.ACLDatacenter = "primary"
 		c.Build = "1.6.0"
 		c.ACLsEnabled = true
 		c.ACLDefaultPolicy = "deny"
+		c.ACLTokenReplication = true
 	})
 	defer os.RemoveAll(dir2)
 	defer s2.Shutdown()
@@ -57,29 +57,45 @@ func TestLeader_SecondaryCA_Initialize(t *testing.T) {
 	s2.tokens.UpdateAgentToken(masterToken, token.TokenSourceConfig)
 	s2.tokens.UpdateReplicationToken(masterToken, token.TokenSourceConfig)
 
-	// Create the WAN link
-	joinWAN(t, s2, s1)
 	testrpc.WaitForLeader(t, s2.RPC, "secondary")
 
-	_, caRoot := s1.getCAProvider()
+	// Create the WAN link
-	secondaryProvider, _ := s2.getCAProvider()
+	joinWAN(t, s2, s1)
-	intermediatePEM, err := secondaryProvider.ActiveIntermediate()
-	require.NoError(err)
 
-	// Verify the root lists are equal in each DC's state store.
+	waitForNewACLs(t, s1)
-	state1 := s1.fsm.State()
+	waitForNewACLs(t, s2)
-	_, roots1, err := state1.CARoots(nil)
-	require.NoError(err)
 
-	state2 := s2.fsm.State()
+	// Ensure s2 is authoritative.
-	_, roots2, err := state2.CARoots(nil)
+	waitForNewACLReplication(t, s2, structs.ACLReplicateTokens, 1, 1, 0)
-	require.NoError(err)
+
-	require.Equal(roots1[0].ID, roots2[0].ID)
+	// Wait until the providers are fully bootstrapped.
-	require.Equal(roots1[0].RootCert, roots2[0].RootCert)
+	var (
-	require.Equal(1, len(roots1))
+		caRoot            *structs.CARoot
-	require.Equal(len(roots1), len(roots2))
+		secondaryProvider ca.Provider
-	require.Empty(roots1[0].IntermediateCerts)
+		intermediatePEM   string
-	require.NotEmpty(roots2[0].IntermediateCerts)
+		err               error
+	)
+	retry.Run(t, func(r *retry.R) {
+		_, caRoot = s1.getCAProvider()
+		secondaryProvider, _ = s2.getCAProvider()
+		intermediatePEM, err = secondaryProvider.ActiveIntermediate()
+		require.NoError(r, err)
+
+		// Verify the root lists are equal in each DC's state store.
+		state1 := s1.fsm.State()
+		_, roots1, err := state1.CARoots(nil)
+		require.NoError(r, err)
+
+		state2 := s2.fsm.State()
+		_, roots2, err := state2.CARoots(nil)
+		require.NoError(r, err)
+		require.Len(r, roots1, 1)
+		require.Len(r, roots1, 1)
+		require.Equal(r, roots1[0].ID, roots2[0].ID)
+		require.Equal(r, roots1[0].RootCert, roots2[0].RootCert)
+		require.Empty(r, roots1[0].IntermediateCerts)
+		require.NotEmpty(r, roots2[0].IntermediateCerts)
+	})
 
 	// Have secondary sign a leaf cert and make sure the chain is correct.
 	spiffeService := &connect.SpiffeIDService{
@@ -91,13 +107,13 @@ func TestLeader_SecondaryCA_Initialize(t *testing.T) {
 	raw, _ := connect.TestCSR(t, spiffeService)
 
 	leafCsr, err := connect.ParseCSR(raw)
-	require.NoError(err)
+	require.NoError(t, err)
 
 	leafPEM, err := secondaryProvider.Sign(leafCsr)
-	require.NoError(err)
+	require.NoError(t, err)
 
 	cert, err := connect.ParseCert(leafPEM)
-	require.NoError(err)
+	require.NoError(t, err)
 
 	// Check that the leaf signed by the new cert can be verified using the
 	// returned cert chain (signed intermediate + remote root).
@@ -110,7 +126,7 @@ func TestLeader_SecondaryCA_Initialize(t *testing.T) {
 		Intermediates: intermediatePool,
 		Roots:         rootPool,
 	})
-	require.NoError(err)
+	require.NoError(t, err)
 }
 
 func TestLeader_SecondaryCA_IntermediateRefresh(t *testing.T) {
@@ -627,7 +643,6 @@ func TestLeader_ReplicateIntentions(t *testing.T) {
 	require := require.New(t)
 	dir1, s1 := testServerWithConfig(t, func(c *Config) {
 		c.Datacenter = "dc1"
-		c.PrimaryDatacenter = "dc1"
 		c.ACLDatacenter = "dc1"
 		c.ACLsEnabled = true
 		c.ACLMasterToken = "root"
@@ -654,7 +669,6 @@ func TestLeader_ReplicateIntentions(t *testing.T) {
 	// dc2 as a secondary DC
 	dir2, s2 := testServerWithConfig(t, func(c *Config) {
 		c.Datacenter = "dc2"
-		c.PrimaryDatacenter = "dc1"
 		c.ACLDatacenter = "dc1"
 		c.ACLsEnabled = true
 		c.ACLDefaultPolicy = "deny"