open-consul/agent/consul/acl_replication_test.go

987 lines
30 KiB
Go

package consul
import (
"fmt"
"os"
"strconv"
"testing"
"time"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/structs"
tokenStore "github.com/hashicorp/consul/agent/token"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/hashicorp/consul/testrpc"
"github.com/stretchr/testify/require"
)
func TestACLReplication_diffACLPolicies(t *testing.T) {
diffACLPolicies := func(local structs.ACLPolicies, remote structs.ACLPolicyListStubs, lastRemoteIndex uint64) ([]string, []string) {
tr := &aclPolicyReplicator{local: local, remote: remote}
res := diffACLType(tr, lastRemoteIndex)
return res.LocalDeletes, res.LocalUpserts
}
local := structs.ACLPolicies{
&structs.ACLPolicy{
ID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc",
Name: "policy1",
Description: "policy1 - already in sync",
Rules: `acl = "read"`,
Syntax: acl.SyntaxCurrent,
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 2},
},
&structs.ACLPolicy{
ID: "8ea41efb-8519-4091-bc91-c42da0cda9ae",
Name: "policy2",
Description: "policy2 - updated but not changed",
Rules: `acl = "read"`,
Syntax: acl.SyntaxCurrent,
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25},
},
&structs.ACLPolicy{
ID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2",
Name: "policy3",
Description: "policy3 - updated and changed",
Rules: `acl = "read"`,
Syntax: acl.SyntaxCurrent,
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25},
},
&structs.ACLPolicy{
ID: "e9d33298-6490-4466-99cb-ba93af64fa76",
Name: "policy4",
Description: "policy4 - needs deleting",
Rules: `acl = "read"`,
Syntax: acl.SyntaxCurrent,
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25},
},
}
remote := structs.ACLPolicyListStubs{
&structs.ACLPolicyListStub{
ID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc",
Name: "policy1",
Description: "policy1 - already in sync",
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 2,
},
&structs.ACLPolicyListStub{
ID: "8ea41efb-8519-4091-bc91-c42da0cda9ae",
Name: "policy2",
Description: "policy2 - updated but not changed",
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 50,
},
&structs.ACLPolicyListStub{
ID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2",
Name: "policy3",
Description: "policy3 - updated and changed",
Datacenters: nil,
Hash: []byte{5, 6, 7, 8},
CreateIndex: 1,
ModifyIndex: 50,
},
&structs.ACLPolicyListStub{
ID: "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926",
Name: "policy5",
Description: "policy5 - needs adding",
Datacenters: nil,
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 50,
},
}
// Do the full diff. This full exercises the main body of the loop
deletions, updates := diffACLPolicies(local, remote, 28)
require.Len(t, updates, 2)
require.ElementsMatch(t, updates, []string{
"c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926",
"539f1cb6-40aa-464f-ae66-a900d26bc1b2"})
require.Len(t, deletions, 1)
require.Equal(t, "e9d33298-6490-4466-99cb-ba93af64fa76", deletions[0])
deletions, updates = diffACLPolicies(local, nil, 28)
require.Len(t, updates, 0)
require.Len(t, deletions, 4)
require.ElementsMatch(t, deletions, []string{
"44ef9aec-7654-4401-901b-4d4a8b3c80fc",
"8ea41efb-8519-4091-bc91-c42da0cda9ae",
"539f1cb6-40aa-464f-ae66-a900d26bc1b2",
"e9d33298-6490-4466-99cb-ba93af64fa76"})
deletions, updates = diffACLPolicies(nil, remote, 28)
require.Len(t, deletions, 0)
require.Len(t, updates, 4)
require.ElementsMatch(t, updates, []string{
"44ef9aec-7654-4401-901b-4d4a8b3c80fc",
"8ea41efb-8519-4091-bc91-c42da0cda9ae",
"539f1cb6-40aa-464f-ae66-a900d26bc1b2",
"c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926"})
}
func TestACLReplication_diffACLTokens(t *testing.T) {
diffACLTokens := func(
local structs.ACLTokens,
remote structs.ACLTokenListStubs,
lastRemoteIndex uint64,
) itemDiffResults {
tr := &aclTokenReplicator{local: local, remote: remote}
return diffACLType(tr, lastRemoteIndex)
}
local := structs.ACLTokens{
// When a just-upgraded (1.3->1.4+) secondary DC is replicating from an
// upgraded primary DC (1.4+), the local state for tokens predating the
// upgrade will lack AccessorIDs.
//
// The primary DC will lazily perform the update to assign AccessorIDs,
// and that new update will come across the wire locally as a new
// insert.
//
// We simulate that scenario here with 'token0' having no AccessorID in
// the secondary (local) DC and having an AccessorID assigned in the
// payload retrieved from the primary (remote) DC.
&structs.ACLToken{
AccessorID: "",
SecretID: "5128289f-c22c-4d32-936e-7662443f1a55",
Description: "token0 - old and not yet upgraded",
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 3},
},
&structs.ACLToken{
AccessorID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc",
SecretID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc",
Description: "token1 - already in sync",
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 2},
},
&structs.ACLToken{
AccessorID: "8ea41efb-8519-4091-bc91-c42da0cda9ae",
SecretID: "8ea41efb-8519-4091-bc91-c42da0cda9ae",
Description: "token2 - updated but not changed",
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25},
},
&structs.ACLToken{
AccessorID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2",
SecretID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2",
Description: "token3 - updated and changed",
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25},
},
&structs.ACLToken{
AccessorID: "e9d33298-6490-4466-99cb-ba93af64fa76",
SecretID: "e9d33298-6490-4466-99cb-ba93af64fa76",
Description: "token4 - needs deleting",
Hash: []byte{1, 2, 3, 4},
RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25},
},
}
remote := structs.ACLTokenListStubs{
&structs.ACLTokenListStub{
AccessorID: "72fac6a3-a014-41c8-9cb2-8d9a5e935f3d",
//SecretID: "5128289f-c22c-4d32-936e-7662443f1a55", (formerly)
Description: "token0 - old and not yet upgraded locally",
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 3,
},
&structs.ACLTokenListStub{
AccessorID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc",
Description: "token1 - already in sync",
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 2,
},
&structs.ACLTokenListStub{
AccessorID: "8ea41efb-8519-4091-bc91-c42da0cda9ae",
Description: "token2 - updated but not changed",
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 50,
},
&structs.ACLTokenListStub{
AccessorID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2",
Description: "token3 - updated and changed",
Hash: []byte{5, 6, 7, 8},
CreateIndex: 1,
ModifyIndex: 50,
},
&structs.ACLTokenListStub{
AccessorID: "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926",
Description: "token5 - needs adding",
Hash: []byte{1, 2, 3, 4},
CreateIndex: 1,
ModifyIndex: 50,
},
// When a 1.4+ secondary DC is replicating from a 1.4+ primary DC,
// tokens created using the legacy APIs will not initially have
// AccessorIDs assigned. That assignment is lazy (but in quick
// succession).
//
// The secondary (local) will see these in the api response as a stub
// with "" as the AccessorID.
//
// We simulate that here to verify that the secondary does the right
// thing by skipping them until it sees them with nonempty AccessorIDs.
&structs.ACLTokenListStub{
AccessorID: "",
Description: "token6 - pending async AccessorID assignment",
Hash: []byte{1, 2, 3, 4},
CreateIndex: 51,
ModifyIndex: 51,
},
}
// Do the full diff. This full exercises the main body of the loop
t.Run("full-diff", func(t *testing.T) {
res := diffACLTokens(local, remote, 28)
require.Equal(t, 1, res.LocalSkipped)
require.Equal(t, 1, res.RemoteSkipped)
require.Len(t, res.LocalUpserts, 3)
require.ElementsMatch(t, res.LocalUpserts, []string{
"72fac6a3-a014-41c8-9cb2-8d9a5e935f3d",
"c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926",
"539f1cb6-40aa-464f-ae66-a900d26bc1b2"})
require.Len(t, res.LocalDeletes, 1)
require.Equal(t, "e9d33298-6490-4466-99cb-ba93af64fa76", res.LocalDeletes[0])
})
t.Run("only-local", func(t *testing.T) {
res := diffACLTokens(local, nil, 28)
require.Equal(t, 1, res.LocalSkipped)
require.Equal(t, 0, res.RemoteSkipped)
require.Len(t, res.LocalUpserts, 0)
require.Len(t, res.LocalDeletes, 4)
require.ElementsMatch(t, res.LocalDeletes, []string{
"44ef9aec-7654-4401-901b-4d4a8b3c80fc",
"8ea41efb-8519-4091-bc91-c42da0cda9ae",
"539f1cb6-40aa-464f-ae66-a900d26bc1b2",
"e9d33298-6490-4466-99cb-ba93af64fa76"})
})
t.Run("only-remote", func(t *testing.T) {
res := diffACLTokens(nil, remote, 28)
require.Equal(t, 0, res.LocalSkipped)
require.Equal(t, 1, res.RemoteSkipped)
require.Len(t, res.LocalDeletes, 0)
require.Len(t, res.LocalUpserts, 5)
require.ElementsMatch(t, res.LocalUpserts, []string{
"72fac6a3-a014-41c8-9cb2-8d9a5e935f3d",
"44ef9aec-7654-4401-901b-4d4a8b3c80fc",
"8ea41efb-8519-4091-bc91-c42da0cda9ae",
"539f1cb6-40aa-464f-ae66-a900d26bc1b2",
"c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926"})
})
}
func TestACLReplication_Tokens(t *testing.T) {
t.Parallel()
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLMasterToken = "root"
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
testrpc.WaitForLeader(t, s1.RPC, "dc1")
client := rpcClient(t, s1)
defer client.Close()
dir2, s2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLTokenReplication = true
c.ACLReplicationRate = 100
c.ACLReplicationBurst = 100
c.ACLReplicationApplyLimit = 1000000
})
s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
testrpc.WaitForLeader(t, s2.RPC, "dc2")
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join.
joinWAN(t, s2, s1)
testrpc.WaitForLeader(t, s1.RPC, "dc1")
testrpc.WaitForLeader(t, s1.RPC, "dc2")
// Wait for legacy acls to be disabled so we are clear that
// legacy replication isn't meddling.
waitForNewACLs(t, s1)
waitForNewACLs(t, s2)
waitForNewACLReplication(t, s2, structs.ACLReplicateTokens, 1, 1, 0)
// Create a bunch of new tokens and policies
var tokens structs.ACLTokens
for i := 0; i < 50; i++ {
arg := structs.ACLTokenSetRequest{
Datacenter: "dc1",
ACLToken: structs.ACLToken{
Description: fmt.Sprintf("token-%d", i),
Policies: []structs.ACLTokenPolicyLink{
structs.ACLTokenPolicyLink{
ID: structs.ACLPolicyGlobalManagementID,
},
},
Local: false,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var token structs.ACLToken
require.NoError(t, s1.RPC("ACL.TokenSet", &arg, &token))
tokens = append(tokens, &token)
}
checkSame := func(t *retry.R) {
// only account for global tokens - local tokens shouldn't be replicated
index, remote, err := s1.fsm.State().ACLTokenList(nil, false, true, "", "", "")
require.NoError(t, err)
_, local, err := s2.fsm.State().ACLTokenList(nil, false, true, "", "", "")
require.NoError(t, err)
require.Len(t, local, len(remote))
for i, token := range remote {
require.Equal(t, token.Hash, local[i].Hash)
}
s2.aclReplicationStatusLock.RLock()
status := s2.aclReplicationStatus
s2.aclReplicationStatusLock.RUnlock()
require.True(t, status.Enabled)
require.True(t, status.Running)
require.Equal(t, status.ReplicationType, structs.ACLReplicateTokens)
require.Equal(t, status.ReplicatedTokenIndex, index)
require.Equal(t, status.SourceDatacenter, "dc1")
}
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
// Wait for s2 global-management policy
retry.Run(t, func(r *retry.R) {
_, policy, err := s2.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID)
require.NoError(r, err)
require.NotNil(r, policy)
})
// add some local tokens to the secondary DC
// these shouldn't be deleted by replication
for i := 0; i < 50; i++ {
arg := structs.ACLTokenSetRequest{
Datacenter: "dc2",
ACLToken: structs.ACLToken{
Description: fmt.Sprintf("token-%d", i),
Policies: []structs.ACLTokenPolicyLink{
structs.ACLTokenPolicyLink{
ID: structs.ACLPolicyGlobalManagementID,
},
},
Local: true,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var token structs.ACLToken
require.NoError(t, s2.RPC("ACL.TokenSet", &arg, &token))
}
// add some local tokens to the primary DC
// these shouldn't be replicated to the secondary DC
for i := 0; i < 50; i++ {
arg := structs.ACLTokenSetRequest{
Datacenter: "dc1",
ACLToken: structs.ACLToken{
Description: fmt.Sprintf("token-%d", i),
Policies: []structs.ACLTokenPolicyLink{
structs.ACLTokenPolicyLink{
ID: structs.ACLPolicyGlobalManagementID,
},
},
Local: true,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var token structs.ACLToken
require.NoError(t, s1.RPC("ACL.TokenSet", &arg, &token))
}
// Update those other tokens
for i := 0; i < 50; i++ {
arg := structs.ACLTokenSetRequest{
Datacenter: "dc1",
ACLToken: structs.ACLToken{
AccessorID: tokens[i].AccessorID,
SecretID: tokens[i].SecretID,
Description: fmt.Sprintf("token-%d-modified", i),
Policies: []structs.ACLTokenPolicyLink{
structs.ACLTokenPolicyLink{
ID: structs.ACLPolicyGlobalManagementID,
},
},
Local: false,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var token structs.ACLToken
require.NoError(t, s1.RPC("ACL.TokenSet", &arg, &token))
}
// Wait for the replica to converge.
// this time it also verifies the local tokens from the primary were not replicated.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
// verify dc2 local tokens didn't get blown away
_, local, err := s2.fsm.State().ACLTokenList(nil, true, false, "", "", "")
require.NoError(t, err)
require.Len(t, local, 50)
for _, token := range tokens {
arg := structs.ACLTokenDeleteRequest{
Datacenter: "dc1",
TokenID: token.AccessorID,
WriteRequest: structs.WriteRequest{Token: "root"},
}
var dontCare string
require.NoError(t, s1.RPC("ACL.TokenDelete", &arg, &dontCare))
}
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
}
func TestACLReplication_Policies(t *testing.T) {
t.Parallel()
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLMasterToken = "root"
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
testrpc.WaitForLeader(t, s1.RPC, "dc1")
client := rpcClient(t, s1)
defer client.Close()
dir2, s2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLTokenReplication = false
c.ACLReplicationRate = 100
c.ACLReplicationBurst = 100
c.ACLReplicationApplyLimit = 1000000
})
s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
testrpc.WaitForLeader(t, s2.RPC, "dc2")
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join.
joinWAN(t, s2, s1)
testrpc.WaitForLeader(t, s1.RPC, "dc1")
testrpc.WaitForLeader(t, s1.RPC, "dc2")
// Wait for legacy acls to be disabled so we are clear that
// legacy replication isn't meddling.
waitForNewACLs(t, s1)
waitForNewACLs(t, s2)
waitForNewACLReplication(t, s2, structs.ACLReplicatePolicies, 1, 0, 0)
// Create a bunch of new policies
var policies structs.ACLPolicies
for i := 0; i < 50; i++ {
arg := structs.ACLPolicySetRequest{
Datacenter: "dc1",
Policy: structs.ACLPolicy{
Name: fmt.Sprintf("token-%d", i),
Description: fmt.Sprintf("token-%d", i),
Rules: fmt.Sprintf(`service "app-%d" { policy = "read" }`, i),
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var policy structs.ACLPolicy
require.NoError(t, s1.RPC("ACL.PolicySet", &arg, &policy))
policies = append(policies, &policy)
}
checkSame := func(t *retry.R) {
// only account for global tokens - local tokens shouldn't be replicated
index, remote, err := s1.fsm.State().ACLPolicyList(nil)
require.NoError(t, err)
_, local, err := s2.fsm.State().ACLPolicyList(nil)
require.NoError(t, err)
require.Len(t, local, len(remote))
for i, policy := range remote {
require.Equal(t, policy.Hash, local[i].Hash)
}
s2.aclReplicationStatusLock.RLock()
status := s2.aclReplicationStatus
s2.aclReplicationStatusLock.RUnlock()
require.True(t, status.Enabled)
require.True(t, status.Running)
require.Equal(t, status.ReplicationType, structs.ACLReplicatePolicies)
require.Equal(t, status.ReplicatedIndex, index)
require.Equal(t, status.SourceDatacenter, "dc1")
}
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
// Update those policies
for i := 0; i < 50; i++ {
arg := structs.ACLPolicySetRequest{
Datacenter: "dc1",
Policy: structs.ACLPolicy{
ID: policies[i].ID,
Name: fmt.Sprintf("token-%d-modified", i),
Description: fmt.Sprintf("token-%d-modified", i),
Rules: policies[i].Rules,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var policy structs.ACLPolicy
require.NoError(t, s1.RPC("ACL.PolicySet", &arg, &policy))
}
// Wait for the replica to converge.
// this time it also verifies the local tokens from the primary were not replicated.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
for _, policy := range policies {
arg := structs.ACLPolicyDeleteRequest{
Datacenter: "dc1",
PolicyID: policy.ID,
WriteRequest: structs.WriteRequest{Token: "root"},
}
var dontCare string
require.NoError(t, s1.RPC("ACL.PolicyDelete", &arg, &dontCare))
}
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
}
func TestACLReplication_TokensRedacted(t *testing.T) {
t.Parallel()
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLMasterToken = "root"
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
testrpc.WaitForLeader(t, s1.RPC, "dc1")
client := rpcClient(t, s1)
defer client.Close()
// Create the ACL Write Policy
policyArg := structs.ACLPolicySetRequest{
Datacenter: "dc1",
Policy: structs.ACLPolicy{
Name: "token-replication-redacted",
Description: "token-replication-redacted",
Rules: `acl = "write"`,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var policy structs.ACLPolicy
require.NoError(t, s1.RPC("ACL.PolicySet", &policyArg, &policy))
// Create the dc2 replication token
tokenArg := structs.ACLTokenSetRequest{
Datacenter: "dc1",
ACLToken: structs.ACLToken{
Description: "dc2-replication",
Policies: []structs.ACLTokenPolicyLink{
structs.ACLTokenPolicyLink{
ID: policy.ID,
},
},
Local: false,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var token structs.ACLToken
require.NoError(t, s1.RPC("ACL.TokenSet", &tokenArg, &token))
dir2, s2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLTokenReplication = true
c.ACLReplicationRate = 100
c.ACLReplicationBurst = 100
c.ACLReplicationApplyLimit = 1000000
})
s2.tokens.UpdateReplicationToken(token.SecretID, tokenStore.TokenSourceConfig)
testrpc.WaitForLeader(t, s2.RPC, "dc2")
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join.
joinWAN(t, s2, s1)
testrpc.WaitForLeader(t, s2.RPC, "dc2")
testrpc.WaitForLeader(t, s2.RPC, "dc1")
waitForNewACLs(t, s2)
// ensures replication is working ok
retry.Run(t, func(r *retry.R) {
var tokenResp structs.ACLTokenResponse
req := structs.ACLTokenGetRequest{
Datacenter: "dc2",
TokenID: "root",
TokenIDType: structs.ACLTokenSecret,
QueryOptions: structs.QueryOptions{Token: "root"},
}
err := s2.RPC("ACL.TokenRead", &req, &tokenResp)
require.NoError(r, err)
require.NotNil(r, tokenResp.Token)
require.Equal(r, "root", tokenResp.Token.SecretID)
var status structs.ACLReplicationStatus
statusReq := structs.DCSpecificRequest{
Datacenter: "dc2",
}
require.NoError(r, s2.RPC("ACL.ReplicationStatus", &statusReq, &status))
// ensures that tokens are not being synced
require.True(r, status.ReplicatedTokenIndex > 0, "ReplicatedTokenIndex not greater than 0")
})
// modify the replication policy to change to only granting read privileges
policyArg = structs.ACLPolicySetRequest{
Datacenter: "dc1",
Policy: structs.ACLPolicy{
ID: policy.ID,
Name: "token-replication-redacted",
Description: "token-replication-redacted",
Rules: `acl = "read"`,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
require.NoError(t, s1.RPC("ACL.PolicySet", &policyArg, &policy))
// Create the another token so that replication will attempt to read it.
tokenArg = structs.ACLTokenSetRequest{
Datacenter: "dc1",
ACLToken: structs.ACLToken{
Description: "management",
Policies: []structs.ACLTokenPolicyLink{
structs.ACLTokenPolicyLink{
ID: structs.ACLPolicyGlobalManagementID,
},
},
Local: false,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var token2 structs.ACLToken
// record the time right before we are touching the token
minErrorTime := time.Now()
require.NoError(t, s1.RPC("ACL.TokenSet", &tokenArg, &token2))
retry.Run(t, func(r *retry.R) {
var tokenResp structs.ACLTokenResponse
req := structs.ACLTokenGetRequest{
Datacenter: "dc2",
TokenID: redactedToken,
TokenIDType: structs.ACLTokenSecret,
QueryOptions: structs.QueryOptions{Token: redactedToken},
}
err := s2.RPC("ACL.TokenRead", &req, &tokenResp)
// its not an error for the secret to not be found.
require.NoError(r, err)
require.Nil(r, tokenResp.Token)
var status structs.ACLReplicationStatus
statusReq := structs.DCSpecificRequest{
Datacenter: "dc2",
}
require.NoError(r, s2.RPC("ACL.ReplicationStatus", &statusReq, &status))
// ensures that tokens are not being synced
require.True(r, status.ReplicatedTokenIndex < token2.CreateIndex, "ReplicatedTokenIndex is not less than the token2s create index")
// ensures that token replication is erroring
require.True(r, status.LastError.After(minErrorTime), "Replication LastError not after the minErrorTime")
})
}
func TestACLReplication_AllTypes(t *testing.T) {
t.Parallel()
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLMasterToken = "root"
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
testrpc.WaitForLeader(t, s1.RPC, "dc1")
client := rpcClient(t, s1)
defer client.Close()
dir2, s2 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc2"
c.ACLDatacenter = "dc1"
c.ACLsEnabled = true
c.ACLTokenReplication = true
c.ACLReplicationRate = 100
c.ACLReplicationBurst = 100
c.ACLReplicationApplyLimit = 1000000
})
s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
testrpc.WaitForLeader(t, s2.RPC, "dc2")
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join.
joinWAN(t, s2, s1)
testrpc.WaitForLeader(t, s1.RPC, "dc1")
testrpc.WaitForLeader(t, s1.RPC, "dc2")
// Wait for legacy acls to be disabled so we are clear that
// legacy replication isn't meddling.
waitForNewACLs(t, s1)
waitForNewACLs(t, s2)
waitForNewACLReplication(t, s2, structs.ACLReplicateTokens, 1, 1, 0)
const (
numItems = 50
numItemsThatAreLocal = 10
)
// Create some data.
policyIDs, roleIDs, tokenIDs := createACLTestData(t, s1, "b1", numItems, numItemsThatAreLocal)
checkSameTokens := func(t *retry.R) {
// only account for global tokens - local tokens shouldn't be replicated
index, remote, err := s1.fsm.State().ACLTokenList(nil, false, true, "", "", "")
require.NoError(t, err)
// Query for all of them, so that we can prove that no globals snuck in.
_, local, err := s2.fsm.State().ACLTokenList(nil, true, true, "", "", "")
require.NoError(t, err)
require.Len(t, remote, len(local))
for i, token := range remote {
require.Equal(t, token.Hash, local[i].Hash)
}
s2.aclReplicationStatusLock.RLock()
status := s2.aclReplicationStatus
s2.aclReplicationStatusLock.RUnlock()
require.True(t, status.Enabled)
require.True(t, status.Running)
require.Equal(t, status.ReplicationType, structs.ACLReplicateTokens)
require.Equal(t, status.ReplicatedTokenIndex, index)
require.Equal(t, status.SourceDatacenter, "dc1")
}
checkSamePolicies := func(t *retry.R) {
index, remote, err := s1.fsm.State().ACLPolicyList(nil)
require.NoError(t, err)
_, local, err := s2.fsm.State().ACLPolicyList(nil)
require.NoError(t, err)
require.Len(t, remote, len(local))
for i, policy := range remote {
require.Equal(t, policy.Hash, local[i].Hash)
}
s2.aclReplicationStatusLock.RLock()
status := s2.aclReplicationStatus
s2.aclReplicationStatusLock.RUnlock()
require.True(t, status.Enabled)
require.True(t, status.Running)
require.Equal(t, status.ReplicationType, structs.ACLReplicateTokens)
require.Equal(t, status.ReplicatedIndex, index)
require.Equal(t, status.SourceDatacenter, "dc1")
}
checkSameRoles := func(t *retry.R) {
index, remote, err := s1.fsm.State().ACLRoleList(nil, "")
require.NoError(t, err)
_, local, err := s2.fsm.State().ACLRoleList(nil, "")
require.NoError(t, err)
require.Len(t, remote, len(local))
for i, role := range remote {
require.Equal(t, role.Hash, local[i].Hash)
}
s2.aclReplicationStatusLock.RLock()
status := s2.aclReplicationStatus
s2.aclReplicationStatusLock.RUnlock()
require.True(t, status.Enabled)
require.True(t, status.Running)
require.Equal(t, status.ReplicationType, structs.ACLReplicateTokens)
require.Equal(t, status.ReplicatedRoleIndex, index)
require.Equal(t, status.SourceDatacenter, "dc1")
}
checkSame := func(t *retry.R) {
checkSameTokens(t)
checkSamePolicies(t)
checkSameRoles(t)
}
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
// Create additional data to replicate.
_, _, _ = createACLTestData(t, s1, "b2", numItems, numItemsThatAreLocal)
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
// Delete one piece of each type of data from batch 1.
const itemToDelete = numItems - 1
{
id := tokenIDs[itemToDelete]
arg := structs.ACLTokenDeleteRequest{
Datacenter: "dc1",
TokenID: id,
WriteRequest: structs.WriteRequest{Token: "root"},
}
var dontCare string
if err := s1.RPC("ACL.TokenDelete", &arg, &dontCare); err != nil {
t.Fatalf("err: %v", err)
}
}
{
id := roleIDs[itemToDelete]
arg := structs.ACLRoleDeleteRequest{
Datacenter: "dc1",
RoleID: id,
WriteRequest: structs.WriteRequest{Token: "root"},
}
var dontCare string
if err := s1.RPC("ACL.RoleDelete", &arg, &dontCare); err != nil {
t.Fatalf("err: %v", err)
}
}
{
id := policyIDs[itemToDelete]
arg := structs.ACLPolicyDeleteRequest{
Datacenter: "dc1",
PolicyID: id,
WriteRequest: structs.WriteRequest{Token: "root"},
}
var dontCare string
if err := s1.RPC("ACL.PolicyDelete", &arg, &dontCare); err != nil {
t.Fatalf("err: %v", err)
}
}
// Wait for the replica to converge.
retry.Run(t, func(r *retry.R) {
checkSame(r)
})
}
func createACLTestData(t *testing.T, srv *Server, namePrefix string, numObjects, numItemsThatAreLocal int) (policyIDs, roleIDs, tokenIDs []string) {
require.True(t, numItemsThatAreLocal <= numObjects, 0, "numItemsThatAreLocal <= numObjects")
// Create some policies.
for i := 0; i < numObjects; i++ {
str := strconv.Itoa(i)
arg := structs.ACLPolicySetRequest{
Datacenter: "dc1",
Policy: structs.ACLPolicy{
Name: namePrefix + "-policy-" + str,
Description: namePrefix + "-policy " + str,
Rules: testACLPolicyNew,
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var out structs.ACLPolicy
if err := srv.RPC("ACL.PolicySet", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
policyIDs = append(policyIDs, out.ID)
}
// Create some roles.
for i := 0; i < numObjects; i++ {
str := strconv.Itoa(i)
arg := structs.ACLRoleSetRequest{
Datacenter: "dc1",
Role: structs.ACLRole{
Name: namePrefix + "-role-" + str,
Description: namePrefix + "-role " + str,
Policies: []structs.ACLRolePolicyLink{
{ID: policyIDs[i]},
},
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var out structs.ACLRole
if err := srv.RPC("ACL.RoleSet", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
roleIDs = append(roleIDs, out.ID)
}
// Create a bunch of new tokens.
for i := 0; i < numObjects; i++ {
str := strconv.Itoa(i)
arg := structs.ACLTokenSetRequest{
Datacenter: "dc1",
ACLToken: structs.ACLToken{
Description: namePrefix + "-token " + str,
Policies: []structs.ACLTokenPolicyLink{
{ID: policyIDs[i]},
},
Roles: []structs.ACLTokenRoleLink{
{ID: roleIDs[i]},
},
Local: (i < numItemsThatAreLocal),
},
WriteRequest: structs.WriteRequest{Token: "root"},
}
var out structs.ACLToken
if err := srv.RPC("ACL.TokenSet", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
tokenIDs = append(tokenIDs, out.AccessorID)
}
return policyIDs, roleIDs, tokenIDs
}