Snapshot restore tests (#16647)

* add snapshot restore test

* add logstore as test parameter

* Use the correct image version

* make sure we read the logs from a followers to test the follower snapshot install path.

* update to raf-wal v0.3.0

* add changelog.

* updating changelog for bug description and removed integration test.

* setting up test container builder to only set logStore for 1.15 and higher

---------

Co-authored-by: Paul Banks <pbanks@hashicorp.com>
Co-authored-by: John Murret <john.murret@hashicorp.com>
This commit is contained in:
Dhia Ayachi 2023-03-18 16:43:22 -04:00 committed by GitHub
parent adbd0626af
commit 5a9948fab7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 150 additions and 16 deletions

3
.changelog/16647.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:bug
raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic.
```

10
go.mod
View File

@ -19,7 +19,7 @@ require (
github.com/NYTimes/gziphandler v1.0.1
github.com/aliyun/alibaba-cloud-sdk-go v1.62.156
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e
github.com/armon/go-metrics v0.3.10
github.com/armon/go-metrics v0.4.1
github.com/armon/go-radix v1.0.0
github.com/aws/aws-sdk-go v1.42.34
github.com/coredns/coredns v1.6.6
@ -64,10 +64,10 @@ require (
github.com/hashicorp/hcp-sdk-go v0.23.1-0.20220921131124-49168300a7dc
github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038
github.com/hashicorp/memberlist v0.5.0
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft v1.4.0
github.com/hashicorp/raft-autopilot v0.1.6
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/hashicorp/raft-wal v0.2.4
github.com/hashicorp/raft-wal v0.3.0
github.com/hashicorp/serf v0.10.1
github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0
github.com/hashicorp/vault/api v1.8.3
@ -92,7 +92,7 @@ require (
github.com/rboyer/safeio v0.2.1
github.com/ryanuber/columnize v2.1.2+incompatible
github.com/shirou/gopsutil/v3 v3.22.8
github.com/stretchr/testify v1.8.0
github.com/stretchr/testify v1.8.2
go.etcd.io/bbolt v1.3.6
go.uber.org/goleak v1.1.10
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
@ -215,7 +215,7 @@ require (
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.4.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect

21
go.sum
View File

@ -144,8 +144,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@ -610,8 +610,9 @@ github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 h1:kBpVVl1sl3MaSrs97e0+pDQhSrq
github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56O3N18qBrAz/XRIcfINfTbwo=
github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A=
github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4=
github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY=
github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew=
github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I=
github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw=
github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
@ -619,8 +620,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qR
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc=
github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw=
github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY=
github.com/hashicorp/raft-wal v0.2.4 h1:Ke0ytMj8XyOVKQqFDmmgs/6hqkTJg0b/GO2a2XQBZ6A=
github.com/hashicorp/raft-wal v0.2.4/go.mod h1:JQ/4RbnKFi5Q/4rA73CekaYtHCJhU7qM7AQ4X5Y6q4M=
github.com/hashicorp/raft-wal v0.3.0 h1:Mi6RPoRbsxIIYZryI+bSTXHD97Ua6rIYO51ibYV9bkY=
github.com/hashicorp/raft-wal v0.3.0/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE=
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0 h1:O6tNk0s/arubLUbLeCyaRs5xGo9VwmbQazISY/BfPK4=
@ -703,8 +704,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -928,8 +929,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rs/zerolog v1.4.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@ -984,8 +985,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@ -994,8 +996,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 h1:8fDzz4GuVg4skjY2B0nMN7h6uN61EDVkuLyI2+qGHhI=
github.com/tencentcloud/tencentcloud-sdk-go v1.0.162/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=

View File

@ -20,6 +20,13 @@ const (
ConsulCACertKey = "consul-agent-ca-key.pem"
)
type LogStore string
const (
LogStore_WAL LogStore = "wal"
LogStore_BoltDB LogStore = "boltdb"
)
// BuildContext provides a reusable object meant to share common configuration settings
// between agent configuration builders.
type BuildContext struct {
@ -41,6 +48,7 @@ type BuildContext struct {
tlsCertIndex int // keeps track of the certificates issued for naming purposes
aclEnabled bool
logStore LogStore
}
func (c *BuildContext) DockerImage() string {
@ -89,6 +97,9 @@ type BuildOptions struct {
// ACLEnabled configures acl in agent configuration
ACLEnabled bool
//StoreLog define which LogStore to use
LogStore LogStore
}
func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext {
@ -103,6 +114,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext {
useAPIWithTLS: opts.UseAPIWithTLS,
useGRPCWithTLS: opts.UseGRPCWithTLS,
aclEnabled: opts.ACLEnabled,
logStore: opts.LogStore,
}
if ctx.consulImageName == "" {
@ -202,6 +214,18 @@ func NewConfigBuilder(ctx *BuildContext) *Builder {
b.conf.Set("acl.enable_token_persistence", true)
}
ls := string(ctx.logStore)
if ls != "" && (ctx.consulVersion == "local" ||
semver.Compare("v"+ctx.consulVersion, "v1.15.0") >= 0) {
// Enable logstore backend for version after v1.15.0
if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) {
ls = string(LogStore_BoltDB)
}
b.conf.Set("raft_logstore.backend", ls)
} else {
b.conf.Unset("raft_logstore.backend")
}
return b
}

View File

@ -154,11 +154,17 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
info AgentInfo
)
if httpPort > 0 {
uri, err := podContainer.PortEndpoint(ctx, "8500", "http")
for i := 0; i < 10; i++ {
uri, err := podContainer.PortEndpoint(ctx, "8500", "http")
if err != nil {
time.Sleep(500 * time.Millisecond)
continue
}
clientAddr = uri
}
if err != nil {
return nil, err
}
clientAddr = uri
} else if httpsPort > 0 {
uri, err := podContainer.PortEndpoint(ctx, "8501", "https")

View File

@ -198,6 +198,7 @@ func NewCluster(
AllowHTTPAnyway: true,
ConsulVersion: config.BuildOpts.ConsulVersion,
ACLEnabled: config.BuildOpts.ACLEnabled,
LogStore: config.BuildOpts.LogStore,
}
ctx := libcluster.NewBuildContext(t, opts)

View File

@ -0,0 +1,97 @@
package snapshot
import (
"fmt"
"github.com/hashicorp/consul/api"
libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
"github.com/stretchr/testify/require"
"io"
"testing"
)
func TestSnapshotRestore(t *testing.T) {
cases := []libcluster.LogStore{libcluster.LogStore_WAL, libcluster.LogStore_BoltDB}
for _, c := range cases {
t.Run(fmt.Sprintf("test log store: %s", c), func(t *testing.T) {
testSnapShotRestoreForLogStore(t, c)
})
}
}
func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) {
const (
numServers = 3
)
// Create initial cluster
cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{
NumServers: numServers,
NumClients: 0,
BuildOpts: &libcluster.BuildOptions{
Datacenter: "dc1",
ConsulImageName: utils.TargetImageName,
ConsulVersion: utils.TargetVersion,
LogStore: logStore,
},
ApplyDefaultProxySettings: true,
})
client := cluster.APIClient(0)
libcluster.WaitForLeader(t, cluster, client)
libcluster.WaitForMembers(t, client, 3)
for i := 0; i < 100; i++ {
_, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil)
require.NoError(t, err)
}
var snapshot io.ReadCloser
var err error
snapshot, _, err = client.Snapshot().Save(nil)
require.NoError(t, err)
err = cluster.Terminate()
require.NoError(t, err)
// Create a fresh cluster from scratch
cluster2, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{
NumServers: numServers,
NumClients: 0,
BuildOpts: &libcluster.BuildOptions{
Datacenter: "dc1",
ConsulImageName: utils.TargetImageName,
ConsulVersion: utils.TargetVersion,
LogStore: logStore,
},
ApplyDefaultProxySettings: true,
})
client2 := cluster2.APIClient(0)
libcluster.WaitForLeader(t, cluster2, client2)
libcluster.WaitForMembers(t, client2, 3)
// Restore the saved snapshot
require.NoError(t, client2.Snapshot().Restore(nil, snapshot))
libcluster.WaitForLeader(t, cluster2, client2)
followers, err := cluster2.Followers()
require.NoError(t, err)
require.Len(t, followers, 2)
// use a follower api client and set `AllowStale` to true
// to test the follower snapshot install code path as well.
fc := followers[0].GetClient()
for i := 0; i < 100; i++ {
kv, _, err := fc.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true})
require.NoError(t, err)
require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i))
require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i)))
}
}