From 5a9948fab7948f81d5036de26ebc8591717d6842 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Sat, 18 Mar 2023 16:43:22 -0400 Subject: [PATCH] Snapshot restore tests (#16647) * add snapshot restore test * add logstore as test parameter * Use the correct image version * make sure we read the logs from a followers to test the follower snapshot install path. * update to raf-wal v0.3.0 * add changelog. * updating changelog for bug description and removed integration test. * setting up test container builder to only set logStore for 1.15 and higher --------- Co-authored-by: Paul Banks Co-authored-by: John Murret --- .changelog/16647.txt | 3 + go.mod | 10 +- go.sum | 21 ++-- .../consul-container/libs/cluster/builder.go | 24 +++++ .../libs/cluster/container.go | 10 +- .../libs/topology/peering_topology.go | 1 + .../test/snapshot/snapshot_restore_test.go | 97 +++++++++++++++++++ 7 files changed, 150 insertions(+), 16 deletions(-) create mode 100644 .changelog/16647.txt create mode 100644 test/integration/consul-container/test/snapshot/snapshot_restore_test.go diff --git a/.changelog/16647.txt b/.changelog/16647.txt new file mode 100644 index 000000000..cbe38b3ed --- /dev/null +++ b/.changelog/16647.txt @@ -0,0 +1,3 @@ +```release-note:bug +raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic. +``` diff --git a/go.mod b/go.mod index 570f6692a..4b76c79a4 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/NYTimes/gziphandler v1.0.1 github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e - github.com/armon/go-metrics v0.3.10 + github.com/armon/go-metrics v0.4.1 github.com/armon/go-radix v1.0.0 github.com/aws/aws-sdk-go v1.42.34 github.com/coredns/coredns v1.6.6 @@ -64,10 +64,10 @@ require ( github.com/hashicorp/hcp-sdk-go v0.23.1-0.20220921131124-49168300a7dc github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038 github.com/hashicorp/memberlist v0.5.0 - github.com/hashicorp/raft v1.3.11 + github.com/hashicorp/raft v1.4.0 github.com/hashicorp/raft-autopilot v0.1.6 github.com/hashicorp/raft-boltdb/v2 v2.2.2 - github.com/hashicorp/raft-wal v0.2.4 + github.com/hashicorp/raft-wal v0.3.0 github.com/hashicorp/serf v0.10.1 github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0 github.com/hashicorp/vault/api v1.8.3 @@ -92,7 +92,7 @@ require ( github.com/rboyer/safeio v0.2.1 github.com/ryanuber/columnize v2.1.2+incompatible github.com/shirou/gopsutil/v3 v3.22.8 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 go.uber.org/goleak v1.1.10 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -215,7 +215,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect diff --git a/go.sum b/go.sum index 0ed6defcb..4667b8247 100644 --- a/go.sum +++ b/go.sum @@ -144,8 +144,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -610,8 +610,9 @@ github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 h1:kBpVVl1sl3MaSrs97e0+pDQhSrq github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56O3N18qBrAz/XRIcfINfTbwo= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= -github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= +github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY= +github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew= github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I= github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= @@ -619,8 +620,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qR github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc= github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw= github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY= -github.com/hashicorp/raft-wal v0.2.4 h1:Ke0ytMj8XyOVKQqFDmmgs/6hqkTJg0b/GO2a2XQBZ6A= -github.com/hashicorp/raft-wal v0.2.4/go.mod h1:JQ/4RbnKFi5Q/4rA73CekaYtHCJhU7qM7AQ4X5Y6q4M= +github.com/hashicorp/raft-wal v0.3.0 h1:Mi6RPoRbsxIIYZryI+bSTXHD97Ua6rIYO51ibYV9bkY= +github.com/hashicorp/raft-wal v0.3.0/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0 h1:O6tNk0s/arubLUbLeCyaRs5xGo9VwmbQazISY/BfPK4= @@ -703,8 +704,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -928,8 +929,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/zerolog v1.4.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -984,8 +985,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -994,8 +996,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 h1:8fDzz4GuVg4skjY2B0nMN7h6uN61EDVkuLyI2+qGHhI= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index 9ce5bf5c8..2807985e1 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -20,6 +20,13 @@ const ( ConsulCACertKey = "consul-agent-ca-key.pem" ) +type LogStore string + +const ( + LogStore_WAL LogStore = "wal" + LogStore_BoltDB LogStore = "boltdb" +) + // BuildContext provides a reusable object meant to share common configuration settings // between agent configuration builders. type BuildContext struct { @@ -41,6 +48,7 @@ type BuildContext struct { tlsCertIndex int // keeps track of the certificates issued for naming purposes aclEnabled bool + logStore LogStore } func (c *BuildContext) DockerImage() string { @@ -89,6 +97,9 @@ type BuildOptions struct { // ACLEnabled configures acl in agent configuration ACLEnabled bool + + //StoreLog define which LogStore to use + LogStore LogStore } func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { @@ -103,6 +114,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { useAPIWithTLS: opts.UseAPIWithTLS, useGRPCWithTLS: opts.UseGRPCWithTLS, aclEnabled: opts.ACLEnabled, + logStore: opts.LogStore, } if ctx.consulImageName == "" { @@ -202,6 +214,18 @@ func NewConfigBuilder(ctx *BuildContext) *Builder { b.conf.Set("acl.enable_token_persistence", true) } + ls := string(ctx.logStore) + if ls != "" && (ctx.consulVersion == "local" || + semver.Compare("v"+ctx.consulVersion, "v1.15.0") >= 0) { + // Enable logstore backend for version after v1.15.0 + if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) { + ls = string(LogStore_BoltDB) + } + b.conf.Set("raft_logstore.backend", ls) + } else { + b.conf.Unset("raft_logstore.backend") + } + return b } diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 3ad1b2d35..4e3e1c399 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -154,11 +154,17 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) if httpPort > 0 { - uri, err := podContainer.PortEndpoint(ctx, "8500", "http") + for i := 0; i < 10; i++ { + uri, err := podContainer.PortEndpoint(ctx, "8500", "http") + if err != nil { + time.Sleep(500 * time.Millisecond) + continue + } + clientAddr = uri + } if err != nil { return nil, err } - clientAddr = uri } else if httpsPort > 0 { uri, err := podContainer.PortEndpoint(ctx, "8501", "https") diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 0e853f7ad..3e7ac8d06 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -198,6 +198,7 @@ func NewCluster( AllowHTTPAnyway: true, ConsulVersion: config.BuildOpts.ConsulVersion, ACLEnabled: config.BuildOpts.ACLEnabled, + LogStore: config.BuildOpts.LogStore, } ctx := libcluster.NewBuildContext(t, opts) diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go new file mode 100644 index 000000000..1d82b1cfb --- /dev/null +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -0,0 +1,97 @@ +package snapshot + +import ( + "fmt" + "github.com/hashicorp/consul/api" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/stretchr/testify/require" + "io" + "testing" +) + +func TestSnapshotRestore(t *testing.T) { + + cases := []libcluster.LogStore{libcluster.LogStore_WAL, libcluster.LogStore_BoltDB} + + for _, c := range cases { + t.Run(fmt.Sprintf("test log store: %s", c), func(t *testing.T) { + testSnapShotRestoreForLogStore(t, c) + }) + } +} + +func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) { + + const ( + numServers = 3 + ) + + // Create initial cluster + cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: numServers, + NumClients: 0, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulImageName: utils.TargetImageName, + ConsulVersion: utils.TargetVersion, + LogStore: logStore, + }, + ApplyDefaultProxySettings: true, + }) + + client := cluster.APIClient(0) + libcluster.WaitForLeader(t, cluster, client) + libcluster.WaitForMembers(t, client, 3) + + for i := 0; i < 100; i++ { + _, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil) + require.NoError(t, err) + } + + var snapshot io.ReadCloser + var err error + snapshot, _, err = client.Snapshot().Save(nil) + require.NoError(t, err) + + err = cluster.Terminate() + require.NoError(t, err) + // Create a fresh cluster from scratch + cluster2, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: numServers, + NumClients: 0, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulImageName: utils.TargetImageName, + ConsulVersion: utils.TargetVersion, + LogStore: logStore, + }, + ApplyDefaultProxySettings: true, + }) + client2 := cluster2.APIClient(0) + + libcluster.WaitForLeader(t, cluster2, client2) + libcluster.WaitForMembers(t, client2, 3) + + // Restore the saved snapshot + require.NoError(t, client2.Snapshot().Restore(nil, snapshot)) + + libcluster.WaitForLeader(t, cluster2, client2) + + followers, err := cluster2.Followers() + require.NoError(t, err) + require.Len(t, followers, 2) + + // use a follower api client and set `AllowStale` to true + // to test the follower snapshot install code path as well. + fc := followers[0].GetClient() + + for i := 0; i < 100; i++ { + kv, _, err := fc.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true}) + require.NoError(t, err) + require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i)) + require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i))) + } + +}