From 3b22ab248676f94e0e1f1f9af9e4a823c9b2df49 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Mon, 22 Jul 2019 12:02:48 -0400 Subject: [PATCH] Add chunking support to raft --- go.mod | 5 +++-- go.sum | 6 ++++++ physical/raft/raft.go | 21 ++++++++------------- physical/raft/raft_test.go | 28 ---------------------------- physical/raft/types.proto | 6 +++--- 5 files changed, 20 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index d592c9bc1..300717f01 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-errors/errors v1.0.1 github.com/go-sql-driver/mysql v1.4.1 - github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31 + github.com/go-test/deep v1.0.2 github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.3.1 @@ -59,6 +59,7 @@ require ( github.com/hashicorp/go-memdb v1.0.2 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/go-multierror v1.0.0 + github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e github.com/hashicorp/go-rootcerts v1.0.1 github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/go-syslog v1.0.0 @@ -66,7 +67,7 @@ require ( github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf - github.com/hashicorp/raft v1.1.1-0.20190719143500-dcbf3052ba60 + github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 github.com/hashicorp/raft-snapshot v1.0.1 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.2-0.20190719145746-998f93644e9d github.com/hashicorp/vault-plugin-auth-azure v0.5.2-0.20190719145754-00bc60f1941d diff --git a/go.sum b/go.sum index 328880dea..81dd72caf 100644 --- a/go.sum +++ b/go.sum @@ -161,6 +161,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31 h1:28FVBuwkwowZMjbA7M0wXsI6t3PYulRTMio3SO+eKCM= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df h1:fwXmhM0OqixzJDOGgTSyNH9eEDij9uGTXwsyWXvyR0A= github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -262,6 +264,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e h1:cwjgfF2tRzfpGXS+sIdGSq8tQo8Zjwakvq6kMcEifpI= +github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e/go.mod h1:gjvnPBjZ7yydFYq3AaKGlrwW8p7BEIeV1OljPYpJzto= github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= @@ -297,6 +301,8 @@ github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf/go.mod h1:BDng github.com/hashicorp/raft v1.0.1/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= github.com/hashicorp/raft v1.1.1-0.20190719143500-dcbf3052ba60 h1:yH2Ks+0cEH2GzxvT5J/xYk26f31bKF9EM8K3RjJfIKE= github.com/hashicorp/raft v1.1.1-0.20190719143500-dcbf3052ba60/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 h1:/VZgMcVsZOKyA5SeKHjARffb/5qd6ZX5TEB0i+FuL3M= +github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= github.com/hashicorp/raft-snapshot v1.0.1 h1:cx002JsTEAfAP0pIuANlDtTXg/pi2Db6YbRRmLQTQKw= github.com/hashicorp/raft-snapshot v1.0.1/go.mod h1:5sL9eUn72lH5DzsFIJ9jaysITbHksSSszImWSOTC8Ic= diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 0f7042b1b..1eef094f6 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -15,6 +15,7 @@ import ( proto "github.com/golang/protobuf/proto" "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-raftchunking" uuid "github.com/hashicorp/go-uuid" "github.com/hashicorp/raft" snapshot "github.com/hashicorp/raft-snapshot" @@ -39,13 +40,6 @@ var ( peersFileName = "peers.json" snapshotsRetained = 2 - // Set a max size of 512kb - maxCommandSizeBytes = 512 * 1024 - - // ErrCommandTooLarge is returned when the backend tries to apply a log - // greater than the max allowed size. - ErrCommandTooLarge = fmt.Errorf("%s: exceeds %d byte limit", physical.ErrValueTooLarge, maxCommandSizeBytes) - restoreOpDelayDuration = 5 * time.Second ) @@ -460,7 +454,7 @@ func (b *RaftBackend) SetupCluster(ctx context.Context, raftTLSKeyring *RaftTLSK b.logger.Info("raft recovery deleted peers.json") } - raftObj, err := raft.NewRaft(raftConfig, b.fsm, b.logStore, b.stableStore, b.snapStore, b.raftTransport) + raftObj, err := raft.NewRaft(raftConfig, raftchunking.NewChunkingFSM(b.fsm), b.logStore, b.stableStore, b.snapStore, b.raftTransport) b.fsm.SetNoopRestore(false) if err != nil { return err @@ -783,12 +777,13 @@ func (b *RaftBackend) applyLog(ctx context.Context, command *LogData) error { return err } - // Restrict the value to maxCommandSizeBytes in length - if len(commandBytes) > maxCommandSizeBytes { - return ErrCommandTooLarge + var applyFuture raft.ApplyFuture + switch { + case len(commandBytes) <= raft.SuggestedMaxDataSize: + applyFuture = b.raft.Apply(commandBytes, 0) + default: + applyFuture = raftchunking.ChunkingApply(commandBytes, nil, 0, b.raft.ApplyLog) } - - applyFuture := b.raft.Apply(commandBytes, 0) err = applyFuture.Error() if err != nil { return err diff --git a/physical/raft/raft_test.go b/physical/raft/raft_test.go index b5eb7130e..a93a228a5 100644 --- a/physical/raft/raft_test.go +++ b/physical/raft/raft_test.go @@ -295,34 +295,6 @@ func TestRaft_TransactionalBackend_ThreeNode(t *testing.T) { compareFSMs(t, raft1.fsm, raft3.fsm) } -func TestRaft_Backend_MaxSize(t *testing.T) { - // Set the max size a little lower for the test - maxCommandSizeBytes = 10 * 1024 - - b, dir := getRaft(t, true, true) - defer os.RemoveAll(dir) - - // Test a value slightly below the max size - value := make([]byte, maxCommandSizeBytes-100) - err := b.Put(context.Background(), &physical.Entry{ - Key: "key", - Value: value, - }) - if err != nil { - t.Fatal(err) - } - - // Test value at max size, should error - value = make([]byte, maxCommandSizeBytes) - err = b.Put(context.Background(), &physical.Entry{ - Key: "key", - Value: value, - }) - if err != ErrCommandTooLarge { - t.Fatal(err) - } -} - func TestRaft_Backend_Performance(t *testing.T) { b, dir := getRaft(t, true, false) defer os.RemoveAll(dir) diff --git a/physical/raft/types.proto b/physical/raft/types.proto index bec32ad71..1bb923fad 100644 --- a/physical/raft/types.proto +++ b/physical/raft/types.proto @@ -9,13 +9,13 @@ message LogOperation { uint32 op_type = 1; // Flags is an opaque value, currently unused. Reserved. - uint64 flags = 2; + uint64 flags = 2; // Key that is being affected - string key = 3; + string key = 3; // Value is optional, corresponds to the key - bytes value = 4; + bytes value = 4; } message LogData {