luxolus
/
open-vault
2
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

Add chunking support to raft

This commit is contained in:
Jeff Mitchell 2019-07-22 12:02:48 -04:00
parent 0ee85c06b8
commit 3b22ab2486
5 changed files with 20 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@ -44,7 +44,7 @@ require (
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/go-errors/errors v1.0.1
github.com/go-sql-driver/mysql v1.4.1
github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31
github.com/go-test/deep v1.0.2
github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.3.1
@ -59,6 +59,7 @@ require (
github.com/hashicorp/go-memdb v1.0.2
github.com/hashicorp/go-msgpack v0.5.5
github.com/hashicorp/go-multierror v1.0.0
github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e
github.com/hashicorp/go-rootcerts v1.0.1
github.com/hashicorp/go-sockaddr v1.0.2
github.com/hashicorp/go-syslog v1.0.0
@ -66,7 +67,7 @@ require (
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/hcl v1.0.0
github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf
github.com/hashicorp/raft v1.1.1-0.20190719143500-dcbf3052ba60
github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3
github.com/hashicorp/raft-snapshot v1.0.1
github.com/hashicorp/vault-plugin-auth-alicloud v0.5.2-0.20190719145746-998f93644e9d
github.com/hashicorp/vault-plugin-auth-azure v0.5.2-0.20190719145754-00bc60f1941d

6
go.sum
View File

@ -161,6 +161,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31 h1:28FVBuwkwowZMjbA7M0wXsI6t3PYulRTMio3SO+eKCM=
github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df h1:fwXmhM0OqixzJDOGgTSyNH9eEDij9uGTXwsyWXvyR0A=
github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@ -262,6 +264,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE=
github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e h1:cwjgfF2tRzfpGXS+sIdGSq8tQo8Zjwakvq6kMcEifpI=
github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e/go.mod h1:gjvnPBjZ7yydFYq3AaKGlrwW8p7BEIeV1OljPYpJzto=
github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE=
@ -297,6 +301,8 @@ github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf/go.mod h1:BDng
github.com/hashicorp/raft v1.0.1/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI=
github.com/hashicorp/raft v1.1.1-0.20190719143500-dcbf3052ba60 h1:yH2Ks+0cEH2GzxvT5J/xYk26f31bKF9EM8K3RjJfIKE=
github.com/hashicorp/raft v1.1.1-0.20190719143500-dcbf3052ba60/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 h1:/VZgMcVsZOKyA5SeKHjARffb/5qd6ZX5TEB0i+FuL3M=
github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
github.com/hashicorp/raft-snapshot v1.0.1 h1:cx002JsTEAfAP0pIuANlDtTXg/pi2Db6YbRRmLQTQKw=
github.com/hashicorp/raft-snapshot v1.0.1/go.mod h1:5sL9eUn72lH5DzsFIJ9jaysITbHksSSszImWSOTC8Ic=

21
physical/raft/raft.go
View File

@ -15,6 +15,7 @@ import (
proto "github.com/golang/protobuf/proto"
"github.com/hashicorp/errwrap"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-raftchunking"
uuid "github.com/hashicorp/go-uuid"
"github.com/hashicorp/raft"
snapshot "github.com/hashicorp/raft-snapshot"
@ -39,13 +40,6 @@ var (
peersFileName = "peers.json"
snapshotsRetained = 2
// Set a max size of 512kb
maxCommandSizeBytes = 512 * 1024
// ErrCommandTooLarge is returned when the backend tries to apply a log
// greater than the max allowed size.
ErrCommandTooLarge = fmt.Errorf("%s: exceeds %d byte limit", physical.ErrValueTooLarge, maxCommandSizeBytes)
restoreOpDelayDuration = 5 * time.Second
)
@ -460,7 +454,7 @@ func (b *RaftBackend) SetupCluster(ctx context.Context, raftTLSKeyring *RaftTLSK
b.logger.Info("raft recovery deleted peers.json")
}
raftObj, err := raft.NewRaft(raftConfig, b.fsm, b.logStore, b.stableStore, b.snapStore, b.raftTransport)
raftObj, err := raft.NewRaft(raftConfig, raftchunking.NewChunkingFSM(b.fsm), b.logStore, b.stableStore, b.snapStore, b.raftTransport)
b.fsm.SetNoopRestore(false)
if err != nil {
return err
@ -783,12 +777,13 @@ func (b *RaftBackend) applyLog(ctx context.Context, command *LogData) error {
return err
}
// Restrict the value to maxCommandSizeBytes in length
if len(commandBytes) > maxCommandSizeBytes {
return ErrCommandTooLarge
var applyFuture raft.ApplyFuture
switch {
case len(commandBytes) <= raft.SuggestedMaxDataSize:
applyFuture = b.raft.Apply(commandBytes, 0)
default:
applyFuture = raftchunking.ChunkingApply(commandBytes, nil, 0, b.raft.ApplyLog)
}
applyFuture := b.raft.Apply(commandBytes, 0)
err = applyFuture.Error()
if err != nil {
return err

28
physical/raft/raft_test.go
View File

@ -295,34 +295,6 @@ func TestRaft_TransactionalBackend_ThreeNode(t *testing.T) {
compareFSMs(t, raft1.fsm, raft3.fsm)
}
func TestRaft_Backend_MaxSize(t *testing.T) {
// Set the max size a little lower for the test
maxCommandSizeBytes = 10 * 1024
b, dir := getRaft(t, true, true)
defer os.RemoveAll(dir)
// Test a value slightly below the max size
value := make([]byte, maxCommandSizeBytes-100)
err := b.Put(context.Background(), &physical.Entry{
Key: "key",
Value: value,
})
if err != nil {
t.Fatal(err)
}
// Test value at max size, should error
value = make([]byte, maxCommandSizeBytes)
err = b.Put(context.Background(), &physical.Entry{
Key: "key",
Value: value,
})
if err != ErrCommandTooLarge {
t.Fatal(err)
}
}
func TestRaft_Backend_Performance(t *testing.T) {
b, dir := getRaft(t, true, false)
defer os.RemoveAll(dir)

6
physical/raft/types.proto
View File

@ -9,13 +9,13 @@ message LogOperation {
uint32 op_type = 1;
// Flags is an opaque value, currently unused. Reserved.
uint64 flags = 2;
uint64 flags = 2;
// Key that is being affected
string key = 3;
string key = 3;
// Value is optional, corresponds to the key
bytes value = 4;
bytes value = 4;
}
message LogData {