diff --git a/go.mod b/go.mod index bd88d5d3f..1be2b332e 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/hashicorp/go-memdb v1.0.2 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/go-multierror v1.0.0 - github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e + github.com/hashicorp/go-raftchunking v0.5.1 github.com/hashicorp/go-rootcerts v1.0.1 github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/go-syslog v1.0.0 @@ -67,7 +67,7 @@ require ( github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf - github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 + github.com/hashicorp/raft v1.1.1 github.com/hashicorp/raft-snapshot v1.0.1 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.2-0.20190722164529-3bdfad02a66f github.com/hashicorp/vault-plugin-auth-azure v0.5.2-0.20190722164620-5255fa1e475b diff --git a/go.sum b/go.sum index d56a2d7a5..0aec8f94c 100644 --- a/go.sum +++ b/go.sum @@ -264,8 +264,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e h1:cwjgfF2tRzfpGXS+sIdGSq8tQo8Zjwakvq6kMcEifpI= -github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e/go.mod h1:gjvnPBjZ7yydFYq3AaKGlrwW8p7BEIeV1OljPYpJzto= +github.com/hashicorp/go-raftchunking v0.5.1 h1:IgWlAZiks8JaJ9NHkdBaml2ECubzvugrz0ANgbrk8nk= +github.com/hashicorp/go-raftchunking v0.5.1/go.mod h1:R5Nlfvg9MLL6ArwsSh9AVFsRpCtPaPCKYstXfU5GKeY= github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= @@ -299,8 +299,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf h1:U/40PQvWkaXCDdK9QHKf1pVDVcA+NIDVbzzonFGkgIA= github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf/go.mod h1:BDngVi1f4UA6aJq9WYTgxhfWSE1+42xshvstLU2fRGk= github.com/hashicorp/raft v1.0.1/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= -github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 h1:/VZgMcVsZOKyA5SeKHjARffb/5qd6ZX5TEB0i+FuL3M= -github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft v1.1.1 h1:HJr7UE1x/JrJSc9Oy6aDBHtNHUUBHjcQjTgvUVihoZs= +github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= github.com/hashicorp/raft-snapshot v1.0.1 h1:cx002JsTEAfAP0pIuANlDtTXg/pi2Db6YbRRmLQTQKw= github.com/hashicorp/raft-snapshot v1.0.1/go.mod h1:5sL9eUn72lH5DzsFIJ9jaysITbHksSSszImWSOTC8Ic= diff --git a/physical/raft/raft.go b/physical/raft/raft.go index ac4407f4a..99409e7e6 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -455,7 +455,7 @@ func (b *RaftBackend) SetupCluster(ctx context.Context, raftTLSKeyring *RaftTLSK b.logger.Info("raft recovery deleted peers.json") } - raftObj, err := raft.NewRaft(raftConfig, raftchunking.NewChunkingFSM(b.fsm), b.logStore, b.stableStore, b.snapStore, b.raftTransport) + raftObj, err := raft.NewRaft(raftConfig, raftchunking.NewChunkingConfigurationStore(b.fsm), b.logStore, b.stableStore, b.snapStore, b.raftTransport) b.fsm.SetNoopRestore(false) if err != nil { return err diff --git a/vendor/github.com/hashicorp/go-raftchunking/fsm.go b/vendor/github.com/hashicorp/go-raftchunking/fsm.go index cf61ea693..2106a9803 100644 --- a/vendor/github.com/hashicorp/go-raftchunking/fsm.go +++ b/vendor/github.com/hashicorp/go-raftchunking/fsm.go @@ -28,18 +28,36 @@ type chunkInfo struct { data []byte } +var _ raft.FSM = (*ChunkingFSM)(nil) +var _ raft.ConfigurationStore = (*ChunkingConfigurationStore)(nil) + type ChunkingFSM struct { underlying raft.FSM opMap map[uint64][]chunkInfo } -func NewChunkingFSM(underlying raft.FSM) *ChunkingFSM { +type ChunkingConfigurationStore struct { + *ChunkingFSM + underlyingConfigurationStore raft.ConfigurationStore +} + +func NewChunkingFSM(underlying raft.FSM) raft.FSM { return &ChunkingFSM{ underlying: underlying, opMap: make(map[uint64][]chunkInfo), } } +func NewChunkingConfigurationStore(underlying raft.ConfigurationStore) raft.ConfigurationStore { + return &ChunkingConfigurationStore{ + ChunkingFSM: &ChunkingFSM{ + underlying: underlying, + opMap: make(map[uint64][]chunkInfo), + }, + underlyingConfigurationStore: underlying, + } +} + // Apply applies the log, handling chunking as needed. The return value will // either be an error or whatever is returned from the underlying Apply. func (c *ChunkingFSM) Apply(l *raft.Log) interface{} { @@ -131,3 +149,7 @@ func (c *ChunkingFSM) Restore(rc io.ReadCloser) error { func (c *ChunkingFSM) Underlying() raft.FSM { return c.underlying } + +func (c *ChunkingConfigurationStore) StoreConfiguration(index uint64, configuration raft.Configuration) { + c.underlyingConfigurationStore.StoreConfiguration(index, configuration) +} diff --git a/vendor/github.com/hashicorp/go-raftchunking/go.mod b/vendor/github.com/hashicorp/go-raftchunking/go.mod index bc5de7a65..27bcdacf8 100644 --- a/vendor/github.com/hashicorp/go-raftchunking/go.mod +++ b/vendor/github.com/hashicorp/go-raftchunking/go.mod @@ -6,6 +6,6 @@ require ( github.com/go-test/deep v1.0.2 github.com/golang/protobuf v1.3.1 github.com/hashicorp/errwrap v1.0.0 - github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 + github.com/hashicorp/raft v1.1.1 github.com/kr/pretty v0.1.0 ) diff --git a/vendor/github.com/hashicorp/go-raftchunking/go.sum b/vendor/github.com/hashicorp/go-raftchunking/go.sum index 746d7f033..ee52eda9d 100644 --- a/vendor/github.com/hashicorp/go-raftchunking/go.sum +++ b/vendor/github.com/hashicorp/go-raftchunking/go.sum @@ -28,8 +28,8 @@ github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCS github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 h1:/VZgMcVsZOKyA5SeKHjARffb/5qd6ZX5TEB0i+FuL3M= -github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft v1.1.1 h1:HJr7UE1x/JrJSc9Oy6aDBHtNHUUBHjcQjTgvUVihoZs= +github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/vendor/github.com/hashicorp/raft/CHANGELOG.md b/vendor/github.com/hashicorp/raft/CHANGELOG.md index 5e771c548..efd56b0fe 100644 --- a/vendor/github.com/hashicorp/raft/CHANGELOG.md +++ b/vendor/github.com/hashicorp/raft/CHANGELOG.md @@ -1,5 +1,22 @@ # UNRELEASED +# 1.1.1 (July 23rd, 2019) + +FEATURES + +* Add support for extensions to be sent on log entries [[GH-353](https://github.com/hashicorp/raft/pull/353)] +* Add config option to skip snapshot restore on startup [[GH-340](https://github.com/hashicorp/raft/pull/340)] +* Add optional configuration store interface [[GH-339](https://github.com/hashicorp/raft/pull/339)] + +IMPROVEMENTS + +* Break out of group commit early when no logs are present [[GH-341](https://github.com/hashicorp/raft/pull/341)] + +BUGFIXES + +* Fix 64-bit counters on 32-bit platforms [[GH-344](https://github.com/hashicorp/raft/pull/344)] +* Don't defer closing source in recover/restore operations since it's in a loop [[GH-337](https://github.com/hashicorp/raft/pull/337)] + # 1.1.0 (May 23rd, 2019) FEATURES diff --git a/vendor/github.com/hashicorp/raft/api.go b/vendor/github.com/hashicorp/raft/api.go index 779d0e334..9c61fefbe 100644 --- a/vendor/github.com/hashicorp/raft/api.go +++ b/vendor/github.com/hashicorp/raft/api.go @@ -20,6 +20,10 @@ const ( // ignore this value if they want as there is no actual hard checking // within the library. As the library is enhanced this value may change // over time to reflect current suggested maximums. + // + // Increasing beyond this risks RPC IO taking too long and preventing + // timely heartbeat signals which are sent in serial in current transports, + // potentially causing leadership instability. SuggestedMaxDataSize = 512 * 1024 ) @@ -645,15 +649,13 @@ func (r *Raft) Leader() ServerAddress { // for the command to be started. This must be run on the leader or it // will fail. func (r *Raft) Apply(cmd []byte, timeout time.Duration) ApplyFuture { - metrics.IncrCounter([]string{"raft", "apply"}, 1) - return r.ApplyLog(Log{Data: cmd}, timeout) } // ApplyLog performs Apply but takes in a Log directly. The only values // currently taken from the submitted Log are Data and Extensions. func (r *Raft) ApplyLog(log Log, timeout time.Duration) ApplyFuture { - metrics.IncrCounter([]string{"raft", "apply_with_log"}, 1) + metrics.IncrCounter([]string{"raft", "apply"}, 1) var timer <-chan time.Time if timeout > 0 { diff --git a/vendor/github.com/hashicorp/raft/log.go b/vendor/github.com/hashicorp/raft/log.go index aa92a2d29..c15a84472 100644 --- a/vendor/github.com/hashicorp/raft/log.go +++ b/vendor/github.com/hashicorp/raft/log.go @@ -52,6 +52,15 @@ type Log struct { // is up to the client of the library to properly modify this as it adds // layers and remove those layers when appropriate. This value is a part of // the log, so very large values could cause timing issues. + // + // N.B. It is _up to the client_ to handle upgrade paths. For instance if + // using this with go-raftchunking, the client should ensure that all Raft + // peers are using a version that can handle that extension before ever + // actually triggering chunking behavior. It is sometimes sufficient to + // ensure that non-leaders are upgraded first, then the current leader is + // upgraded, but a leader changeover during this process could lead to + // trouble, so gating extension behavior via some flag in the client + // program is also a good idea. Extensions []byte } diff --git a/vendor/modules.txt b/vendor/modules.txt index d793a1e43..cff29e433 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -287,7 +287,7 @@ github.com/hashicorp/go-multierror # github.com/hashicorp/go-plugin v1.0.1 github.com/hashicorp/go-plugin github.com/hashicorp/go-plugin/internal/plugin -# github.com/hashicorp/go-raftchunking v0.0.0-20190722150955-a5774da47e6e +# github.com/hashicorp/go-raftchunking v0.5.1 github.com/hashicorp/go-raftchunking github.com/hashicorp/go-raftchunking/types # github.com/hashicorp/go-retryablehttp v0.5.4 @@ -319,7 +319,7 @@ github.com/hashicorp/hcl/json/token # github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf github.com/hashicorp/nomad/api github.com/hashicorp/nomad/api/contexts -# github.com/hashicorp/raft v1.1.1-0.20190722150907-447155de1bd3 +# github.com/hashicorp/raft v1.1.1 github.com/hashicorp/raft # github.com/hashicorp/raft-snapshot v1.0.1 github.com/hashicorp/raft-snapshot