ed14061578
* Work on raft backend * Add logstore locally * Add encryptor and unsealable interfaces * Add clustering support to raft * Remove client and handler * Bootstrap raft on init * Cleanup raft logic a bit * More raft work * Work on TLS config * More work on bootstrapping * Fix build * More work on bootstrapping * More bootstrapping work * fix build * Remove consul dep * Fix build * merged oss/master into raft-storage * Work on bootstrapping * Get bootstrapping to work * Clean up FMS and node-id * Update local node ID logic * Cleanup node-id change * Work on snapshotting * Raft: Add remove peer API (#906) * Add remove peer API * Add some comments * Fix existing snapshotting (#909) * Raft get peers API (#912) * Read raft configuration * address review feedback * Use the Leadership Transfer API to step-down the active node (#918) * Raft join and unseal using Shamir keys (#917) * Raft join using shamir * Store AEAD instead of master key * Split the raft join process to answer the challenge after a successful unseal * get the follower to standby state * Make unseal work * minor changes * Some input checks * reuse the shamir seal access instead of new default seal access * refactor joinRaftSendAnswer function * Synchronously send answer in auto-unseal case * Address review feedback * Raft snapshots (#910) * Fix existing snapshotting * implement the noop snapshotting * Add comments and switch log libraries * add some snapshot tests * add snapshot test file * add TODO * More work on raft snapshotting * progress on the ConfigStore strategy * Don't use two buckets * Update the snapshot store logic to hide the file logic * Add more backend tests * Cleanup code a bit * [WIP] Raft recovery (#938) * Add recovery functionality * remove fmt.Printfs * Fix a few fsm bugs * Add max size value for raft backend (#942) * Add max size value for raft backend * Include physical.ErrValueTooLarge in the message * Raft snapshot Take/Restore API (#926) * Inital work on raft snapshot APIs * Always redirect snapshot install/download requests * More work on the snapshot APIs * Cleanup code a bit * On restore handle special cases * Use the seal to encrypt the sha sum file * Add sealer mechanism and fix some bugs * Call restore while state lock is held * Send restore cb trigger through raft log * Make error messages nicer * Add test helpers * Add snapshot test * Add shamir unseal test * Add more raft snapshot API tests * Fix locking * Change working to initalize * Add underlying raw object to test cluster core * Move leaderUUID to core * Add raft TLS rotation logic (#950) * Add TLS rotation logic * Cleanup logic a bit * Add/Remove from follower state on add/remove peer * add comments * Update more comments * Update request_forwarding_service.proto * Make sure we populate all nodes in the followerstate obj * Update times * Apply review feedback * Add more raft config setting (#947) * Add performance config setting * Add more config options and fix tests * Test Raft Recovery (#944) * Test raft recovery * Leave out a node during recovery * remove unused struct * Update physical/raft/snapshot_test.go * Update physical/raft/snapshot_test.go * fix vendoring * Switch to new raft interface * Remove unused files * Switch a gogo -> proto instance * Remove unneeded vault dep in go.sum * Update helper/testhelpers/testhelpers.go Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com> * Update vault/cluster/cluster.go * track active key within the keyring itself (#6915) * track active key within the keyring itself * lookup and store using the active key ID * update docstring * minor refactor * Small text fixes (#6912) * Update physical/raft/raft.go Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com> * review feedback * Move raft logical system into separate file * Update help text a bit * Enforce cluster addr is set and use it for raft bootstrapping * Fix tests * fix http test panic * Pull in latest raft-snapshot library * Add comment
179 lines
3.7 KiB
Go
179 lines
3.7 KiB
Go
package bbolt
|
|
|
|
import "sort"
|
|
|
|
// hashmapFreeCount returns count of free pages(hashmap version)
|
|
func (f *freelist) hashmapFreeCount() int {
|
|
// use the forwardmap to get the total count
|
|
count := 0
|
|
for _, size := range f.forwardMap {
|
|
count += int(size)
|
|
}
|
|
return count
|
|
}
|
|
|
|
// hashmapAllocate serves the same purpose as arrayAllocate, but use hashmap as backend
|
|
func (f *freelist) hashmapAllocate(txid txid, n int) pgid {
|
|
if n == 0 {
|
|
return 0
|
|
}
|
|
|
|
// if we have a exact size match just return short path
|
|
if bm, ok := f.freemaps[uint64(n)]; ok {
|
|
for pid := range bm {
|
|
// remove the span
|
|
f.delSpan(pid, uint64(n))
|
|
|
|
f.allocs[pid] = txid
|
|
|
|
for i := pgid(0); i < pgid(n); i++ {
|
|
delete(f.cache, pid+pgid(i))
|
|
}
|
|
return pid
|
|
}
|
|
}
|
|
|
|
// lookup the map to find larger span
|
|
for size, bm := range f.freemaps {
|
|
if size < uint64(n) {
|
|
continue
|
|
}
|
|
|
|
for pid := range bm {
|
|
// remove the initial
|
|
f.delSpan(pid, uint64(size))
|
|
|
|
f.allocs[pid] = txid
|
|
|
|
remain := size - uint64(n)
|
|
|
|
// add remain span
|
|
f.addSpan(pid+pgid(n), remain)
|
|
|
|
for i := pgid(0); i < pgid(n); i++ {
|
|
delete(f.cache, pid+pgid(i))
|
|
}
|
|
return pid
|
|
}
|
|
}
|
|
|
|
return 0
|
|
}
|
|
|
|
// hashmapReadIDs reads pgids as input an initial the freelist(hashmap version)
|
|
func (f *freelist) hashmapReadIDs(pgids []pgid) {
|
|
f.init(pgids)
|
|
|
|
// Rebuild the page cache.
|
|
f.reindex()
|
|
}
|
|
|
|
// hashmapGetFreePageIDs returns the sorted free page ids
|
|
func (f *freelist) hashmapGetFreePageIDs() []pgid {
|
|
count := f.free_count()
|
|
if count == 0 {
|
|
return nil
|
|
}
|
|
|
|
m := make([]pgid, 0, count)
|
|
for start, size := range f.forwardMap {
|
|
for i := 0; i < int(size); i++ {
|
|
m = append(m, start+pgid(i))
|
|
}
|
|
}
|
|
sort.Sort(pgids(m))
|
|
|
|
return m
|
|
}
|
|
|
|
// hashmapMergeSpans try to merge list of pages(represented by pgids) with existing spans
|
|
func (f *freelist) hashmapMergeSpans(ids pgids) {
|
|
for _, id := range ids {
|
|
// try to see if we can merge and update
|
|
f.mergeWithExistingSpan(id)
|
|
}
|
|
}
|
|
|
|
// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward
|
|
func (f *freelist) mergeWithExistingSpan(pid pgid) {
|
|
prev := pid - 1
|
|
next := pid + 1
|
|
|
|
preSize, mergeWithPrev := f.backwardMap[prev]
|
|
nextSize, mergeWithNext := f.forwardMap[next]
|
|
newStart := pid
|
|
newSize := uint64(1)
|
|
|
|
if mergeWithPrev {
|
|
//merge with previous span
|
|
start := prev + 1 - pgid(preSize)
|
|
f.delSpan(start, preSize)
|
|
|
|
newStart -= pgid(preSize)
|
|
newSize += preSize
|
|
}
|
|
|
|
if mergeWithNext {
|
|
// merge with next span
|
|
f.delSpan(next, nextSize)
|
|
newSize += nextSize
|
|
}
|
|
|
|
f.addSpan(newStart, newSize)
|
|
}
|
|
|
|
func (f *freelist) addSpan(start pgid, size uint64) {
|
|
f.backwardMap[start-1+pgid(size)] = size
|
|
f.forwardMap[start] = size
|
|
if _, ok := f.freemaps[size]; !ok {
|
|
f.freemaps[size] = make(map[pgid]struct{})
|
|
}
|
|
|
|
f.freemaps[size][start] = struct{}{}
|
|
}
|
|
|
|
func (f *freelist) delSpan(start pgid, size uint64) {
|
|
delete(f.forwardMap, start)
|
|
delete(f.backwardMap, start+pgid(size-1))
|
|
delete(f.freemaps[size], start)
|
|
if len(f.freemaps[size]) == 0 {
|
|
delete(f.freemaps, size)
|
|
}
|
|
}
|
|
|
|
// initial from pgids using when use hashmap version
|
|
// pgids must be sorted
|
|
func (f *freelist) init(pgids []pgid) {
|
|
if len(pgids) == 0 {
|
|
return
|
|
}
|
|
|
|
size := uint64(1)
|
|
start := pgids[0]
|
|
|
|
if !sort.SliceIsSorted([]pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) {
|
|
panic("pgids not sorted")
|
|
}
|
|
|
|
f.freemaps = make(map[uint64]pidSet)
|
|
f.forwardMap = make(map[pgid]uint64)
|
|
f.backwardMap = make(map[pgid]uint64)
|
|
|
|
for i := 1; i < len(pgids); i++ {
|
|
// continuous page
|
|
if pgids[i] == pgids[i-1]+1 {
|
|
size++
|
|
} else {
|
|
f.addSpan(start, size)
|
|
|
|
size = 1
|
|
start = pgids[i]
|
|
}
|
|
}
|
|
|
|
// init the tail
|
|
if size != 0 && start != 0 {
|
|
f.addSpan(start, size)
|
|
}
|
|
}
|