ed14061578
* Work on raft backend * Add logstore locally * Add encryptor and unsealable interfaces * Add clustering support to raft * Remove client and handler * Bootstrap raft on init * Cleanup raft logic a bit * More raft work * Work on TLS config * More work on bootstrapping * Fix build * More work on bootstrapping * More bootstrapping work * fix build * Remove consul dep * Fix build * merged oss/master into raft-storage * Work on bootstrapping * Get bootstrapping to work * Clean up FMS and node-id * Update local node ID logic * Cleanup node-id change * Work on snapshotting * Raft: Add remove peer API (#906) * Add remove peer API * Add some comments * Fix existing snapshotting (#909) * Raft get peers API (#912) * Read raft configuration * address review feedback * Use the Leadership Transfer API to step-down the active node (#918) * Raft join and unseal using Shamir keys (#917) * Raft join using shamir * Store AEAD instead of master key * Split the raft join process to answer the challenge after a successful unseal * get the follower to standby state * Make unseal work * minor changes * Some input checks * reuse the shamir seal access instead of new default seal access * refactor joinRaftSendAnswer function * Synchronously send answer in auto-unseal case * Address review feedback * Raft snapshots (#910) * Fix existing snapshotting * implement the noop snapshotting * Add comments and switch log libraries * add some snapshot tests * add snapshot test file * add TODO * More work on raft snapshotting * progress on the ConfigStore strategy * Don't use two buckets * Update the snapshot store logic to hide the file logic * Add more backend tests * Cleanup code a bit * [WIP] Raft recovery (#938) * Add recovery functionality * remove fmt.Printfs * Fix a few fsm bugs * Add max size value for raft backend (#942) * Add max size value for raft backend * Include physical.ErrValueTooLarge in the message * Raft snapshot Take/Restore API (#926) * Inital work on raft snapshot APIs * Always redirect snapshot install/download requests * More work on the snapshot APIs * Cleanup code a bit * On restore handle special cases * Use the seal to encrypt the sha sum file * Add sealer mechanism and fix some bugs * Call restore while state lock is held * Send restore cb trigger through raft log * Make error messages nicer * Add test helpers * Add snapshot test * Add shamir unseal test * Add more raft snapshot API tests * Fix locking * Change working to initalize * Add underlying raw object to test cluster core * Move leaderUUID to core * Add raft TLS rotation logic (#950) * Add TLS rotation logic * Cleanup logic a bit * Add/Remove from follower state on add/remove peer * add comments * Update more comments * Update request_forwarding_service.proto * Make sure we populate all nodes in the followerstate obj * Update times * Apply review feedback * Add more raft config setting (#947) * Add performance config setting * Add more config options and fix tests * Test Raft Recovery (#944) * Test raft recovery * Leave out a node during recovery * remove unused struct * Update physical/raft/snapshot_test.go * Update physical/raft/snapshot_test.go * fix vendoring * Switch to new raft interface * Remove unused files * Switch a gogo -> proto instance * Remove unneeded vault dep in go.sum * Update helper/testhelpers/testhelpers.go Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com> * Update vault/cluster/cluster.go * track active key within the keyring itself (#6915) * track active key within the keyring itself * lookup and store using the active key ID * update docstring * minor refactor * Small text fixes (#6912) * Update physical/raft/raft.go Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com> * review feedback * Move raft logical system into separate file * Update help text a bit * Enforce cluster addr is set and use it for raft bootstrapping * Fix tests * fix http test panic * Pull in latest raft-snapshot library * Add comment
198 lines
4.8 KiB
Go
198 lines
4.8 KiB
Go
package bbolt
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"unsafe"
|
|
)
|
|
|
|
const pageHeaderSize = int(unsafe.Offsetof(((*page)(nil)).ptr))
|
|
|
|
const minKeysPerPage = 2
|
|
|
|
const branchPageElementSize = int(unsafe.Sizeof(branchPageElement{}))
|
|
const leafPageElementSize = int(unsafe.Sizeof(leafPageElement{}))
|
|
|
|
const (
|
|
branchPageFlag = 0x01
|
|
leafPageFlag = 0x02
|
|
metaPageFlag = 0x04
|
|
freelistPageFlag = 0x10
|
|
)
|
|
|
|
const (
|
|
bucketLeafFlag = 0x01
|
|
)
|
|
|
|
type pgid uint64
|
|
|
|
type page struct {
|
|
id pgid
|
|
flags uint16
|
|
count uint16
|
|
overflow uint32
|
|
ptr uintptr
|
|
}
|
|
|
|
// typ returns a human readable page type string used for debugging.
|
|
func (p *page) typ() string {
|
|
if (p.flags & branchPageFlag) != 0 {
|
|
return "branch"
|
|
} else if (p.flags & leafPageFlag) != 0 {
|
|
return "leaf"
|
|
} else if (p.flags & metaPageFlag) != 0 {
|
|
return "meta"
|
|
} else if (p.flags & freelistPageFlag) != 0 {
|
|
return "freelist"
|
|
}
|
|
return fmt.Sprintf("unknown<%02x>", p.flags)
|
|
}
|
|
|
|
// meta returns a pointer to the metadata section of the page.
|
|
func (p *page) meta() *meta {
|
|
return (*meta)(unsafe.Pointer(&p.ptr))
|
|
}
|
|
|
|
// leafPageElement retrieves the leaf node by index
|
|
func (p *page) leafPageElement(index uint16) *leafPageElement {
|
|
n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index]
|
|
return n
|
|
}
|
|
|
|
// leafPageElements retrieves a list of leaf nodes.
|
|
func (p *page) leafPageElements() []leafPageElement {
|
|
if p.count == 0 {
|
|
return nil
|
|
}
|
|
return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:]
|
|
}
|
|
|
|
// branchPageElement retrieves the branch node by index
|
|
func (p *page) branchPageElement(index uint16) *branchPageElement {
|
|
return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index]
|
|
}
|
|
|
|
// branchPageElements retrieves a list of branch nodes.
|
|
func (p *page) branchPageElements() []branchPageElement {
|
|
if p.count == 0 {
|
|
return nil
|
|
}
|
|
return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:]
|
|
}
|
|
|
|
// dump writes n bytes of the page to STDERR as hex output.
|
|
func (p *page) hexdump(n int) {
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:n]
|
|
fmt.Fprintf(os.Stderr, "%x\n", buf)
|
|
}
|
|
|
|
type pages []*page
|
|
|
|
func (s pages) Len() int { return len(s) }
|
|
func (s pages) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
|
func (s pages) Less(i, j int) bool { return s[i].id < s[j].id }
|
|
|
|
// branchPageElement represents a node on a branch page.
|
|
type branchPageElement struct {
|
|
pos uint32
|
|
ksize uint32
|
|
pgid pgid
|
|
}
|
|
|
|
// key returns a byte slice of the node key.
|
|
func (n *branchPageElement) key() []byte {
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
|
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize]
|
|
}
|
|
|
|
// leafPageElement represents a node on a leaf page.
|
|
type leafPageElement struct {
|
|
flags uint32
|
|
pos uint32
|
|
ksize uint32
|
|
vsize uint32
|
|
}
|
|
|
|
// key returns a byte slice of the node key.
|
|
func (n *leafPageElement) key() []byte {
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
|
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize]
|
|
}
|
|
|
|
// value returns a byte slice of the node value.
|
|
func (n *leafPageElement) value() []byte {
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
|
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize]
|
|
}
|
|
|
|
// PageInfo represents human readable information about a page.
|
|
type PageInfo struct {
|
|
ID int
|
|
Type string
|
|
Count int
|
|
OverflowCount int
|
|
}
|
|
|
|
type pgids []pgid
|
|
|
|
func (s pgids) Len() int { return len(s) }
|
|
func (s pgids) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
|
func (s pgids) Less(i, j int) bool { return s[i] < s[j] }
|
|
|
|
// merge returns the sorted union of a and b.
|
|
func (a pgids) merge(b pgids) pgids {
|
|
// Return the opposite slice if one is nil.
|
|
if len(a) == 0 {
|
|
return b
|
|
}
|
|
if len(b) == 0 {
|
|
return a
|
|
}
|
|
merged := make(pgids, len(a)+len(b))
|
|
mergepgids(merged, a, b)
|
|
return merged
|
|
}
|
|
|
|
// mergepgids copies the sorted union of a and b into dst.
|
|
// If dst is too small, it panics.
|
|
func mergepgids(dst, a, b pgids) {
|
|
if len(dst) < len(a)+len(b) {
|
|
panic(fmt.Errorf("mergepgids bad len %d < %d + %d", len(dst), len(a), len(b)))
|
|
}
|
|
// Copy in the opposite slice if one is nil.
|
|
if len(a) == 0 {
|
|
copy(dst, b)
|
|
return
|
|
}
|
|
if len(b) == 0 {
|
|
copy(dst, a)
|
|
return
|
|
}
|
|
|
|
// Merged will hold all elements from both lists.
|
|
merged := dst[:0]
|
|
|
|
// Assign lead to the slice with a lower starting value, follow to the higher value.
|
|
lead, follow := a, b
|
|
if b[0] < a[0] {
|
|
lead, follow = b, a
|
|
}
|
|
|
|
// Continue while there are elements in the lead.
|
|
for len(lead) > 0 {
|
|
// Merge largest prefix of lead that is ahead of follow[0].
|
|
n := sort.Search(len(lead), func(i int) bool { return lead[i] > follow[0] })
|
|
merged = append(merged, lead[:n]...)
|
|
if n >= len(lead) {
|
|
break
|
|
}
|
|
|
|
// Swap lead and follow.
|
|
lead, follow = follow, lead[n:]
|
|
}
|
|
|
|
// Append what's left in follow.
|
|
_ = append(merged, follow...)
|
|
}
|