From c1be6c0093210a01245d1e2f3b5ccc275c93d402 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Mon, 25 Jul 2016 16:27:31 -0700 Subject: [PATCH] Updates Bolt DB to v1.2.1. This fixes #2203 which was a consistency problem on Windows. --- Godeps/Godeps.json | 4 +- vendor/github.com/boltdb/bolt/README.md | 12 +++- vendor/github.com/boltdb/bolt/db.go | 79 +++++++++++++++++++------ vendor/github.com/boltdb/bolt/errors.go | 3 +- vendor/github.com/boltdb/bolt/page.go | 4 +- vendor/github.com/boltdb/bolt/tx.go | 20 ++++++- 6 files changed, 94 insertions(+), 28 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 865744092..9b94e73df 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -33,8 +33,8 @@ }, { "ImportPath": "github.com/boltdb/bolt", - "Comment": "v1.2.0", - "Rev": "c6ba97b89e0454fec9aa92e1d33a4e2c5fc1f631" + "Comment": "v1.2.1", + "Rev": "dfb21201d9270c1082d5fb0f07f500311ff72f18" }, { "ImportPath": "github.com/circonus-labs/circonus-gometrics", diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/github.com/boltdb/bolt/README.md index 66b19ace8..3bff3cce4 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/github.com/boltdb/bolt/README.md @@ -1,4 +1,4 @@ -Bolt [![Build Status](https://drone.io/github.com/boltdb/bolt/status.png)](https://drone.io/github.com/boltdb/bolt/latest) [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg) +Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg) ==== Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] @@ -427,6 +427,8 @@ db.View(func(tx *bolt.Tx) error { }) ``` +Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable. + #### ForEach() @@ -437,7 +439,7 @@ all the keys in a bucket: db.View(func(tx *bolt.Tx) error { // Assume bucket exists and has keys b := tx.Bucket([]byte("MyBucket")) - + b.ForEach(func(k, v []byte) error { fmt.Printf("key=%s, value=%s\n", k, v) return nil @@ -617,7 +619,7 @@ Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path) { NSURL* URL= [NSURL fileURLWithPath: filePathString]; assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]); - + NSError *error = nil; BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES] forKey: NSURLIsExcludedFromBackupKey error: &error]; @@ -840,5 +842,9 @@ Below is a list of public, open source projects that use Bolt: * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. * [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [Storm](https://github.com/asdine/storm) - A simple ORM around BoltDB. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/github.com/boltdb/bolt/db.go index 501d36aac..1223493ca 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/github.com/boltdb/bolt/db.go @@ -36,6 +36,9 @@ const ( DefaultAllocSize = 16 * 1024 * 1024 ) +// default page size for db is set to the OS page size. +var defaultPageSize = os.Getpagesize() + // DB represents a collection of buckets persisted to a file on disk. // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. @@ -94,7 +97,7 @@ type DB struct { path string file *os.File lockfile *os.File // windows only - dataref []byte // mmap'ed readonly, write throws SEGV + dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int filesz int // current on disk file size @@ -107,6 +110,8 @@ type DB struct { freelist *freelist stats Stats + pagePool sync.Pool + batchMu sync.Mutex batch *batch @@ -200,12 +205,27 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { if _, err := db.file.ReadAt(buf[:], 0); err == nil { m := db.pageInBuffer(buf[:], 0).meta() if err := m.validate(); err != nil { - return nil, err + // If we can't read the page size, we can assume it's the same + // as the OS -- since that's how the page size was chosen in the + // first place. + // + // If the first page is invalid and this OS uses a different + // page size than what the database was created with then we + // are out of luck and cannot access the database. + db.pageSize = os.Getpagesize() + } else { + db.pageSize = int(m.pageSize) } - db.pageSize = int(m.pageSize) } } + // Initialize page pool. + db.pagePool = sync.Pool{ + New: func() interface{} { + return make([]byte, db.pageSize) + }, + } + // Memory map the data file. if err := db.mmap(options.InitialMmapSize); err != nil { _ = db.close() @@ -262,12 +282,13 @@ func (db *DB) mmap(minsz int) error { db.meta0 = db.page(0).meta() db.meta1 = db.page(1).meta() - // Validate the meta pages. - if err := db.meta0.validate(); err != nil { - return err - } - if err := db.meta1.validate(); err != nil { - return err + // Validate the meta pages. We only return an error if both meta pages fail + // validation, since meta0 failing validation means that it wasn't saved + // properly -- but we can recover using meta1. And vice-versa. + err0 := db.meta0.validate() + err1 := db.meta1.validate() + if err0 != nil && err1 != nil { + return err0 } return nil @@ -339,6 +360,7 @@ func (db *DB) init() error { m.root = bucket{root: 3} m.pgid = 4 m.txid = txid(i) + m.checksum = m.sum64() } // Write an empty freelist at page 3. @@ -383,11 +405,10 @@ func (db *DB) close() error { if !db.opened { return nil } - + db.opened = false db.freelist = nil - db.path = "" // Clear ops. db.ops.writeAt = nil @@ -414,6 +435,7 @@ func (db *DB) close() error { db.file = nil } + db.path = "" return nil } @@ -778,16 +800,37 @@ func (db *DB) pageInBuffer(b []byte, id pgid) *page { // meta retrieves the current meta page reference. func (db *DB) meta() *meta { - if db.meta0.txid > db.meta1.txid { - return db.meta0 + // We have to return the meta with the highest txid which doesn't fail + // validation. Otherwise, we can cause errors when in fact the database is + // in a consistent state. metaA is the one with the higher txid. + metaA := db.meta0 + metaB := db.meta1 + if db.meta1.txid > db.meta0.txid { + metaA = db.meta1 + metaB = db.meta0 } - return db.meta1 + + // Use higher meta page if valid. Otherwise fallback to previous, if valid. + if err := metaA.validate(); err == nil { + return metaA + } else if err := metaB.validate(); err == nil { + return metaB + } + + // This should never be reached, because both meta1 and meta0 were validated + // on mmap() and we do fsync() on every write. + panic("bolt.DB.meta(): invalid meta pages") } // allocate returns a contiguous block of memory starting at a given page. func (db *DB) allocate(count int) (*page, error) { // Allocate a temporary buffer for the page. - buf := make([]byte, count*db.pageSize) + var buf []byte + if count == 1 { + buf = db.pagePool.Get().([]byte) + } else { + buf = make([]byte, count*db.pageSize) + } p := (*page)(unsafe.Pointer(&buf[0])) p.overflow = uint32(count - 1) @@ -937,12 +980,12 @@ type meta struct { // validate checks the marker bytes and version of the meta page to ensure it matches this binary. func (m *meta) validate() error { - if m.checksum != 0 && m.checksum != m.sum64() { - return ErrChecksum - } else if m.magic != magic { + if m.magic != magic { return ErrInvalid } else if m.version != version { return ErrVersionMismatch + } else if m.checksum != 0 && m.checksum != m.sum64() { + return ErrChecksum } return nil } diff --git a/vendor/github.com/boltdb/bolt/errors.go b/vendor/github.com/boltdb/bolt/errors.go index 6883786d5..a3620a3eb 100644 --- a/vendor/github.com/boltdb/bolt/errors.go +++ b/vendor/github.com/boltdb/bolt/errors.go @@ -12,7 +12,8 @@ var ( // already open. ErrDatabaseOpen = errors.New("database already open") - // ErrInvalid is returned when a data file is not a Bolt-formatted database. + // ErrInvalid is returned when both meta pages on a database are invalid. + // This typically occurs when a file is not a bolt database. ErrInvalid = errors.New("invalid database") // ErrVersionMismatch is returned when the data file was created with a diff --git a/vendor/github.com/boltdb/bolt/page.go b/vendor/github.com/boltdb/bolt/page.go index 818aa1b15..4a555286a 100644 --- a/vendor/github.com/boltdb/bolt/page.go +++ b/vendor/github.com/boltdb/bolt/page.go @@ -111,13 +111,13 @@ type leafPageElement struct { // key returns a byte slice of the node key. func (n *leafPageElement) key() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize] } // value returns a byte slice of the node value. func (n *leafPageElement) value() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize] } // PageInfo represents human readable information about a page. diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/github.com/boltdb/bolt/tx.go index b8510fdb8..1cfb4cde8 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/github.com/boltdb/bolt/tx.go @@ -473,6 +473,8 @@ func (tx *Tx) write() error { for _, p := range tx.pages { pages = append(pages, p) } + // Clear out page cache early. + tx.pages = make(map[pgid]*page) sort.Sort(pages) // Write pages to disk in order. @@ -517,8 +519,22 @@ func (tx *Tx) write() error { } } - // Clear out page cache. - tx.pages = make(map[pgid]*page) + // Put small pages back to page pool. + for _, p := range pages { + // Ignore page sizes over 1 page. + // These are allocated using make() instead of the page pool. + if int(p.overflow) != 0 { + continue + } + + buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:tx.db.pageSize] + + // See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1 + for i := range buf { + buf[i] = 0 + } + tx.db.pagePool.Put(buf) + } return nil }