Merge pull request #2211 from hashicorp/b-bolt-1.2.1

Updates Bolt DB to v1.2.1.

Godeps/Godeps.json

@@ -33,8 +33,8 @@
 		},
 		{
 			"ImportPath": "github.com/boltdb/bolt",
-			"Comment": "v1.2.0",
+			"Comment": "v1.2.1",
-			"Rev": "c6ba97b89e0454fec9aa92e1d33a4e2c5fc1f631"
+			"Rev": "dfb21201d9270c1082d5fb0f07f500311ff72f18"
 		},
 		{
 			"ImportPath": "github.com/circonus-labs/circonus-gometrics",

vendor/github.com/boltdb/bolt/README.md

@@ -1,4 +1,4 @@
-Bolt [![Build Status](https://drone.io/github.com/boltdb/bolt/status.png)](https://drone.io/github.com/boltdb/bolt/latest) [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg)
+Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg)
 ====
 
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
@@ -427,6 +427,8 @@ db.View(func(tx *bolt.Tx) error {
 })
 ```
 
+Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable.
+
 
 #### ForEach()
 
@@ -437,7 +439,7 @@ all the keys in a bucket:
 db.View(func(tx *bolt.Tx) error {
 	// Assume bucket exists and has keys
 	b := tx.Bucket([]byte("MyBucket"))
-	
+
 	b.ForEach(func(k, v []byte) error {
 		fmt.Printf("key=%s, value=%s\n", k, v)
 		return nil
@@ -617,7 +619,7 @@ Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path)
 {
     NSURL* URL= [NSURL fileURLWithPath: filePathString];
     assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]);
-    
+
     NSError *error = nil;
     BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES]
                                   forKey: NSURLIsExcludedFromBackupKey error: &error];
@@ -840,5 +842,9 @@ Below is a list of public, open source projects that use Bolt:
 * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
 * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
 * [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
+* [Storm](https://github.com/asdine/storm) - A simple ORM around BoltDB.
+* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB.
+* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
+* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
 
 If you are using Bolt in a project please send a pull request to add it to the list.

vendor/github.com/boltdb/bolt/db.go

@@ -36,6 +36,9 @@ const (
 	DefaultAllocSize         = 16 * 1024 * 1024
 )
 
+// default page size for db is set to the OS page size.
+var defaultPageSize = os.Getpagesize()
+
 // DB represents a collection of buckets persisted to a file on disk.
 // All data access is performed through transactions which can be obtained through the DB.
 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
@@ -94,7 +97,7 @@ type DB struct {
 	path     string
 	file     *os.File
 	lockfile *os.File // windows only
-	dataref  []byte // mmap'ed readonly, write throws SEGV
+	dataref  []byte   // mmap'ed readonly, write throws SEGV
 	data     *[maxMapSize]byte
 	datasz   int
 	filesz   int // current on disk file size
@@ -107,6 +110,8 @@ type DB struct {
 	freelist *freelist
 	stats    Stats
 
+	pagePool sync.Pool
+
 	batchMu sync.Mutex
 	batch   *batch
 
@@ -200,12 +205,27 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		if _, err := db.file.ReadAt(buf[:], 0); err == nil {
 			m := db.pageInBuffer(buf[:], 0).meta()
 			if err := m.validate(); err != nil {
-				return nil, err
+				// If we can't read the page size, we can assume it's the same
+				// as the OS -- since that's how the page size was chosen in the
+				// first place.
+				//
+				// If the first page is invalid and this OS uses a different
+				// page size than what the database was created with then we
+				// are out of luck and cannot access the database.
+				db.pageSize = os.Getpagesize()
+			} else {
+				db.pageSize = int(m.pageSize)
 			}
-			db.pageSize = int(m.pageSize)
 		}
 	}
 
+	// Initialize page pool.
+	db.pagePool = sync.Pool{
+		New: func() interface{} {
+			return make([]byte, db.pageSize)
+		},
+	}
+
 	// Memory map the data file.
 	if err := db.mmap(options.InitialMmapSize); err != nil {
 		_ = db.close()
@@ -262,12 +282,13 @@ func (db *DB) mmap(minsz int) error {
 	db.meta0 = db.page(0).meta()
 	db.meta1 = db.page(1).meta()
 
-	// Validate the meta pages.
+	// Validate the meta pages. We only return an error if both meta pages fail
-	if err := db.meta0.validate(); err != nil {
+	// validation, since meta0 failing validation means that it wasn't saved
-		return err
+	// properly -- but we can recover using meta1. And vice-versa.
-	}
+	err0 := db.meta0.validate()
-	if err := db.meta1.validate(); err != nil {
+	err1 := db.meta1.validate()
-		return err
+	if err0 != nil && err1 != nil {
+		return err0
 	}
 
 	return nil
@@ -339,6 +360,7 @@ func (db *DB) init() error {
 		m.root = bucket{root: 3}
 		m.pgid = 4
 		m.txid = txid(i)
+		m.checksum = m.sum64()
 	}
 
 	// Write an empty freelist at page 3.
@@ -383,11 +405,10 @@ func (db *DB) close() error {
 	if !db.opened {
 		return nil
 	}
-	
+
 	db.opened = false
 
 	db.freelist = nil
-	db.path = ""
 
 	// Clear ops.
 	db.ops.writeAt = nil
@@ -414,6 +435,7 @@ func (db *DB) close() error {
 		db.file = nil
 	}
 
+	db.path = ""
 	return nil
 }
 
@@ -778,16 +800,37 @@ func (db *DB) pageInBuffer(b []byte, id pgid) *page {
 
 // meta retrieves the current meta page reference.
 func (db *DB) meta() *meta {
-	if db.meta0.txid > db.meta1.txid {
+	// We have to return the meta with the highest txid which doesn't fail
-		return db.meta0
+	// validation. Otherwise, we can cause errors when in fact the database is
+	// in a consistent state. metaA is the one with the higher txid.
+	metaA := db.meta0
+	metaB := db.meta1
+	if db.meta1.txid > db.meta0.txid {
+		metaA = db.meta1
+		metaB = db.meta0
 	}
-	return db.meta1
+
+	// Use higher meta page if valid. Otherwise fallback to previous, if valid.
+	if err := metaA.validate(); err == nil {
+		return metaA
+	} else if err := metaB.validate(); err == nil {
+		return metaB
+	}
+
+	// This should never be reached, because both meta1 and meta0 were validated
+	// on mmap() and we do fsync() on every write.
+	panic("bolt.DB.meta(): invalid meta pages")
 }
 
 // allocate returns a contiguous block of memory starting at a given page.
 func (db *DB) allocate(count int) (*page, error) {
 	// Allocate a temporary buffer for the page.
-	buf := make([]byte, count*db.pageSize)
+	var buf []byte
+	if count == 1 {
+		buf = db.pagePool.Get().([]byte)
+	} else {
+		buf = make([]byte, count*db.pageSize)
+	}
 	p := (*page)(unsafe.Pointer(&buf[0]))
 	p.overflow = uint32(count - 1)
 
@@ -937,12 +980,12 @@ type meta struct {
 
 // validate checks the marker bytes and version of the meta page to ensure it matches this binary.
 func (m *meta) validate() error {
-	if m.checksum != 0 && m.checksum != m.sum64() {
+	if m.magic != magic {
-		return ErrChecksum
-	} else if m.magic != magic {
 		return ErrInvalid
 	} else if m.version != version {
 		return ErrVersionMismatch
+	} else if m.checksum != 0 && m.checksum != m.sum64() {
+		return ErrChecksum
 	}
 	return nil
 }

vendor/github.com/boltdb/bolt/errors.go

@@ -12,7 +12,8 @@ var (
 	// already open.
 	ErrDatabaseOpen = errors.New("database already open")
 
-	// ErrInvalid is returned when a data file is not a Bolt-formatted database.
+	// ErrInvalid is returned when both meta pages on a database are invalid.
+	// This typically occurs when a file is not a bolt database.
 	ErrInvalid = errors.New("invalid database")
 
 	// ErrVersionMismatch is returned when the data file was created with a

vendor/github.com/boltdb/bolt/page.go

@@ -111,13 +111,13 @@ type leafPageElement struct {
 // key returns a byte slice of the node key.
 func (n *leafPageElement) key() []byte {
 	buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
-	return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize]
+	return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize]
 }
 
 // value returns a byte slice of the node value.
 func (n *leafPageElement) value() []byte {
 	buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
-	return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize]
+	return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize]
 }
 
 // PageInfo represents human readable information about a page.

vendor/github.com/boltdb/bolt/tx.go

@@ -473,6 +473,8 @@ func (tx *Tx) write() error {
 	for _, p := range tx.pages {
 		pages = append(pages, p)
 	}
+	// Clear out page cache early.
+	tx.pages = make(map[pgid]*page)
 	sort.Sort(pages)
 
 	// Write pages to disk in order.
@@ -517,8 +519,22 @@ func (tx *Tx) write() error {
 		}
 	}
 
-	// Clear out page cache.
+	// Put small pages back to page pool.
-	tx.pages = make(map[pgid]*page)
+	for _, p := range pages {
+		// Ignore page sizes over 1 page.
+		// These are allocated using make() instead of the page pool.
+		if int(p.overflow) != 0 {
+			continue
+		}
+
+		buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:tx.db.pageSize]
+
+		// See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1
+		for i := range buf {
+			buf[i] = 0
+		}
+		tx.db.pagePool.Put(buf)
+	}
 
 	return nil
 }