Add new boltdb options (#11895)

changelog/11895.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+raft: change freelist type to map and set nofreelistsync to true
+```

physical/raft/fsm.go

@@ -154,10 +154,19 @@ func (f *FSM) openDBFile(dbPath string) error {
 		return errors.New("can not open empty filename")
 	}
 
-	boltDB, err := bolt.Open(dbPath, 0o666, &bolt.Options{Timeout: 1 * time.Second})
+	freelistType, noFreelistSync := freelistOptions()
+	start := time.Now()
+	boltDB, err := bolt.Open(dbPath, 0o666, &bolt.Options{
+		Timeout:        1 * time.Second,
+		FreelistType:   freelistType,
+		NoFreelistSync: noFreelistSync,
+	})
 	if err != nil {
 		return err
 	}
+	elapsed := time.Now().Sub(start)
+	f.logger.Debug("time to open database", "elapsed", elapsed, "path", dbPath)
+	metrics.MeasureSince([]string{"raft_storage", "fsm", "open_db_file"}, start)
 
 	err = boltDB.Update(func(tx *bolt.Tx) error {
 		// make sure we have the necessary buckets created

physical/raft/raft.go

@@ -364,7 +364,15 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		}
 
 		// Create the backend raft store for logs and stable storage.
-		store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db"))
+		freelistType, noFreelistSync := freelistOptions()
+		raftOptions := raftboltdb.Options{
+			Path: filepath.Join(path, "raft.db"),
+			BoltOptions: &bolt.Options{
+				FreelistType:   freelistType,
+				NoFreelistSync: noFreelistSync,
+			},
+		}
+		store, err := raftboltdb.New(raftOptions)
 		if err != nil {
 			return nil, err
 		}
@@ -1573,3 +1581,21 @@ func (s sealer) Open(ctx context.Context, ct []byte) ([]byte, error) {
 
 	return s.access.Decrypt(ctx, &eblob, nil)
 }
+
+// freelistOptions returns the freelist type and nofreelistsync values to use
+// when opening boltdb files, based on our preferred defaults, and the possible
+// presence of overriding environment variables.
+func freelistOptions() (bolt.FreelistType, bool) {
+	freelistType := bolt.FreelistMapType
+	noFreelistSync := true
+
+	if os.Getenv("VAULT_RAFT_FREELIST_TYPE") == "array" {
+		freelistType = bolt.FreelistArrayType
+	}
+
+	if os.Getenv("VAULT_RAFT_FREELIST_SYNC") != "" {
+		noFreelistSync = false
+	}
+
+	return freelistType, noFreelistSync
+}