diff --git a/changelog/13573.txt b/changelog/13573.txt new file mode 100644 index 000000000..f90885481 --- /dev/null +++ b/changelog/13573.txt @@ -0,0 +1,3 @@ +```release-note:bug +storage/raft: On linux, use map_populate for bolt files to improve startup time. +``` diff --git a/physical/raft/bolt_32bit_test.go b/physical/raft/bolt_32bit_test.go index 4b7ed4b8b..ccb1641ea 100644 --- a/physical/raft/bolt_32bit_test.go +++ b/physical/raft/bolt_32bit_test.go @@ -32,7 +32,7 @@ func Test_BoltOptions(t *testing.T) { os.Setenv(key, tc.env) } - o := boltOptions() + o := boltOptions("") if o.InitialMmapSize != tc.expectedSize { t.Errorf("expected InitialMmapSize to be %d but it was %d", tc.expectedSize, o.InitialMmapSize) diff --git a/physical/raft/bolt_64bit_test.go b/physical/raft/bolt_64bit_test.go index 173f8aa71..d88c01eed 100644 --- a/physical/raft/bolt_64bit_test.go +++ b/physical/raft/bolt_64bit_test.go @@ -32,7 +32,7 @@ func Test_BoltOptions(t *testing.T) { os.Setenv(key, tc.env) } - o := boltOptions() + o := boltOptions("") if o.InitialMmapSize != tc.expectedSize { t.Errorf("expected InitialMmapSize to be %d but it was %d", tc.expectedSize, o.InitialMmapSize) diff --git a/physical/raft/bolt_linux.go b/physical/raft/bolt_linux.go new file mode 100644 index 000000000..03db74bc4 --- /dev/null +++ b/physical/raft/bolt_linux.go @@ -0,0 +1,36 @@ +package raft + +import ( + "context" + "os" + + "github.com/shirou/gopsutil/mem" + "golang.org/x/sys/unix" +) + +func init() { + getMmapFlags = getMmapFlagsLinux +} + +func getMmapFlagsLinux(dbPath string) int { + if os.Getenv("VAULT_RAFT_DISABLE_MAP_POPULATE") != "" { + return 0 + } + stat, err := os.Stat(dbPath) + if err != nil { + return 0 + } + size := stat.Size() + + v, err := mem.VirtualMemoryWithContext(context.Background()) + if err != nil { + return 0 + } + + // We won't worry about swap, since we already tell people not to use it. + if v.Total > uint64(size) { + return unix.MAP_POPULATE + } + + return 0 +} diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index 8c2e79fbd..ab8d75a6f 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -168,7 +168,7 @@ func (f *FSM) openDBFile(dbPath string) error { } } - opts := boltOptions() + opts := boltOptions(dbPath) start := time.Now() boltDB, err := bolt.Open(dbPath, 0o600, opts) if err != nil { diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 52826c09c..2b2de78ac 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -40,6 +40,8 @@ const EnvVaultRaftNodeID = "VAULT_RAFT_NODE_ID" // EnvVaultRaftPath is used to fetch the path where Raft data is stored from the environment. const EnvVaultRaftPath = "VAULT_RAFT_PATH" +var getMmapFlags = func(string) int { return 0 } + // Verify RaftBackend satisfies the correct interfaces var ( _ physical.Backend = (*RaftBackend)(nil) @@ -364,9 +366,10 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend } // Create the backend raft store for logs and stable storage. - opts := boltOptions() + dbPath := filepath.Join(path, "raft.db") + opts := boltOptions(dbPath) raftOptions := raftboltdb.Options{ - Path: filepath.Join(path, "raft.db"), + Path: dbPath, BoltOptions: opts, } store, err := raftboltdb.New(raftOptions) @@ -1649,11 +1652,12 @@ func (s sealer) Open(ctx context.Context, ct []byte) ([]byte, error) { // boltOptions returns a bolt.Options struct, suitable for passing to // bolt.Open(), pre-configured with all of our preferred defaults. -func boltOptions() *bolt.Options { +func boltOptions(path string) *bolt.Options { o := &bolt.Options{ Timeout: 1 * time.Second, FreelistType: bolt.FreelistMapType, NoFreelistSync: true, + MmapFlags: getMmapFlags(path), } if os.Getenv("VAULT_RAFT_FREELIST_TYPE") == "array" {