2013-10-16 21:59:46 +00:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 22:37:00 +00:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2014-12-02 20:09:20 +00:00
|
|
|
// Arena is an implementation of Allocator class. For a request of small size,
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 19:42:23 +00:00
|
|
|
// it allocates a block with pre-defined block size. For a request of big
|
|
|
|
// size, it uses malloc to directly get the requested size.
|
|
|
|
|
2013-10-05 05:32:05 +00:00
|
|
|
#pragma once
|
2015-09-28 18:49:48 +00:00
|
|
|
#ifndef OS_WIN
|
|
|
|
#include <sys/mman.h>
|
|
|
|
#endif
|
2011-03-18 22:37:00 +00:00
|
|
|
#include <cstddef>
|
2014-05-15 04:31:03 +00:00
|
|
|
#include <cerrno>
|
2011-03-18 22:37:00 +00:00
|
|
|
#include <vector>
|
|
|
|
#include <assert.h>
|
|
|
|
#include <stdint.h>
|
2014-12-02 20:09:20 +00:00
|
|
|
#include "util/allocator.h"
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
#include "util/mutexlock.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
namespace rocksdb {
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-12-02 20:09:20 +00:00
|
|
|
class Arena : public Allocator {
|
2011-03-18 22:37:00 +00:00
|
|
|
public:
|
2014-01-08 23:06:07 +00:00
|
|
|
// No copying allowed
|
2014-01-31 01:18:17 +00:00
|
|
|
Arena(const Arena&) = delete;
|
|
|
|
void operator=(const Arena&) = delete;
|
2014-01-08 23:06:07 +00:00
|
|
|
|
2014-05-14 19:36:21 +00:00
|
|
|
static const size_t kInlineSize = 2048;
|
2014-01-08 23:06:07 +00:00
|
|
|
static const size_t kMinBlockSize;
|
|
|
|
static const size_t kMaxBlockSize;
|
|
|
|
|
2014-11-21 22:11:22 +00:00
|
|
|
// huge_page_size: if 0, don't use huge page TLB. If > 0 (should set to the
|
|
|
|
// supported hugepage size of the system), block allocation will try huge
|
|
|
|
// page TLB first. If allocation fails, will fall back to normal case.
|
|
|
|
explicit Arena(size_t block_size = kMinBlockSize, size_t huge_page_size = 0);
|
2014-01-31 01:18:17 +00:00
|
|
|
~Arena();
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-12-02 20:09:20 +00:00
|
|
|
char* Allocate(size_t bytes) override;
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-05-09 18:01:54 +00:00
|
|
|
// huge_page_size: if >0, will try to allocate from huage page TLB.
|
|
|
|
// The argument will be the size of the page size for huge page TLB. Bytes
|
|
|
|
// will be rounded up to multiple of the page size to allocate through mmap
|
|
|
|
// anonymous option with huge page on. The extra space allocated will be
|
|
|
|
// wasted. If allocation fails, will fall back to normal case. To enable it,
|
|
|
|
// need to reserve huge pages for it to be allocated, like:
|
2014-05-04 20:55:53 +00:00
|
|
|
// sysctl -w vm.nr_hugepages=20
|
|
|
|
// See linux doc Documentation/vm/hugetlbpage.txt for details.
|
2014-05-04 22:52:23 +00:00
|
|
|
// huge page allocation can fail. In this case it will fail back to
|
|
|
|
// normal cases. The messages will be logged to logger. So when calling with
|
|
|
|
// huge_page_tlb_size > 0, we highly recommend a logger is passed in.
|
|
|
|
// Otherwise, the error message will be printed out to stderr directly.
|
2014-05-09 18:01:54 +00:00
|
|
|
char* AllocateAligned(size_t bytes, size_t huge_page_size = 0,
|
2014-12-02 20:09:20 +00:00
|
|
|
Logger* logger = nullptr) override;
|
2011-03-18 22:37:00 +00:00
|
|
|
|
|
|
|
// Returns an estimate of the total memory usage of data allocated
|
2014-01-08 23:06:07 +00:00
|
|
|
// by the arena (exclude the space allocated but not yet used for future
|
2011-03-18 22:37:00 +00:00
|
|
|
// allocations).
|
2014-02-18 22:58:55 +00:00
|
|
|
size_t ApproximateMemoryUsage() const {
|
2014-01-08 23:06:07 +00:00
|
|
|
return blocks_memory_ + blocks_.capacity() * sizeof(char*) -
|
|
|
|
alloc_bytes_remaining_;
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
|
2014-02-18 22:58:55 +00:00
|
|
|
size_t MemoryAllocatedBytes() const { return blocks_memory_; }
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 19:42:23 +00:00
|
|
|
|
2014-03-12 23:40:14 +00:00
|
|
|
size_t AllocatedAndUnused() const { return alloc_bytes_remaining_; }
|
|
|
|
|
|
|
|
// If an allocation is too big, we'll allocate an irregular block with the
|
|
|
|
// same size of that allocation.
|
2014-11-10 22:39:38 +00:00
|
|
|
size_t IrregularBlockNum() const { return irregular_block_num; }
|
2014-03-12 23:40:14 +00:00
|
|
|
|
2014-12-02 20:09:20 +00:00
|
|
|
size_t BlockSize() const override { return kBlockSize; }
|
2014-03-14 17:02:04 +00:00
|
|
|
|
2011-03-18 22:37:00 +00:00
|
|
|
private:
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
char inline_block_[kInlineSize] __attribute__((__aligned__(sizeof(void*))));
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 19:42:23 +00:00
|
|
|
// Number of bytes allocated in one block
|
2014-01-08 23:06:07 +00:00
|
|
|
const size_t kBlockSize;
|
2011-03-18 22:37:00 +00:00
|
|
|
// Array of new[] allocated memory blocks
|
2014-01-08 23:06:07 +00:00
|
|
|
typedef std::vector<char*> Blocks;
|
|
|
|
Blocks blocks_;
|
2014-05-04 20:55:53 +00:00
|
|
|
|
|
|
|
struct MmapInfo {
|
|
|
|
void* addr_;
|
|
|
|
size_t length_;
|
|
|
|
|
|
|
|
MmapInfo(void* addr, size_t length) : addr_(addr), length_(length) {}
|
|
|
|
};
|
|
|
|
std::vector<MmapInfo> huge_blocks_;
|
2014-03-12 23:40:14 +00:00
|
|
|
size_t irregular_block_num = 0;
|
2014-01-08 23:06:07 +00:00
|
|
|
|
|
|
|
// Stats for current active block.
|
|
|
|
// For each block, we allocate aligned memory chucks from one end and
|
|
|
|
// allocate unaligned memory chucks from the other end. Otherwise the
|
|
|
|
// memory waste for alignment will be higher if we allocate both types of
|
|
|
|
// memory from one direction.
|
|
|
|
char* unaligned_alloc_ptr_ = nullptr;
|
|
|
|
char* aligned_alloc_ptr_ = nullptr;
|
|
|
|
// How many bytes left in currently active block?
|
|
|
|
size_t alloc_bytes_remaining_ = 0;
|
|
|
|
|
2015-09-28 18:40:29 +00:00
|
|
|
#ifdef MAP_HUGETLB
|
2014-11-21 22:11:22 +00:00
|
|
|
size_t hugetlb_size_ = 0;
|
2015-09-28 18:40:29 +00:00
|
|
|
#endif // MAP_HUGETLB
|
2014-11-21 22:11:22 +00:00
|
|
|
char* AllocateFromHugePage(size_t bytes);
|
2014-01-08 23:06:07 +00:00
|
|
|
char* AllocateFallback(size_t bytes, bool aligned);
|
|
|
|
char* AllocateNewBlock(size_t block_bytes);
|
2011-03-18 22:37:00 +00:00
|
|
|
|
|
|
|
// Bytes of memory in blocks allocated so far
|
2014-01-08 23:06:07 +00:00
|
|
|
size_t blocks_memory_ = 0;
|
2011-03-18 22:37:00 +00:00
|
|
|
};
|
|
|
|
|
2014-01-31 01:18:17 +00:00
|
|
|
inline char* Arena::Allocate(size_t bytes) {
|
2011-03-18 22:37:00 +00:00
|
|
|
// The semantics of what to return are a bit messy if we allow
|
|
|
|
// 0-byte allocations, so we disallow them here (we don't need
|
|
|
|
// them for our internal use).
|
|
|
|
assert(bytes > 0);
|
|
|
|
if (bytes <= alloc_bytes_remaining_) {
|
2014-01-08 23:06:07 +00:00
|
|
|
unaligned_alloc_ptr_ -= bytes;
|
2011-03-18 22:37:00 +00:00
|
|
|
alloc_bytes_remaining_ -= bytes;
|
2014-01-08 23:06:07 +00:00
|
|
|
return unaligned_alloc_ptr_;
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
2014-01-08 23:06:07 +00:00
|
|
|
return AllocateFallback(bytes, false /* unaligned */);
|
2011-03-18 22:37:00 +00:00
|
|
|
}
|
|
|
|
|
2014-01-08 23:06:07 +00:00
|
|
|
// check and adjust the block_size so that the return value is
|
|
|
|
// 1. in the range of [kMinBlockSize, kMaxBlockSize].
|
|
|
|
// 2. the multiple of align unit.
|
|
|
|
extern size_t OptimizeBlockSize(size_t block_size);
|
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
} // namespace rocksdb
|