dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-25 22:44:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable allow_concurrent_memtable_write and enable_write_thread_adaptive_yield by default

Browse source 
Summary: Closes https://github.com/facebook/rocksdb/pull/1496

Differential Revision: D4168080

Pulled By: siying

fbshipit-source-id: 056ae62
This commit is contained in:
Siying Dong 2016-11-16 09:24:52 -08:00 committed by Facebook Github Bot
parent 420bdb42e7
commit 972e3ff295
19 changed files with 59 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
HISTORY.md
View file

@ -4,6 +4,8 @@
* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
* Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
* Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys.
* Support dynamically change `delayed_write_rate` option via SetDBOptions().
* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default.
### New Features
* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions().

10
db/c.cc
View file

@ -1703,6 +1703,16 @@ void rocksdb_options_set_bytes_per_sync(
opt->rep.bytes_per_sync = v;
}
void rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t* opt,
unsigned char v) {
opt->rep.allow_concurrent_memtable_write = v;
}
void rocksdb_options_set_enable_write_thread_adaptive_yield(
rocksdb_options_t* opt, unsigned char v) {
opt->rep.enable_write_thread_adaptive_yield = v;
}
void rocksdb_options_set_verify_checksums_in_compaction(
rocksdb_options_t* opt, unsigned char v) {
opt->rep.verify_checksums_in_compaction = v;

1
db/c_test.c
View file

@ -895,6 +895,7 @@ int main(int argc, char** argv) {
rocksdb_options_set_prefix_extractor(options, rocksdb_slicetransform_create_fixed_prefix(3));
rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
db = rocksdb_open(options, dbname, &err);
CheckNoError(err);

1
db/column_family_test.cc
View file

@ -1056,6 +1056,7 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
#ifndef ROCKSDB_LITE // Cuckoo is not supported in lite
TEST_F(ColumnFamilyTest, MemtableNotSupportSnapshot) {
db_options_.allow_concurrent_memtable_write = false;
Open();
auto* s1 = dbfull()->GetSnapshot();
ASSERT_TRUE(s1 != nullptr);

1
db/cuckoo_table_db_test.cc
View file

@ -41,6 +41,7 @@ class CuckooTableDBTest : public testing::Test {
options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));
options.allow_mmap_reads = true;
options.create_if_missing = true;
options.allow_concurrent_memtable_write = false;
return options;
}

1
db/db_bloom_filter_test.cc
View file

@ -810,6 +810,7 @@ TEST_F(DBBloomFilterTest, PrefixScan) {
options.max_background_compactions = 2;
options.create_if_missing = true;
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
options.allow_concurrent_memtable_write = false;
BlockBasedTableOptions table_options;
table_options.no_block_cache = true;

14
db/db_inplace_update_test.cc
View file

@ -23,6 +23,8 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
options.inplace_update_support = true;
options.env = env_;
options.write_buffer_size = 100000;
options.allow_concurrent_memtable_write = false;
Reopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of smaller size
@ -45,6 +47,8 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
options.inplace_update_support = true;
options.env = env_;
options.write_buffer_size = 100000;
options.allow_concurrent_memtable_write = false;
Reopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of larger size
@ -70,6 +74,8 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
options.write_buffer_size = 100000;
options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
options.allow_concurrent_memtable_write = false;
Reopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of smaller size
@ -97,6 +103,8 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
options.write_buffer_size = 100000;
options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
options.allow_concurrent_memtable_write = false;
Reopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of smaller varint size
@ -124,6 +132,8 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
options.write_buffer_size = 100000;
options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
options.allow_concurrent_memtable_write = false;
Reopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of larger size
@ -148,7 +158,9 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
options.env = env_;
options.write_buffer_size = 100000;
options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
options.allow_concurrent_memtable_write = false;
Reopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
// Callback function requests no actions from db

3
db/db_memtable_test.cc
View file

@ -86,6 +86,8 @@ class MockMemTableRepFactory : public MemTableRepFactory {
MockMemTableRep* rep() { return mock_rep_; }
bool IsInsertConcurrentlySupported() const override { return false; }
private:
MockMemTableRep* mock_rep_;
};
@ -116,6 +118,7 @@ class TestPrefixExtractor : public SliceTransform {
TEST_F(DBMemTableTest, InsertWithHint) {
Options options;
options.allow_concurrent_memtable_write = false;
options.create_if_missing = true;
options.memtable_factory.reset(new MockMemTableRepFactory());
options.memtable_insert_with_hint_prefix_extractor.reset(

1
db/db_properties_test.cc
View file

@ -34,6 +34,7 @@ TEST_F(DBPropertiesTest, Empty) {
Options options;
options.env = env_;
options.write_buffer_size = 100000; // Small write buffer
options.allow_concurrent_memtable_write = false;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options);

2
db/db_tailing_iter_test.cc
View file

@ -305,6 +305,7 @@ TEST_F(DBTestTailingIterator, TailingIteratorPrefixSeek) {
options.disable_auto_compactions = true;
options.prefix_extractor.reset(NewFixedPrefixTransform(2));
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
options.allow_concurrent_memtable_write = false;
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
@ -625,6 +626,7 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorPrefixSeek) {
options.disable_auto_compactions = true;
options.prefix_extractor.reset(NewFixedPrefixTransform(2));
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
options.allow_concurrent_memtable_write = false;
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);

4
db/db_test.cc
View file

@ -2469,11 +2469,13 @@ class MultiThreadedDBTest : public DBTest,
TEST_P(MultiThreadedDBTest, MultiThreaded) {
anon::OptionsOverride options_override;
options_override.skip_policy = kSkipNoSnapshot;
Options options = CurrentOptions(options_override);
std::vector<std::string> cfs;
for (int i = 1; i < kColumnFamilies; ++i) {
cfs.push_back(ToString(i));
}
CreateAndReopenWithCF(cfs, CurrentOptions(options_override));
Reopen(options);
CreateAndReopenWithCF(cfs, options);
// Initialize state
MTState mt;
mt.test = this;

1
db/db_test2.cc
View file

@ -956,6 +956,7 @@ TEST_F(DBTest2, PresetCompressionDict) {
const int kNumL0Files = 5;
Options options;
options.allow_concurrent_memtable_write = false;
options.arena_block_size = kBlockSizeBytes;
options.compaction_style = kCompactionStyleUniversal;
options.create_if_missing = true;

4
db/db_test_util.cc
View file

@ -233,6 +233,7 @@ Options DBTestBase::CurrentOptions(
case kHashSkipList:
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
options.allow_concurrent_memtable_write = false;
break;
case kPlainTableFirstBytePrefix:
options.table_factory.reset(new PlainTableFactory());
@ -264,15 +265,18 @@ Options DBTestBase::CurrentOptions(
break;
case kVectorRep:
options.memtable_factory.reset(new VectorRepFactory(100));
options.allow_concurrent_memtable_write = false;
break;
case kHashLinkList:
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
options.memtable_factory.reset(
NewHashLinkListRepFactory(4, 0, 3, true, 4));
options.allow_concurrent_memtable_write = false;
break;
case kHashCuckoo:
options.memtable_factory.reset(
NewHashCuckooRepFactory(options.write_buffer_size));
options.allow_concurrent_memtable_write = false;
break;
#endif // ROCKSDB_LITE
case kMergePut:

4
db/db_test_util.h
View file

@ -193,6 +193,10 @@ class SpecialSkipListFactory : public MemTableRepFactory {
}
virtual const char* Name() const override { return "SkipListFactory"; }
bool IsInsertConcurrentlySupported() const override {
return factory_.IsInsertConcurrentlySupported();
}
private:
SkipListFactory factory_;
int num_entries_flush_;

1
db/plain_table_db_test.cc
View file

@ -142,6 +142,7 @@ class PlainTableDBTest : public testing::Test,
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
options.allow_mmap_reads = mmap_mode_;
options.allow_concurrent_memtable_write = false;
return options;
}

1
db/prefix_test.cc
View file

@ -222,6 +222,7 @@ class PrefixTest : public testing::Test {
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
bbto.whole_key_filtering = false;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.allow_concurrent_memtable_write = false;
Status s = DB::Open(options, kDbName, &db);
EXPECT_OK(s);

6
include/rocksdb/c.h
View file

@ -647,6 +647,12 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_adaptive_mutex(
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bytes_per_sync(
rocksdb_options_t*, uint64_t);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_verify_checksums_in_compaction(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void

4
include/rocksdb/options.h
View file

@ -1298,7 +1298,7 @@ struct DBOptions {
// It is strongly recommended to set enable_write_thread_adaptive_yield
// if you are going to use this feature.
//
// Default: false
// Default: true
bool allow_concurrent_memtable_write;
// If true, threads synchronizing with the write batch group leader will
@ -1306,7 +1306,7 @@ struct DBOptions {
// This can substantially improve throughput for concurrent workloads,
// regardless of whether allow_concurrent_memtable_write is enabled.
//
// Default: false
// Default: true
bool enable_write_thread_adaptive_yield;
// The maximum number of microseconds that a write operation will use

4
util/options.cc
View file

@ -219,8 +219,8 @@ DBOptions::DBOptions()
listeners(),
enable_thread_tracking(false),
delayed_write_rate(2 * 1024U * 1024U),
allow_concurrent_memtable_write(false),
enable_write_thread_adaptive_yield(false),
allow_concurrent_memtable_write(true),
enable_write_thread_adaptive_yield(true),
write_thread_max_yield_usec(100),
write_thread_slow_yield_usec(3),
skip_stats_update_on_db_open(false),