From beed86473a078f58e6a8f214068a142bfc53d6aa Mon Sep 17 00:00:00 2001 From: mrambacher Date: Wed, 8 Sep 2021 07:45:59 -0700 Subject: [PATCH] Make MemTableRepFactory into a Customizable class (#8419) Summary: This PR does the following: -> Makes the MemTableRepFactory into a Customizable class and creatable/configurable via CreateFromString -> Makes the existing implementations compatible with configurations -> Moves the "SpecialRepFactory" test class into testutil, accessible via the ObjectRegistry or a NewSpecial API New tests were added to validate the functionality and all existing tests pass. db_bench and memtablerep_bench were hand-tested to verify the functionality in those tools. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8419 Reviewed By: zhichao-cao Differential Revision: D29558961 Pulled By: mrambacher fbshipit-source-id: 81b7229636e4e649a0c914e73ac7b0f8454c931c --- db/column_family.cc | 1 - db/column_family_test.cc | 12 +- db/compaction/compaction_job_stats_test.cc | 1 - db/db_bloom_filter_test.cc | 4 +- db/db_compaction_test.cc | 25 ++- db/db_flush_test.cc | 5 +- db/db_impl/db_impl.cc | 2 - db/db_io_failure_test.cc | 7 +- db/db_range_del_test.cc | 28 +-- db/db_secondary_test.cc | 7 +- db/db_test.cc | 11 +- db/db_test2.cc | 12 +- db/db_test_util.h | 89 --------- db/db_universal_compaction_test.cc | 14 +- db/db_with_timestamp_basic_test.cc | 28 ++- db/db_with_timestamp_compaction_test.cc | 4 +- db/listener_test.cc | 17 +- include/rocksdb/memtablerep.h | 35 +++- include/rocksdb/utilities/options_type.h | 1 - memtable/hash_linklist_rep.cc | 75 +++++++- memtable/hash_linklist_rep.h | 49 ----- memtable/hash_skiplist_rep.cc | 56 +++++- memtable/hash_skiplist_rep.h | 44 ----- memtable/memtablerep_bench.cc | 19 +- memtable/skiplistrep.cc | 23 +++ memtable/vectorrep.cc | 20 +- options/cf_options.cc | 34 ++-- options/customizable_test.cc | 17 ++ options/options_helper.cc | 6 - options/options_test.cc | 108 ++++++++++- table/plain/plain_table_factory.cc | 206 ++++++++++++++------- test_util/testutil.cc | 126 +++++++++++++ test_util/testutil.h | 8 + tools/db_bench_tool.cc | 122 +++++------- 34 files changed, 767 insertions(+), 449 deletions(-) delete mode 100644 memtable/hash_linklist_rep.h delete mode 100644 memtable/hash_skiplist_rep.h diff --git a/db/column_family.cc b/db/column_family.cc index ab9db0950b..6a8e0c7a46 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -29,7 +29,6 @@ #include "db/version_set.h" #include "db/write_controller.h" #include "file/sst_file_manager_impl.h" -#include "memtable/hash_skiplist_rep.h" #include "monitoring/thread_status_util.h" #include "options/options_helper.h" #include "port/port.h" diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 2db49813ab..b772fb1ce0 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -2975,7 +2975,8 @@ TEST_P(ColumnFamilyTest, FlushCloseWALFiles) { SpecialEnv env(Env::Default()); db_options_.env = &env; db_options_.max_background_flushes = 1; - column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2)); + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(2)); Open(); CreateColumnFamilies({"one"}); ASSERT_OK(Put(1, "fodor", "mirko")); @@ -3020,7 +3021,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) { SpecialEnv env(Env::Default()); db_options_.env = &env; db_options_.max_background_flushes = 1; - column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2)); + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(2)); Open(); CreateColumnFamilies({"one"}); ASSERT_OK(Put(1, "fodor", "mirko")); @@ -3071,7 +3073,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile2) { env.SetBackgroundThreads(2, Env::HIGH); db_options_.env = &env; db_options_.max_background_flushes = 1; - column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2)); + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(2)); Open(); CreateColumnFamilies({"one"}); ASSERT_OK(Put(1, "fodor", "mirko")); @@ -3129,7 +3132,8 @@ TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) { env.SetBackgroundThreads(2, Env::HIGH); db_options_.env = &env; db_options_.max_background_flushes = 1; - column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(3)); + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(3)); column_family_options_.level0_file_num_compaction_trigger = 2; Open(); CreateColumnFamilies({"one"}); diff --git a/db/compaction/compaction_job_stats_test.cc b/db/compaction/compaction_job_stats_test.cc index 7c3879558d..4aba6897cb 100644 --- a/db/compaction/compaction_job_stats_test.cc +++ b/db/compaction/compaction_job_stats_test.cc @@ -24,7 +24,6 @@ #include "db/write_batch_internal.h" #include "env/mock_env.h" #include "file/filename.h" -#include "memtable/hash_linklist_rep.h" #include "monitoring/statistics.h" #include "monitoring/thread_status_util.h" #include "port/stack_trace.h" diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index e4ef68597c..882bf21094 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -15,6 +15,7 @@ #include "port/stack_trace.h" #include "rocksdb/perf_context.h" #include "table/block_based/filter_policy_internal.h" +#include "test_util/testutil.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { @@ -2171,7 +2172,8 @@ TEST_F(DBBloomFilterTest, SeekForPrevWithPartitionedFilters) { Options options = CurrentOptions(); constexpr size_t kNumKeys = 10000; static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000"); - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeys + 10)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeys + 10)); options.create_if_missing = true; constexpr size_t kPrefixLength = 4; options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength)); diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index 53dadde717..5eac7a9298 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -19,6 +19,7 @@ #include "rocksdb/sst_file_writer.h" #include "rocksdb/utilities/convenience.h" #include "test_util/sync_point.h" +#include "test_util/testutil.h" #include "util/concurrent_task_limiter_impl.h" #include "util/random.h" #include "utilities/fault_injection_env.h" @@ -805,7 +806,8 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) { options.num_levels = 3; options.level0_file_num_compaction_trigger = 3; options.max_subcompactions = max_subcompactions_; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); CreateAndReopenWithCF({"pikachu"}, options); Random rnd(301); @@ -852,7 +854,8 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) { options.level0_slowdown_writes_trigger = 20; options.soft_pending_compaction_bytes_limit = 1 << 30; // Infinitely large options.max_background_compactions = 3; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); // Block all threads in thread pool. const size_t kTotalTasks = 4; @@ -2094,7 +2097,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) { options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_style = kCompactionStyleLevel; options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; @@ -2203,7 +2206,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) { options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_style = kCompactionStyleLevel; options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; @@ -2313,7 +2316,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionCFPathUse) { options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_style = kCompactionStyleLevel; options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; @@ -3012,7 +3015,7 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) { } Options options = CurrentOptions(); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_style = kCompactionStyleLevel; options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; @@ -4815,7 +4818,8 @@ TEST_F(DBCompactionTest, CompactionLimiter) { options.level0_slowdown_writes_trigger = 64; options.level0_stop_writes_trigger = 64; options.max_background_jobs = kMaxBackgroundThreads; // Enough threads - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); options.max_write_buffer_number = 10; // Enough memtables DestroyAndReopen(options); @@ -5338,7 +5342,8 @@ TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) { options.level0_file_num_compaction_trigger = options.level0_stop_writes_trigger; options.max_subcompactions = max_subcompactions_; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); DestroyAndReopen(options); Random rnd(301); @@ -5864,7 +5869,7 @@ TEST_F(DBCompactionTest, ChangeLevelCompactRangeConflictsWithManual) { // `Status::Incomplete`. Options options = CurrentOptions(); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.level0_file_num_compaction_trigger = 2; options.num_levels = 3; Reopen(options); @@ -5952,7 +5957,7 @@ TEST_F(DBCompactionTest, ChangeLevelErrorPathTest) { // succeeds Options options = CurrentOptions(); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.level0_file_num_compaction_trigger = 2; options.num_levels = 3; Reopen(options); diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index 41a9a983ed..73cdd813bd 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -18,6 +18,7 @@ #include "port/stack_trace.h" #include "rocksdb/utilities/transaction_db.h" #include "test_util/sync_point.h" +#include "test_util/testutil.h" #include "util/cast_util.h" #include "util/mutexlock.h" #include "utilities/fault_injection_env.h" @@ -139,7 +140,7 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) { // scheduled in the low-pri (compaction) thread pool. Options options = CurrentOptions(); options.level0_file_num_compaction_trigger = 4; - options.memtable_factory.reset(new SpecialSkipListFactory(1)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); Reopen(options); env_->SetBackgroundThreads(0, Env::HIGH); @@ -2320,7 +2321,7 @@ TEST_P(DBAtomicFlushTest, TriggerFlushAndClose) { options.create_if_missing = true; options.atomic_flush = atomic_flush; options.memtable_factory.reset( - new SpecialSkipListFactory(kNumKeysTriggerFlush)); + test::NewSpecialSkipListFactory(kNumKeysTriggerFlush)); CreateAndReopenWithCF({"pikachu"}, options); for (int i = 0; i != kNumKeysTriggerFlush; ++i) { diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index c05bde8572..f23502b474 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -61,8 +61,6 @@ #include "logging/auto_roll_logger.h" #include "logging/log_buffer.h" #include "logging/logging.h" -#include "memtable/hash_linklist_rep.h" -#include "memtable/hash_skiplist_rep.h" #include "monitoring/in_memory_stats_history.h" #include "monitoring/instrumented_mutex.h" #include "monitoring/iostats_context_imp.h" diff --git a/db/db_io_failure_test.cc b/db/db_io_failure_test.cc index da435a3b58..75228b4b1f 100644 --- a/db/db_io_failure_test.cc +++ b/db/db_io_failure_test.cc @@ -9,6 +9,7 @@ #include "db/db_test_util.h" #include "port/stack_trace.h" +#include "test_util/testutil.h" #include "util/random.h" namespace ROCKSDB_NAMESPACE { @@ -269,7 +270,7 @@ TEST_F(DBIOFailureTest, FlushSstRangeSyncError) { options.writable_file_max_buffer_size = 128 * 1024; options.bytes_per_sync = 128 * 1024; options.level0_file_num_compaction_trigger = 4; - options.memtable_factory.reset(new SpecialSkipListFactory(10)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(10)); BlockBasedTableOptions table_options; table_options.filter_policy.reset(NewBloomFilterPolicy(10)); options.table_factory.reset(NewBlockBasedTableFactory(table_options)); @@ -396,7 +397,7 @@ TEST_F(DBIOFailureTest, FlushSstCloseError) { options.error_if_exists = false; options.paranoid_checks = true; options.level0_file_num_compaction_trigger = 4; - options.memtable_factory.reset(new SpecialSkipListFactory(2)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(2)); DestroyAndReopen(options); CreateAndReopenWithCF({"pikachu"}, options); @@ -492,7 +493,7 @@ TEST_F(DBIOFailureTest, FlushSstSyncError) { options.paranoid_checks = true; options.use_fsync = false; options.level0_file_num_compaction_trigger = 4; - options.memtable_factory.reset(new SpecialSkipListFactory(2)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(2)); DestroyAndReopen(options); CreateAndReopenWithCF({"pikachu"}, options); diff --git a/db/db_range_del_test.cc b/db/db_range_del_test.cc index 918fadbaf9..fc2e05ad46 100644 --- a/db/db_range_del_test.cc +++ b/db/db_range_del_test.cc @@ -118,7 +118,7 @@ TEST_F(DBRangeDelTest, CompactionOutputFilesExactlyFilled) { Options options = CurrentOptions(); options.disable_auto_compactions = true; options.level0_file_num_compaction_trigger = kNumFiles; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); options.num_levels = 2; options.target_file_size_base = kFileBytes; BlockBasedTableOptions table_options; @@ -166,7 +166,7 @@ TEST_F(DBRangeDelTest, MaxCompactionBytesCutsOutputFiles) { opts.disable_auto_compactions = true; opts.level0_file_num_compaction_trigger = kNumFiles; opts.max_compaction_bytes = kNumPerFile * kBytesPerVal; - opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); // Want max_compaction_bytes to trigger the end of compaction output file, not // target_file_size_base, so make the latter much bigger opts.target_file_size_base = 100 * opts.max_compaction_bytes; @@ -319,7 +319,7 @@ TEST_F(DBRangeDelTest, CompactionRemovesCoveredKeys) { Options opts = CurrentOptions(); opts.comparator = test::Uint64Comparator(); opts.disable_auto_compactions = true; - opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); opts.num_levels = 2; opts.statistics = CreateDBStatistics(); DestroyAndReopen(opts); @@ -377,7 +377,7 @@ TEST_F(DBRangeDelTest, ValidLevelSubcompactionBoundaries) { options.level0_file_num_compaction_trigger = kNumFiles; options.max_bytes_for_level_base = 2 * kFileBytes; options.max_subcompactions = 4; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); options.num_levels = 3; options.target_file_size_base = kFileBytes; options.target_file_size_multiplier = 1; @@ -436,7 +436,7 @@ TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) { options.compaction_style = kCompactionStyleUniversal; options.level0_file_num_compaction_trigger = kFilesPerLevel; options.max_subcompactions = 4; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); options.num_levels = kNumLevels; options.target_file_size_base = kNumPerFile << 10; options.target_file_size_multiplier = 1; @@ -489,7 +489,7 @@ TEST_F(DBRangeDelTest, CompactionRemovesCoveredMergeOperands) { const int kNumPerFile = 3, kNumFiles = 3; Options opts = CurrentOptions(); opts.disable_auto_compactions = true; - opts.memtable_factory.reset(new SpecialSkipListFactory(2 * kNumPerFile)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(2 * kNumPerFile)); opts.merge_operator = MergeOperators::CreateUInt64AddOperator(); opts.num_levels = 2; Reopen(opts); @@ -613,7 +613,7 @@ TEST_F(DBRangeDelTest, TableEvictedDuringScan) { opts.comparator = test::Uint64Comparator(); opts.level0_file_num_compaction_trigger = 4; opts.level0_stop_writes_trigger = 4; - opts.memtable_factory.reset(new SpecialSkipListFactory(1)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); opts.num_levels = 2; BlockBasedTableOptions bbto; bbto.cache_index_and_filter_blocks = true; @@ -694,7 +694,7 @@ TEST_F(DBRangeDelTest, GetCoveredKeyFromImmutableMemtable) { // memtable can hold. It switches the active memtable to immutable (flush is // prevented by the above options) upon inserting an element that would // overflow the memtable. - opts.memtable_factory.reset(new SpecialSkipListFactory(1)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); DestroyAndReopen(opts); ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); @@ -759,7 +759,7 @@ TEST_F(DBRangeDelTest, GetIgnoresRangeDeletions) { Options opts = CurrentOptions(); opts.max_write_buffer_number = 4; opts.min_write_buffer_number_to_merge = 3; - opts.memtable_factory.reset(new SpecialSkipListFactory(1)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); Reopen(opts); ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val")); @@ -788,7 +788,7 @@ TEST_F(DBRangeDelTest, IteratorRemovesCoveredKeys) { const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25; Options opts = CurrentOptions(); opts.comparator = test::Uint64Comparator(); - opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); DestroyAndReopen(opts); // Write half of the keys before the tombstone and half after the tombstone. @@ -823,7 +823,7 @@ TEST_F(DBRangeDelTest, IteratorOverUserSnapshot) { const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25; Options opts = CurrentOptions(); opts.comparator = test::Uint64Comparator(); - opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); DestroyAndReopen(opts); const Snapshot* snapshot = nullptr; @@ -857,7 +857,7 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) { Options opts = CurrentOptions(); opts.max_write_buffer_number = 4; opts.min_write_buffer_number_to_merge = 3; - opts.memtable_factory.reset(new SpecialSkipListFactory(1)); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); Reopen(opts); ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val")); @@ -1005,7 +1005,7 @@ TEST_F(DBRangeDelTest, CompactionTreatsSplitInputLevelDeletionAtomically) { options.compression = kNoCompression; options.level0_file_num_compaction_trigger = kNumFilesPerLevel; options.memtable_factory.reset( - new SpecialSkipListFactory(2 /* num_entries_flush */)); + test::NewSpecialSkipListFactory(2 /* num_entries_flush */)); options.target_file_size_base = kValueBytes; // i == 0: CompactFiles // i == 1: CompactRange @@ -1073,7 +1073,7 @@ TEST_F(DBRangeDelTest, RangeTombstoneEndKeyAsSstableUpperBound) { options.compression = kNoCompression; options.level0_file_num_compaction_trigger = kNumFilesPerLevel; options.memtable_factory.reset( - new SpecialSkipListFactory(2 /* num_entries_flush */)); + test::NewSpecialSkipListFactory(2 /* num_entries_flush */)); options.target_file_size_base = kValueBytes; options.disable_auto_compactions = true; diff --git a/db/db_secondary_test.cc b/db/db_secondary_test.cc index 902f1ec2cd..35a7bd2cf3 100644 --- a/db/db_secondary_test.cc +++ b/db/db_secondary_test.cc @@ -12,6 +12,7 @@ #include "port/stack_trace.h" #include "rocksdb/utilities/transaction_db.h" #include "test_util/sync_point.h" +#include "test_util/testutil.h" #include "utilities/fault_injection_env.h" namespace ROCKSDB_NAMESPACE { @@ -937,7 +938,7 @@ TEST_F(DBSecondaryTest, DISABLED_SwitchWAL) { options.max_write_buffer_number = 4; options.min_write_buffer_number_to_merge = 2; options.memtable_factory.reset( - new SpecialSkipListFactory(kNumKeysPerMemtable)); + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); Reopen(options); Options options1; @@ -992,7 +993,7 @@ TEST_F(DBSecondaryTest, DISABLED_SwitchWALMultiColumnFamilies) { options.max_write_buffer_number = 4; options.min_write_buffer_number_to_merge = 2; options.memtable_factory.reset( - new SpecialSkipListFactory(kNumKeysPerMemtable)); + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); CreateAndReopenWithCF({kCFName1}, options); Options options1; @@ -1056,7 +1057,7 @@ TEST_F(DBSecondaryTest, CatchUpAfterFlush) { options.max_write_buffer_number = 4; options.min_write_buffer_number_to_merge = 2; options.memtable_factory.reset( - new SpecialSkipListFactory(kNumKeysPerMemtable)); + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); Reopen(options); Options options1; diff --git a/db/db_test.cc b/db/db_test.cc index 0d8e479f41..fca8b2880d 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -34,7 +34,6 @@ #include "db/write_batch_internal.h" #include "env/mock_env.h" #include "file/filename.h" -#include "memtable/hash_linklist_rep.h" #include "monitoring/thread_status_util.h" #include "port/port.h" #include "port/stack_trace.h" @@ -5732,8 +5731,8 @@ TEST_F(DBTest, DISABLED_SuggestCompactRangeTest) { }; Options options = CurrentOptions(); - options.memtable_factory.reset( - new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); options.compaction_style = kCompactionStyleLevel; options.compaction_filter_factory.reset( new CompactionFilterFactoryGetContext()); @@ -6138,7 +6137,7 @@ TEST_F(DBTest, DelayedWriteRate) { options.level0_stop_writes_trigger = 999999; options.delayed_write_rate = 20000000; // Start with 200MB/s options.memtable_factory.reset( - new SpecialSkipListFactory(kEntriesPerMemTable)); + test::NewSpecialSkipListFactory(kEntriesPerMemTable)); SetTimeElapseOnlySleepOnReopen(&options); CreateAndReopenWithCF({"pikachu"}, options); @@ -6201,7 +6200,7 @@ TEST_F(DBTest, HardLimit) { options.max_bytes_for_level_base = 10000000000u; options.max_background_compactions = 1; options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); env_->SetBackgroundThreads(1, Env::LOW); test::SleepingBackgroundTask sleeping_task_low; @@ -6450,7 +6449,7 @@ TEST_F(DBTest, LastWriteBufferDelay) { options.disable_auto_compactions = true; int kNumKeysPerMemtable = 3; options.memtable_factory.reset( - new SpecialSkipListFactory(kNumKeysPerMemtable)); + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); Reopen(options); test::SleepingBackgroundTask sleeping_task; diff --git a/db/db_test2.cc b/db/db_test2.cc index 9dd40f4e16..1e0352e982 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -22,6 +22,7 @@ #include "rocksdb/trace_record_result.h" #include "rocksdb/utilities/replayer.h" #include "rocksdb/wal_filter.h" +#include "test_util/testutil.h" #include "util/random.h" #include "utilities/fault_injection_env.h" @@ -1264,7 +1265,7 @@ TEST_F(DBTest2, PresetCompressionDict) { options.disable_auto_compactions = true; options.level0_file_num_compaction_trigger = kNumL0Files; options.memtable_factory.reset( - new SpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes)); + test::NewSpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes)); options.num_levels = 2; options.target_file_size_base = kL0FileBytes; options.target_file_size_multiplier = 2; @@ -1484,7 +1485,7 @@ TEST_P(PresetCompressionDictTest, Flush) { options.compression_opts.max_dict_bytes = kDictLen; options.compression_opts.max_dict_buffer_bytes = kBlockLen; } - options.memtable_factory.reset(new SpecialSkipListFactory(kKeysPerFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kKeysPerFile)); options.statistics = CreateDBStatistics(); BlockBasedTableOptions bbto; bbto.block_size = kBlockLen; @@ -2319,8 +2320,8 @@ INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest, #ifndef ROCKSDB_LITE TEST_F(DBTest2, MaxCompactionBytesTest) { Options options = CurrentOptions(); - options.memtable_factory.reset( - new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); options.compaction_style = kCompactionStyleLevel; options.write_buffer_size = 200 << 10; options.arena_block_size = 4 << 10; @@ -3945,7 +3946,8 @@ TEST_F(DBTest2, RateLimitedCompactionReads) { Options options = CurrentOptions(); options.compression = kNoCompression; options.level0_file_num_compaction_trigger = kNumL0Files; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); options.new_table_reader_for_compaction_inputs = true; // takes roughly one second, split into 100 x 10ms intervals. Each interval // permits 5.12KB, which is smaller than the block size, so this test diff --git a/db/db_test_util.h b/db/db_test_util.h index 1b28009311..36dde492f6 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -24,7 +24,6 @@ #include "db/db_impl/db_impl.h" #include "db/dbformat.h" #include "file/filename.h" -#include "memtable/hash_linklist_rep.h" #include "rocksdb/cache.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/convenience.h" @@ -112,94 +111,6 @@ struct OptionsOverride { enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 }; -// A hacky skip list mem table that triggers flush after number of entries. -class SpecialMemTableRep : public MemTableRep { - public: - explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable, - int num_entries_flush) - : MemTableRep(allocator), - memtable_(memtable), - num_entries_flush_(num_entries_flush), - num_entries_(0) {} - - virtual KeyHandle Allocate(const size_t len, char** buf) override { - return memtable_->Allocate(len, buf); - } - - // Insert key into the list. - // REQUIRES: nothing that compares equal to key is currently in the list. - virtual void Insert(KeyHandle handle) override { - num_entries_++; - memtable_->Insert(handle); - } - - void InsertConcurrently(KeyHandle handle) override { - num_entries_++; - memtable_->Insert(handle); - } - - // Returns true iff an entry that compares equal to key is in the list. - virtual bool Contains(const char* key) const override { - return memtable_->Contains(key); - } - - virtual size_t ApproximateMemoryUsage() override { - // Return a high memory usage when number of entries exceeds the threshold - // to trigger a flush. - return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024; - } - - virtual void Get(const LookupKey& k, void* callback_args, - bool (*callback_func)(void* arg, - const char* entry)) override { - memtable_->Get(k, callback_args, callback_func); - } - - uint64_t ApproximateNumEntries(const Slice& start_ikey, - const Slice& end_ikey) override { - return memtable_->ApproximateNumEntries(start_ikey, end_ikey); - } - - virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override { - return memtable_->GetIterator(arena); - } - - virtual ~SpecialMemTableRep() override {} - - private: - std::unique_ptr memtable_; - int num_entries_flush_; - int num_entries_; -}; - -// The factory for the hacky skip list mem table that triggers flush after -// number of entries exceeds a threshold. -class SpecialSkipListFactory : public MemTableRepFactory { - public: - // After number of inserts exceeds `num_entries_flush` in a mem table, trigger - // flush. - explicit SpecialSkipListFactory(int num_entries_flush) - : num_entries_flush_(num_entries_flush) {} - - using MemTableRepFactory::CreateMemTableRep; - virtual MemTableRep* CreateMemTableRep( - const MemTableRep::KeyComparator& compare, Allocator* allocator, - const SliceTransform* transform, Logger* /*logger*/) override { - return new SpecialMemTableRep( - allocator, factory_.CreateMemTableRep(compare, allocator, transform, 0), - num_entries_flush_); - } - virtual const char* Name() const override { return "SkipListFactory"; } - - bool IsInsertConcurrentlySupported() const override { - return factory_.IsInsertConcurrentlySupported(); - } - - private: - SkipListFactory factory_; - int num_entries_flush_; -}; - // Special Env used to delay background operations class SpecialEnv : public EnvWrapper { public: diff --git a/db/db_universal_compaction_test.cc b/db/db_universal_compaction_test.cc index e7dceb8129..d3414e2e76 100644 --- a/db/db_universal_compaction_test.cc +++ b/db/db_universal_compaction_test.cc @@ -12,6 +12,7 @@ #if !defined(ROCKSDB_LITE) #include "rocksdb/utilities/table_properties_collectors.h" #include "test_util/sync_point.h" +#include "test_util/testutil.h" #include "util/random.h" namespace ROCKSDB_NAMESPACE { @@ -151,7 +152,7 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) { options.table_factory.reset(NewBlockBasedTableFactory(bbto)); options.optimize_filters_for_hits = true; options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); - options.memtable_factory.reset(new SpecialSkipListFactory(3)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(3)); DestroyAndReopen(options); @@ -1230,7 +1231,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) { options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024); options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_style = kCompactionStyleUniversal; options.compaction_options_universal.size_ratio = 5; options.write_buffer_size = 111 << 10; // 114KB @@ -1334,7 +1335,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) { options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024); options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_style = kCompactionStyleUniversal; options.compaction_options_universal.size_ratio = 10; options.write_buffer_size = 111 << 10; // 114KB @@ -1498,7 +1499,8 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { options.num_levels = 1; options.write_buffer_size = 200 << 10; // 200KB options.level0_file_num_compaction_trigger = 3; - options.memtable_factory.reset(new SpecialSkipListFactory(KNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysPerFile)); options = CurrentOptions(options); CreateAndReopenWithCF({"pikachu"}, options); @@ -1579,7 +1581,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) { options.level0_file_num_compaction_trigger = 2; options.num_levels = 1; options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); std::vector filenames; if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) { @@ -1729,7 +1731,7 @@ TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) { const int kNumFilesTrigger = 8; Options options = CurrentOptions(); options.memtable_factory.reset( - new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2; options.compaction_options_universal.max_size_amplification_percent = static_cast(-1); diff --git a/db/db_with_timestamp_basic_test.cc b/db/db_with_timestamp_basic_test.cc index f7aebd2633..792a2e4809 100644 --- a/db/db_with_timestamp_basic_test.cc +++ b/db/db_with_timestamp_basic_test.cc @@ -16,6 +16,7 @@ #if !defined(ROCKSDB_LITE) #include "test_util/sync_point.h" #endif +#include "test_util/testutil.h" #include "utilities/fault_injection_env.h" namespace ROCKSDB_NAMESPACE { @@ -415,7 +416,8 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) { const size_t kTimestampSize = Timestamp(0, 0).size(); TestComparator test_cmp(kTimestampSize); options.comparator = &test_cmp; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); DestroyAndReopen(options); const std::vector start_keys = {1, 0}; const std::vector write_timestamps = {Timestamp(1, 0), @@ -842,7 +844,8 @@ TEST_F(DBBasicTestWithTimestamp, SimpleForwardIterateLowerTsBound) { const size_t kTimestampSize = Timestamp(0, 0).size(); TestComparator test_cmp(kTimestampSize); options.comparator = &test_cmp; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); DestroyAndReopen(options); const std::vector write_timestamps = {Timestamp(1, 0), Timestamp(3, 0)}; @@ -927,7 +930,8 @@ TEST_F(DBBasicTestWithTimestamp, ForwardIterateStartSeqnum) { const size_t kTimestampSize = Timestamp(0, 0).size(); TestComparator test_cmp(kTimestampSize); options.comparator = &test_cmp; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); DestroyAndReopen(options); std::vector start_seqs; @@ -2235,7 +2239,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGet) { Options options = CurrentOptions(); options.create_if_missing = true; options.env = env_; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); size_t ts_sz = Timestamp(0, 0).size(); TestComparator test_cmp(ts_sz); options.comparator = &test_cmp; @@ -2313,7 +2318,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutDeleteGet) { TestComparator test_cmp(kTimestampSize); options.comparator = &test_cmp; const int kNumKeysPerFile = 1024; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); BlockBasedTableOptions bbto; bbto.filter_policy = std::get<0>(GetParam()); bbto.whole_key_filtering = true; @@ -2442,7 +2448,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGetWithCompaction) { Options options = CurrentOptions(); options.create_if_missing = true; options.env = env_; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); FlushedFileCollector* collector = new FlushedFileCollector(); options.listeners.emplace_back(collector); @@ -2558,7 +2565,8 @@ TEST_F(DBBasicTestWithTimestamp, BatchWriteAndMultiGet) { Options options = CurrentOptions(); options.create_if_missing = true; options.env = env_; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); options.memtable_prefix_bloom_size_ratio = 0.1; options.memtable_whole_key_filtering = true; @@ -2725,7 +2733,8 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) { TestComparator test_cmp(kTimestampSize); options.comparator = &test_cmp; options.prefix_extractor = std::get<0>(GetParam()); - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); BlockBasedTableOptions bbto; bbto.filter_policy = std::get<1>(GetParam()); bbto.index_type = std::get<3>(GetParam()); @@ -2879,7 +2888,8 @@ TEST_P(DBBasicTestWithTsIterTombstones, IterWithDelete) { TestComparator test_cmp(kTimestampSize); options.comparator = &test_cmp; options.prefix_extractor = std::get<0>(GetParam()); - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); BlockBasedTableOptions bbto; bbto.filter_policy = std::get<1>(GetParam()); bbto.index_type = std::get<3>(GetParam()); diff --git a/db/db_with_timestamp_compaction_test.cc b/db/db_with_timestamp_compaction_test.cc index de449e9622..9c8e3057f9 100644 --- a/db/db_with_timestamp_compaction_test.cc +++ b/db/db_with_timestamp_compaction_test.cc @@ -10,6 +10,7 @@ #include "db/compaction/compaction.h" #include "db/db_test_util.h" #include "port/stack_trace.h" +#include "test_util/testutil.h" namespace ROCKSDB_NAMESPACE { @@ -56,7 +57,8 @@ TEST_F(TimestampCompatibleCompactionTest, UserKeyCrossFileBoundary) { options.comparator = test::ComparatorWithU64Ts(); options.level0_file_num_compaction_trigger = 3; constexpr size_t kNumKeysPerFile = 101; - options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); DestroyAndReopen(options); SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); diff --git a/db/listener_test.cc b/db/listener_test.cc index c1e68d42cc..e9e6bc7ff5 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -10,7 +10,6 @@ #include "db/version_set.h" #include "db/write_batch_internal.h" #include "file/filename.h" -#include "memtable/hash_linklist_rep.h" #include "monitoring/statistics.h" #include "rocksdb/cache.h" #include "rocksdb/compaction_filter.h" @@ -529,8 +528,8 @@ TEST_F(EventListenerTest, CompactionReasonLevel) { Options options; options.env = CurrentOptions().env; options.create_if_missing = true; - options.memtable_factory.reset( - new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); TestCompactionReasonListener* listener = new TestCompactionReasonListener(); options.listeners.emplace_back(listener); @@ -595,8 +594,8 @@ TEST_F(EventListenerTest, CompactionReasonUniversal) { Options options; options.env = CurrentOptions().env; options.create_if_missing = true; - options.memtable_factory.reset( - new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); TestCompactionReasonListener* listener = new TestCompactionReasonListener(); options.listeners.emplace_back(listener); @@ -657,8 +656,8 @@ TEST_F(EventListenerTest, CompactionReasonFIFO) { Options options; options.env = CurrentOptions().env; options.create_if_missing = true; - options.memtable_factory.reset( - new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); TestCompactionReasonListener* listener = new TestCompactionReasonListener(); options.listeners.emplace_back(listener); @@ -925,7 +924,7 @@ TEST_F(EventListenerTest, BackgroundErrorListenerFailedFlushTest) { options.create_if_missing = true; options.env = env_; options.listeners.push_back(listener); - options.memtable_factory.reset(new SpecialSkipListFactory(1)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); options.paranoid_checks = true; DestroyAndReopen(options); @@ -956,7 +955,7 @@ TEST_F(EventListenerTest, BackgroundErrorListenerFailedCompactionTest) { options.env = env_; options.level0_file_num_compaction_trigger = 2; options.listeners.push_back(listener); - options.memtable_factory.reset(new SpecialSkipListFactory(2)); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(2)); options.paranoid_checks = true; DestroyAndReopen(options); diff --git a/include/rocksdb/memtablerep.h b/include/rocksdb/memtablerep.h index 3e4a786168..952d5d41f9 100644 --- a/include/rocksdb/memtablerep.h +++ b/include/rocksdb/memtablerep.h @@ -42,6 +42,7 @@ #include #include +#include "rocksdb/customizable.h" #include "rocksdb/slice.h" namespace ROCKSDB_NAMESPACE { @@ -51,6 +52,7 @@ class Allocator; class LookupKey; class SliceTransform; class Logger; +struct DBOptions; using KeyHandle = void*; @@ -290,10 +292,15 @@ class MemTableRep { // This is the base class for all factories that are used by RocksDB to create // new MemTableRep objects -class MemTableRepFactory { +class MemTableRepFactory : public Customizable { public: virtual ~MemTableRepFactory() {} + static const char* Type() { return "MemTableRepFactory"; } + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::unique_ptr* factory); + virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, Allocator*, const SliceTransform*, Logger* logger) = 0; @@ -326,20 +333,27 @@ class MemTableRepFactory { // seeks with consecutive keys. class SkipListFactory : public MemTableRepFactory { public: - explicit SkipListFactory(size_t lookahead = 0) : lookahead_(lookahead) {} + explicit SkipListFactory(size_t lookahead = 0); + // Methods for Configurable/Customizable class overrides + static const char* kClassName() { return "SkipListFactory"; } + static const char* kNickName() { return "skip_list"; } + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + std::string GetId() const override; + + // Methods for MemTableRepFactory class overrides using MemTableRepFactory::CreateMemTableRep; virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, Allocator*, const SliceTransform*, Logger* logger) override; - virtual const char* Name() const override { return "SkipListFactory"; } bool IsInsertConcurrentlySupported() const override { return true; } bool CanHandleDuplicatedKey() const override { return true; } private: - const size_t lookahead_; + size_t lookahead_; }; #ifndef ROCKSDB_LITE @@ -352,17 +366,22 @@ class SkipListFactory : public MemTableRepFactory { // VectorRep. On initialization, the underlying array will be at least count // bytes reserved for usage. class VectorRepFactory : public MemTableRepFactory { - const size_t count_; + size_t count_; public: - explicit VectorRepFactory(size_t count = 0) : count_(count) {} + explicit VectorRepFactory(size_t count = 0); + // Methods for Configurable/Customizable class overrides + static const char* kClassName() { return "VectorRepFactory"; } + static const char* kNickName() { return "vector"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kNickName(); } + + // Methods for MemTableRepFactory class overrides using MemTableRepFactory::CreateMemTableRep; virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, Allocator*, const SliceTransform*, Logger* logger) override; - - virtual const char* Name() const override { return "VectorRepFactory"; } }; // This class contains a fixed array of buckets, each diff --git a/include/rocksdb/utilities/options_type.h b/include/rocksdb/utilities/options_type.h index 1762ec70c1..906e2d433d 100644 --- a/include/rocksdb/utilities/options_type.h +++ b/include/rocksdb/utilities/options_type.h @@ -36,7 +36,6 @@ enum class OptionType { kSliceTransform, kCompressionType, kCompactionStopStyle, - kMemTableRepFactory, kFilterPolicy, kChecksumType, kEncodingType, diff --git a/memtable/hash_linklist_rep.cc b/memtable/hash_linklist_rep.cc index 15bc648a27..cf1f1f85fe 100644 --- a/memtable/hash_linklist_rep.cc +++ b/memtable/hash_linklist_rep.cc @@ -5,10 +5,10 @@ // #ifndef ROCKSDB_LITE -#include "memtable/hash_linklist_rep.h" #include #include + #include "db/memtable.h" #include "memory/arena.h" #include "memtable/skiplist.h" @@ -17,6 +17,7 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" +#include "rocksdb/utilities/options_type.h" #include "util/hash.h" namespace ROCKSDB_NAMESPACE { @@ -820,15 +821,77 @@ Node* HashLinkListRep::FindGreaterOrEqualInBucket(Node* head, return x; } -} // anon namespace +struct HashLinkListRepOptions { + static const char* kName() { return "HashLinkListRepFactoryOptions"; } + size_t bucket_count; + uint32_t threshold_use_skiplist; + size_t huge_page_tlb_size; + int bucket_entries_logging_threshold; + bool if_log_bucket_dist_when_flash; +}; + +static std::unordered_map hash_linklist_info = { + {"bucket_count", + {offsetof(struct HashLinkListRepOptions, bucket_count), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"threshold", + {offsetof(struct HashLinkListRepOptions, threshold_use_skiplist), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"huge_page_size", + {offsetof(struct HashLinkListRepOptions, huge_page_tlb_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"logging_threshold", + {offsetof(struct HashLinkListRepOptions, bucket_entries_logging_threshold), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"log_when_flash", + {offsetof(struct HashLinkListRepOptions, if_log_bucket_dist_when_flash), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +class HashLinkListRepFactory : public MemTableRepFactory { + public: + explicit HashLinkListRepFactory(size_t bucket_count, + uint32_t threshold_use_skiplist, + size_t huge_page_tlb_size, + int bucket_entries_logging_threshold, + bool if_log_bucket_dist_when_flash) { + options_.bucket_count = bucket_count; + options_.threshold_use_skiplist = threshold_use_skiplist; + options_.huge_page_tlb_size = huge_page_tlb_size; + options_.bucket_entries_logging_threshold = + bucket_entries_logging_threshold; + options_.if_log_bucket_dist_when_flash = if_log_bucket_dist_when_flash; + RegisterOptions(&options_, &hash_linklist_info); + } + + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* logger) override; + + static const char* kClassName() { return "HashLinkListRepFactory"; } + static const char* kNickName() { return "hash_linkedlist"; } + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + + private: + HashLinkListRepOptions options_; +}; + +} // namespace MemTableRep* HashLinkListRepFactory::CreateMemTableRep( const MemTableRep::KeyComparator& compare, Allocator* allocator, const SliceTransform* transform, Logger* logger) { - return new HashLinkListRep(compare, allocator, transform, bucket_count_, - threshold_use_skiplist_, huge_page_tlb_size_, - logger, bucket_entries_logging_threshold_, - if_log_bucket_dist_when_flash_); + return new HashLinkListRep( + compare, allocator, transform, options_.bucket_count, + options_.threshold_use_skiplist, options_.huge_page_tlb_size, logger, + options_.bucket_entries_logging_threshold, + options_.if_log_bucket_dist_when_flash); } MemTableRepFactory* NewHashLinkListRepFactory( diff --git a/memtable/hash_linklist_rep.h b/memtable/hash_linklist_rep.h deleted file mode 100644 index 7e7195526b..0000000000 --- a/memtable/hash_linklist_rep.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#pragma once -#ifndef ROCKSDB_LITE -#include "rocksdb/slice_transform.h" -#include "rocksdb/memtablerep.h" - -namespace ROCKSDB_NAMESPACE { - -class HashLinkListRepFactory : public MemTableRepFactory { - public: - explicit HashLinkListRepFactory(size_t bucket_count, - uint32_t threshold_use_skiplist, - size_t huge_page_tlb_size, - int bucket_entries_logging_threshold, - bool if_log_bucket_dist_when_flash) - : bucket_count_(bucket_count), - threshold_use_skiplist_(threshold_use_skiplist), - huge_page_tlb_size_(huge_page_tlb_size), - bucket_entries_logging_threshold_(bucket_entries_logging_threshold), - if_log_bucket_dist_when_flash_(if_log_bucket_dist_when_flash) {} - - virtual ~HashLinkListRepFactory() {} - - using MemTableRepFactory::CreateMemTableRep; - virtual MemTableRep* CreateMemTableRep( - const MemTableRep::KeyComparator& compare, Allocator* allocator, - const SliceTransform* transform, Logger* logger) override; - - virtual const char* Name() const override { - return "HashLinkListRepFactory"; - } - - private: - const size_t bucket_count_; - const uint32_t threshold_use_skiplist_; - const size_t huge_page_tlb_size_; - int bucket_entries_logging_threshold_; - bool if_log_bucket_dist_when_flash_; -}; - -} // namespace ROCKSDB_NAMESPACE -#endif // ROCKSDB_LITE diff --git a/memtable/hash_skiplist_rep.cc b/memtable/hash_skiplist_rep.cc index 72fc391e17..dc58046a45 100644 --- a/memtable/hash_skiplist_rep.cc +++ b/memtable/hash_skiplist_rep.cc @@ -5,8 +5,6 @@ // #ifndef ROCKSDB_LITE -#include "memtable/hash_skiplist_rep.h" - #include #include "db/memtable.h" @@ -16,6 +14,7 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" +#include "rocksdb/utilities/options_type.h" #include "util/murmurhash.h" namespace ROCKSDB_NAMESPACE { @@ -329,13 +328,60 @@ MemTableRep::Iterator* HashSkipListRep::GetDynamicPrefixIterator(Arena* arena) { } } -} // anon namespace +struct HashSkipListRepOptions { + static const char* kName() { return "HashSkipListRepFactoryOptions"; } + size_t bucket_count; + int32_t skiplist_height; + int32_t skiplist_branching_factor; +}; + +static std::unordered_map hash_skiplist_info = { + {"bucket_count", + {offsetof(struct HashSkipListRepOptions, bucket_count), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"skiplist_height", + {offsetof(struct HashSkipListRepOptions, skiplist_height), + OptionType::kInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"branching_factor", + {offsetof(struct HashSkipListRepOptions, skiplist_branching_factor), + OptionType::kInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +class HashSkipListRepFactory : public MemTableRepFactory { + public: + explicit HashSkipListRepFactory(size_t bucket_count, int32_t skiplist_height, + int32_t skiplist_branching_factor) { + options_.bucket_count = bucket_count; + options_.skiplist_height = skiplist_height; + options_.skiplist_branching_factor = skiplist_branching_factor; + RegisterOptions(&options_, &hash_skiplist_info); + } + + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* logger) override; + + static const char* kClassName() { return "HashSkipListRepFactory"; } + static const char* kNickName() { return "prefix_hash"; } + + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + + private: + HashSkipListRepOptions options_; +}; + +} // namespace MemTableRep* HashSkipListRepFactory::CreateMemTableRep( const MemTableRep::KeyComparator& compare, Allocator* allocator, const SliceTransform* transform, Logger* /*logger*/) { - return new HashSkipListRep(compare, allocator, transform, bucket_count_, - skiplist_height_, skiplist_branching_factor_); + return new HashSkipListRep(compare, allocator, transform, + options_.bucket_count, options_.skiplist_height, + options_.skiplist_branching_factor); } MemTableRepFactory* NewHashSkipListRepFactory( diff --git a/memtable/hash_skiplist_rep.h b/memtable/hash_skiplist_rep.h deleted file mode 100644 index 6da5a4e943..0000000000 --- a/memtable/hash_skiplist_rep.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#pragma once -#ifndef ROCKSDB_LITE -#include "rocksdb/slice_transform.h" -#include "rocksdb/memtablerep.h" - -namespace ROCKSDB_NAMESPACE { - -class HashSkipListRepFactory : public MemTableRepFactory { - public: - explicit HashSkipListRepFactory( - size_t bucket_count, - int32_t skiplist_height, - int32_t skiplist_branching_factor) - : bucket_count_(bucket_count), - skiplist_height_(skiplist_height), - skiplist_branching_factor_(skiplist_branching_factor) { } - - virtual ~HashSkipListRepFactory() {} - - using MemTableRepFactory::CreateMemTableRep; - virtual MemTableRep* CreateMemTableRep( - const MemTableRep::KeyComparator& compare, Allocator* allocator, - const SliceTransform* transform, Logger* logger) override; - - virtual const char* Name() const override { - return "HashSkipListRepFactory"; - } - - private: - const size_t bucket_count_; - const int32_t skiplist_height_; - const int32_t skiplist_branching_factor_; -}; - -} // namespace ROCKSDB_NAMESPACE -#endif // ROCKSDB_LITE diff --git a/memtable/memtablerep_bench.cc b/memtable/memtablerep_bench.cc index d476d03fb1..1eaa7658f0 100644 --- a/memtable/memtablerep_bench.cc +++ b/memtable/memtablerep_bench.cc @@ -28,6 +28,7 @@ int main() { #include "port/port.h" #include "port/stack_trace.h" #include "rocksdb/comparator.h" +#include "rocksdb/convenience.h" #include "rocksdb/memtablerep.h" #include "rocksdb/options.h" #include "rocksdb/slice_transform.h" @@ -581,13 +582,15 @@ int main(int argc, char** argv) { #ifndef ROCKSDB_LITE } else if (FLAGS_memtablerep == "vector") { factory.reset(new ROCKSDB_NAMESPACE::VectorRepFactory); - } else if (FLAGS_memtablerep == "hashskiplist") { + } else if (FLAGS_memtablerep == "hashskiplist" || + FLAGS_memtablerep == "prefix_hash") { factory.reset(ROCKSDB_NAMESPACE::NewHashSkipListRepFactory( FLAGS_bucket_count, FLAGS_hashskiplist_height, FLAGS_hashskiplist_branching_factor)); options.prefix_extractor.reset( ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length)); - } else if (FLAGS_memtablerep == "hashlinklist") { + } else if (FLAGS_memtablerep == "hashlinklist" || + FLAGS_memtablerep == "hash_linkedlist") { factory.reset(ROCKSDB_NAMESPACE::NewHashLinkListRepFactory( FLAGS_bucket_count, FLAGS_huge_page_tlb_size, FLAGS_bucket_entries_logging_threshold, @@ -596,8 +599,16 @@ int main(int argc, char** argv) { ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length)); #endif // ROCKSDB_LITE } else { - fprintf(stdout, "Unknown memtablerep: %s\n", FLAGS_memtablerep.c_str()); - exit(1); + ROCKSDB_NAMESPACE::ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::MemTableRepFactory::CreateFromString( + config_options, FLAGS_memtablerep, &factory); + if (!s.ok()) { + fprintf(stdout, "Unknown memtablerep: %s\n", s.ToString().c_str()); + exit(1); + } } ROCKSDB_NAMESPACE::InternalKeyComparator internal_key_comp( diff --git a/memtable/skiplistrep.cc b/memtable/skiplistrep.cc index d7f78672fc..016515c44a 100644 --- a/memtable/skiplistrep.cc +++ b/memtable/skiplistrep.cc @@ -9,6 +9,8 @@ #include "memory/arena.h" #include "memtable/inlineskiplist.h" #include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" namespace ROCKSDB_NAMESPACE { namespace { @@ -335,6 +337,27 @@ public: }; } +static std::unordered_map skiplist_factory_info = { +#ifndef ROCKSDB_LITE + {"lookahead", + {0, OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kDontSerialize /*Since it is part of the ID*/}}, +#endif +}; + +SkipListFactory::SkipListFactory(size_t lookahead) : lookahead_(lookahead) { + RegisterOptions("SkipListFactoryOptions", &lookahead_, + &skiplist_factory_info); +} + +std::string SkipListFactory::GetId() const { + std::string id = Name(); + if (lookahead_ > 0) { + id.append(":").append(ROCKSDB_NAMESPACE::ToString(lookahead_)); + } + return id; +} + MemTableRep* SkipListFactory::CreateMemTableRep( const MemTableRep::KeyComparator& compare, Allocator* allocator, const SliceTransform* transform, Logger* /*logger*/) { diff --git a/memtable/vectorrep.cc b/memtable/vectorrep.cc index 7dde80b229..2386e46ab5 100644 --- a/memtable/vectorrep.cc +++ b/memtable/vectorrep.cc @@ -4,18 +4,18 @@ // (found in the LICENSE.Apache file in the root directory). // #ifndef ROCKSDB_LITE -#include "rocksdb/memtablerep.h" - -#include -#include -#include #include +#include +#include #include +#include #include "db/memtable.h" #include "memory/arena.h" #include "memtable/stl_wrappers.h" #include "port/port.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/options_type.h" #include "util/mutexlock.h" namespace ROCKSDB_NAMESPACE { @@ -292,6 +292,16 @@ MemTableRep::Iterator* VectorRep::GetIterator(Arena* arena) { } } // anon namespace +static std::unordered_map vector_rep_table_info = { + {"count", + {0, OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +VectorRepFactory::VectorRepFactory(size_t count) : count_(count) { + RegisterOptions("VectorRepFactoryOptions", &count_, &vector_rep_table_info); +} + MemTableRep* VectorRepFactory::CreateMemTableRep( const MemTableRep::KeyComparator& compare, Allocator* allocator, const SliceTransform*, Logger* /*logger*/) { diff --git a/options/cf_options.cc b/options/cf_options.cc index 996aefab59..823f432743 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -573,21 +573,33 @@ static std::unordered_map OptionTypeFlags::kNone}}, {"memtable_factory", {offset_of(&ImmutableCFOptions::memtable_factory), - OptionType::kMemTableRepFactory, OptionVerificationType::kByName, - OptionTypeFlags::kNone}}, + OptionType::kCustomizable, OptionVerificationType::kByName, + OptionTypeFlags::kShared, + [](const ConfigOptions& opts, const std::string&, + const std::string& value, void* addr) { + std::unique_ptr factory; + auto* shared = + static_cast*>(addr); + Status s = + MemTableRepFactory::CreateFromString(opts, value, &factory); + if (s.ok()) { + shared->reset(factory.release()); + } + return s; + }}}, {"memtable", {offset_of(&ImmutableCFOptions::memtable_factory), - OptionType::kMemTableRepFactory, OptionVerificationType::kAlias, - OptionTypeFlags::kNone, - // Parses the value string and updates the memtable_factory - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + OptionType::kCustomizable, OptionVerificationType::kAlias, + OptionTypeFlags::kShared, + [](const ConfigOptions& opts, const std::string&, const std::string& value, void* addr) { - std::unique_ptr new_mem_factory; - Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory); + std::unique_ptr factory; + auto* shared = + static_cast*>(addr); + Status s = + MemTableRepFactory::CreateFromString(opts, value, &factory); if (s.ok()) { - auto memtable_factory = - static_cast*>(addr); - memtable_factory->reset(new_mem_factory.release()); + shared->reset(factory.release()); } return s; }}}, diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 193bfe5f67..20cd3c0c5b 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -1302,6 +1302,23 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) { } } +TEST_F(LoadCustomizableTest, LoadMemTableRepFactoryTest) { + std::unique_ptr result; + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options_, "SpecialSkipListFactory", &result)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options_, SkipListFactory::kClassName(), &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf(SkipListFactory::kClassName())); + + if (RegisterTests("Test")) { + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options_, "SpecialSkipListFactory", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "SpecialSkipListFactory"); + } +} + TEST_F(LoadCustomizableTest, LoadMergeOperatorTest) { std::shared_ptr result; diff --git a/options/options_helper.cc b/options/options_helper.cc index aee19c9b22..215c2f1ba2 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -562,12 +562,6 @@ bool SerializeSingleOptionHelper(const void* opt_address, : kNullptrString; break; } - case OptionType::kMemTableRepFactory: { - const auto* ptr = - static_cast*>(opt_address); - *value = ptr->get() ? ptr->get()->Name() : kNullptrString; - break; - } case OptionType::kFilterPolicy: { const auto* ptr = static_cast*>(opt_address); diff --git a/options/options_test.cc b/options/options_test.cc index 2a478a0e06..bc9d8aed9f 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -575,6 +575,7 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { &new_cf_opt)); ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); + ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory")); } TEST_F(OptionsTest, CompressionOptionsFromString) { @@ -1137,14 +1138,14 @@ TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); - ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory"); + ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory"); ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", &new_mem_factory)); ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", &new_mem_factory)); - ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory"); + ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory"); ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", &new_mem_factory)); @@ -1170,6 +1171,99 @@ TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { } #endif // !ROCKSDB_LITE +TEST_F(OptionsTest, MemTableRepFactoryCreateFromString) { + std::unique_ptr new_mem_factory = nullptr; + ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + config_options.ignore_unknown_options = false; + + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list", + &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list:16", + &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("skip_list")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("SkipListFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "skip_list:16:invalid_opt", &new_mem_factory)); + + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "invalid_opt=10", &new_mem_factory)); + + // Test a reset + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "", + &new_mem_factory)); + ASSERT_EQ(new_mem_factory, nullptr); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "invalid_opt=10", &new_mem_factory)); + +#ifndef ROCKSDB_LITE + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "id=skip_list; lookahead=32", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "prefix_hash", + &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "prefix_hash:1000", &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("prefix_hash")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashSkipListRepFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "prefix_hash:1000:invalid_opt", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, + "id=prefix_hash; bucket_count=32; skiplist_height=64; " + "branching_factor=16", + &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, + "id=prefix_hash; bucket_count=32; skiplist_height=64; " + "branching_factor=16; invalid=unknown", + &new_mem_factory)); + + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "hash_linkedlist", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "hash_linkedlist:1000", &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "HashLinkListRepFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("hash_linkedlist")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashLinkListRepFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "hash_linkedlist:1000:invalid_opt", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, + "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; " + "logging_threshold=12; log_when_flash=true", + &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, + "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; " + "logging_threshold=12; log_when_flash=true; invalid=unknown", + &new_mem_factory)); + + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector", + &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector:1024", + &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "VectorRepFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("vector")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("VectorRepFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "vector:1024:invalid_opt", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "id=vector; count=42", &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "id=vector; invalid=unknown", &new_mem_factory)); +#endif // ROCKSDB_LITE + ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo", + &new_mem_factory)); + // CuckooHash memtable is already removed. + ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo:1024", + &new_mem_factory)); + + ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "bad_factory", + &new_mem_factory)); +} + #ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite TEST_F(OptionsTest, GetOptionsFromStringTest) { Options base_options, new_options; @@ -2454,7 +2548,7 @@ TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) { "memtable=skip_list:10;arena_block_size=1024", &new_cf_opt)); ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); - ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); + ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory")); } TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) { @@ -2650,6 +2744,14 @@ TEST_F(OptionsOldApiTest, GetPlainTableOptionsFromString) { ASSERT_TRUE(new_opt.full_scan_mode); ASSERT_TRUE(new_opt.store_index_in_file); + std::unordered_map opt_map; + ASSERT_OK(StringToMap( + "user_key_len=55;bloom_bits_per_key=10;huge_page_tlb_size=8;", &opt_map)); + ASSERT_OK(GetPlainTableOptionsFromMap(table_opt, opt_map, &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 55u); + ASSERT_EQ(new_opt.bloom_bits_per_key, 10); + ASSERT_EQ(new_opt.huge_page_tlb_size, 8); + // unknown option ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index 90e0629518..22844bee6a 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -3,7 +3,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#ifndef ROCKSDB_LITE #include "table/plain/plain_table_factory.h" #include @@ -11,15 +10,17 @@ #include #include "db/dbformat.h" -#include "options/configurable_helper.h" #include "port/port.h" #include "rocksdb/convenience.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/options_type.h" #include "table/plain/plain_table_builder.h" #include "table/plain/plain_table_reader.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { +#ifndef ROCKSDB_LITE static std::unordered_map plain_table_type_info = { {"user_key_len", {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, @@ -151,73 +152,146 @@ Status GetPlainTableOptionsFromString(const ConfigOptions& config_options, return Status::InvalidArgument(s.getState()); } } +#endif // ROCKSDB_LITE +#ifndef ROCKSDB_LITE +static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library, + const std::string& /*arg*/) { + // The MemTableRepFactory built-in classes will be either a class + // (VectorRepFactory) or a nickname (vector), followed optionally by ":#", + // where # is the "size" of the factory. + auto AsRegex = [](const std::string& name, const std::string& alt) { + std::string regex; + regex.append("(").append(name); + regex.append("|").append(alt).append(")(:[0-9]*)?"); + return regex; + }; + + library.Register( + AsRegex(VectorRepFactory::kClassName(), VectorRepFactory::kNickName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t count = ParseSizeT(uri.substr(colon + 1)); + guard->reset(new VectorRepFactory(count)); + } else { + guard->reset(new VectorRepFactory()); + } + return guard->get(); + }); + library.Register( + AsRegex(SkipListFactory::kClassName(), SkipListFactory::kNickName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t lookahead = ParseSizeT(uri.substr(colon + 1)); + guard->reset(new SkipListFactory(lookahead)); + } else { + guard->reset(new SkipListFactory()); + } + return guard->get(); + }); + library.Register( + AsRegex("HashLinkListRepFactory", "hash_linkedlist"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + // Expecting format: hash_linkedlist: + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1)); + guard->reset(NewHashLinkListRepFactory(hash_bucket_count)); + } else { + guard->reset(NewHashLinkListRepFactory()); + } + return guard->get(); + }); + library.Register( + AsRegex("HashSkipListRepFactory", "prefix_hash"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + // Expecting format: prefix_hash: + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1)); + guard->reset(NewHashSkipListRepFactory(hash_bucket_count)); + } else { + guard->reset(NewHashSkipListRepFactory()); + } + return guard->get(); + }); + library.Register( + "cuckoo", + [](const std::string& /*uri*/, + std::unique_ptr* /*guard*/, std::string* errmsg) { + *errmsg = "cuckoo hash memtable is not supported anymore."; + return nullptr; + }); + + return 5; +} +#endif // ROCKSDB_LITE Status GetMemTableRepFactoryFromString( - const std::string& opts_str, - std::unique_ptr* new_mem_factory) { - std::vector opts_list = StringSplit(opts_str, ':'); - size_t len = opts_list.size(); - - if (opts_list.empty() || opts_list.size() > 2) { - return Status::InvalidArgument("Can't parse memtable_factory option ", - opts_str); - } - - MemTableRepFactory* mem_factory = nullptr; - - if (opts_list[0] == "skip_list" || opts_list[0] == "SkipListFactory") { - // Expecting format - // skip_list: - if (2 == len) { - size_t lookahead = ParseSizeT(opts_list[1]); - mem_factory = new SkipListFactory(lookahead); - } else if (1 == len) { - mem_factory = new SkipListFactory(); - } - } else if (opts_list[0] == "prefix_hash" || - opts_list[0] == "HashSkipListRepFactory") { - // Expecting format - // prfix_hash: - if (2 == len) { - size_t hash_bucket_count = ParseSizeT(opts_list[1]); - mem_factory = NewHashSkipListRepFactory(hash_bucket_count); - } else if (1 == len) { - mem_factory = NewHashSkipListRepFactory(); - } - } else if (opts_list[0] == "hash_linkedlist" || - opts_list[0] == "HashLinkListRepFactory") { - // Expecting format - // hash_linkedlist: - if (2 == len) { - size_t hash_bucket_count = ParseSizeT(opts_list[1]); - mem_factory = NewHashLinkListRepFactory(hash_bucket_count); - } else if (1 == len) { - mem_factory = NewHashLinkListRepFactory(); - } - } else if (opts_list[0] == "vector" || opts_list[0] == "VectorRepFactory") { - // Expecting format - // vector: - if (2 == len) { - size_t count = ParseSizeT(opts_list[1]); - mem_factory = new VectorRepFactory(count); - } else if (1 == len) { - mem_factory = new VectorRepFactory(); - } - } else if (opts_list[0] == "cuckoo") { - return Status::NotSupported( - "cuckoo hash memtable is not supported anymore."); - } else { - return Status::InvalidArgument("Unrecognized memtable_factory option ", - opts_str); - } - - if (mem_factory != nullptr) { - new_mem_factory->reset(mem_factory); - } - - return Status::OK(); + const std::string& opts_str, std::unique_ptr* result) { + ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + config_options.ignore_unknown_options = false; + return MemTableRepFactory::CreateFromString(config_options, opts_str, result); } +Status MemTableRepFactory::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::unique_ptr* result) { +#ifndef ROCKSDB_LITE + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinMemTableRepFactory(*(ObjectLibrary::Default().get()), ""); + }); +#endif // ROCKSDB_LITE + std::string id; + std::unordered_map opt_map; + Status status = Customizable::GetOptionsMap(config_options, result->get(), + value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else if (value.empty()) { + // No Id and no options. Clear the object + result->reset(); + return Status::OK(); + } else if (id.empty()) { // We have no Id but have options. Not good + return Status::NotSupported("Cannot reset object ", id); + } else { +#ifndef ROCKSDB_LITE + status = NewUniqueObject(config_options, id, opt_map, + result); +#else + // To make it possible to configure the memtables in LITE mode, the ID + // is of the form :, where name is the name of the class and + // is the length of the object (e.g. skip_list:10). + std::vector opts_list = StringSplit(id, ':'); + if (opts_list.empty() || opts_list.size() > 2 || !opt_map.empty()) { + status = Status::InvalidArgument("Can't parse memtable_factory option ", + value); + } else if (opts_list[0] == "skip_list" || + opts_list[0] == SkipListFactory::kClassName()) { + // Expecting format + // skip_list: + if (opts_list.size() == 2) { + size_t lookahead = ParseSizeT(opts_list[1]); + result->reset(new SkipListFactory(lookahead)); + } else { + result->reset(new SkipListFactory()); + } + } else if (!config_options.ignore_unsupported_options) { + status = Status::NotSupported("Cannot load object in LITE mode ", id); + } +#endif // ROCKSDB_LITE + } + return status; +} + +#ifndef ROCKSDB_LITE Status GetPlainTableOptionsFromMap( const PlainTableOptions& table_options, const std::unordered_map& opts_map, @@ -259,5 +333,5 @@ const std::string PlainTablePropertyNames::kBloomVersion = const std::string PlainTablePropertyNames::kNumBloomBlocks = "rocksdb.plain.table.bloom.numblocks"; -} // namespace ROCKSDB_NAMESPACE #endif // ROCKSDB_LITE +} // namespace ROCKSDB_NAMESPACE diff --git a/test_util/testutil.cc b/test_util/testutil.cc index eb403fb73e..add1920e04 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -601,6 +601,106 @@ Status CreateEnvFromSystem(const ConfigOptions& config_options, Env** result, return Status::OK(); } } +namespace { +// A hacky skip list mem table that triggers flush after number of entries. +class SpecialMemTableRep : public MemTableRep { + public: + explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable, + int num_entries_flush) + : MemTableRep(allocator), + memtable_(memtable), + num_entries_flush_(num_entries_flush), + num_entries_(0) {} + + virtual KeyHandle Allocate(const size_t len, char** buf) override { + return memtable_->Allocate(len, buf); + } + + // Insert key into the list. + // REQUIRES: nothing that compares equal to key is currently in the list. + virtual void Insert(KeyHandle handle) override { + num_entries_++; + memtable_->Insert(handle); + } + + void InsertConcurrently(KeyHandle handle) override { + num_entries_++; + memtable_->Insert(handle); + } + + // Returns true iff an entry that compares equal to key is in the list. + virtual bool Contains(const char* key) const override { + return memtable_->Contains(key); + } + + virtual size_t ApproximateMemoryUsage() override { + // Return a high memory usage when number of entries exceeds the threshold + // to trigger a flush. + return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024; + } + + virtual void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, + const char* entry)) override { + memtable_->Get(k, callback_args, callback_func); + } + + uint64_t ApproximateNumEntries(const Slice& start_ikey, + const Slice& end_ikey) override { + return memtable_->ApproximateNumEntries(start_ikey, end_ikey); + } + + virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override { + return memtable_->GetIterator(arena); + } + + virtual ~SpecialMemTableRep() override {} + + private: + std::unique_ptr memtable_; + int num_entries_flush_; + int num_entries_; +}; +class SpecialSkipListFactory : public MemTableRepFactory { + public: + // After number of inserts exceeds `num_entries_flush` in a mem table, trigger + // flush. + explicit SpecialSkipListFactory(int num_entries_flush) + : num_entries_flush_(num_entries_flush) {} + + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* /*logger*/) override { + return new SpecialMemTableRep( + allocator, + factory_.CreateMemTableRep(compare, allocator, transform, nullptr), + num_entries_flush_); + } + static const char* kClassName() { return "SpecialSkipListFactory"; } + virtual const char* Name() const override { return kClassName(); } + std::string GetId() const override { + std::string id = Name(); + if (num_entries_flush_ > 0) { + id.append(":").append(ROCKSDB_NAMESPACE::ToString(num_entries_flush_)); + } + return id; + } + + bool IsInsertConcurrentlySupported() const override { + return factory_.IsInsertConcurrentlySupported(); + } + + private: + SkipListFactory factory_; + int num_entries_flush_; +}; +} // namespace + +MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush) { + RegisterTestLibrary(); + return new SpecialSkipListFactory(num_entries_per_flush); +} #ifndef ROCKSDB_LITE // This method loads existing test classes into the ObjectRegistry @@ -614,6 +714,19 @@ int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/) { static test::SimpleSuffixReverseComparator ssrc; return &ssrc; }); + library.Register( + std::string(SpecialSkipListFactory::kClassName()) + "(:[0-9]*)?", + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + auto colon = uri.find(":"); + if (colon != std::string::npos) { + auto count = ParseInt(uri.substr(colon + 1)); + guard->reset(new SpecialSkipListFactory(count)); + } else { + guard->reset(new SpecialSkipListFactory(2)); + } + return guard->get(); + }); library.Register( "Changling", [](const std::string& uri, std::unique_ptr* guard, @@ -637,6 +750,19 @@ int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/) { return static_cast(library.GetFactoryCount(&num_types)); } + #endif // ROCKSDB_LITE + +void RegisterTestLibrary(const std::string& arg) { + static bool registered = false; + if (!registered) { + registered = true; +#ifndef ROCKSDB_LITE + ObjectRegistry::Default()->AddLibrary("test", RegisterTestObjects, arg); +#else + (void)arg; +#endif // ROCKSDB_LITE + } +} } // namespace test } // namespace ROCKSDB_NAMESPACE diff --git a/test_util/testutil.h b/test_util/testutil.h index ef5d161392..0c358b2542 100644 --- a/test_util/testutil.h +++ b/test_util/testutil.h @@ -27,6 +27,7 @@ namespace ROCKSDB_NAMESPACE { class FileSystem; +class MemTableRepFactory; class ObjectLibrary; class Random; class SequentialFile; @@ -853,6 +854,10 @@ class ChanglingCompactionFilterFactory : public CompactionFilterFactory { std::string name_; }; +// The factory for the hacky skip list mem table that triggers flush after +// number of entries exceeds a threshold. +extern MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush); + extern const Comparator* ComparatorWithU64Ts(); CompressionType RandomCompressionType(Random* rnd); @@ -901,5 +906,8 @@ Status CreateEnvFromSystem(const ConfigOptions& options, Env** result, // Registers the testutil classes with the ObjectLibrary int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/); #endif // ROCKSDB_LITE + +// Register the testutil classes with the default ObjectRegistry/Library +void RegisterTestLibrary(const std::string& arg = ""); } // namespace test } // namespace ROCKSDB_NAMESPACE diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index bf6e0c1c06..5ff8c4e250 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -1447,30 +1447,6 @@ DEFINE_int64(multiread_stride, 0, "Stride length for the keys in a MultiGet batch"); DEFINE_bool(multiread_batched, false, "Use the new MultiGet API"); -enum RepFactory { - kSkipList, - kPrefixHash, - kVectorRep, - kHashLinkedList, -}; - -static enum RepFactory StringToRepFactory(const char* ctype) { - assert(ctype); - - if (!strcasecmp(ctype, "skip_list")) - return kSkipList; - else if (!strcasecmp(ctype, "prefix_hash")) - return kPrefixHash; - else if (!strcasecmp(ctype, "vector")) - return kVectorRep; - else if (!strcasecmp(ctype, "hash_linkedlist")) - return kHashLinkedList; - - fprintf(stdout, "Cannot parse memreptable %s\n", ctype); - return kSkipList; -} - -static enum RepFactory FLAGS_rep_factory; DEFINE_string(memtablerep, "skip_list", ""); DEFINE_int64(hash_bucket_count, 1024 * 1024, "hash bucket count"); DEFINE_bool(use_plain_table, false, "if use plain table " @@ -1534,8 +1510,33 @@ static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((__unused__ &ValidateTableCacheNumshardbits); namespace ROCKSDB_NAMESPACE { - namespace { +static Status CreateMemTableRepFactory( + const ConfigOptions& config_options, + std::shared_ptr* factory) { + Status s; + if (!strcasecmp(FLAGS_memtablerep.c_str(), SkipListFactory::kNickName())) { + factory->reset(new SkipListFactory(FLAGS_skip_list_lookahead)); +#ifndef ROCKSDB_LITE + } else if (!strcasecmp(FLAGS_memtablerep.c_str(), "prefix_hash")) { + factory->reset(NewHashSkipListRepFactory(FLAGS_hash_bucket_count)); + } else if (!strcasecmp(FLAGS_memtablerep.c_str(), + VectorRepFactory::kNickName())) { + factory->reset(new VectorRepFactory()); + } else if (!strcasecmp(FLAGS_memtablerep.c_str(), "hash_linkedlist")) { + factory->reset(NewHashLinkListRepFactory(FLAGS_hash_bucket_count)); +#endif // ROCKSDB_LITE + } else { + std::unique_ptr unique; + s = MemTableRepFactory::CreateFromString(config_options, FLAGS_memtablerep, + &unique); + if (s.ok()) { + factory->reset(unique.release()); + } + } + return s; +} + struct ReportFileOpCounters { std::atomic open_counter_; std::atomic delete_counter_; @@ -2681,7 +2682,7 @@ class Benchmark { compressed); } - void PrintHeader() { + void PrintHeader(const Options& options) { PrintEnvironment(); fprintf(stdout, "Keys: %d bytes each (+ %d bytes user-defined timestamp)\n", @@ -2731,20 +2732,9 @@ class Benchmark { fprintf(stdout, "Compression: %s\n", compression.c_str()); fprintf(stdout, "Compression sampling rate: %" PRId64 "\n", FLAGS_sample_for_compression); - - switch (FLAGS_rep_factory) { - case kPrefixHash: - fprintf(stdout, "Memtablerep: prefix_hash\n"); - break; - case kSkipList: - fprintf(stdout, "Memtablerep: skip_list\n"); - break; - case kVectorRep: - fprintf(stdout, "Memtablerep: vector\n"); - break; - case kHashLinkedList: - fprintf(stdout, "Memtablerep: hash_linkedlist\n"); - break; + if (options.memtable_factory != nullptr) { + fprintf(stdout, "Memtablerep: %s\n", + options.memtable_factory->GetId().c_str()); } fprintf(stdout, "Perf Level: %d\n", FLAGS_perf_level); @@ -3209,7 +3199,7 @@ class Benchmark { ErrorExit(); } Open(&open_options_); - PrintHeader(); + PrintHeader(open_options_); std::stringstream benchmark_stream(FLAGS_benchmarks); std::string name; std::unique_ptr filter; @@ -3932,6 +3922,7 @@ class Benchmark { printf("Initializing RocksDB Options from command-line flags\n"); Options& options = *opts; ConfigOptions config_options(options); + config_options.ignore_unsupported_options = false; assert(db_.db == nullptr); @@ -4007,42 +3998,25 @@ class Benchmark { FLAGS_level_compaction_dynamic_level_bytes; options.max_bytes_for_level_multiplier = FLAGS_max_bytes_for_level_multiplier; - if ((FLAGS_prefix_size == 0) && (FLAGS_rep_factory == kPrefixHash || - FLAGS_rep_factory == kHashLinkedList)) { + Status s = + CreateMemTableRepFactory(config_options, &options.memtable_factory); + if (!s.ok()) { + fprintf(stderr, "Could not create memtable factory: %s\n", + s.ToString().c_str()); + exit(1); + } else if ((FLAGS_prefix_size == 0) && + (options.memtable_factory->IsInstanceOf("prefix_hash") || + options.memtable_factory->IsInstanceOf("hash_linkedlist"))) { fprintf(stderr, "prefix_size should be non-zero if PrefixHash or " "HashLinkedList memtablerep is used\n"); exit(1); } - switch (FLAGS_rep_factory) { - case kSkipList: - options.memtable_factory.reset(new SkipListFactory( - FLAGS_skip_list_lookahead)); - break; -#ifndef ROCKSDB_LITE - case kPrefixHash: - options.memtable_factory.reset( - NewHashSkipListRepFactory(FLAGS_hash_bucket_count)); - break; - case kHashLinkedList: - options.memtable_factory.reset(NewHashLinkListRepFactory( - FLAGS_hash_bucket_count)); - break; - case kVectorRep: - options.memtable_factory.reset( - new VectorRepFactory - ); - break; -#else - default: - fprintf(stderr, "Only skip list is supported in lite mode\n"); - exit(1); -#endif // ROCKSDB_LITE - } if (FLAGS_use_plain_table) { #ifndef ROCKSDB_LITE - if (FLAGS_rep_factory != kPrefixHash && - FLAGS_rep_factory != kHashLinkedList) { - fprintf(stderr, "Waring: plain table is used with skipList\n"); + if (!options.memtable_factory->IsInstanceOf("prefix_hash") && + !options.memtable_factory->IsInstanceOf("hash_linkedlist")) { + fprintf(stderr, "Warning: plain table is used with %s\n", + options.memtable_factory->Name()); } int bloom_bits_per_key = FLAGS_bloom_bits; @@ -4298,8 +4272,8 @@ class Benchmark { // merge operator options if (!FLAGS_merge_operator.empty()) { - Status s = MergeOperator::CreateFromString( - config_options, FLAGS_merge_operator, &options.merge_operator); + s = MergeOperator::CreateFromString(config_options, FLAGS_merge_operator, + &options.merge_operator); if (!s.ok()) { fprintf(stderr, "invalid merge operator[%s]: %s\n", FLAGS_merge_operator.c_str(), s.ToString().c_str()); @@ -8148,8 +8122,6 @@ int db_bench_tool(int argc, char** argv) { FLAGS_value_size_distribution_type_e = StringToDistributionType(FLAGS_value_size_distribution_type.c_str()); - FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str()); - // Note options sanitization may increase thread pool sizes according to // max_background_flushes/max_background_compactions/max_background_jobs FLAGS_env->SetBackgroundThreads(FLAGS_num_high_pri_threads,