2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2013-10-16 21:59:46 +00:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
//
|
2011-03-18 22:37:00 +00:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/options.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-05 06:14:37 +00:00
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-04-30 18:33:40 +00:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 06:14:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
2014-04-30 18:33:40 +00:00
|
|
|
#include <inttypes.h>
|
2013-01-11 01:18:50 +00:00
|
|
|
#include <limits>
|
|
|
|
|
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
|
#include "rocksdb/env.h"
|
2016-01-29 02:35:01 +00:00
|
|
|
#include "rocksdb/sst_file_manager.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/memtablerep.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/table.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/table_properties.h"
|
2015-10-13 00:03:03 +00:00
|
|
|
#include "rocksdb/wal_filter.h"
|
2013-10-29 00:54:09 +00:00
|
|
|
#include "table/block_based_table_factory.h"
|
2015-04-06 19:50:44 +00:00
|
|
|
#include "util/compression.h"
|
2014-07-28 19:05:36 +00:00
|
|
|
#include "util/statistics.h"
|
2015-02-18 19:49:31 +00:00
|
|
|
#include "util/xfunc.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
namespace rocksdb {
|
2011-03-18 22:37:00 +00:00
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions()
|
2011-03-18 22:37:00 +00:00
|
|
|
: comparator(BytewiseComparator()),
|
2013-03-21 22:59:47 +00:00
|
|
|
merge_operator(nullptr),
|
2013-05-12 09:36:59 +00:00
|
|
|
compaction_filter(nullptr),
|
2015-06-08 23:34:26 +00:00
|
|
|
compaction_filter_factory(nullptr),
|
2016-03-24 19:45:50 +00:00
|
|
|
write_buffer_size(64 << 20),
|
2012-10-19 21:00:53 +00:00
|
|
|
max_write_buffer_number(2),
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge(1),
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain(0),
|
2015-06-18 21:55:05 +00:00
|
|
|
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
|
2016-05-09 22:57:19 +00:00
|
|
|
bottommost_compression(kDisableCompressionOption),
|
2013-08-13 21:04:56 +00:00
|
|
|
prefix_extractor(nullptr),
|
2012-06-23 02:30:03 +00:00
|
|
|
num_levels(7),
|
|
|
|
|
level0_file_num_compaction_trigger(4),
|
2014-03-28 23:57:04 +00:00
|
|
|
level0_slowdown_writes_trigger(20),
|
|
|
|
|
level0_stop_writes_trigger(24),
|
2016-03-24 19:45:50 +00:00
|
|
|
target_file_size_base(64 * 1048576),
|
2012-08-22 00:33:46 +00:00
|
|
|
target_file_size_multiplier(1),
|
2016-03-24 19:45:50 +00:00
|
|
|
max_bytes_for_level_base(256 * 1048576),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes(false),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_bytes_for_level_multiplier(10),
|
2013-05-21 18:37:06 +00:00
|
|
|
max_bytes_for_level_multiplier_additional(num_levels, 1),
|
2016-06-16 23:02:52 +00:00
|
|
|
max_compaction_bytes(0),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
soft_rate_limit(0.0),
|
2015-09-15 00:41:40 +00:00
|
|
|
hard_rate_limit(0.0),
|
2016-04-08 20:09:19 +00:00
|
|
|
soft_pending_compaction_bytes_limit(64 * 1073741824ull),
|
|
|
|
|
hard_pending_compaction_bytes_limit(256 * 1073741824ull),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
rate_limit_delay_max_milliseconds(1000),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(0),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
disable_auto_compactions(false),
|
|
|
|
|
purge_redundant_kvs_while_flush(true),
|
|
|
|
|
compaction_style(kCompactionStyleLevel),
|
2015-12-05 01:03:55 +00:00
|
|
|
compaction_pri(kByCompensatedSize),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
max_sequential_skip_in_iterations(8),
|
|
|
|
|
memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)),
|
|
|
|
|
table_factory(
|
2014-03-25 18:09:40 +00:00
|
|
|
std::shared_ptr<TableFactory>(new BlockBasedTableFactory())),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
inplace_update_support(false),
|
2014-01-13 21:27:43 +00:00
|
|
|
inplace_update_num_locks(10000),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(nullptr),
|
2016-06-04 00:02:10 +00:00
|
|
|
memtable_prefix_bloom_size_ratio(0.0),
|
2016-07-27 01:05:30 +00:00
|
|
|
memtable_huge_page_size(0),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(0),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(0),
|
2015-02-17 16:03:45 +00:00
|
|
|
min_partial_merge_operands(2),
|
2015-04-17 22:26:50 +00:00
|
|
|
optimize_filters_for_hits(false),
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 00:24:45 +00:00
|
|
|
paranoid_file_checks(false),
|
2016-10-08 00:21:45 +00:00
|
|
|
force_consistency_checks(false),
|
2016-04-14 20:56:29 +00:00
|
|
|
report_bg_io_stats(false) {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
|
|
|
|
|
: comparator(options.comparator),
|
|
|
|
|
merge_operator(options.merge_operator),
|
|
|
|
|
compaction_filter(options.compaction_filter),
|
|
|
|
|
compaction_filter_factory(options.compaction_filter_factory),
|
|
|
|
|
write_buffer_size(options.write_buffer_size),
|
|
|
|
|
max_write_buffer_number(options.max_write_buffer_number),
|
|
|
|
|
min_write_buffer_number_to_merge(
|
|
|
|
|
options.min_write_buffer_number_to_merge),
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain(
|
|
|
|
|
options.max_write_buffer_number_to_maintain),
|
2014-01-06 21:31:06 +00:00
|
|
|
compression(options.compression),
|
|
|
|
|
compression_per_level(options.compression_per_level),
|
2016-05-09 22:57:19 +00:00
|
|
|
bottommost_compression(options.bottommost_compression),
|
2014-01-06 21:31:06 +00:00
|
|
|
compression_opts(options.compression_opts),
|
|
|
|
|
prefix_extractor(options.prefix_extractor),
|
|
|
|
|
num_levels(options.num_levels),
|
|
|
|
|
level0_file_num_compaction_trigger(
|
|
|
|
|
options.level0_file_num_compaction_trigger),
|
|
|
|
|
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
|
|
|
|
|
level0_stop_writes_trigger(options.level0_stop_writes_trigger),
|
|
|
|
|
target_file_size_base(options.target_file_size_base),
|
|
|
|
|
target_file_size_multiplier(options.target_file_size_multiplier),
|
|
|
|
|
max_bytes_for_level_base(options.max_bytes_for_level_base),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes(
|
|
|
|
|
options.level_compaction_dynamic_level_bytes),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
|
|
|
|
|
max_bytes_for_level_multiplier_additional(
|
|
|
|
|
options.max_bytes_for_level_multiplier_additional),
|
2016-06-16 23:02:52 +00:00
|
|
|
max_compaction_bytes(options.max_compaction_bytes),
|
2014-01-06 21:31:06 +00:00
|
|
|
soft_rate_limit(options.soft_rate_limit),
|
2015-11-19 02:10:20 +00:00
|
|
|
soft_pending_compaction_bytes_limit(
|
|
|
|
|
options.soft_pending_compaction_bytes_limit),
|
2015-09-11 21:31:23 +00:00
|
|
|
hard_pending_compaction_bytes_limit(
|
|
|
|
|
options.hard_pending_compaction_bytes_limit),
|
2014-01-06 21:31:06 +00:00
|
|
|
rate_limit_delay_max_milliseconds(
|
|
|
|
|
options.rate_limit_delay_max_milliseconds),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(options.arena_block_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
disable_auto_compactions(options.disable_auto_compactions),
|
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
|
compaction_style(options.compaction_style),
|
2015-09-22 00:16:31 +00:00
|
|
|
compaction_pri(options.compaction_pri),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
|
2014-01-06 21:31:06 +00:00
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo(options.compaction_options_fifo),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_sequential_skip_in_iterations(
|
|
|
|
|
options.max_sequential_skip_in_iterations),
|
|
|
|
|
memtable_factory(options.memtable_factory),
|
|
|
|
|
table_factory(options.table_factory),
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
table_properties_collector_factories(
|
|
|
|
|
options.table_properties_collector_factories),
|
2014-01-06 21:31:06 +00:00
|
|
|
inplace_update_support(options.inplace_update_support),
|
2014-01-13 17:06:44 +00:00
|
|
|
inplace_update_num_locks(options.inplace_update_num_locks),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(options.inplace_callback),
|
2016-06-04 00:02:10 +00:00
|
|
|
memtable_prefix_bloom_size_ratio(
|
|
|
|
|
options.memtable_prefix_bloom_size_ratio),
|
2016-07-27 01:05:30 +00:00
|
|
|
memtable_huge_page_size(options.memtable_huge_page_size),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(options.bloom_locality),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(options.max_successive_merges),
|
2015-02-17 16:03:45 +00:00
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands),
|
2015-04-17 22:26:50 +00:00
|
|
|
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 00:24:45 +00:00
|
|
|
paranoid_file_checks(options.paranoid_file_checks),
|
2016-10-08 00:21:45 +00:00
|
|
|
force_consistency_checks(options.force_consistency_checks),
|
2016-04-14 20:56:29 +00:00
|
|
|
report_bg_io_stats(options.report_bg_io_stats) {
|
2014-01-06 21:31:06 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
2014-06-25 21:31:30 +00:00
|
|
|
if (max_bytes_for_level_multiplier_additional.size() <
|
|
|
|
|
static_cast<unsigned int>(num_levels)) {
|
2014-06-25 20:11:12 +00:00
|
|
|
max_bytes_for_level_multiplier_additional.resize(num_levels, 1);
|
|
|
|
|
}
|
2014-01-06 21:31:06 +00:00
|
|
|
}
|
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
DBOptions::DBOptions()
|
|
|
|
|
: create_if_missing(false),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(false),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
error_if_exists(false),
|
2014-03-31 19:44:54 +00:00
|
|
|
paranoid_checks(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
env(Env::Default()),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(nullptr),
|
2016-01-29 02:35:01 +00:00
|
|
|
sst_file_manager(nullptr),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(nullptr),
|
2015-02-06 16:44:30 +00:00
|
|
|
#ifdef NDEBUG
|
2014-04-10 22:27:42 +00:00
|
|
|
info_log_level(INFO_LEVEL),
|
2015-02-06 16:44:30 +00:00
|
|
|
#else
|
|
|
|
|
info_log_level(DEBUG_LEVEL),
|
|
|
|
|
#endif // NDEBUG
|
2016-04-08 00:40:42 +00:00
|
|
|
max_open_files(-1),
|
2016-04-07 20:51:47 +00:00
|
|
|
max_file_opening_threads(16),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(0),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(nullptr),
|
2012-08-14 22:20:36 +00:00
|
|
|
disableDataSync(false),
|
2012-08-27 19:10:26 +00:00
|
|
|
use_fsync(false),
|
2012-09-17 20:35:57 +00:00
|
|
|
db_log_dir(""),
|
2013-10-01 21:46:52 +00:00
|
|
|
wal_dir(""),
|
2015-11-06 19:34:06 +00:00
|
|
|
delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000),
|
2016-04-08 00:40:42 +00:00
|
|
|
base_background_compactions(1),
|
2012-11-06 00:51:55 +00:00
|
|
|
max_background_compactions(1),
|
2015-08-21 21:25:34 +00:00
|
|
|
max_subcompactions(1),
|
2014-02-06 20:59:16 +00:00
|
|
|
max_background_flushes(1),
|
2012-10-29 21:18:00 +00:00
|
|
|
max_log_file_size(0),
|
2013-02-05 03:42:40 +00:00
|
|
|
log_file_time_to_roll(0),
|
|
|
|
|
keep_log_file_num(1000),
|
2015-10-08 01:06:28 +00:00
|
|
|
recycle_log_file_num(0),
|
2013-01-11 01:18:50 +00:00
|
|
|
max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
|
2016-04-07 20:51:47 +00:00
|
|
|
table_cache_numshardbits(6),
|
2013-01-15 22:05:42 +00:00
|
|
|
WAL_ttl_seconds(0),
|
2013-11-07 02:46:28 +00:00
|
|
|
WAL_size_limit_MB(0),
|
2013-02-28 22:09:30 +00:00
|
|
|
manifest_preallocation_size(4 * 1024 * 1024),
|
2013-03-15 00:00:04 +00:00
|
|
|
allow_os_buffer(true),
|
|
|
|
|
allow_mmap_reads(false),
|
2014-03-28 23:57:04 +00:00
|
|
|
allow_mmap_writes(false),
|
2016-10-28 17:36:05 +00:00
|
|
|
use_direct_reads(false),
|
2015-10-07 17:04:05 +00:00
|
|
|
allow_fallocate(true),
|
2013-05-21 18:53:33 +00:00
|
|
|
is_fd_close_on_exec(true),
|
2013-05-10 22:21:04 +00:00
|
|
|
skip_log_error_on_recovery(false),
|
2015-05-21 18:22:16 +00:00
|
|
|
stats_dump_period_sec(600),
|
2013-05-17 22:53:01 +00:00
|
|
|
advise_random_on_open(true),
|
2014-12-02 20:09:20 +00:00
|
|
|
db_write_buffer_size(0),
|
2013-05-31 23:30:17 +00:00
|
|
|
access_hint_on_compaction_start(NORMAL),
|
2015-08-21 05:33:25 +00:00
|
|
|
new_table_reader_for_compaction_inputs(false),
|
2015-08-26 22:25:59 +00:00
|
|
|
compaction_readahead_size(0),
|
2015-10-27 21:44:16 +00:00
|
|
|
random_access_max_buffer_size(1024 * 1024),
|
2015-10-30 05:10:25 +00:00
|
|
|
writable_file_max_buffer_size(1024 * 1024),
|
2013-06-14 05:49:46 +00:00
|
|
|
use_adaptive_mutex(false),
|
2014-11-21 05:13:18 +00:00
|
|
|
bytes_per_sync(0),
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync(0),
|
2015-05-28 20:21:39 +00:00
|
|
|
listeners(),
|
2015-05-15 22:52:51 +00:00
|
|
|
enable_thread_tracking(false),
|
2015-12-23 21:49:53 +00:00
|
|
|
delayed_write_rate(2 * 1024U * 1024U),
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
allow_concurrent_memtable_write(false),
|
|
|
|
|
enable_write_thread_adaptive_yield(false),
|
|
|
|
|
write_thread_max_yield_usec(100),
|
|
|
|
|
write_thread_slow_yield_usec(3),
|
2015-08-04 20:48:16 +00:00
|
|
|
skip_stats_update_on_db_open(false),
|
2016-04-08 00:40:42 +00:00
|
|
|
wal_recovery_mode(WALRecoveryMode::kPointInTimeRecovery),
|
2015-11-30 18:30:22 +00:00
|
|
|
row_cache(nullptr),
|
2015-10-13 18:10:14 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2015-11-11 06:58:01 +00:00
|
|
|
wal_filter(nullptr),
|
2015-10-29 22:52:32 +00:00
|
|
|
#endif // ROCKSDB_LITE
|
2016-04-27 23:23:33 +00:00
|
|
|
fail_if_options_file_error(false),
|
2016-06-13 18:34:16 +00:00
|
|
|
dump_malloc_stats(false),
|
|
|
|
|
avoid_flush_during_recovery(false) {
|
2015-02-17 16:03:45 +00:00
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
DBOptions::DBOptions(const Options& options)
|
|
|
|
|
: create_if_missing(options.create_if_missing),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(options.create_missing_column_families),
|
2014-01-06 21:31:06 +00:00
|
|
|
error_if_exists(options.error_if_exists),
|
|
|
|
|
paranoid_checks(options.paranoid_checks),
|
|
|
|
|
env(options.env),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(options.rate_limiter),
|
2016-01-29 02:35:01 +00:00
|
|
|
sst_file_manager(options.sst_file_manager),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(options.info_log),
|
2014-03-06 00:55:51 +00:00
|
|
|
info_log_level(options.info_log_level),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_open_files(options.max_open_files),
|
2015-08-11 19:19:56 +00:00
|
|
|
max_file_opening_threads(options.max_file_opening_threads),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(options.max_total_wal_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(options.statistics),
|
2014-01-06 21:31:06 +00:00
|
|
|
disableDataSync(options.disableDataSync),
|
|
|
|
|
use_fsync(options.use_fsync),
|
2014-07-02 16:54:20 +00:00
|
|
|
db_paths(options.db_paths),
|
2014-01-06 21:31:06 +00:00
|
|
|
db_log_dir(options.db_log_dir),
|
|
|
|
|
wal_dir(options.wal_dir),
|
|
|
|
|
delete_obsolete_files_period_micros(
|
|
|
|
|
options.delete_obsolete_files_period_micros),
|
Add options.base_background_compactions as a number of compaction threads for low compaction debt
Summary:
If options.base_background_compactions is given, we try to schedule number of compactions not existing this number, only when L0 files increase to certain number, or pending compaction bytes more than certain threshold, we schedule compactions based on options.max_background_compactions.
The watermarks are calculated based on slowdown thresholds.
Test Plan:
Add new test cases in column_family_test.
Adding more unit tests.
Reviewers: IslamAbdelRahman, yhchiang, kradhakrishnan, rven, anthony
Reviewed By: anthony
Subscribers: leveldb, dhruba, yoshinorim
Differential Revision: https://reviews.facebook.net/D53409
2016-01-28 19:56:16 +00:00
|
|
|
base_background_compactions(options.base_background_compactions),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_background_compactions(options.max_background_compactions),
|
2015-08-21 21:25:34 +00:00
|
|
|
max_subcompactions(options.max_subcompactions),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_background_flushes(options.max_background_flushes),
|
|
|
|
|
max_log_file_size(options.max_log_file_size),
|
|
|
|
|
log_file_time_to_roll(options.log_file_time_to_roll),
|
|
|
|
|
keep_log_file_num(options.keep_log_file_num),
|
2015-10-08 01:06:28 +00:00
|
|
|
recycle_log_file_num(options.recycle_log_file_num),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_manifest_file_size(options.max_manifest_file_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(options.table_cache_numshardbits),
|
2014-01-06 21:31:06 +00:00
|
|
|
WAL_ttl_seconds(options.WAL_ttl_seconds),
|
|
|
|
|
WAL_size_limit_MB(options.WAL_size_limit_MB),
|
|
|
|
|
manifest_preallocation_size(options.manifest_preallocation_size),
|
|
|
|
|
allow_os_buffer(options.allow_os_buffer),
|
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
2016-10-28 17:36:05 +00:00
|
|
|
use_direct_reads(options.use_direct_reads),
|
2015-10-07 17:04:05 +00:00
|
|
|
allow_fallocate(options.allow_fallocate),
|
2014-01-06 21:31:06 +00:00
|
|
|
is_fd_close_on_exec(options.is_fd_close_on_exec),
|
|
|
|
|
skip_log_error_on_recovery(options.skip_log_error_on_recovery),
|
|
|
|
|
stats_dump_period_sec(options.stats_dump_period_sec),
|
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
2014-12-02 20:09:20 +00:00
|
|
|
db_write_buffer_size(options.db_write_buffer_size),
|
2016-06-21 01:01:03 +00:00
|
|
|
write_buffer_manager(options.write_buffer_manager),
|
2014-01-06 21:31:06 +00:00
|
|
|
access_hint_on_compaction_start(options.access_hint_on_compaction_start),
|
2015-08-21 05:33:25 +00:00
|
|
|
new_table_reader_for_compaction_inputs(
|
|
|
|
|
options.new_table_reader_for_compaction_inputs),
|
2015-08-26 22:25:59 +00:00
|
|
|
compaction_readahead_size(options.compaction_readahead_size),
|
2015-10-27 21:44:16 +00:00
|
|
|
random_access_max_buffer_size(options.random_access_max_buffer_size),
|
2015-10-30 05:10:25 +00:00
|
|
|
writable_file_max_buffer_size(options.writable_file_max_buffer_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
use_adaptive_mutex(options.use_adaptive_mutex),
|
2014-11-21 05:13:18 +00:00
|
|
|
bytes_per_sync(options.bytes_per_sync),
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync(options.wal_bytes_per_sync),
|
2015-05-28 20:21:39 +00:00
|
|
|
listeners(options.listeners),
|
2015-05-15 22:52:51 +00:00
|
|
|
enable_thread_tracking(options.enable_thread_tracking),
|
2015-06-15 19:03:13 +00:00
|
|
|
delayed_write_rate(options.delayed_write_rate),
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
allow_concurrent_memtable_write(options.allow_concurrent_memtable_write),
|
|
|
|
|
enable_write_thread_adaptive_yield(
|
|
|
|
|
options.enable_write_thread_adaptive_yield),
|
|
|
|
|
write_thread_max_yield_usec(options.write_thread_max_yield_usec),
|
|
|
|
|
write_thread_slow_yield_usec(options.write_thread_slow_yield_usec),
|
2015-08-04 20:48:16 +00:00
|
|
|
skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
|
2015-06-23 17:25:45 +00:00
|
|
|
wal_recovery_mode(options.wal_recovery_mode),
|
2015-11-11 06:58:01 +00:00
|
|
|
row_cache(options.row_cache),
|
2015-10-13 18:10:14 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2015-11-11 06:58:01 +00:00
|
|
|
wal_filter(options.wal_filter),
|
2015-10-29 22:52:32 +00:00
|
|
|
#endif // ROCKSDB_LITE
|
2016-04-27 23:23:33 +00:00
|
|
|
fail_if_options_file_error(options.fail_if_options_file_error),
|
2016-06-13 18:34:16 +00:00
|
|
|
dump_malloc_stats(options.dump_malloc_stats),
|
|
|
|
|
avoid_flush_during_recovery(options.avoid_flush_during_recovery) {
|
2015-10-13 18:10:14 +00:00
|
|
|
}
|
2014-01-06 21:31:06 +00:00
|
|
|
|
2013-05-17 22:53:01 +00:00
|
|
|
static const char* const access_hints[] = {
|
|
|
|
|
"NONE", "NORMAL", "SEQUENTIAL", "WILLNEED"
|
|
|
|
|
};
|
|
|
|
|
|
2014-02-07 05:39:20 +00:00
|
|
|
void DBOptions::Dump(Logger* log) const {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.error_if_exists: %d", error_if_exists);
|
|
|
|
|
Header(log, " Options.create_if_missing: %d", create_if_missing);
|
|
|
|
|
Header(log, " Options.paranoid_checks: %d", paranoid_checks);
|
|
|
|
|
Header(log, " Options.env: %p", env);
|
|
|
|
|
Header(log, " Options.info_log: %p", info_log.get());
|
|
|
|
|
Header(log, " Options.max_open_files: %d", max_open_files);
|
|
|
|
|
Header(log,
|
|
|
|
|
"Options.max_file_opening_threads: %d", max_file_opening_threads);
|
|
|
|
|
Header(log,
|
|
|
|
|
" Options.max_total_wal_size: %" PRIu64, max_total_wal_size);
|
|
|
|
|
Header(log, " Options.disableDataSync: %d", disableDataSync);
|
|
|
|
|
Header(log, " Options.use_fsync: %d", use_fsync);
|
|
|
|
|
Header(log, " Options.max_log_file_size: %" ROCKSDB_PRIszt,
|
2015-07-13 19:11:05 +00:00
|
|
|
max_log_file_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, "Options.max_manifest_file_size: %" PRIu64,
|
2015-07-13 19:11:05 +00:00
|
|
|
max_manifest_file_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt,
|
2015-07-13 19:11:05 +00:00
|
|
|
log_file_time_to_roll);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt,
|
2015-07-13 19:11:05 +00:00
|
|
|
keep_log_file_num);
|
2015-10-08 01:06:28 +00:00
|
|
|
Header(log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt,
|
|
|
|
|
recycle_log_file_num);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.allow_os_buffer: %d", allow_os_buffer);
|
|
|
|
|
Header(log, " Options.allow_mmap_reads: %d", allow_mmap_reads);
|
2015-10-07 17:04:05 +00:00
|
|
|
Header(log, " Options.allow_fallocate: %d", allow_fallocate);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.allow_mmap_writes: %d", allow_mmap_writes);
|
2016-10-28 17:36:05 +00:00
|
|
|
Header(log, " Options.use_direct_reads: %d", use_direct_reads);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.create_missing_column_families: %d",
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.db_log_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
db_log_dir.c_str());
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.wal_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
wal_dir.c_str());
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.table_cache_numshardbits: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
table_cache_numshardbits);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
delete_obsolete_files_period_micros);
|
Add options.base_background_compactions as a number of compaction threads for low compaction debt
Summary:
If options.base_background_compactions is given, we try to schedule number of compactions not existing this number, only when L0 files increase to certain number, or pending compaction bytes more than certain threshold, we schedule compactions based on options.max_background_compactions.
The watermarks are calculated based on slowdown thresholds.
Test Plan:
Add new test cases in column_family_test.
Adding more unit tests.
Reviewers: IslamAbdelRahman, yhchiang, kradhakrishnan, rven, anthony
Reviewed By: anthony
Subscribers: leveldb, dhruba, yoshinorim
Differential Revision: https://reviews.facebook.net/D53409
2016-01-28 19:56:16 +00:00
|
|
|
Header(log, " Options.base_background_compactions: %d",
|
|
|
|
|
base_background_compactions);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_background_compactions: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
max_background_compactions);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_subcompactions: %" PRIu32,
|
2015-09-15 18:41:46 +00:00
|
|
|
max_subcompactions);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_background_flushes: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
max_background_flushes);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.WAL_ttl_seconds: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
WAL_ttl_seconds);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.WAL_size_limit_MB: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
WAL_size_limit_MB);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2015-07-13 19:11:05 +00:00
|
|
|
" Options.manifest_preallocation_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
manifest_preallocation_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.allow_os_buffer: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_os_buffer);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.allow_mmap_reads: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_mmap_reads);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.allow_mmap_writes: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_mmap_writes);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.is_fd_close_on_exec: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
is_fd_close_on_exec);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.stats_dump_period_sec: %u",
|
2014-02-07 05:39:20 +00:00
|
|
|
stats_dump_period_sec);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.advise_random_on_open: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
advise_random_on_open);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2015-07-13 19:11:05 +00:00
|
|
|
" Options.db_write_buffer_size: %" ROCKSDB_PRIszt
|
|
|
|
|
"d",
|
|
|
|
|
db_write_buffer_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.access_hint_on_compaction_start: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
access_hints[access_hint_on_compaction_start]);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.new_table_reader_for_compaction_inputs: %d",
|
2015-08-21 05:33:25 +00:00
|
|
|
new_table_reader_for_compaction_inputs);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2015-08-26 22:25:59 +00:00
|
|
|
" Options.compaction_readahead_size: %" ROCKSDB_PRIszt
|
|
|
|
|
"d",
|
|
|
|
|
compaction_readahead_size);
|
2015-10-29 22:52:32 +00:00
|
|
|
Header(
|
|
|
|
|
log,
|
2015-10-27 21:44:16 +00:00
|
|
|
" Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt
|
|
|
|
|
"d",
|
|
|
|
|
random_access_max_buffer_size);
|
2015-10-30 05:10:25 +00:00
|
|
|
Header(log,
|
|
|
|
|
" Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt
|
|
|
|
|
"d",
|
|
|
|
|
writable_file_max_buffer_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.use_adaptive_mutex: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
use_adaptive_mutex);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.rate_limiter: %p",
|
2014-08-12 23:42:18 +00:00
|
|
|
rate_limiter.get());
|
2016-01-29 02:35:01 +00:00
|
|
|
Header(
|
|
|
|
|
log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64,
|
|
|
|
|
sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.bytes_per_sync: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
bytes_per_sync);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.wal_bytes_per_sync: %" PRIu64,
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.wal_recovery_mode: %d",
|
2015-08-10 16:21:04 +00:00
|
|
|
wal_recovery_mode);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.enable_thread_tracking: %d",
|
2014-11-21 05:13:18 +00:00
|
|
|
enable_thread_tracking);
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-14 23:59:07 +00:00
|
|
|
Header(log, " Options.allow_concurrent_memtable_write: %d",
|
|
|
|
|
allow_concurrent_memtable_write);
|
|
|
|
|
Header(log, " Options.enable_write_thread_adaptive_yield: %d",
|
|
|
|
|
enable_write_thread_adaptive_yield);
|
|
|
|
|
Header(log, " Options.write_thread_max_yield_usec: %" PRIu64,
|
|
|
|
|
write_thread_max_yield_usec);
|
|
|
|
|
Header(log, " Options.write_thread_slow_yield_usec: %" PRIu64,
|
|
|
|
|
write_thread_slow_yield_usec);
|
2015-06-23 17:25:45 +00:00
|
|
|
if (row_cache) {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.row_cache: %" PRIu64,
|
2015-06-23 17:25:45 +00:00
|
|
|
row_cache->GetCapacity());
|
|
|
|
|
} else {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.row_cache: None");
|
2015-06-23 17:25:45 +00:00
|
|
|
}
|
2015-10-13 18:10:14 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2015-10-13 00:03:03 +00:00
|
|
|
Header(log, " Options.wal_filter: %s",
|
2015-10-29 22:52:32 +00:00
|
|
|
wal_filter ? wal_filter->Name() : "None");
|
|
|
|
|
#endif // ROCKDB_LITE
|
2016-06-13 18:34:16 +00:00
|
|
|
Header(log, " Options.avoid_flush_during_recovery: %d",
|
|
|
|
|
avoid_flush_during_recovery);
|
2014-02-07 05:39:20 +00:00
|
|
|
} // DBOptions::Dump
|
|
|
|
|
|
|
|
|
|
void ColumnFamilyOptions::Dump(Logger* log) const {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.comparator: %s", comparator->Name());
|
|
|
|
|
Header(log, " Options.merge_operator: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
merge_operator ? merge_operator->Name() : "None");
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compaction_filter: %s",
|
2014-08-13 22:58:31 +00:00
|
|
|
compaction_filter ? compaction_filter->Name() : "None");
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compaction_filter_factory: %s",
|
2015-06-08 23:34:26 +00:00
|
|
|
compaction_filter_factory ? compaction_filter_factory->Name() : "None");
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.memtable_factory: %s", memtable_factory->Name());
|
|
|
|
|
Header(log, " Options.table_factory: %s", table_factory->Name());
|
|
|
|
|
Header(log, " table_factory options: %s",
|
2014-08-25 21:24:09 +00:00
|
|
|
table_factory->GetPrintableTableOptions().c_str());
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt,
|
2015-07-13 19:11:05 +00:00
|
|
|
write_buffer_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_write_buffer_number: %d", max_write_buffer_number);
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!compression_per_level.empty()) {
|
2013-03-15 01:32:01 +00:00
|
|
|
for (unsigned int i = 0; i < compression_per_level.size(); i++) {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compression[%d]: %s", i,
|
2015-06-18 21:55:05 +00:00
|
|
|
CompressionTypeToString(compression_per_level[i]).c_str());
|
2015-04-06 19:50:44 +00:00
|
|
|
}
|
2012-10-28 06:13:17 +00:00
|
|
|
} else {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compression: %s",
|
2015-06-18 21:55:05 +00:00
|
|
|
CompressionTypeToString(compression).c_str());
|
2012-10-28 06:13:17 +00:00
|
|
|
}
|
2016-05-09 22:57:19 +00:00
|
|
|
Header(log, " Options.bottommost_compression: %s",
|
|
|
|
|
bottommost_compression == kDisableCompressionOption
|
|
|
|
|
? "Disabled"
|
|
|
|
|
: CompressionTypeToString(bottommost_compression).c_str());
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.prefix_extractor: %s",
|
2013-08-14 16:06:10 +00:00
|
|
|
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.num_levels: %d", num_levels);
|
|
|
|
|
Header(log, " Options.min_write_buffer_number_to_merge: %d",
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_write_buffer_number_to_maintain: %d",
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compression_opts.window_bits: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.window_bits);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compression_opts.level: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.level);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compression_opts.strategy: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.strategy);
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 00:36:03 +00:00
|
|
|
Header(log,
|
|
|
|
|
" Options.compression_opts.max_dict_bytes: %" ROCKSDB_PRIszt,
|
|
|
|
|
compression_opts.max_dict_bytes);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.level0_file_num_compaction_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_file_num_compaction_trigger);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.level0_slowdown_writes_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_slowdown_writes_trigger);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.level0_stop_writes_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_stop_writes_trigger);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.target_file_size_base: %" PRIu64,
|
2012-08-27 19:10:26 +00:00
|
|
|
target_file_size_base);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.target_file_size_multiplier: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
target_file_size_multiplier);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_bytes_for_level_base: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
max_bytes_for_level_base);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, "Options.level_compaction_dynamic_level_bytes: %d",
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_bytes_for_level_multiplier: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
max_bytes_for_level_multiplier);
|
2015-03-30 21:04:21 +00:00
|
|
|
for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
|
|
|
|
|
i++) {
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
|
|
|
|
"Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt
|
2015-07-13 19:11:05 +00:00
|
|
|
"]: %d",
|
|
|
|
|
i, max_bytes_for_level_multiplier_additional[i]);
|
2013-05-21 18:37:06 +00:00
|
|
|
}
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.max_sequential_skip_in_iterations: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
max_sequential_skip_in_iterations);
|
2016-06-16 23:02:52 +00:00
|
|
|
Header(log, " Options.max_compaction_bytes: %" PRIu64,
|
|
|
|
|
max_compaction_bytes);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2015-07-13 19:11:05 +00:00
|
|
|
" Options.arena_block_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
arena_block_size);
|
2015-11-19 02:10:20 +00:00
|
|
|
Header(log, " Options.soft_pending_compaction_bytes_limit: %" PRIu64,
|
|
|
|
|
soft_pending_compaction_bytes_limit);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.hard_pending_compaction_bytes_limit: %" PRIu64,
|
2015-09-11 21:31:23 +00:00
|
|
|
hard_pending_compaction_bytes_limit);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.rate_limit_delay_max_milliseconds: %u",
|
2013-08-05 22:43:49 +00:00
|
|
|
rate_limit_delay_max_milliseconds);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.disable_auto_compactions: %d",
|
2012-11-20 23:45:41 +00:00
|
|
|
disable_auto_compactions);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.verify_checksums_in_compaction: %d",
|
2014-03-10 17:06:34 +00:00
|
|
|
verify_checksums_in_compaction);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compaction_style: %d",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_style);
|
2015-09-22 00:16:31 +00:00
|
|
|
Header(log, " Options.compaction_pri: %d",
|
|
|
|
|
compaction_pri);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.compaction_options_universal.size_ratio: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.size_ratio);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, "Options.compaction_options_universal.min_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.min_merge_width);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, "Options.compaction_options_universal.max_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.max_merge_width);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, "Options.compaction_options_universal."
|
2013-09-09 23:06:10 +00:00
|
|
|
"max_size_amplification_percent: %u",
|
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2014-09-19 20:09:25 +00:00
|
|
|
"Options.compaction_options_universal.compression_size_percent: %d",
|
2013-12-03 20:32:07 +00:00
|
|
|
compaction_options_universal.compression_size_percent);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
|
|
|
|
"Options.compaction_options_fifo.max_table_files_size: %" PRIu64,
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo.max_table_files_size);
|
2013-10-16 18:50:50 +00:00
|
|
|
std::string collector_names;
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
for (const auto& collector_factory : table_properties_collector_factories) {
|
|
|
|
|
collector_names.append(collector_factory->Name());
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.append("; ");
|
|
|
|
|
}
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.table_properties_collectors: %s",
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.c_str());
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.inplace_update_support: %d",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
inplace_update_support);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2015-07-13 19:11:05 +00:00
|
|
|
" Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
|
|
|
|
inplace_update_num_locks);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.min_partial_merge_operands: %u",
|
2014-03-25 00:57:13 +00:00
|
|
|
min_partial_merge_operands);
|
2013-11-27 22:27:02 +00:00
|
|
|
// TODO: easier config for bloom (maybe based on avg key/value size)
|
2016-06-04 00:02:10 +00:00
|
|
|
Header(log, " Options.memtable_prefix_bloom_size_ratio: %f",
|
|
|
|
|
memtable_prefix_bloom_size_ratio);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2016-07-27 01:05:30 +00:00
|
|
|
Header(log, " Options.memtable_huge_page_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
memtable_huge_page_size);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.bloom_locality: %d",
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log,
|
2015-07-13 19:11:05 +00:00
|
|
|
" Options.max_successive_merges: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_successive_merges);
|
2016-02-15 23:55:29 +00:00
|
|
|
Header(log, " Options.optimize_filters_for_hits: %d",
|
2015-02-17 16:03:45 +00:00
|
|
|
optimize_filters_for_hits);
|
2015-09-16 18:31:45 +00:00
|
|
|
Header(log, " Options.paranoid_file_checks: %d",
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 00:24:45 +00:00
|
|
|
paranoid_file_checks);
|
2016-10-08 00:21:45 +00:00
|
|
|
Header(log, " Options.force_consistency_checks: %d",
|
|
|
|
|
force_consistency_checks);
|
2016-04-14 20:56:29 +00:00
|
|
|
Header(log, " Options.report_bg_io_stats: %d",
|
|
|
|
|
report_bg_io_stats);
|
2014-02-07 05:39:20 +00:00
|
|
|
} // ColumnFamilyOptions::Dump
|
|
|
|
|
|
|
|
|
|
void Options::Dump(Logger* log) const {
|
|
|
|
|
DBOptions::Dump(log);
|
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
2012-08-22 18:43:53 +00:00
|
|
|
} // Options::Dump
|
|
|
|
|
|
2015-06-18 17:15:54 +00:00
|
|
|
void Options::DumpCFOptions(Logger* log) const {
|
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
|
|
|
|
} // Options::DumpCFOptions
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
//
|
|
|
|
|
// The goal of this method is to create a configuration that
|
|
|
|
|
// allows an application to write all files into L0 and
|
|
|
|
|
// then do a single compaction to output all files into L1.
|
2013-02-26 06:57:37 +00:00
|
|
|
Options*
|
|
|
|
|
Options::PrepareForBulkLoad()
|
|
|
|
|
{
|
2013-03-08 17:19:24 +00:00
|
|
|
// never slowdown ingest.
|
2013-02-26 06:57:37 +00:00
|
|
|
level0_file_num_compaction_trigger = (1<<30);
|
|
|
|
|
level0_slowdown_writes_trigger = (1<<30);
|
|
|
|
|
level0_stop_writes_trigger = (1<<30);
|
2016-06-16 21:05:34 +00:00
|
|
|
soft_pending_compaction_bytes_limit = 0;
|
|
|
|
|
hard_pending_compaction_bytes_limit = 0;
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
|
// no auto compactions please. The application should issue a
|
|
|
|
|
// manual compaction after all data is loaded into L0.
|
2013-02-26 06:57:37 +00:00
|
|
|
disable_auto_compactions = true;
|
|
|
|
|
disableDataSync = true;
|
2013-03-08 17:19:24 +00:00
|
|
|
// A manual compaction run should pick all files in L0 in
|
|
|
|
|
// a single compaction run.
|
2016-06-16 23:02:52 +00:00
|
|
|
max_compaction_bytes = (static_cast<uint64_t>(1) << 60);
|
2013-02-26 06:57:37 +00:00
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// It is better to have only 2 levels, otherwise a manual
|
|
|
|
|
// compaction would compact at every possible level, thereby
|
|
|
|
|
// increasing the total time needed for compactions.
|
|
|
|
|
num_levels = 2;
|
|
|
|
|
|
2015-02-02 19:09:21 +00:00
|
|
|
// Need to allow more write buffers to allow more parallism
|
|
|
|
|
// of flushes.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
min_write_buffer_number_to_merge = 1;
|
|
|
|
|
|
|
|
|
|
// When compaction is disabled, more parallel flush threads can
|
|
|
|
|
// help with write throughput.
|
|
|
|
|
max_background_flushes = 4;
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// Prevent a memtable flush to automatically promote files
|
|
|
|
|
// to L1. This is helpful so that all files that are
|
|
|
|
|
// input to the manual compaction are all at L0.
|
|
|
|
|
max_background_compactions = 2;
|
Add options.base_background_compactions as a number of compaction threads for low compaction debt
Summary:
If options.base_background_compactions is given, we try to schedule number of compactions not existing this number, only when L0 files increase to certain number, or pending compaction bytes more than certain threshold, we schedule compactions based on options.max_background_compactions.
The watermarks are calculated based on slowdown thresholds.
Test Plan:
Add new test cases in column_family_test.
Adding more unit tests.
Reviewers: IslamAbdelRahman, yhchiang, kradhakrishnan, rven, anthony
Reviewed By: anthony
Subscribers: leveldb, dhruba, yoshinorim
Differential Revision: https://reviews.facebook.net/D53409
2016-01-28 19:56:16 +00:00
|
|
|
base_background_compactions = 2;
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
|
// The compaction would create large files in L1.
|
|
|
|
|
target_file_size_base = 256 * 1024 * 1024;
|
2013-02-26 06:57:37 +00:00
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
2016-05-05 23:50:32 +00:00
|
|
|
Options* Options::OptimizeForSmallDb() {
|
|
|
|
|
ColumnFamilyOptions::OptimizeForSmallDb();
|
|
|
|
|
DBOptions::OptimizeForSmallDb();
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-24 19:45:50 +00:00
|
|
|
Options* Options::OldDefaults(int rocksdb_major_version,
|
|
|
|
|
int rocksdb_minor_version) {
|
|
|
|
|
ColumnFamilyOptions::OldDefaults(rocksdb_major_version,
|
|
|
|
|
rocksdb_minor_version);
|
|
|
|
|
DBOptions::OldDefaults(rocksdb_major_version, rocksdb_minor_version);
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DBOptions* DBOptions::OldDefaults(int rocksdb_major_version,
|
|
|
|
|
int rocksdb_minor_version) {
|
2016-04-08 00:40:42 +00:00
|
|
|
if (rocksdb_major_version < 4 ||
|
|
|
|
|
(rocksdb_major_version == 4 && rocksdb_minor_version < 7)) {
|
|
|
|
|
max_file_opening_threads = 1;
|
|
|
|
|
table_cache_numshardbits = 4;
|
|
|
|
|
}
|
|
|
|
|
max_open_files = 5000;
|
|
|
|
|
base_background_compactions = -1;
|
|
|
|
|
wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
|
2016-03-24 19:45:50 +00:00
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OldDefaults(
|
|
|
|
|
int rocksdb_major_version, int rocksdb_minor_version) {
|
2016-04-08 00:40:42 +00:00
|
|
|
if (rocksdb_major_version < 4 ||
|
|
|
|
|
(rocksdb_major_version == 4 && rocksdb_minor_version < 7)) {
|
|
|
|
|
write_buffer_size = 4 << 20;
|
|
|
|
|
target_file_size_base = 2 * 1048576;
|
|
|
|
|
max_bytes_for_level_base = 10 * 1048576;
|
|
|
|
|
soft_pending_compaction_bytes_limit = 0;
|
|
|
|
|
hard_pending_compaction_bytes_limit = 0;
|
|
|
|
|
}
|
|
|
|
|
compaction_pri = CompactionPri::kByCompensatedSize;
|
|
|
|
|
|
2016-03-24 19:45:50 +00:00
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-10 17:49:33 +00:00
|
|
|
// Optimization functions
|
2016-05-05 23:50:32 +00:00
|
|
|
DBOptions* DBOptions::OptimizeForSmallDb() {
|
|
|
|
|
max_file_opening_threads = 1;
|
|
|
|
|
max_open_files = 5000;
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-24 19:45:50 +00:00
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForSmallDb() {
|
|
|
|
|
write_buffer_size = 2 << 20;
|
|
|
|
|
target_file_size_base = 2 * 1048576;
|
|
|
|
|
max_bytes_for_level_base = 10 * 1048576;
|
|
|
|
|
soft_pending_compaction_bytes_limit = 256 * 1048576;
|
|
|
|
|
hard_pending_compaction_bytes_limit = 1073741824ul;
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
2014-08-26 21:15:00 +00:00
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
|
|
|
|
|
uint64_t block_cache_size_mb) {
|
2014-05-10 17:49:33 +00:00
|
|
|
prefix_extractor.reset(NewNoopTransform());
|
|
|
|
|
BlockBasedTableOptions block_based_options;
|
2014-08-26 21:15:00 +00:00
|
|
|
block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
|
block_based_options.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
|
|
|
block_based_options.block_cache =
|
2014-11-13 19:39:30 +00:00
|
|
|
NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024));
|
2014-05-10 17:49:33 +00:00
|
|
|
table_factory.reset(new BlockBasedTableFactory(block_based_options));
|
|
|
|
|
memtable_factory.reset(NewHashLinkListRepFactory());
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction(
|
|
|
|
|
uint64_t memtable_memory_budget) {
|
2014-11-13 19:39:30 +00:00
|
|
|
write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
|
2014-05-10 17:49:33 +00:00
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
|
// write stalls.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
// start flushing L0->L1 as soon as possible. each file on level0 is
|
|
|
|
|
// (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
|
|
|
|
|
// memtable_memory_budget.
|
|
|
|
|
level0_file_num_compaction_trigger = 2;
|
|
|
|
|
// doesn't really matter much, but we don't want to create too many files
|
|
|
|
|
target_file_size_base = memtable_memory_budget / 8;
|
|
|
|
|
// make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
|
|
|
|
|
max_bytes_for_level_base = memtable_memory_budget;
|
|
|
|
|
|
|
|
|
|
// level style compaction
|
|
|
|
|
compaction_style = kCompactionStyleLevel;
|
|
|
|
|
|
|
|
|
|
// only compress levels >= 2
|
|
|
|
|
compression_per_level.resize(num_levels);
|
|
|
|
|
for (int i = 0; i < num_levels; ++i) {
|
|
|
|
|
if (i < 2) {
|
|
|
|
|
compression_per_level[i] = kNoCompression;
|
|
|
|
|
} else {
|
|
|
|
|
compression_per_level[i] = kSnappyCompression;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction(
|
|
|
|
|
uint64_t memtable_memory_budget) {
|
2014-11-13 19:39:30 +00:00
|
|
|
write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
|
2014-05-10 17:49:33 +00:00
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
|
// write stalls.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
// universal style compaction
|
|
|
|
|
compaction_style = kCompactionStyleUniversal;
|
|
|
|
|
compaction_options_universal.compression_size_percent = 80;
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
|
|
|
|
|
max_background_compactions = total_threads - 1;
|
|
|
|
|
max_background_flushes = 1;
|
|
|
|
|
env->SetBackgroundThreads(total_threads, Env::LOW);
|
|
|
|
|
env->SetBackgroundThreads(1, Env::HIGH);
|
|
|
|
|
return this;
|
|
|
|
|
}
|
2015-02-18 19:49:31 +00:00
|
|
|
|
2015-04-10 04:05:09 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
2015-02-18 19:49:31 +00:00
|
|
|
ReadOptions::ReadOptions()
|
|
|
|
|
: verify_checksums(true),
|
|
|
|
|
fill_cache(true),
|
|
|
|
|
snapshot(nullptr),
|
|
|
|
|
iterate_upper_bound(nullptr),
|
|
|
|
|
read_tier(kReadAllTier),
|
|
|
|
|
tailing(false),
|
|
|
|
|
managed(false),
|
2015-11-05 21:24:05 +00:00
|
|
|
total_order_seek(false),
|
2015-12-16 20:08:30 +00:00
|
|
|
prefix_same_as_start(false),
|
2016-05-04 22:25:58 +00:00
|
|
|
pin_data(false),
|
2016-06-22 01:41:23 +00:00
|
|
|
background_purge_on_iterator_cleanup(false),
|
2016-05-04 22:25:58 +00:00
|
|
|
readahead_size(0) {
|
2015-02-18 19:49:31 +00:00
|
|
|
XFUNC_TEST("", "managed_options", managed_options, xf_manage_options,
|
|
|
|
|
reinterpret_cast<ReadOptions*>(this));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ReadOptions::ReadOptions(bool cksum, bool cache)
|
|
|
|
|
: verify_checksums(cksum),
|
|
|
|
|
fill_cache(cache),
|
|
|
|
|
snapshot(nullptr),
|
|
|
|
|
iterate_upper_bound(nullptr),
|
|
|
|
|
read_tier(kReadAllTier),
|
|
|
|
|
tailing(false),
|
|
|
|
|
managed(false),
|
2015-11-05 21:24:05 +00:00
|
|
|
total_order_seek(false),
|
2015-12-16 20:08:30 +00:00
|
|
|
prefix_same_as_start(false),
|
2016-05-04 22:25:58 +00:00
|
|
|
pin_data(false),
|
2016-06-22 01:41:23 +00:00
|
|
|
background_purge_on_iterator_cleanup(false),
|
2016-05-04 22:25:58 +00:00
|
|
|
readahead_size(0) {
|
2015-02-18 19:49:31 +00:00
|
|
|
XFUNC_TEST("", "managed_options", managed_options, xf_manage_options,
|
|
|
|
|
reinterpret_cast<ReadOptions*>(this));
|
|
|
|
|
}
|
|
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
} // namespace rocksdb
|