2013-10-16 21:59:46 +00:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
//
|
2011-03-18 22:37:00 +00:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/options.h"
|
2014-09-04 23:18:36 +00:00
|
|
|
#include "rocksdb/immutable_options.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-05 06:14:37 +00:00
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-04-30 18:33:40 +00:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 06:14:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
2014-04-30 18:33:40 +00:00
|
|
|
#include <inttypes.h>
|
2013-01-11 01:18:50 +00:00
|
|
|
#include <limits>
|
|
|
|
|
|
2014-12-02 20:09:20 +00:00
|
|
|
#include "db/writebuffer.h"
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
|
#include "rocksdb/comparator.h"
|
2015-08-05 03:45:27 +00:00
|
|
|
#include "rocksdb/delete_scheduler.h"
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/env.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/memtablerep.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/table.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/table_properties.h"
|
2013-10-29 00:54:09 +00:00
|
|
|
#include "table/block_based_table_factory.h"
|
2015-04-06 19:50:44 +00:00
|
|
|
#include "util/compression.h"
|
2014-07-28 19:05:36 +00:00
|
|
|
#include "util/statistics.h"
|
2015-02-18 19:49:31 +00:00
|
|
|
#include "util/xfunc.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
namespace rocksdb {
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-04 23:18:36 +00:00
|
|
|
ImmutableCFOptions::ImmutableCFOptions(const Options& options)
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
: compaction_style(options.compaction_style),
|
|
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
|
|
|
|
compaction_options_fifo(options.compaction_options_fifo),
|
|
|
|
|
prefix_extractor(options.prefix_extractor.get()),
|
|
|
|
|
comparator(options.comparator),
|
|
|
|
|
merge_operator(options.merge_operator.get()),
|
|
|
|
|
compaction_filter(options.compaction_filter),
|
|
|
|
|
compaction_filter_factory(options.compaction_filter_factory.get()),
|
|
|
|
|
inplace_update_support(options.inplace_update_support),
|
|
|
|
|
inplace_callback(options.inplace_callback),
|
|
|
|
|
info_log(options.info_log.get()),
|
|
|
|
|
statistics(options.statistics.get()),
|
|
|
|
|
env(options.env),
|
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
|
|
|
|
db_paths(options.db_paths),
|
|
|
|
|
memtable_factory(options.memtable_factory.get()),
|
|
|
|
|
table_factory(options.table_factory.get()),
|
|
|
|
|
table_properties_collector_factories(
|
|
|
|
|
options.table_properties_collector_factories),
|
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
|
|
|
|
bloom_locality(options.bloom_locality),
|
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands),
|
|
|
|
|
disable_data_sync(options.disableDataSync),
|
|
|
|
|
use_fsync(options.use_fsync),
|
|
|
|
|
compression(options.compression),
|
|
|
|
|
compression_per_level(options.compression_per_level),
|
|
|
|
|
compression_opts(options.compression_opts),
|
|
|
|
|
level_compaction_dynamic_level_bytes(
|
|
|
|
|
options.level_compaction_dynamic_level_bytes),
|
|
|
|
|
access_hint_on_compaction_start(options.access_hint_on_compaction_start),
|
|
|
|
|
num_levels(options.num_levels),
|
2015-05-29 20:17:49 +00:00
|
|
|
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
2015-06-23 17:25:45 +00:00
|
|
|
listeners(options.listeners),
|
|
|
|
|
row_cache(options.row_cache) {}
|
2014-09-04 23:18:36 +00:00
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions()
|
2011-03-18 22:37:00 +00:00
|
|
|
: comparator(BytewiseComparator()),
|
2013-03-21 22:59:47 +00:00
|
|
|
merge_operator(nullptr),
|
2013-05-12 09:36:59 +00:00
|
|
|
compaction_filter(nullptr),
|
2015-06-08 23:34:26 +00:00
|
|
|
compaction_filter_factory(nullptr),
|
2014-03-25 00:57:13 +00:00
|
|
|
write_buffer_size(4 << 20),
|
2012-10-19 21:00:53 +00:00
|
|
|
max_write_buffer_number(2),
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge(1),
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain(0),
|
2015-06-18 21:55:05 +00:00
|
|
|
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
|
2013-08-13 21:04:56 +00:00
|
|
|
prefix_extractor(nullptr),
|
2012-06-23 02:30:03 +00:00
|
|
|
num_levels(7),
|
|
|
|
|
level0_file_num_compaction_trigger(4),
|
2014-03-28 23:57:04 +00:00
|
|
|
level0_slowdown_writes_trigger(20),
|
|
|
|
|
level0_stop_writes_trigger(24),
|
2012-06-23 02:30:03 +00:00
|
|
|
target_file_size_base(2 * 1048576),
|
2012-08-22 00:33:46 +00:00
|
|
|
target_file_size_multiplier(1),
|
2012-07-03 00:30:32 +00:00
|
|
|
max_bytes_for_level_base(10 * 1048576),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes(false),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_bytes_for_level_multiplier(10),
|
2013-05-21 18:37:06 +00:00
|
|
|
max_bytes_for_level_multiplier_additional(num_levels, 1),
|
2012-06-23 02:30:03 +00:00
|
|
|
expanded_compaction_factor(25),
|
2012-11-21 07:07:41 +00:00
|
|
|
source_compaction_factor(1),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_grandparent_overlap_factor(10),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
soft_rate_limit(0.0),
|
|
|
|
|
hard_rate_limit(0.0),
|
|
|
|
|
rate_limit_delay_max_milliseconds(1000),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(0),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
disable_auto_compactions(false),
|
|
|
|
|
purge_redundant_kvs_while_flush(true),
|
|
|
|
|
compaction_style(kCompactionStyleLevel),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
filter_deletes(false),
|
|
|
|
|
max_sequential_skip_in_iterations(8),
|
|
|
|
|
memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)),
|
|
|
|
|
table_factory(
|
2014-03-25 18:09:40 +00:00
|
|
|
std::shared_ptr<TableFactory>(new BlockBasedTableFactory())),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
inplace_update_support(false),
|
2014-01-13 21:27:43 +00:00
|
|
|
inplace_update_num_locks(10000),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(nullptr),
|
|
|
|
|
memtable_prefix_bloom_bits(0),
|
|
|
|
|
memtable_prefix_bloom_probes(6),
|
2014-05-04 20:55:53 +00:00
|
|
|
memtable_prefix_bloom_huge_page_tlb_size(0),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(0),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(0),
|
2015-02-17 16:03:45 +00:00
|
|
|
min_partial_merge_operands(2),
|
2015-04-17 22:26:50 +00:00
|
|
|
optimize_filters_for_hits(false),
|
2015-05-28 20:21:39 +00:00
|
|
|
paranoid_file_checks(false) {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
|
|
|
|
|
: comparator(options.comparator),
|
|
|
|
|
merge_operator(options.merge_operator),
|
|
|
|
|
compaction_filter(options.compaction_filter),
|
|
|
|
|
compaction_filter_factory(options.compaction_filter_factory),
|
|
|
|
|
write_buffer_size(options.write_buffer_size),
|
|
|
|
|
max_write_buffer_number(options.max_write_buffer_number),
|
|
|
|
|
min_write_buffer_number_to_merge(
|
|
|
|
|
options.min_write_buffer_number_to_merge),
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain(
|
|
|
|
|
options.max_write_buffer_number_to_maintain),
|
2014-01-06 21:31:06 +00:00
|
|
|
compression(options.compression),
|
|
|
|
|
compression_per_level(options.compression_per_level),
|
|
|
|
|
compression_opts(options.compression_opts),
|
|
|
|
|
prefix_extractor(options.prefix_extractor),
|
|
|
|
|
num_levels(options.num_levels),
|
|
|
|
|
level0_file_num_compaction_trigger(
|
|
|
|
|
options.level0_file_num_compaction_trigger),
|
|
|
|
|
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
|
|
|
|
|
level0_stop_writes_trigger(options.level0_stop_writes_trigger),
|
|
|
|
|
target_file_size_base(options.target_file_size_base),
|
|
|
|
|
target_file_size_multiplier(options.target_file_size_multiplier),
|
|
|
|
|
max_bytes_for_level_base(options.max_bytes_for_level_base),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes(
|
|
|
|
|
options.level_compaction_dynamic_level_bytes),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
|
|
|
|
|
max_bytes_for_level_multiplier_additional(
|
|
|
|
|
options.max_bytes_for_level_multiplier_additional),
|
|
|
|
|
expanded_compaction_factor(options.expanded_compaction_factor),
|
|
|
|
|
source_compaction_factor(options.source_compaction_factor),
|
|
|
|
|
max_grandparent_overlap_factor(options.max_grandparent_overlap_factor),
|
|
|
|
|
soft_rate_limit(options.soft_rate_limit),
|
|
|
|
|
hard_rate_limit(options.hard_rate_limit),
|
|
|
|
|
rate_limit_delay_max_milliseconds(
|
|
|
|
|
options.rate_limit_delay_max_milliseconds),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(options.arena_block_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
disable_auto_compactions(options.disable_auto_compactions),
|
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
|
compaction_style(options.compaction_style),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
|
2014-01-06 21:31:06 +00:00
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo(options.compaction_options_fifo),
|
2014-01-06 21:31:06 +00:00
|
|
|
filter_deletes(options.filter_deletes),
|
|
|
|
|
max_sequential_skip_in_iterations(
|
|
|
|
|
options.max_sequential_skip_in_iterations),
|
|
|
|
|
memtable_factory(options.memtable_factory),
|
|
|
|
|
table_factory(options.table_factory),
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
table_properties_collector_factories(
|
|
|
|
|
options.table_properties_collector_factories),
|
2014-01-06 21:31:06 +00:00
|
|
|
inplace_update_support(options.inplace_update_support),
|
2014-01-13 17:06:44 +00:00
|
|
|
inplace_update_num_locks(options.inplace_update_num_locks),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(options.inplace_callback),
|
|
|
|
|
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
|
|
|
|
|
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
|
2014-05-04 20:55:53 +00:00
|
|
|
memtable_prefix_bloom_huge_page_tlb_size(
|
|
|
|
|
options.memtable_prefix_bloom_huge_page_tlb_size),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(options.bloom_locality),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(options.max_successive_merges),
|
2015-02-17 16:03:45 +00:00
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands),
|
2015-04-17 22:26:50 +00:00
|
|
|
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
2015-05-28 20:21:39 +00:00
|
|
|
paranoid_file_checks(options.paranoid_file_checks) {
|
2014-01-06 21:31:06 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
2014-06-25 21:31:30 +00:00
|
|
|
if (max_bytes_for_level_multiplier_additional.size() <
|
|
|
|
|
static_cast<unsigned int>(num_levels)) {
|
2014-06-25 20:11:12 +00:00
|
|
|
max_bytes_for_level_multiplier_additional.resize(num_levels, 1);
|
|
|
|
|
}
|
2014-01-06 21:31:06 +00:00
|
|
|
}
|
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
DBOptions::DBOptions()
|
|
|
|
|
: create_if_missing(false),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(false),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
error_if_exists(false),
|
2014-03-31 19:44:54 +00:00
|
|
|
paranoid_checks(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
env(Env::Default()),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(nullptr),
|
2015-08-05 03:45:27 +00:00
|
|
|
delete_scheduler(nullptr),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(nullptr),
|
2015-02-06 16:44:30 +00:00
|
|
|
#ifdef NDEBUG
|
2014-04-10 22:27:42 +00:00
|
|
|
info_log_level(INFO_LEVEL),
|
2015-02-06 16:44:30 +00:00
|
|
|
#else
|
|
|
|
|
info_log_level(DEBUG_LEVEL),
|
|
|
|
|
#endif // NDEBUG
|
2014-03-31 19:44:54 +00:00
|
|
|
max_open_files(5000),
|
2015-08-11 19:19:56 +00:00
|
|
|
max_file_opening_threads(1),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(0),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(nullptr),
|
2012-08-14 22:20:36 +00:00
|
|
|
disableDataSync(false),
|
2012-08-27 19:10:26 +00:00
|
|
|
use_fsync(false),
|
2012-09-17 20:35:57 +00:00
|
|
|
db_log_dir(""),
|
2013-10-01 21:46:52 +00:00
|
|
|
wal_dir(""),
|
2013-11-08 23:23:46 +00:00
|
|
|
delete_obsolete_files_period_micros(6 * 60 * 60 * 1000000UL),
|
2012-11-06 00:51:55 +00:00
|
|
|
max_background_compactions(1),
|
Parallelize L0-L1 Compaction: Restructure Compaction Job
Summary:
As of now compactions involving files from Level 0 and Level 1 are single
threaded because the files in L0, although sorted, are not range partitioned like
the other levels. This means that during L0-L1 compaction each file from L1
needs to be merged with potentially all the files from L0.
This attempt to parallelize the L0-L1 compaction assigns a thread and a
corresponding iterator to each L1 file that then considers only the key range
found in that L1 file and only the L0 files that have those keys (and only the
specific portion of those L0 files in which those keys are found). In this way
the overlap is minimized and potentially eliminated between different iterators
focusing on the same files.
The first step is to restructure the compaction logic to break L0-L1 compactions
into multiple, smaller, sequential compactions. Eventually each of these smaller
jobs will be run simultaneously. Areas to pay extra attention to are
# Correct aggregation of compaction job statistics across multiple threads
# Proper opening/closing of output files (make sure each thread's is unique)
# Keys that span multiple L1 files
# Skewed distributions of keys within L0 files
Test Plan: Make and run db_test (newer version has separate compaction tests) and compaction_job_stats_test
Reviewers: igor, noetzli, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D42699
2015-08-03 18:32:14 +00:00
|
|
|
num_subcompactions(1),
|
2014-02-06 20:59:16 +00:00
|
|
|
max_background_flushes(1),
|
2012-10-29 21:18:00 +00:00
|
|
|
max_log_file_size(0),
|
2013-02-05 03:42:40 +00:00
|
|
|
log_file_time_to_roll(0),
|
|
|
|
|
keep_log_file_num(1000),
|
2013-01-11 01:18:50 +00:00
|
|
|
max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(4),
|
2013-01-15 22:05:42 +00:00
|
|
|
WAL_ttl_seconds(0),
|
2013-11-07 02:46:28 +00:00
|
|
|
WAL_size_limit_MB(0),
|
2013-02-28 22:09:30 +00:00
|
|
|
manifest_preallocation_size(4 * 1024 * 1024),
|
2013-03-15 00:00:04 +00:00
|
|
|
allow_os_buffer(true),
|
|
|
|
|
allow_mmap_reads(false),
|
2014-03-28 23:57:04 +00:00
|
|
|
allow_mmap_writes(false),
|
2013-05-21 18:53:33 +00:00
|
|
|
is_fd_close_on_exec(true),
|
2013-05-10 22:21:04 +00:00
|
|
|
skip_log_error_on_recovery(false),
|
2015-05-21 18:22:16 +00:00
|
|
|
stats_dump_period_sec(600),
|
2013-05-17 22:53:01 +00:00
|
|
|
advise_random_on_open(true),
|
2014-12-02 20:09:20 +00:00
|
|
|
db_write_buffer_size(0),
|
2013-05-31 23:30:17 +00:00
|
|
|
access_hint_on_compaction_start(NORMAL),
|
2013-06-14 05:49:46 +00:00
|
|
|
use_adaptive_mutex(false),
|
2014-11-21 05:13:18 +00:00
|
|
|
bytes_per_sync(0),
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync(0),
|
2015-05-28 20:21:39 +00:00
|
|
|
listeners(),
|
2015-05-15 22:52:51 +00:00
|
|
|
enable_thread_tracking(false),
|
2015-06-15 19:03:13 +00:00
|
|
|
delayed_write_rate(1024U * 1024U),
|
2015-08-04 20:48:16 +00:00
|
|
|
skip_stats_update_on_db_open(false),
|
2015-06-15 19:03:13 +00:00
|
|
|
wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
2015-02-17 16:03:45 +00:00
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
DBOptions::DBOptions(const Options& options)
|
|
|
|
|
: create_if_missing(options.create_if_missing),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(options.create_missing_column_families),
|
2014-01-06 21:31:06 +00:00
|
|
|
error_if_exists(options.error_if_exists),
|
|
|
|
|
paranoid_checks(options.paranoid_checks),
|
|
|
|
|
env(options.env),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(options.rate_limiter),
|
2015-08-05 03:45:27 +00:00
|
|
|
delete_scheduler(options.delete_scheduler),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(options.info_log),
|
2014-03-06 00:55:51 +00:00
|
|
|
info_log_level(options.info_log_level),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_open_files(options.max_open_files),
|
2015-08-11 19:19:56 +00:00
|
|
|
max_file_opening_threads(options.max_file_opening_threads),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(options.max_total_wal_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(options.statistics),
|
2014-01-06 21:31:06 +00:00
|
|
|
disableDataSync(options.disableDataSync),
|
|
|
|
|
use_fsync(options.use_fsync),
|
2014-07-02 16:54:20 +00:00
|
|
|
db_paths(options.db_paths),
|
2014-01-06 21:31:06 +00:00
|
|
|
db_log_dir(options.db_log_dir),
|
|
|
|
|
wal_dir(options.wal_dir),
|
|
|
|
|
delete_obsolete_files_period_micros(
|
|
|
|
|
options.delete_obsolete_files_period_micros),
|
|
|
|
|
max_background_compactions(options.max_background_compactions),
|
Parallelize L0-L1 Compaction: Restructure Compaction Job
Summary:
As of now compactions involving files from Level 0 and Level 1 are single
threaded because the files in L0, although sorted, are not range partitioned like
the other levels. This means that during L0-L1 compaction each file from L1
needs to be merged with potentially all the files from L0.
This attempt to parallelize the L0-L1 compaction assigns a thread and a
corresponding iterator to each L1 file that then considers only the key range
found in that L1 file and only the L0 files that have those keys (and only the
specific portion of those L0 files in which those keys are found). In this way
the overlap is minimized and potentially eliminated between different iterators
focusing on the same files.
The first step is to restructure the compaction logic to break L0-L1 compactions
into multiple, smaller, sequential compactions. Eventually each of these smaller
jobs will be run simultaneously. Areas to pay extra attention to are
# Correct aggregation of compaction job statistics across multiple threads
# Proper opening/closing of output files (make sure each thread's is unique)
# Keys that span multiple L1 files
# Skewed distributions of keys within L0 files
Test Plan: Make and run db_test (newer version has separate compaction tests) and compaction_job_stats_test
Reviewers: igor, noetzli, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D42699
2015-08-03 18:32:14 +00:00
|
|
|
num_subcompactions(options.num_subcompactions),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_background_flushes(options.max_background_flushes),
|
|
|
|
|
max_log_file_size(options.max_log_file_size),
|
|
|
|
|
log_file_time_to_roll(options.log_file_time_to_roll),
|
|
|
|
|
keep_log_file_num(options.keep_log_file_num),
|
|
|
|
|
max_manifest_file_size(options.max_manifest_file_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(options.table_cache_numshardbits),
|
2014-01-06 21:31:06 +00:00
|
|
|
WAL_ttl_seconds(options.WAL_ttl_seconds),
|
|
|
|
|
WAL_size_limit_MB(options.WAL_size_limit_MB),
|
|
|
|
|
manifest_preallocation_size(options.manifest_preallocation_size),
|
|
|
|
|
allow_os_buffer(options.allow_os_buffer),
|
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
|
|
|
|
is_fd_close_on_exec(options.is_fd_close_on_exec),
|
|
|
|
|
skip_log_error_on_recovery(options.skip_log_error_on_recovery),
|
|
|
|
|
stats_dump_period_sec(options.stats_dump_period_sec),
|
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
2014-12-02 20:09:20 +00:00
|
|
|
db_write_buffer_size(options.db_write_buffer_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
access_hint_on_compaction_start(options.access_hint_on_compaction_start),
|
|
|
|
|
use_adaptive_mutex(options.use_adaptive_mutex),
|
2014-11-21 05:13:18 +00:00
|
|
|
bytes_per_sync(options.bytes_per_sync),
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync(options.wal_bytes_per_sync),
|
2015-05-28 20:21:39 +00:00
|
|
|
listeners(options.listeners),
|
2015-05-15 22:52:51 +00:00
|
|
|
enable_thread_tracking(options.enable_thread_tracking),
|
2015-06-15 19:03:13 +00:00
|
|
|
delayed_write_rate(options.delayed_write_rate),
|
2015-08-04 20:48:16 +00:00
|
|
|
skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
|
2015-06-23 17:25:45 +00:00
|
|
|
wal_recovery_mode(options.wal_recovery_mode),
|
|
|
|
|
row_cache(options.row_cache) {}
|
2014-01-06 21:31:06 +00:00
|
|
|
|
2013-05-17 22:53:01 +00:00
|
|
|
static const char* const access_hints[] = {
|
|
|
|
|
"NONE", "NORMAL", "SEQUENTIAL", "WILLNEED"
|
|
|
|
|
};
|
|
|
|
|
|
2014-02-07 05:39:20 +00:00
|
|
|
void DBOptions::Dump(Logger* log) const {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.error_if_exists: %d", error_if_exists);
|
|
|
|
|
Warn(log, " Options.create_if_missing: %d", create_if_missing);
|
|
|
|
|
Warn(log, " Options.paranoid_checks: %d", paranoid_checks);
|
|
|
|
|
Warn(log, " Options.env: %p", env);
|
|
|
|
|
Warn(log, " Options.info_log: %p", info_log.get());
|
|
|
|
|
Warn(log, " Options.max_open_files: %d", max_open_files);
|
2015-08-11 19:19:56 +00:00
|
|
|
Warn(log, "Options.max_file_opening_threads: %d", max_file_opening_threads);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_total_wal_size: %" PRIu64, max_total_wal_size);
|
|
|
|
|
Warn(log, " Options.disableDataSync: %d", disableDataSync);
|
|
|
|
|
Warn(log, " Options.use_fsync: %d", use_fsync);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log, " Options.max_log_file_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_log_file_size);
|
|
|
|
|
Warn(log, "Options.max_manifest_file_size: %" PRIu64,
|
|
|
|
|
max_manifest_file_size);
|
|
|
|
|
Warn(log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt,
|
|
|
|
|
log_file_time_to_roll);
|
|
|
|
|
Warn(log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt,
|
|
|
|
|
keep_log_file_num);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_os_buffer: %d", allow_os_buffer);
|
|
|
|
|
Warn(log, " Options.allow_mmap_reads: %d", allow_mmap_reads);
|
|
|
|
|
Warn(log, " Options.allow_mmap_writes: %d", allow_mmap_writes);
|
|
|
|
|
Warn(log, " Options.create_missing_column_families: %d",
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.db_log_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
db_log_dir.c_str());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.wal_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
wal_dir.c_str());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.table_cache_numshardbits: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
table_cache_numshardbits);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
delete_obsolete_files_period_micros);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_background_compactions: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
max_background_compactions);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_background_flushes: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
max_background_flushes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.WAL_ttl_seconds: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
WAL_ttl_seconds);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.WAL_size_limit_MB: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
WAL_size_limit_MB);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.manifest_preallocation_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
manifest_preallocation_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_os_buffer: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_os_buffer);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_mmap_reads: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_mmap_reads);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_mmap_writes: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_mmap_writes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.is_fd_close_on_exec: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
is_fd_close_on_exec);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.stats_dump_period_sec: %u",
|
2014-02-07 05:39:20 +00:00
|
|
|
stats_dump_period_sec);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.advise_random_on_open: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
advise_random_on_open);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.db_write_buffer_size: %" ROCKSDB_PRIszt
|
|
|
|
|
"d",
|
|
|
|
|
db_write_buffer_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.access_hint_on_compaction_start: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
access_hints[access_hint_on_compaction_start]);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.use_adaptive_mutex: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
use_adaptive_mutex);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.rate_limiter: %p",
|
2014-08-12 23:42:18 +00:00
|
|
|
rate_limiter.get());
|
2015-08-05 03:45:27 +00:00
|
|
|
Warn(log, " Options.delete_scheduler.rate_bytes_per_sec: %" PRIi64,
|
|
|
|
|
delete_scheduler ? delete_scheduler->GetRateBytesPerSecond() : 0);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.bytes_per_sync: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
bytes_per_sync);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.wal_bytes_per_sync: %" PRIu64,
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync);
|
2015-08-10 16:21:04 +00:00
|
|
|
Warn(log, " Options.wal_recovery_mode: %d",
|
|
|
|
|
wal_recovery_mode);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.enable_thread_tracking: %d",
|
2014-11-21 05:13:18 +00:00
|
|
|
enable_thread_tracking);
|
2015-06-23 17:25:45 +00:00
|
|
|
if (row_cache) {
|
|
|
|
|
Warn(log, " Options.row_cache: %" PRIu64,
|
|
|
|
|
row_cache->GetCapacity());
|
|
|
|
|
} else {
|
|
|
|
|
Warn(log, " Options.row_cache: None");
|
|
|
|
|
}
|
2014-02-07 05:39:20 +00:00
|
|
|
} // DBOptions::Dump
|
|
|
|
|
|
|
|
|
|
void ColumnFamilyOptions::Dump(Logger* log) const {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.comparator: %s", comparator->Name());
|
|
|
|
|
Warn(log, " Options.merge_operator: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
merge_operator ? merge_operator->Name() : "None");
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_filter: %s",
|
2014-08-13 22:58:31 +00:00
|
|
|
compaction_filter ? compaction_filter->Name() : "None");
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_filter_factory: %s",
|
2015-06-08 23:34:26 +00:00
|
|
|
compaction_filter_factory ? compaction_filter_factory->Name() : "None");
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.memtable_factory: %s", memtable_factory->Name());
|
|
|
|
|
Warn(log, " Options.table_factory: %s", table_factory->Name());
|
|
|
|
|
Warn(log, " table_factory options: %s",
|
2014-08-25 21:24:09 +00:00
|
|
|
table_factory->GetPrintableTableOptions().c_str());
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
write_buffer_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_write_buffer_number: %d", max_write_buffer_number);
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!compression_per_level.empty()) {
|
2013-03-15 01:32:01 +00:00
|
|
|
for (unsigned int i = 0; i < compression_per_level.size(); i++) {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression[%d]: %s", i,
|
2015-06-18 21:55:05 +00:00
|
|
|
CompressionTypeToString(compression_per_level[i]).c_str());
|
2015-04-06 19:50:44 +00:00
|
|
|
}
|
2012-10-28 06:13:17 +00:00
|
|
|
} else {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression: %s",
|
2015-06-18 21:55:05 +00:00
|
|
|
CompressionTypeToString(compression).c_str());
|
2012-10-28 06:13:17 +00:00
|
|
|
}
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.prefix_extractor: %s",
|
2013-08-14 16:06:10 +00:00
|
|
|
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.num_levels: %d", num_levels);
|
|
|
|
|
Warn(log, " Options.min_write_buffer_number_to_merge: %d",
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge);
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
Warn(log, " Options.max_write_buffer_number_to_maintain: %d",
|
|
|
|
|
max_write_buffer_number_to_maintain);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression_opts.window_bits: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.window_bits);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression_opts.level: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.level);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression_opts.strategy: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.strategy);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.level0_file_num_compaction_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_file_num_compaction_trigger);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.level0_slowdown_writes_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_slowdown_writes_trigger);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.level0_stop_writes_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_stop_writes_trigger);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.target_file_size_base: %" PRIu64,
|
2012-08-27 19:10:26 +00:00
|
|
|
target_file_size_base);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.target_file_size_multiplier: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
target_file_size_multiplier);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_bytes_for_level_base: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
max_bytes_for_level_base);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.level_compaction_dynamic_level_bytes: %d",
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_bytes_for_level_multiplier: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
max_bytes_for_level_multiplier);
|
2015-03-30 21:04:21 +00:00
|
|
|
for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
|
|
|
|
|
i++) {
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log, "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt
|
|
|
|
|
"]: %d",
|
|
|
|
|
i, max_bytes_for_level_multiplier_additional[i]);
|
2013-05-21 18:37:06 +00:00
|
|
|
}
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_sequential_skip_in_iterations: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
max_sequential_skip_in_iterations);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.expanded_compaction_factor: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
expanded_compaction_factor);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.source_compaction_factor: %d",
|
2012-11-21 07:07:41 +00:00
|
|
|
source_compaction_factor);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_grandparent_overlap_factor: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
max_grandparent_overlap_factor);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.arena_block_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
arena_block_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.soft_rate_limit: %.2f",
|
2013-08-14 16:06:10 +00:00
|
|
|
soft_rate_limit);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.hard_rate_limit: %.2f",
|
2013-08-05 22:43:49 +00:00
|
|
|
hard_rate_limit);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.rate_limit_delay_max_milliseconds: %u",
|
2013-08-05 22:43:49 +00:00
|
|
|
rate_limit_delay_max_milliseconds);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.disable_auto_compactions: %d",
|
2012-11-20 23:45:41 +00:00
|
|
|
disable_auto_compactions);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.filter_deletes: %d",
|
2013-07-12 23:56:52 +00:00
|
|
|
filter_deletes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.verify_checksums_in_compaction: %d",
|
2014-03-10 17:06:34 +00:00
|
|
|
verify_checksums_in_compaction);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_style: %d",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_style);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_options_universal.size_ratio: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.size_ratio);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_universal.min_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.min_merge_width);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_universal.max_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.max_merge_width);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_universal."
|
2013-09-09 23:06:10 +00:00
|
|
|
"max_size_amplification_percent: %u",
|
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log,
|
2014-09-19 20:09:25 +00:00
|
|
|
"Options.compaction_options_universal.compression_size_percent: %d",
|
2013-12-03 20:32:07 +00:00
|
|
|
compaction_options_universal.compression_size_percent);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64,
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo.max_table_files_size);
|
2013-10-16 18:50:50 +00:00
|
|
|
std::string collector_names;
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
for (const auto& collector_factory : table_properties_collector_factories) {
|
|
|
|
|
collector_names.append(collector_factory->Name());
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.append("; ");
|
|
|
|
|
}
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.table_properties_collectors: %s",
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.c_str());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.inplace_update_support: %d",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
inplace_update_support);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
|
|
|
|
inplace_update_num_locks);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.min_partial_merge_operands: %u",
|
2014-03-25 00:57:13 +00:00
|
|
|
min_partial_merge_operands);
|
2013-11-27 22:27:02 +00:00
|
|
|
// TODO: easier config for bloom (maybe based on avg key/value size)
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.memtable_prefix_bloom_bits: %d",
|
2013-11-27 22:27:02 +00:00
|
|
|
memtable_prefix_bloom_bits);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.memtable_prefix_bloom_probes: %d",
|
2013-11-27 22:27:02 +00:00
|
|
|
memtable_prefix_bloom_probes);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
memtable_prefix_bloom_huge_page_tlb_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.bloom_locality: %d",
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.max_successive_merges: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_successive_merges);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.optimize_fllters_for_hits: %d",
|
2015-02-17 16:03:45 +00:00
|
|
|
optimize_filters_for_hits);
|
2014-02-07 05:39:20 +00:00
|
|
|
} // ColumnFamilyOptions::Dump
|
|
|
|
|
|
|
|
|
|
void Options::Dump(Logger* log) const {
|
|
|
|
|
DBOptions::Dump(log);
|
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
2012-08-22 18:43:53 +00:00
|
|
|
} // Options::Dump
|
|
|
|
|
|
2015-06-18 17:15:54 +00:00
|
|
|
void Options::DumpCFOptions(Logger* log) const {
|
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
|
|
|
|
} // Options::DumpCFOptions
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
//
|
|
|
|
|
// The goal of this method is to create a configuration that
|
|
|
|
|
// allows an application to write all files into L0 and
|
|
|
|
|
// then do a single compaction to output all files into L1.
|
2013-02-26 06:57:37 +00:00
|
|
|
Options*
|
|
|
|
|
Options::PrepareForBulkLoad()
|
|
|
|
|
{
|
2013-03-08 17:19:24 +00:00
|
|
|
// never slowdown ingest.
|
2013-02-26 06:57:37 +00:00
|
|
|
level0_file_num_compaction_trigger = (1<<30);
|
|
|
|
|
level0_slowdown_writes_trigger = (1<<30);
|
|
|
|
|
level0_stop_writes_trigger = (1<<30);
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
|
// no auto compactions please. The application should issue a
|
|
|
|
|
// manual compaction after all data is loaded into L0.
|
2013-02-26 06:57:37 +00:00
|
|
|
disable_auto_compactions = true;
|
|
|
|
|
disableDataSync = true;
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
|
// A manual compaction run should pick all files in L0 in
|
|
|
|
|
// a single compaction run.
|
2013-02-26 06:57:37 +00:00
|
|
|
source_compaction_factor = (1<<30);
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// It is better to have only 2 levels, otherwise a manual
|
|
|
|
|
// compaction would compact at every possible level, thereby
|
|
|
|
|
// increasing the total time needed for compactions.
|
|
|
|
|
num_levels = 2;
|
|
|
|
|
|
2015-02-02 19:09:21 +00:00
|
|
|
// Need to allow more write buffers to allow more parallism
|
|
|
|
|
// of flushes.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
min_write_buffer_number_to_merge = 1;
|
|
|
|
|
|
|
|
|
|
// When compaction is disabled, more parallel flush threads can
|
|
|
|
|
// help with write throughput.
|
|
|
|
|
max_background_flushes = 4;
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// Prevent a memtable flush to automatically promote files
|
|
|
|
|
// to L1. This is helpful so that all files that are
|
|
|
|
|
// input to the manual compaction are all at L0.
|
|
|
|
|
max_background_compactions = 2;
|
|
|
|
|
|
|
|
|
|
// The compaction would create large files in L1.
|
|
|
|
|
target_file_size_base = 256 * 1024 * 1024;
|
2013-02-26 06:57:37 +00:00
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
2014-11-13 19:39:30 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2014-05-10 17:49:33 +00:00
|
|
|
// Optimization functions
|
2014-08-26 21:15:00 +00:00
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
|
|
|
|
|
uint64_t block_cache_size_mb) {
|
2014-05-10 17:49:33 +00:00
|
|
|
prefix_extractor.reset(NewNoopTransform());
|
|
|
|
|
BlockBasedTableOptions block_based_options;
|
2014-08-26 21:15:00 +00:00
|
|
|
block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
|
block_based_options.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
|
|
|
block_based_options.block_cache =
|
2014-11-13 19:39:30 +00:00
|
|
|
NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024));
|
2014-05-10 17:49:33 +00:00
|
|
|
table_factory.reset(new BlockBasedTableFactory(block_based_options));
|
|
|
|
|
memtable_factory.reset(NewHashLinkListRepFactory());
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction(
|
|
|
|
|
uint64_t memtable_memory_budget) {
|
2014-11-13 19:39:30 +00:00
|
|
|
write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
|
2014-05-10 17:49:33 +00:00
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
|
// write stalls.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
// start flushing L0->L1 as soon as possible. each file on level0 is
|
|
|
|
|
// (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
|
|
|
|
|
// memtable_memory_budget.
|
|
|
|
|
level0_file_num_compaction_trigger = 2;
|
|
|
|
|
// doesn't really matter much, but we don't want to create too many files
|
|
|
|
|
target_file_size_base = memtable_memory_budget / 8;
|
|
|
|
|
// make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
|
|
|
|
|
max_bytes_for_level_base = memtable_memory_budget;
|
|
|
|
|
|
|
|
|
|
// level style compaction
|
|
|
|
|
compaction_style = kCompactionStyleLevel;
|
|
|
|
|
|
|
|
|
|
// only compress levels >= 2
|
|
|
|
|
compression_per_level.resize(num_levels);
|
|
|
|
|
for (int i = 0; i < num_levels; ++i) {
|
|
|
|
|
if (i < 2) {
|
|
|
|
|
compression_per_level[i] = kNoCompression;
|
|
|
|
|
} else {
|
|
|
|
|
compression_per_level[i] = kSnappyCompression;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction(
|
|
|
|
|
uint64_t memtable_memory_budget) {
|
2014-11-13 19:39:30 +00:00
|
|
|
write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
|
2014-05-10 17:49:33 +00:00
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
|
// write stalls.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
// universal style compaction
|
|
|
|
|
compaction_style = kCompactionStyleUniversal;
|
|
|
|
|
compaction_options_universal.compression_size_percent = 80;
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
|
|
|
|
|
max_background_compactions = total_threads - 1;
|
|
|
|
|
max_background_flushes = 1;
|
|
|
|
|
env->SetBackgroundThreads(total_threads, Env::LOW);
|
|
|
|
|
env->SetBackgroundThreads(1, Env::HIGH);
|
|
|
|
|
return this;
|
|
|
|
|
}
|
2015-02-18 19:49:31 +00:00
|
|
|
|
2015-04-10 04:05:09 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
2015-02-18 19:49:31 +00:00
|
|
|
ReadOptions::ReadOptions()
|
|
|
|
|
: verify_checksums(true),
|
|
|
|
|
fill_cache(true),
|
|
|
|
|
snapshot(nullptr),
|
|
|
|
|
iterate_upper_bound(nullptr),
|
|
|
|
|
read_tier(kReadAllTier),
|
|
|
|
|
tailing(false),
|
|
|
|
|
managed(false),
|
|
|
|
|
total_order_seek(false) {
|
|
|
|
|
XFUNC_TEST("", "managed_options", managed_options, xf_manage_options,
|
|
|
|
|
reinterpret_cast<ReadOptions*>(this));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ReadOptions::ReadOptions(bool cksum, bool cache)
|
|
|
|
|
: verify_checksums(cksum),
|
|
|
|
|
fill_cache(cache),
|
|
|
|
|
snapshot(nullptr),
|
|
|
|
|
iterate_upper_bound(nullptr),
|
|
|
|
|
read_tier(kReadAllTier),
|
|
|
|
|
tailing(false),
|
|
|
|
|
managed(false),
|
|
|
|
|
total_order_seek(false) {
|
|
|
|
|
XFUNC_TEST("", "managed_options", managed_options, xf_manage_options,
|
|
|
|
|
reinterpret_cast<ReadOptions*>(this));
|
|
|
|
|
}
|
|
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
} // namespace rocksdb
|