2013-10-16 21:59:46 +00:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
//
|
2011-03-18 22:37:00 +00:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/options.h"
|
2014-09-04 23:18:36 +00:00
|
|
|
#include "rocksdb/immutable_options.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-05 06:14:37 +00:00
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-04-30 18:33:40 +00:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 06:14:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
2014-04-30 18:33:40 +00:00
|
|
|
#include <inttypes.h>
|
2013-01-11 01:18:50 +00:00
|
|
|
#include <limits>
|
|
|
|
|
|
2014-12-02 20:09:20 +00:00
|
|
|
#include "db/writebuffer.h"
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
|
#include "rocksdb/comparator.h"
|
2015-08-05 03:45:27 +00:00
|
|
|
#include "rocksdb/delete_scheduler.h"
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/env.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/memtablerep.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/table.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/table_properties.h"
|
2013-10-29 00:54:09 +00:00
|
|
|
#include "table/block_based_table_factory.h"
|
2015-04-06 19:50:44 +00:00
|
|
|
#include "util/compression.h"
|
2014-07-28 19:05:36 +00:00
|
|
|
#include "util/statistics.h"
|
2015-02-18 19:49:31 +00:00
|
|
|
#include "util/xfunc.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
namespace rocksdb {
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-04 23:18:36 +00:00
|
|
|
ImmutableCFOptions::ImmutableCFOptions(const Options& options)
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
: compaction_style(options.compaction_style),
|
|
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
|
|
|
|
compaction_options_fifo(options.compaction_options_fifo),
|
|
|
|
|
prefix_extractor(options.prefix_extractor.get()),
|
|
|
|
|
comparator(options.comparator),
|
|
|
|
|
merge_operator(options.merge_operator.get()),
|
|
|
|
|
compaction_filter(options.compaction_filter),
|
|
|
|
|
compaction_filter_factory(options.compaction_filter_factory.get()),
|
|
|
|
|
inplace_update_support(options.inplace_update_support),
|
|
|
|
|
inplace_callback(options.inplace_callback),
|
|
|
|
|
info_log(options.info_log.get()),
|
|
|
|
|
statistics(options.statistics.get()),
|
|
|
|
|
env(options.env),
|
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
|
|
|
|
db_paths(options.db_paths),
|
|
|
|
|
memtable_factory(options.memtable_factory.get()),
|
|
|
|
|
table_factory(options.table_factory.get()),
|
|
|
|
|
table_properties_collector_factories(
|
|
|
|
|
options.table_properties_collector_factories),
|
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
|
|
|
|
bloom_locality(options.bloom_locality),
|
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands),
|
|
|
|
|
disable_data_sync(options.disableDataSync),
|
|
|
|
|
use_fsync(options.use_fsync),
|
|
|
|
|
compression(options.compression),
|
|
|
|
|
compression_per_level(options.compression_per_level),
|
|
|
|
|
compression_opts(options.compression_opts),
|
|
|
|
|
level_compaction_dynamic_level_bytes(
|
|
|
|
|
options.level_compaction_dynamic_level_bytes),
|
|
|
|
|
access_hint_on_compaction_start(options.access_hint_on_compaction_start),
|
2015-08-21 05:33:25 +00:00
|
|
|
new_table_reader_for_compaction_inputs(
|
|
|
|
|
options.new_table_reader_for_compaction_inputs),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
num_levels(options.num_levels),
|
2015-05-29 20:17:49 +00:00
|
|
|
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
2015-06-23 17:25:45 +00:00
|
|
|
listeners(options.listeners),
|
|
|
|
|
row_cache(options.row_cache) {}
|
2014-09-04 23:18:36 +00:00
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions()
|
2011-03-18 22:37:00 +00:00
|
|
|
: comparator(BytewiseComparator()),
|
2013-03-21 22:59:47 +00:00
|
|
|
merge_operator(nullptr),
|
2013-05-12 09:36:59 +00:00
|
|
|
compaction_filter(nullptr),
|
2015-06-08 23:34:26 +00:00
|
|
|
compaction_filter_factory(nullptr),
|
2014-03-25 00:57:13 +00:00
|
|
|
write_buffer_size(4 << 20),
|
2012-10-19 21:00:53 +00:00
|
|
|
max_write_buffer_number(2),
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge(1),
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain(0),
|
2015-06-18 21:55:05 +00:00
|
|
|
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
|
2013-08-13 21:04:56 +00:00
|
|
|
prefix_extractor(nullptr),
|
2012-06-23 02:30:03 +00:00
|
|
|
num_levels(7),
|
|
|
|
|
level0_file_num_compaction_trigger(4),
|
2014-03-28 23:57:04 +00:00
|
|
|
level0_slowdown_writes_trigger(20),
|
|
|
|
|
level0_stop_writes_trigger(24),
|
2012-06-23 02:30:03 +00:00
|
|
|
target_file_size_base(2 * 1048576),
|
2012-08-22 00:33:46 +00:00
|
|
|
target_file_size_multiplier(1),
|
2012-07-03 00:30:32 +00:00
|
|
|
max_bytes_for_level_base(10 * 1048576),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes(false),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_bytes_for_level_multiplier(10),
|
2013-05-21 18:37:06 +00:00
|
|
|
max_bytes_for_level_multiplier_additional(num_levels, 1),
|
2012-06-23 02:30:03 +00:00
|
|
|
expanded_compaction_factor(25),
|
2012-11-21 07:07:41 +00:00
|
|
|
source_compaction_factor(1),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_grandparent_overlap_factor(10),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
soft_rate_limit(0.0),
|
|
|
|
|
hard_rate_limit(0.0),
|
|
|
|
|
rate_limit_delay_max_milliseconds(1000),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(0),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
disable_auto_compactions(false),
|
|
|
|
|
purge_redundant_kvs_while_flush(true),
|
|
|
|
|
compaction_style(kCompactionStyleLevel),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
filter_deletes(false),
|
|
|
|
|
max_sequential_skip_in_iterations(8),
|
|
|
|
|
memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)),
|
|
|
|
|
table_factory(
|
2014-03-25 18:09:40 +00:00
|
|
|
std::shared_ptr<TableFactory>(new BlockBasedTableFactory())),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
inplace_update_support(false),
|
2014-01-13 21:27:43 +00:00
|
|
|
inplace_update_num_locks(10000),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(nullptr),
|
|
|
|
|
memtable_prefix_bloom_bits(0),
|
|
|
|
|
memtable_prefix_bloom_probes(6),
|
2014-05-04 20:55:53 +00:00
|
|
|
memtable_prefix_bloom_huge_page_tlb_size(0),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(0),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(0),
|
2015-02-17 16:03:45 +00:00
|
|
|
min_partial_merge_operands(2),
|
2015-04-17 22:26:50 +00:00
|
|
|
optimize_filters_for_hits(false),
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 00:24:45 +00:00
|
|
|
paranoid_file_checks(false),
|
|
|
|
|
compaction_measure_io_stats(false) {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
|
|
|
|
|
: comparator(options.comparator),
|
|
|
|
|
merge_operator(options.merge_operator),
|
|
|
|
|
compaction_filter(options.compaction_filter),
|
|
|
|
|
compaction_filter_factory(options.compaction_filter_factory),
|
|
|
|
|
write_buffer_size(options.write_buffer_size),
|
|
|
|
|
max_write_buffer_number(options.max_write_buffer_number),
|
|
|
|
|
min_write_buffer_number_to_merge(
|
|
|
|
|
options.min_write_buffer_number_to_merge),
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
max_write_buffer_number_to_maintain(
|
|
|
|
|
options.max_write_buffer_number_to_maintain),
|
2014-01-06 21:31:06 +00:00
|
|
|
compression(options.compression),
|
|
|
|
|
compression_per_level(options.compression_per_level),
|
|
|
|
|
compression_opts(options.compression_opts),
|
|
|
|
|
prefix_extractor(options.prefix_extractor),
|
|
|
|
|
num_levels(options.num_levels),
|
|
|
|
|
level0_file_num_compaction_trigger(
|
|
|
|
|
options.level0_file_num_compaction_trigger),
|
|
|
|
|
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
|
|
|
|
|
level0_stop_writes_trigger(options.level0_stop_writes_trigger),
|
|
|
|
|
target_file_size_base(options.target_file_size_base),
|
|
|
|
|
target_file_size_multiplier(options.target_file_size_multiplier),
|
|
|
|
|
max_bytes_for_level_base(options.max_bytes_for_level_base),
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes(
|
|
|
|
|
options.level_compaction_dynamic_level_bytes),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
|
|
|
|
|
max_bytes_for_level_multiplier_additional(
|
|
|
|
|
options.max_bytes_for_level_multiplier_additional),
|
|
|
|
|
expanded_compaction_factor(options.expanded_compaction_factor),
|
|
|
|
|
source_compaction_factor(options.source_compaction_factor),
|
|
|
|
|
max_grandparent_overlap_factor(options.max_grandparent_overlap_factor),
|
|
|
|
|
soft_rate_limit(options.soft_rate_limit),
|
|
|
|
|
hard_rate_limit(options.hard_rate_limit),
|
|
|
|
|
rate_limit_delay_max_milliseconds(
|
|
|
|
|
options.rate_limit_delay_max_milliseconds),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(options.arena_block_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
disable_auto_compactions(options.disable_auto_compactions),
|
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
|
compaction_style(options.compaction_style),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
|
2014-01-06 21:31:06 +00:00
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo(options.compaction_options_fifo),
|
2014-01-06 21:31:06 +00:00
|
|
|
filter_deletes(options.filter_deletes),
|
|
|
|
|
max_sequential_skip_in_iterations(
|
|
|
|
|
options.max_sequential_skip_in_iterations),
|
|
|
|
|
memtable_factory(options.memtable_factory),
|
|
|
|
|
table_factory(options.table_factory),
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
table_properties_collector_factories(
|
|
|
|
|
options.table_properties_collector_factories),
|
2014-01-06 21:31:06 +00:00
|
|
|
inplace_update_support(options.inplace_update_support),
|
2014-01-13 17:06:44 +00:00
|
|
|
inplace_update_num_locks(options.inplace_update_num_locks),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(options.inplace_callback),
|
|
|
|
|
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
|
|
|
|
|
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
|
2014-05-04 20:55:53 +00:00
|
|
|
memtable_prefix_bloom_huge_page_tlb_size(
|
|
|
|
|
options.memtable_prefix_bloom_huge_page_tlb_size),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(options.bloom_locality),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(options.max_successive_merges),
|
2015-02-17 16:03:45 +00:00
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands),
|
2015-04-17 22:26:50 +00:00
|
|
|
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 00:24:45 +00:00
|
|
|
paranoid_file_checks(options.paranoid_file_checks),
|
|
|
|
|
compaction_measure_io_stats(options.compaction_measure_io_stats) {
|
2014-01-06 21:31:06 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
2014-06-25 21:31:30 +00:00
|
|
|
if (max_bytes_for_level_multiplier_additional.size() <
|
|
|
|
|
static_cast<unsigned int>(num_levels)) {
|
2014-06-25 20:11:12 +00:00
|
|
|
max_bytes_for_level_multiplier_additional.resize(num_levels, 1);
|
|
|
|
|
}
|
2014-01-06 21:31:06 +00:00
|
|
|
}
|
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
DBOptions::DBOptions()
|
|
|
|
|
: create_if_missing(false),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(false),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
error_if_exists(false),
|
2014-03-31 19:44:54 +00:00
|
|
|
paranoid_checks(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
env(Env::Default()),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(nullptr),
|
2015-08-05 03:45:27 +00:00
|
|
|
delete_scheduler(nullptr),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(nullptr),
|
2015-02-06 16:44:30 +00:00
|
|
|
#ifdef NDEBUG
|
2014-04-10 22:27:42 +00:00
|
|
|
info_log_level(INFO_LEVEL),
|
2015-02-06 16:44:30 +00:00
|
|
|
#else
|
|
|
|
|
info_log_level(DEBUG_LEVEL),
|
|
|
|
|
#endif // NDEBUG
|
2014-03-31 19:44:54 +00:00
|
|
|
max_open_files(5000),
|
2015-08-11 19:19:56 +00:00
|
|
|
max_file_opening_threads(1),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(0),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(nullptr),
|
2012-08-14 22:20:36 +00:00
|
|
|
disableDataSync(false),
|
2012-08-27 19:10:26 +00:00
|
|
|
use_fsync(false),
|
2012-09-17 20:35:57 +00:00
|
|
|
db_log_dir(""),
|
2013-10-01 21:46:52 +00:00
|
|
|
wal_dir(""),
|
2013-11-08 23:23:46 +00:00
|
|
|
delete_obsolete_files_period_micros(6 * 60 * 60 * 1000000UL),
|
2012-11-06 00:51:55 +00:00
|
|
|
max_background_compactions(1),
|
2015-08-21 21:25:34 +00:00
|
|
|
max_subcompactions(1),
|
2014-02-06 20:59:16 +00:00
|
|
|
max_background_flushes(1),
|
2012-10-29 21:18:00 +00:00
|
|
|
max_log_file_size(0),
|
2013-02-05 03:42:40 +00:00
|
|
|
log_file_time_to_roll(0),
|
|
|
|
|
keep_log_file_num(1000),
|
2013-01-11 01:18:50 +00:00
|
|
|
max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(4),
|
2013-01-15 22:05:42 +00:00
|
|
|
WAL_ttl_seconds(0),
|
2013-11-07 02:46:28 +00:00
|
|
|
WAL_size_limit_MB(0),
|
2013-02-28 22:09:30 +00:00
|
|
|
manifest_preallocation_size(4 * 1024 * 1024),
|
2013-03-15 00:00:04 +00:00
|
|
|
allow_os_buffer(true),
|
|
|
|
|
allow_mmap_reads(false),
|
2014-03-28 23:57:04 +00:00
|
|
|
allow_mmap_writes(false),
|
2013-05-21 18:53:33 +00:00
|
|
|
is_fd_close_on_exec(true),
|
2013-05-10 22:21:04 +00:00
|
|
|
skip_log_error_on_recovery(false),
|
2015-05-21 18:22:16 +00:00
|
|
|
stats_dump_period_sec(600),
|
2013-05-17 22:53:01 +00:00
|
|
|
advise_random_on_open(true),
|
2014-12-02 20:09:20 +00:00
|
|
|
db_write_buffer_size(0),
|
2013-05-31 23:30:17 +00:00
|
|
|
access_hint_on_compaction_start(NORMAL),
|
2015-08-21 05:33:25 +00:00
|
|
|
new_table_reader_for_compaction_inputs(false),
|
2013-06-14 05:49:46 +00:00
|
|
|
use_adaptive_mutex(false),
|
2014-11-21 05:13:18 +00:00
|
|
|
bytes_per_sync(0),
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync(0),
|
2015-05-28 20:21:39 +00:00
|
|
|
listeners(),
|
2015-05-15 22:52:51 +00:00
|
|
|
enable_thread_tracking(false),
|
2015-06-15 19:03:13 +00:00
|
|
|
delayed_write_rate(1024U * 1024U),
|
2015-08-04 20:48:16 +00:00
|
|
|
skip_stats_update_on_db_open(false),
|
2015-06-15 19:03:13 +00:00
|
|
|
wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords) {
|
2015-02-17 16:03:45 +00:00
|
|
|
}
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
DBOptions::DBOptions(const Options& options)
|
|
|
|
|
: create_if_missing(options.create_if_missing),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(options.create_missing_column_families),
|
2014-01-06 21:31:06 +00:00
|
|
|
error_if_exists(options.error_if_exists),
|
|
|
|
|
paranoid_checks(options.paranoid_checks),
|
|
|
|
|
env(options.env),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(options.rate_limiter),
|
2015-08-05 03:45:27 +00:00
|
|
|
delete_scheduler(options.delete_scheduler),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(options.info_log),
|
2014-03-06 00:55:51 +00:00
|
|
|
info_log_level(options.info_log_level),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_open_files(options.max_open_files),
|
2015-08-11 19:19:56 +00:00
|
|
|
max_file_opening_threads(options.max_file_opening_threads),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(options.max_total_wal_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(options.statistics),
|
2014-01-06 21:31:06 +00:00
|
|
|
disableDataSync(options.disableDataSync),
|
|
|
|
|
use_fsync(options.use_fsync),
|
2014-07-02 16:54:20 +00:00
|
|
|
db_paths(options.db_paths),
|
2014-01-06 21:31:06 +00:00
|
|
|
db_log_dir(options.db_log_dir),
|
|
|
|
|
wal_dir(options.wal_dir),
|
|
|
|
|
delete_obsolete_files_period_micros(
|
|
|
|
|
options.delete_obsolete_files_period_micros),
|
|
|
|
|
max_background_compactions(options.max_background_compactions),
|
2015-08-21 21:25:34 +00:00
|
|
|
max_subcompactions(options.max_subcompactions),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_background_flushes(options.max_background_flushes),
|
|
|
|
|
max_log_file_size(options.max_log_file_size),
|
|
|
|
|
log_file_time_to_roll(options.log_file_time_to_roll),
|
|
|
|
|
keep_log_file_num(options.keep_log_file_num),
|
|
|
|
|
max_manifest_file_size(options.max_manifest_file_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(options.table_cache_numshardbits),
|
2014-01-06 21:31:06 +00:00
|
|
|
WAL_ttl_seconds(options.WAL_ttl_seconds),
|
|
|
|
|
WAL_size_limit_MB(options.WAL_size_limit_MB),
|
|
|
|
|
manifest_preallocation_size(options.manifest_preallocation_size),
|
|
|
|
|
allow_os_buffer(options.allow_os_buffer),
|
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
|
|
|
|
is_fd_close_on_exec(options.is_fd_close_on_exec),
|
|
|
|
|
skip_log_error_on_recovery(options.skip_log_error_on_recovery),
|
|
|
|
|
stats_dump_period_sec(options.stats_dump_period_sec),
|
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
2014-12-02 20:09:20 +00:00
|
|
|
db_write_buffer_size(options.db_write_buffer_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
access_hint_on_compaction_start(options.access_hint_on_compaction_start),
|
2015-08-21 05:33:25 +00:00
|
|
|
new_table_reader_for_compaction_inputs(
|
|
|
|
|
options.new_table_reader_for_compaction_inputs),
|
2014-01-06 21:31:06 +00:00
|
|
|
use_adaptive_mutex(options.use_adaptive_mutex),
|
2014-11-21 05:13:18 +00:00
|
|
|
bytes_per_sync(options.bytes_per_sync),
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync(options.wal_bytes_per_sync),
|
2015-05-28 20:21:39 +00:00
|
|
|
listeners(options.listeners),
|
2015-05-15 22:52:51 +00:00
|
|
|
enable_thread_tracking(options.enable_thread_tracking),
|
2015-06-15 19:03:13 +00:00
|
|
|
delayed_write_rate(options.delayed_write_rate),
|
2015-08-04 20:48:16 +00:00
|
|
|
skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
|
2015-06-23 17:25:45 +00:00
|
|
|
wal_recovery_mode(options.wal_recovery_mode),
|
|
|
|
|
row_cache(options.row_cache) {}
|
2014-01-06 21:31:06 +00:00
|
|
|
|
2013-05-17 22:53:01 +00:00
|
|
|
static const char* const access_hints[] = {
|
|
|
|
|
"NONE", "NORMAL", "SEQUENTIAL", "WILLNEED"
|
|
|
|
|
};
|
|
|
|
|
|
2014-02-07 05:39:20 +00:00
|
|
|
void DBOptions::Dump(Logger* log) const {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.error_if_exists: %d", error_if_exists);
|
|
|
|
|
Warn(log, " Options.create_if_missing: %d", create_if_missing);
|
|
|
|
|
Warn(log, " Options.paranoid_checks: %d", paranoid_checks);
|
|
|
|
|
Warn(log, " Options.env: %p", env);
|
|
|
|
|
Warn(log, " Options.info_log: %p", info_log.get());
|
|
|
|
|
Warn(log, " Options.max_open_files: %d", max_open_files);
|
2015-08-11 19:19:56 +00:00
|
|
|
Warn(log, "Options.max_file_opening_threads: %d", max_file_opening_threads);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_total_wal_size: %" PRIu64, max_total_wal_size);
|
|
|
|
|
Warn(log, " Options.disableDataSync: %d", disableDataSync);
|
|
|
|
|
Warn(log, " Options.use_fsync: %d", use_fsync);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log, " Options.max_log_file_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_log_file_size);
|
|
|
|
|
Warn(log, "Options.max_manifest_file_size: %" PRIu64,
|
|
|
|
|
max_manifest_file_size);
|
|
|
|
|
Warn(log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt,
|
|
|
|
|
log_file_time_to_roll);
|
|
|
|
|
Warn(log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt,
|
|
|
|
|
keep_log_file_num);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_os_buffer: %d", allow_os_buffer);
|
|
|
|
|
Warn(log, " Options.allow_mmap_reads: %d", allow_mmap_reads);
|
|
|
|
|
Warn(log, " Options.allow_mmap_writes: %d", allow_mmap_writes);
|
|
|
|
|
Warn(log, " Options.create_missing_column_families: %d",
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.db_log_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
db_log_dir.c_str());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.wal_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
wal_dir.c_str());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.table_cache_numshardbits: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
table_cache_numshardbits);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
delete_obsolete_files_period_micros);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_background_compactions: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
max_background_compactions);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_background_flushes: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
max_background_flushes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.WAL_ttl_seconds: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
WAL_ttl_seconds);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.WAL_size_limit_MB: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
WAL_size_limit_MB);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.manifest_preallocation_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
manifest_preallocation_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_os_buffer: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_os_buffer);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_mmap_reads: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_mmap_reads);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.allow_mmap_writes: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
allow_mmap_writes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.is_fd_close_on_exec: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
is_fd_close_on_exec);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.stats_dump_period_sec: %u",
|
2014-02-07 05:39:20 +00:00
|
|
|
stats_dump_period_sec);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.advise_random_on_open: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
advise_random_on_open);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.db_write_buffer_size: %" ROCKSDB_PRIszt
|
|
|
|
|
"d",
|
|
|
|
|
db_write_buffer_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.access_hint_on_compaction_start: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
access_hints[access_hint_on_compaction_start]);
|
2015-08-21 05:33:25 +00:00
|
|
|
Warn(log, " Options.new_table_reader_for_compaction_inputs: %d",
|
|
|
|
|
new_table_reader_for_compaction_inputs);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.use_adaptive_mutex: %d",
|
2014-02-07 05:39:20 +00:00
|
|
|
use_adaptive_mutex);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.rate_limiter: %p",
|
2014-08-12 23:42:18 +00:00
|
|
|
rate_limiter.get());
|
2015-08-05 03:45:27 +00:00
|
|
|
Warn(log, " Options.delete_scheduler.rate_bytes_per_sec: %" PRIi64,
|
|
|
|
|
delete_scheduler ? delete_scheduler->GetRateBytesPerSecond() : 0);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.bytes_per_sync: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
bytes_per_sync);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.wal_bytes_per_sync: %" PRIu64,
|
2015-05-19 00:03:59 +00:00
|
|
|
wal_bytes_per_sync);
|
2015-08-10 16:21:04 +00:00
|
|
|
Warn(log, " Options.wal_recovery_mode: %d",
|
|
|
|
|
wal_recovery_mode);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.enable_thread_tracking: %d",
|
2014-11-21 05:13:18 +00:00
|
|
|
enable_thread_tracking);
|
2015-06-23 17:25:45 +00:00
|
|
|
if (row_cache) {
|
|
|
|
|
Warn(log, " Options.row_cache: %" PRIu64,
|
|
|
|
|
row_cache->GetCapacity());
|
|
|
|
|
} else {
|
|
|
|
|
Warn(log, " Options.row_cache: None");
|
|
|
|
|
}
|
2014-02-07 05:39:20 +00:00
|
|
|
} // DBOptions::Dump
|
|
|
|
|
|
|
|
|
|
void ColumnFamilyOptions::Dump(Logger* log) const {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.comparator: %s", comparator->Name());
|
|
|
|
|
Warn(log, " Options.merge_operator: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
merge_operator ? merge_operator->Name() : "None");
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_filter: %s",
|
2014-08-13 22:58:31 +00:00
|
|
|
compaction_filter ? compaction_filter->Name() : "None");
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_filter_factory: %s",
|
2015-06-08 23:34:26 +00:00
|
|
|
compaction_filter_factory ? compaction_filter_factory->Name() : "None");
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.memtable_factory: %s", memtable_factory->Name());
|
|
|
|
|
Warn(log, " Options.table_factory: %s", table_factory->Name());
|
|
|
|
|
Warn(log, " table_factory options: %s",
|
2014-08-25 21:24:09 +00:00
|
|
|
table_factory->GetPrintableTableOptions().c_str());
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
write_buffer_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_write_buffer_number: %d", max_write_buffer_number);
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!compression_per_level.empty()) {
|
2013-03-15 01:32:01 +00:00
|
|
|
for (unsigned int i = 0; i < compression_per_level.size(); i++) {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression[%d]: %s", i,
|
2015-06-18 21:55:05 +00:00
|
|
|
CompressionTypeToString(compression_per_level[i]).c_str());
|
2015-04-06 19:50:44 +00:00
|
|
|
}
|
2012-10-28 06:13:17 +00:00
|
|
|
} else {
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression: %s",
|
2015-06-18 21:55:05 +00:00
|
|
|
CompressionTypeToString(compression).c_str());
|
2012-10-28 06:13:17 +00:00
|
|
|
}
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.prefix_extractor: %s",
|
2013-08-14 16:06:10 +00:00
|
|
|
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.num_levels: %d", num_levels);
|
|
|
|
|
Warn(log, " Options.min_write_buffer_number_to_merge: %d",
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge);
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
2015-05-28 23:34:24 +00:00
|
|
|
Warn(log, " Options.max_write_buffer_number_to_maintain: %d",
|
|
|
|
|
max_write_buffer_number_to_maintain);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression_opts.window_bits: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.window_bits);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression_opts.level: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.level);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compression_opts.strategy: %d",
|
2012-11-01 17:50:08 +00:00
|
|
|
compression_opts.strategy);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.level0_file_num_compaction_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_file_num_compaction_trigger);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.level0_slowdown_writes_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_slowdown_writes_trigger);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.level0_stop_writes_trigger: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
level0_stop_writes_trigger);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.target_file_size_base: %" PRIu64,
|
2012-08-27 19:10:26 +00:00
|
|
|
target_file_size_base);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.target_file_size_multiplier: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
target_file_size_multiplier);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_bytes_for_level_base: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
max_bytes_for_level_base);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.level_compaction_dynamic_level_bytes: %d",
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
level_compaction_dynamic_level_bytes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_bytes_for_level_multiplier: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
max_bytes_for_level_multiplier);
|
2015-03-30 21:04:21 +00:00
|
|
|
for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
|
|
|
|
|
i++) {
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log, "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt
|
|
|
|
|
"]: %d",
|
|
|
|
|
i, max_bytes_for_level_multiplier_additional[i]);
|
2013-05-21 18:37:06 +00:00
|
|
|
}
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_sequential_skip_in_iterations: %" PRIu64,
|
2014-09-22 18:15:03 +00:00
|
|
|
max_sequential_skip_in_iterations);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.expanded_compaction_factor: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
expanded_compaction_factor);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.source_compaction_factor: %d",
|
2012-11-21 07:07:41 +00:00
|
|
|
source_compaction_factor);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.max_grandparent_overlap_factor: %d",
|
2012-08-27 19:10:26 +00:00
|
|
|
max_grandparent_overlap_factor);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.arena_block_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
arena_block_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.soft_rate_limit: %.2f",
|
2013-08-14 16:06:10 +00:00
|
|
|
soft_rate_limit);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.hard_rate_limit: %.2f",
|
2013-08-05 22:43:49 +00:00
|
|
|
hard_rate_limit);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.rate_limit_delay_max_milliseconds: %u",
|
2013-08-05 22:43:49 +00:00
|
|
|
rate_limit_delay_max_milliseconds);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.disable_auto_compactions: %d",
|
2012-11-20 23:45:41 +00:00
|
|
|
disable_auto_compactions);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.filter_deletes: %d",
|
2013-07-12 23:56:52 +00:00
|
|
|
filter_deletes);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.verify_checksums_in_compaction: %d",
|
2014-03-10 17:06:34 +00:00
|
|
|
verify_checksums_in_compaction);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_style: %d",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_style);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.compaction_options_universal.size_ratio: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.size_ratio);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_universal.min_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.min_merge_width);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_universal.max_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.max_merge_width);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_universal."
|
2013-09-09 23:06:10 +00:00
|
|
|
"max_size_amplification_percent: %u",
|
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log,
|
2014-09-19 20:09:25 +00:00
|
|
|
"Options.compaction_options_universal.compression_size_percent: %d",
|
2013-12-03 20:32:07 +00:00
|
|
|
compaction_options_universal.compression_size_percent);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64,
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo.max_table_files_size);
|
2013-10-16 18:50:50 +00:00
|
|
|
std::string collector_names;
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
for (const auto& collector_factory : table_properties_collector_factories) {
|
|
|
|
|
collector_names.append(collector_factory->Name());
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.append("; ");
|
|
|
|
|
}
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.table_properties_collectors: %s",
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.c_str());
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.inplace_update_support: %d",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
inplace_update_support);
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
|
|
|
|
inplace_update_num_locks);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.min_partial_merge_operands: %u",
|
2014-03-25 00:57:13 +00:00
|
|
|
min_partial_merge_operands);
|
2013-11-27 22:27:02 +00:00
|
|
|
// TODO: easier config for bloom (maybe based on avg key/value size)
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.memtable_prefix_bloom_bits: %d",
|
2013-11-27 22:27:02 +00:00
|
|
|
memtable_prefix_bloom_bits);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.memtable_prefix_bloom_probes: %d",
|
2013-11-27 22:27:02 +00:00
|
|
|
memtable_prefix_bloom_probes);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
memtable_prefix_bloom_huge_page_tlb_size);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.bloom_locality: %d",
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality);
|
2015-07-01 23:13:49 +00:00
|
|
|
|
2015-07-13 19:11:05 +00:00
|
|
|
Warn(log,
|
|
|
|
|
" Options.max_successive_merges: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_successive_merges);
|
2015-05-22 18:54:59 +00:00
|
|
|
Warn(log, " Options.optimize_fllters_for_hits: %d",
|
2015-02-17 16:03:45 +00:00
|
|
|
optimize_filters_for_hits);
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 00:24:45 +00:00
|
|
|
Warn(log, " Options.paranoid_file_checks: %d",
|
|
|
|
|
paranoid_file_checks);
|
|
|
|
|
Warn(log, " Options.compaction_measure_io_stats: %d",
|
|
|
|
|
compaction_measure_io_stats);
|
2014-02-07 05:39:20 +00:00
|
|
|
} // ColumnFamilyOptions::Dump
|
|
|
|
|
|
|
|
|
|
void Options::Dump(Logger* log) const {
|
|
|
|
|
DBOptions::Dump(log);
|
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
2012-08-22 18:43:53 +00:00
|
|
|
} // Options::Dump
|
|
|
|
|
|
2015-06-18 17:15:54 +00:00
|
|
|
void Options::DumpCFOptions(Logger* log) const {
|
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
|
|
|
|
} // Options::DumpCFOptions
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
//
|
|
|
|
|
// The goal of this method is to create a configuration that
|
|
|
|
|
// allows an application to write all files into L0 and
|
|
|
|
|
// then do a single compaction to output all files into L1.
|
2013-02-26 06:57:37 +00:00
|
|
|
Options*
|
|
|
|
|
Options::PrepareForBulkLoad()
|
|
|
|
|
{
|
2013-03-08 17:19:24 +00:00
|
|
|
// never slowdown ingest.
|
2013-02-26 06:57:37 +00:00
|
|
|
level0_file_num_compaction_trigger = (1<<30);
|
|
|
|
|
level0_slowdown_writes_trigger = (1<<30);
|
|
|
|
|
level0_stop_writes_trigger = (1<<30);
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
|
// no auto compactions please. The application should issue a
|
|
|
|
|
// manual compaction after all data is loaded into L0.
|
2013-02-26 06:57:37 +00:00
|
|
|
disable_auto_compactions = true;
|
|
|
|
|
disableDataSync = true;
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
|
// A manual compaction run should pick all files in L0 in
|
|
|
|
|
// a single compaction run.
|
2013-02-26 06:57:37 +00:00
|
|
|
source_compaction_factor = (1<<30);
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// It is better to have only 2 levels, otherwise a manual
|
|
|
|
|
// compaction would compact at every possible level, thereby
|
|
|
|
|
// increasing the total time needed for compactions.
|
|
|
|
|
num_levels = 2;
|
|
|
|
|
|
2015-02-02 19:09:21 +00:00
|
|
|
// Need to allow more write buffers to allow more parallism
|
|
|
|
|
// of flushes.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
min_write_buffer_number_to_merge = 1;
|
|
|
|
|
|
|
|
|
|
// When compaction is disabled, more parallel flush threads can
|
|
|
|
|
// help with write throughput.
|
|
|
|
|
max_background_flushes = 4;
|
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// Prevent a memtable flush to automatically promote files
|
|
|
|
|
// to L1. This is helpful so that all files that are
|
|
|
|
|
// input to the manual compaction are all at L0.
|
|
|
|
|
max_background_compactions = 2;
|
|
|
|
|
|
|
|
|
|
// The compaction would create large files in L1.
|
|
|
|
|
target_file_size_base = 256 * 1024 * 1024;
|
2013-02-26 06:57:37 +00:00
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
2014-11-13 19:39:30 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2014-05-10 17:49:33 +00:00
|
|
|
// Optimization functions
|
2014-08-26 21:15:00 +00:00
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
|
|
|
|
|
uint64_t block_cache_size_mb) {
|
2014-05-10 17:49:33 +00:00
|
|
|
prefix_extractor.reset(NewNoopTransform());
|
|
|
|
|
BlockBasedTableOptions block_based_options;
|
2014-08-26 21:15:00 +00:00
|
|
|
block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
|
block_based_options.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
|
|
|
block_based_options.block_cache =
|
2014-11-13 19:39:30 +00:00
|
|
|
NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024));
|
2014-05-10 17:49:33 +00:00
|
|
|
table_factory.reset(new BlockBasedTableFactory(block_based_options));
|
|
|
|
|
memtable_factory.reset(NewHashLinkListRepFactory());
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction(
|
|
|
|
|
uint64_t memtable_memory_budget) {
|
2014-11-13 19:39:30 +00:00
|
|
|
write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
|
2014-05-10 17:49:33 +00:00
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
|
// write stalls.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
// start flushing L0->L1 as soon as possible. each file on level0 is
|
|
|
|
|
// (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
|
|
|
|
|
// memtable_memory_budget.
|
|
|
|
|
level0_file_num_compaction_trigger = 2;
|
|
|
|
|
// doesn't really matter much, but we don't want to create too many files
|
|
|
|
|
target_file_size_base = memtable_memory_budget / 8;
|
|
|
|
|
// make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
|
|
|
|
|
max_bytes_for_level_base = memtable_memory_budget;
|
|
|
|
|
|
|
|
|
|
// level style compaction
|
|
|
|
|
compaction_style = kCompactionStyleLevel;
|
|
|
|
|
|
|
|
|
|
// only compress levels >= 2
|
|
|
|
|
compression_per_level.resize(num_levels);
|
|
|
|
|
for (int i = 0; i < num_levels; ++i) {
|
|
|
|
|
if (i < 2) {
|
|
|
|
|
compression_per_level[i] = kNoCompression;
|
|
|
|
|
} else {
|
|
|
|
|
compression_per_level[i] = kSnappyCompression;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction(
|
|
|
|
|
uint64_t memtable_memory_budget) {
|
2014-11-13 19:39:30 +00:00
|
|
|
write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
|
2014-05-10 17:49:33 +00:00
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
|
// write stalls.
|
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
|
// universal style compaction
|
|
|
|
|
compaction_style = kCompactionStyleUniversal;
|
|
|
|
|
compaction_options_universal.compression_size_percent = 80;
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
|
|
|
|
|
max_background_compactions = total_threads - 1;
|
|
|
|
|
max_background_flushes = 1;
|
|
|
|
|
env->SetBackgroundThreads(total_threads, Env::LOW);
|
|
|
|
|
env->SetBackgroundThreads(1, Env::HIGH);
|
|
|
|
|
return this;
|
|
|
|
|
}
|
2015-02-18 19:49:31 +00:00
|
|
|
|
2015-04-10 04:05:09 +00:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
2015-02-18 19:49:31 +00:00
|
|
|
ReadOptions::ReadOptions()
|
|
|
|
|
: verify_checksums(true),
|
|
|
|
|
fill_cache(true),
|
|
|
|
|
snapshot(nullptr),
|
|
|
|
|
iterate_upper_bound(nullptr),
|
|
|
|
|
read_tier(kReadAllTier),
|
|
|
|
|
tailing(false),
|
|
|
|
|
managed(false),
|
|
|
|
|
total_order_seek(false) {
|
|
|
|
|
XFUNC_TEST("", "managed_options", managed_options, xf_manage_options,
|
|
|
|
|
reinterpret_cast<ReadOptions*>(this));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ReadOptions::ReadOptions(bool cksum, bool cache)
|
|
|
|
|
: verify_checksums(cksum),
|
|
|
|
|
fill_cache(cache),
|
|
|
|
|
snapshot(nullptr),
|
|
|
|
|
iterate_upper_bound(nullptr),
|
|
|
|
|
read_tier(kReadAllTier),
|
|
|
|
|
tailing(false),
|
|
|
|
|
managed(false),
|
|
|
|
|
total_order_seek(false) {
|
|
|
|
|
XFUNC_TEST("", "managed_options", managed_options, xf_manage_options,
|
|
|
|
|
reinterpret_cast<ReadOptions*>(this));
|
|
|
|
|
}
|
|
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
} // namespace rocksdb
|