2013-10-16 21:59:46 +00:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 22:37:00 +00:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/options.h"
|
2014-09-04 23:18:36 +00:00
|
|
|
#include "rocksdb/immutable_options.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-05 06:14:37 +00:00
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-04-30 18:33:40 +00:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 06:14:37 +00:00
|
|
|
#endif
|
|
|
|
|
2014-04-30 18:33:40 +00:00
|
|
|
#include <inttypes.h>
|
2013-01-11 01:18:50 +00:00
|
|
|
#include <limits>
|
|
|
|
|
2013-08-23 15:38:13 +00:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/env.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/memtablerep.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-01-28 05:58:46 +00:00
|
|
|
#include "rocksdb/table.h"
|
2014-01-25 00:15:05 +00:00
|
|
|
#include "rocksdb/table_properties.h"
|
2013-10-29 00:54:09 +00:00
|
|
|
#include "table/block_based_table_factory.h"
|
2014-07-28 19:05:36 +00:00
|
|
|
#include "util/statistics.h"
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
namespace rocksdb {
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-09-04 23:18:36 +00:00
|
|
|
ImmutableCFOptions::ImmutableCFOptions(const Options& options)
|
2014-09-08 22:04:34 +00:00
|
|
|
: compaction_style(options.compaction_style),
|
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
|
|
|
prefix_extractor(options.prefix_extractor.get()),
|
2014-09-04 23:18:36 +00:00
|
|
|
comparator(options.comparator),
|
|
|
|
merge_operator(options.merge_operator.get()),
|
|
|
|
info_log(options.info_log.get()),
|
|
|
|
statistics(options.statistics.get()),
|
|
|
|
env(options.env),
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
|
|
|
db_paths(options.db_paths),
|
|
|
|
table_factory(options.table_factory.get()),
|
|
|
|
table_properties_collector_factories(
|
|
|
|
options.table_properties_collector_factories),
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
|
|
|
bloom_locality(options.bloom_locality),
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands),
|
|
|
|
disable_data_sync(options.disableDataSync),
|
2014-09-08 22:04:34 +00:00
|
|
|
use_fsync(options.use_fsync),
|
|
|
|
compression(options.compression),
|
|
|
|
compression_per_level(options.compression_per_level),
|
|
|
|
compression_opts(options.compression_opts) {}
|
2014-09-04 23:18:36 +00:00
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions()
|
2011-03-18 22:37:00 +00:00
|
|
|
: comparator(BytewiseComparator()),
|
2013-03-21 22:59:47 +00:00
|
|
|
merge_operator(nullptr),
|
2013-05-12 09:36:59 +00:00
|
|
|
compaction_filter(nullptr),
|
2014-03-25 00:57:13 +00:00
|
|
|
compaction_filter_factory(std::shared_ptr<CompactionFilterFactory>(
|
|
|
|
new DefaultCompactionFilterFactory())),
|
2014-05-04 20:55:53 +00:00
|
|
|
compaction_filter_factory_v2(new DefaultCompactionFilterFactoryV2()),
|
2014-03-25 00:57:13 +00:00
|
|
|
write_buffer_size(4 << 20),
|
2012-10-19 21:00:53 +00:00
|
|
|
max_write_buffer_number(2),
|
2013-06-11 21:23:58 +00:00
|
|
|
min_write_buffer_number_to_merge(1),
|
2012-04-17 15:36:46 +00:00
|
|
|
compression(kSnappyCompression),
|
2013-08-13 21:04:56 +00:00
|
|
|
prefix_extractor(nullptr),
|
2012-06-23 02:30:03 +00:00
|
|
|
num_levels(7),
|
|
|
|
level0_file_num_compaction_trigger(4),
|
2014-03-28 23:57:04 +00:00
|
|
|
level0_slowdown_writes_trigger(20),
|
|
|
|
level0_stop_writes_trigger(24),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_mem_compaction_level(2),
|
|
|
|
target_file_size_base(2 * 1048576),
|
2012-08-22 00:33:46 +00:00
|
|
|
target_file_size_multiplier(1),
|
2012-07-03 00:30:32 +00:00
|
|
|
max_bytes_for_level_base(10 * 1048576),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_bytes_for_level_multiplier(10),
|
2013-05-21 18:37:06 +00:00
|
|
|
max_bytes_for_level_multiplier_additional(num_levels, 1),
|
2012-06-23 02:30:03 +00:00
|
|
|
expanded_compaction_factor(25),
|
2012-11-21 07:07:41 +00:00
|
|
|
source_compaction_factor(1),
|
2012-06-23 02:30:03 +00:00
|
|
|
max_grandparent_overlap_factor(10),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
soft_rate_limit(0.0),
|
|
|
|
hard_rate_limit(0.0),
|
|
|
|
rate_limit_delay_max_milliseconds(1000),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(0),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
disable_auto_compactions(false),
|
|
|
|
purge_redundant_kvs_while_flush(true),
|
|
|
|
compaction_style(kCompactionStyleLevel),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
filter_deletes(false),
|
|
|
|
max_sequential_skip_in_iterations(8),
|
|
|
|
memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)),
|
|
|
|
table_factory(
|
2014-03-25 18:09:40 +00:00
|
|
|
std::shared_ptr<TableFactory>(new BlockBasedTableFactory())),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
inplace_update_support(false),
|
2014-01-13 21:27:43 +00:00
|
|
|
inplace_update_num_locks(10000),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(nullptr),
|
|
|
|
memtable_prefix_bloom_bits(0),
|
|
|
|
memtable_prefix_bloom_probes(6),
|
2014-05-04 20:55:53 +00:00
|
|
|
memtable_prefix_bloom_huge_page_tlb_size(0),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(0),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(0),
|
|
|
|
min_partial_merge_operands(2) {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
|
|
|
}
|
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
|
|
|
|
: comparator(options.comparator),
|
|
|
|
merge_operator(options.merge_operator),
|
|
|
|
compaction_filter(options.compaction_filter),
|
|
|
|
compaction_filter_factory(options.compaction_filter_factory),
|
2014-03-25 18:09:40 +00:00
|
|
|
compaction_filter_factory_v2(options.compaction_filter_factory_v2),
|
2014-01-06 21:31:06 +00:00
|
|
|
write_buffer_size(options.write_buffer_size),
|
|
|
|
max_write_buffer_number(options.max_write_buffer_number),
|
|
|
|
min_write_buffer_number_to_merge(
|
|
|
|
options.min_write_buffer_number_to_merge),
|
|
|
|
compression(options.compression),
|
|
|
|
compression_per_level(options.compression_per_level),
|
|
|
|
compression_opts(options.compression_opts),
|
|
|
|
prefix_extractor(options.prefix_extractor),
|
|
|
|
num_levels(options.num_levels),
|
|
|
|
level0_file_num_compaction_trigger(
|
|
|
|
options.level0_file_num_compaction_trigger),
|
|
|
|
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
|
|
|
|
level0_stop_writes_trigger(options.level0_stop_writes_trigger),
|
|
|
|
max_mem_compaction_level(options.max_mem_compaction_level),
|
|
|
|
target_file_size_base(options.target_file_size_base),
|
|
|
|
target_file_size_multiplier(options.target_file_size_multiplier),
|
|
|
|
max_bytes_for_level_base(options.max_bytes_for_level_base),
|
|
|
|
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
|
|
|
|
max_bytes_for_level_multiplier_additional(
|
|
|
|
options.max_bytes_for_level_multiplier_additional),
|
|
|
|
expanded_compaction_factor(options.expanded_compaction_factor),
|
|
|
|
source_compaction_factor(options.source_compaction_factor),
|
|
|
|
max_grandparent_overlap_factor(options.max_grandparent_overlap_factor),
|
|
|
|
soft_rate_limit(options.soft_rate_limit),
|
|
|
|
hard_rate_limit(options.hard_rate_limit),
|
|
|
|
rate_limit_delay_max_milliseconds(
|
|
|
|
options.rate_limit_delay_max_milliseconds),
|
2014-01-24 22:30:28 +00:00
|
|
|
arena_block_size(options.arena_block_size),
|
2014-01-06 21:31:06 +00:00
|
|
|
disable_auto_compactions(options.disable_auto_compactions),
|
|
|
|
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
|
|
|
|
compaction_style(options.compaction_style),
|
2014-03-11 00:25:10 +00:00
|
|
|
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
|
2014-01-06 21:31:06 +00:00
|
|
|
compaction_options_universal(options.compaction_options_universal),
|
2014-05-21 18:43:35 +00:00
|
|
|
compaction_options_fifo(options.compaction_options_fifo),
|
2014-01-06 21:31:06 +00:00
|
|
|
filter_deletes(options.filter_deletes),
|
|
|
|
max_sequential_skip_in_iterations(
|
|
|
|
options.max_sequential_skip_in_iterations),
|
|
|
|
memtable_factory(options.memtable_factory),
|
|
|
|
table_factory(options.table_factory),
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
table_properties_collector_factories(
|
|
|
|
options.table_properties_collector_factories),
|
2014-01-06 21:31:06 +00:00
|
|
|
inplace_update_support(options.inplace_update_support),
|
2014-01-13 17:06:44 +00:00
|
|
|
inplace_update_num_locks(options.inplace_update_num_locks),
|
2014-02-06 23:42:16 +00:00
|
|
|
inplace_callback(options.inplace_callback),
|
|
|
|
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
|
|
|
|
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
|
2014-05-04 20:55:53 +00:00
|
|
|
memtable_prefix_bloom_huge_page_tlb_size(
|
|
|
|
options.memtable_prefix_bloom_huge_page_tlb_size),
|
2014-04-07 23:33:03 +00:00
|
|
|
bloom_locality(options.bloom_locality),
|
2014-03-25 18:09:40 +00:00
|
|
|
max_successive_merges(options.max_successive_merges),
|
|
|
|
min_partial_merge_operands(options.min_partial_merge_operands) {
|
2014-01-06 21:31:06 +00:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
2014-06-25 21:31:30 +00:00
|
|
|
if (max_bytes_for_level_multiplier_additional.size() <
|
|
|
|
static_cast<unsigned int>(num_levels)) {
|
2014-06-25 20:11:12 +00:00
|
|
|
max_bytes_for_level_multiplier_additional.resize(num_levels, 1);
|
|
|
|
}
|
2014-01-06 21:31:06 +00:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
DBOptions::DBOptions()
|
|
|
|
: create_if_missing(false),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(false),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
error_if_exists(false),
|
2014-03-31 19:44:54 +00:00
|
|
|
paranoid_checks(true),
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 19:14:09 +00:00
|
|
|
env(Env::Default()),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(nullptr),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(nullptr),
|
2014-04-10 22:27:42 +00:00
|
|
|
info_log_level(INFO_LEVEL),
|
2014-03-31 19:44:54 +00:00
|
|
|
max_open_files(5000),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(0),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(nullptr),
|
2012-08-14 22:20:36 +00:00
|
|
|
disableDataSync(false),
|
2012-08-27 19:10:26 +00:00
|
|
|
use_fsync(false),
|
2012-09-17 20:35:57 +00:00
|
|
|
db_log_dir(""),
|
2013-10-01 21:46:52 +00:00
|
|
|
wal_dir(""),
|
2013-11-08 23:23:46 +00:00
|
|
|
delete_obsolete_files_period_micros(6 * 60 * 60 * 1000000UL),
|
2012-11-06 00:51:55 +00:00
|
|
|
max_background_compactions(1),
|
2014-02-06 20:59:16 +00:00
|
|
|
max_background_flushes(1),
|
2012-10-29 21:18:00 +00:00
|
|
|
max_log_file_size(0),
|
2013-02-05 03:42:40 +00:00
|
|
|
log_file_time_to_roll(0),
|
|
|
|
keep_log_file_num(1000),
|
2013-01-11 01:18:50 +00:00
|
|
|
max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(4),
|
|
|
|
table_cache_remove_scan_count_limit(16),
|
2013-01-15 22:05:42 +00:00
|
|
|
WAL_ttl_seconds(0),
|
2013-11-07 02:46:28 +00:00
|
|
|
WAL_size_limit_MB(0),
|
2013-02-28 22:09:30 +00:00
|
|
|
manifest_preallocation_size(4 * 1024 * 1024),
|
2013-03-15 00:00:04 +00:00
|
|
|
allow_os_buffer(true),
|
|
|
|
allow_mmap_reads(false),
|
2014-03-28 23:57:04 +00:00
|
|
|
allow_mmap_writes(false),
|
2013-05-21 18:53:33 +00:00
|
|
|
is_fd_close_on_exec(true),
|
2013-05-10 22:21:04 +00:00
|
|
|
skip_log_error_on_recovery(false),
|
2013-05-15 17:34:02 +00:00
|
|
|
stats_dump_period_sec(3600),
|
2013-05-17 22:53:01 +00:00
|
|
|
advise_random_on_open(true),
|
2013-05-31 23:30:17 +00:00
|
|
|
access_hint_on_compaction_start(NORMAL),
|
2013-06-14 05:49:46 +00:00
|
|
|
use_adaptive_mutex(false),
|
2014-08-12 23:42:18 +00:00
|
|
|
allow_thread_local(true),
|
|
|
|
bytes_per_sync(0) {}
|
2011-03-18 22:37:00 +00:00
|
|
|
|
2014-01-06 21:31:06 +00:00
|
|
|
DBOptions::DBOptions(const Options& options)
|
|
|
|
: create_if_missing(options.create_if_missing),
|
2014-06-07 01:04:56 +00:00
|
|
|
create_missing_column_families(options.create_missing_column_families),
|
2014-01-06 21:31:06 +00:00
|
|
|
error_if_exists(options.error_if_exists),
|
|
|
|
paranoid_checks(options.paranoid_checks),
|
|
|
|
env(options.env),
|
2014-07-08 19:31:49 +00:00
|
|
|
rate_limiter(options.rate_limiter),
|
2014-02-05 00:31:18 +00:00
|
|
|
info_log(options.info_log),
|
2014-03-06 00:55:51 +00:00
|
|
|
info_log_level(options.info_log_level),
|
2014-01-06 21:31:06 +00:00
|
|
|
max_open_files(options.max_open_files),
|
2014-04-30 18:33:40 +00:00
|
|
|
max_total_wal_size(options.max_total_wal_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
statistics(options.statistics),
|
2014-01-06 21:31:06 +00:00
|
|
|
disableDataSync(options.disableDataSync),
|
|
|
|
use_fsync(options.use_fsync),
|
2014-07-02 16:54:20 +00:00
|
|
|
db_paths(options.db_paths),
|
2014-01-06 21:31:06 +00:00
|
|
|
db_log_dir(options.db_log_dir),
|
|
|
|
wal_dir(options.wal_dir),
|
|
|
|
delete_obsolete_files_period_micros(
|
|
|
|
options.delete_obsolete_files_period_micros),
|
|
|
|
max_background_compactions(options.max_background_compactions),
|
|
|
|
max_background_flushes(options.max_background_flushes),
|
|
|
|
max_log_file_size(options.max_log_file_size),
|
|
|
|
log_file_time_to_roll(options.log_file_time_to_roll),
|
|
|
|
keep_log_file_num(options.keep_log_file_num),
|
|
|
|
max_manifest_file_size(options.max_manifest_file_size),
|
2014-02-05 00:31:18 +00:00
|
|
|
table_cache_numshardbits(options.table_cache_numshardbits),
|
|
|
|
table_cache_remove_scan_count_limit(
|
|
|
|
options.table_cache_remove_scan_count_limit),
|
2014-01-06 21:31:06 +00:00
|
|
|
WAL_ttl_seconds(options.WAL_ttl_seconds),
|
|
|
|
WAL_size_limit_MB(options.WAL_size_limit_MB),
|
|
|
|
manifest_preallocation_size(options.manifest_preallocation_size),
|
|
|
|
allow_os_buffer(options.allow_os_buffer),
|
|
|
|
allow_mmap_reads(options.allow_mmap_reads),
|
|
|
|
allow_mmap_writes(options.allow_mmap_writes),
|
|
|
|
is_fd_close_on_exec(options.is_fd_close_on_exec),
|
|
|
|
skip_log_error_on_recovery(options.skip_log_error_on_recovery),
|
|
|
|
stats_dump_period_sec(options.stats_dump_period_sec),
|
|
|
|
advise_random_on_open(options.advise_random_on_open),
|
|
|
|
access_hint_on_compaction_start(options.access_hint_on_compaction_start),
|
|
|
|
use_adaptive_mutex(options.use_adaptive_mutex),
|
2014-08-12 23:42:18 +00:00
|
|
|
allow_thread_local(options.allow_thread_local),
|
|
|
|
bytes_per_sync(options.bytes_per_sync) {}
|
2014-01-06 21:31:06 +00:00
|
|
|
|
2013-05-17 22:53:01 +00:00
|
|
|
static const char* const access_hints[] = {
|
|
|
|
"NONE", "NORMAL", "SEQUENTIAL", "WILLNEED"
|
|
|
|
};
|
|
|
|
|
2014-02-07 05:39:20 +00:00
|
|
|
void DBOptions::Dump(Logger* log) const {
|
2012-10-19 21:00:53 +00:00
|
|
|
Log(log," Options.error_if_exists: %d", error_if_exists);
|
2013-08-14 16:06:10 +00:00
|
|
|
Log(log," Options.create_if_missing: %d", create_if_missing);
|
2012-10-19 21:00:53 +00:00
|
|
|
Log(log," Options.paranoid_checks: %d", paranoid_checks);
|
|
|
|
Log(log," Options.env: %p", env);
|
2013-01-20 10:07:13 +00:00
|
|
|
Log(log," Options.info_log: %p", info_log.get());
|
2012-10-19 21:00:53 +00:00
|
|
|
Log(log," Options.max_open_files: %d", max_open_files);
|
2014-04-30 18:33:40 +00:00
|
|
|
Log(log," Options.max_total_wal_size: %" PRIu64, max_total_wal_size);
|
2014-02-07 05:39:20 +00:00
|
|
|
Log(log, " Options.disableDataSync: %d", disableDataSync);
|
|
|
|
Log(log, " Options.use_fsync: %d", use_fsync);
|
|
|
|
Log(log, " Options.max_log_file_size: %zu", max_log_file_size);
|
|
|
|
Log(log, "Options.max_manifest_file_size: %lu",
|
|
|
|
(unsigned long)max_manifest_file_size);
|
|
|
|
Log(log, " Options.log_file_time_to_roll: %zu", log_file_time_to_roll);
|
|
|
|
Log(log, " Options.keep_log_file_num: %zu", keep_log_file_num);
|
|
|
|
Log(log, " Options.allow_os_buffer: %d", allow_os_buffer);
|
|
|
|
Log(log, " Options.allow_mmap_reads: %d", allow_mmap_reads);
|
|
|
|
Log(log, " Options.allow_mmap_writes: %d", allow_mmap_writes);
|
2014-06-07 01:04:56 +00:00
|
|
|
Log(log, " Options.create_missing_column_families: %d",
|
|
|
|
create_missing_column_families);
|
2014-02-07 05:39:20 +00:00
|
|
|
Log(log, " Options.db_log_dir: %s",
|
|
|
|
db_log_dir.c_str());
|
2014-06-07 01:04:56 +00:00
|
|
|
Log(log, " Options.wal_dir: %s",
|
2014-02-07 05:39:20 +00:00
|
|
|
wal_dir.c_str());
|
|
|
|
Log(log, " Options.table_cache_numshardbits: %d",
|
|
|
|
table_cache_numshardbits);
|
|
|
|
Log(log, " Options.table_cache_remove_scan_count_limit: %d",
|
|
|
|
table_cache_remove_scan_count_limit);
|
|
|
|
Log(log, " Options.delete_obsolete_files_period_micros: %lu",
|
|
|
|
(unsigned long)delete_obsolete_files_period_micros);
|
|
|
|
Log(log, " Options.max_background_compactions: %d",
|
|
|
|
max_background_compactions);
|
|
|
|
Log(log, " Options.max_background_flushes: %d",
|
|
|
|
max_background_flushes);
|
|
|
|
Log(log, " Options.WAL_ttl_seconds: %lu",
|
|
|
|
(unsigned long)WAL_ttl_seconds);
|
|
|
|
Log(log, " Options.WAL_size_limit_MB: %lu",
|
|
|
|
(unsigned long)WAL_size_limit_MB);
|
|
|
|
Log(log, " Options.manifest_preallocation_size: %zu",
|
|
|
|
manifest_preallocation_size);
|
|
|
|
Log(log, " Options.allow_os_buffer: %d",
|
|
|
|
allow_os_buffer);
|
|
|
|
Log(log, " Options.allow_mmap_reads: %d",
|
|
|
|
allow_mmap_reads);
|
|
|
|
Log(log, " Options.allow_mmap_writes: %d",
|
|
|
|
allow_mmap_writes);
|
|
|
|
Log(log, " Options.is_fd_close_on_exec: %d",
|
|
|
|
is_fd_close_on_exec);
|
|
|
|
Log(log, " Options.skip_log_error_on_recovery: %d",
|
|
|
|
skip_log_error_on_recovery);
|
|
|
|
Log(log, " Options.stats_dump_period_sec: %u",
|
|
|
|
stats_dump_period_sec);
|
|
|
|
Log(log, " Options.advise_random_on_open: %d",
|
|
|
|
advise_random_on_open);
|
|
|
|
Log(log, " Options.access_hint_on_compaction_start: %s",
|
|
|
|
access_hints[access_hint_on_compaction_start]);
|
|
|
|
Log(log, " Options.use_adaptive_mutex: %d",
|
|
|
|
use_adaptive_mutex);
|
2014-08-12 23:42:18 +00:00
|
|
|
Log(log, " Options.rate_limiter: %p",
|
|
|
|
rate_limiter.get());
|
2014-02-07 05:39:20 +00:00
|
|
|
Log(log, " Options.bytes_per_sync: %lu",
|
|
|
|
(unsigned long)bytes_per_sync);
|
|
|
|
} // DBOptions::Dump
|
|
|
|
|
|
|
|
void ColumnFamilyOptions::Dump(Logger* log) const {
|
|
|
|
Log(log, " Options.comparator: %s", comparator->Name());
|
|
|
|
Log(log, " Options.merge_operator: %s",
|
|
|
|
merge_operator ? merge_operator->Name() : "None");
|
2014-08-13 22:58:31 +00:00
|
|
|
Log(log, " Options.compaction_filter: %s",
|
|
|
|
compaction_filter ? compaction_filter->Name() : "None");
|
2014-02-07 05:39:20 +00:00
|
|
|
Log(log, " Options.compaction_filter_factory: %s",
|
|
|
|
compaction_filter_factory->Name());
|
2014-03-25 18:09:40 +00:00
|
|
|
Log(log, " Options.compaction_filter_factory_v2: %s",
|
|
|
|
compaction_filter_factory_v2->Name());
|
2014-02-07 05:39:20 +00:00
|
|
|
Log(log, " Options.memtable_factory: %s", memtable_factory->Name());
|
|
|
|
Log(log, " Options.table_factory: %s", table_factory->Name());
|
2014-08-25 21:24:09 +00:00
|
|
|
Log(log, " table_factory options: %s",
|
|
|
|
table_factory->GetPrintableTableOptions().c_str());
|
2014-02-07 05:39:20 +00:00
|
|
|
Log(log, " Options.write_buffer_size: %zd", write_buffer_size);
|
|
|
|
Log(log, " Options.max_write_buffer_number: %d", max_write_buffer_number);
|
2013-01-24 18:54:26 +00:00
|
|
|
if (!compression_per_level.empty()) {
|
2013-03-15 01:32:01 +00:00
|
|
|
for (unsigned int i = 0; i < compression_per_level.size(); i++) {
|
2012-11-01 17:50:08 +00:00
|
|
|
Log(log," Options.compression[%d]: %d",
|
2012-10-28 06:13:17 +00:00
|
|
|
i, compression_per_level[i]);
|
|
|
|
}
|
|
|
|
} else {
|
2012-10-29 21:18:00 +00:00
|
|
|
Log(log," Options.compression: %d", compression);
|
2012-10-28 06:13:17 +00:00
|
|
|
}
|
2013-08-14 16:06:10 +00:00
|
|
|
Log(log," Options.prefix_extractor: %s",
|
|
|
|
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
|
2012-08-27 19:10:26 +00:00
|
|
|
Log(log," Options.num_levels: %d", num_levels);
|
2013-06-11 21:23:58 +00:00
|
|
|
Log(log," Options.min_write_buffer_number_to_merge: %d",
|
|
|
|
min_write_buffer_number_to_merge);
|
2013-02-28 22:09:30 +00:00
|
|
|
Log(log," Options.purge_redundant_kvs_while_flush: %d",
|
|
|
|
purge_redundant_kvs_while_flush);
|
2012-11-01 17:50:08 +00:00
|
|
|
Log(log," Options.compression_opts.window_bits: %d",
|
|
|
|
compression_opts.window_bits);
|
|
|
|
Log(log," Options.compression_opts.level: %d",
|
|
|
|
compression_opts.level);
|
|
|
|
Log(log," Options.compression_opts.strategy: %d",
|
|
|
|
compression_opts.strategy);
|
2012-08-27 19:10:26 +00:00
|
|
|
Log(log," Options.level0_file_num_compaction_trigger: %d",
|
|
|
|
level0_file_num_compaction_trigger);
|
|
|
|
Log(log," Options.level0_slowdown_writes_trigger: %d",
|
|
|
|
level0_slowdown_writes_trigger);
|
|
|
|
Log(log," Options.level0_stop_writes_trigger: %d",
|
|
|
|
level0_stop_writes_trigger);
|
|
|
|
Log(log," Options.max_mem_compaction_level: %d",
|
|
|
|
max_mem_compaction_level);
|
|
|
|
Log(log," Options.target_file_size_base: %d",
|
|
|
|
target_file_size_base);
|
|
|
|
Log(log," Options.target_file_size_multiplier: %d",
|
|
|
|
target_file_size_multiplier);
|
2013-11-13 05:02:03 +00:00
|
|
|
Log(log," Options.max_bytes_for_level_base: %lu",
|
|
|
|
(unsigned long)max_bytes_for_level_base);
|
2012-08-27 19:10:26 +00:00
|
|
|
Log(log," Options.max_bytes_for_level_multiplier: %d",
|
|
|
|
max_bytes_for_level_multiplier);
|
2013-05-21 18:37:06 +00:00
|
|
|
for (int i = 0; i < num_levels; i++) {
|
|
|
|
Log(log,"Options.max_bytes_for_level_multiplier_addtl[%d]: %d",
|
|
|
|
i, max_bytes_for_level_multiplier_additional[i]);
|
|
|
|
}
|
2013-11-13 05:02:03 +00:00
|
|
|
Log(log," Options.max_sequential_skip_in_iterations: %lu",
|
|
|
|
(unsigned long)max_sequential_skip_in_iterations);
|
2012-08-27 19:10:26 +00:00
|
|
|
Log(log," Options.expanded_compaction_factor: %d",
|
|
|
|
expanded_compaction_factor);
|
2012-11-21 07:07:41 +00:00
|
|
|
Log(log," Options.source_compaction_factor: %d",
|
|
|
|
source_compaction_factor);
|
2012-08-27 19:10:26 +00:00
|
|
|
Log(log," Options.max_grandparent_overlap_factor: %d",
|
|
|
|
max_grandparent_overlap_factor);
|
2014-02-03 18:21:58 +00:00
|
|
|
Log(log," Options.arena_block_size: %zu",
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 19:42:23 +00:00
|
|
|
arena_block_size);
|
2013-08-14 16:06:10 +00:00
|
|
|
Log(log," Options.soft_rate_limit: %.2f",
|
|
|
|
soft_rate_limit);
|
2013-08-05 22:43:49 +00:00
|
|
|
Log(log," Options.hard_rate_limit: %.2f",
|
|
|
|
hard_rate_limit);
|
2013-08-02 18:46:47 +00:00
|
|
|
Log(log," Options.rate_limit_delay_max_milliseconds: %u",
|
2013-08-05 22:43:49 +00:00
|
|
|
rate_limit_delay_max_milliseconds);
|
2012-11-20 23:45:41 +00:00
|
|
|
Log(log," Options.disable_auto_compactions: %d",
|
|
|
|
disable_auto_compactions);
|
2013-04-10 02:42:07 +00:00
|
|
|
Log(log," Options.purge_redundant_kvs_while_flush: %d",
|
|
|
|
purge_redundant_kvs_while_flush);
|
2013-07-12 23:56:52 +00:00
|
|
|
Log(log," Options.filter_deletes: %d",
|
|
|
|
filter_deletes);
|
2014-03-10 17:06:34 +00:00
|
|
|
Log(log, " Options.verify_checksums_in_compaction: %d",
|
|
|
|
verify_checksums_in_compaction);
|
2013-07-03 22:32:49 +00:00
|
|
|
Log(log," Options.compaction_style: %d",
|
|
|
|
compaction_style);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
Log(log," Options.compaction_options_universal.size_ratio: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.size_ratio);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
Log(log,"Options.compaction_options_universal.min_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.min_merge_width);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
Log(log,"Options.compaction_options_universal.max_merge_width: %u",
|
2013-07-03 22:32:49 +00:00
|
|
|
compaction_options_universal.max_merge_width);
|
2013-09-09 23:06:10 +00:00
|
|
|
Log(log,"Options.compaction_options_universal."
|
|
|
|
"max_size_amplification_percent: %u",
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
2013-12-03 20:32:07 +00:00
|
|
|
Log(log,
|
|
|
|
"Options.compaction_options_universal.compression_size_percent: %u",
|
|
|
|
compaction_options_universal.compression_size_percent);
|
2014-05-21 18:43:35 +00:00
|
|
|
Log(log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64,
|
|
|
|
compaction_options_fifo.max_table_files_size);
|
2013-10-16 18:50:50 +00:00
|
|
|
std::string collector_names;
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 19:30:55 +00:00
|
|
|
for (const auto& collector_factory : table_properties_collector_factories) {
|
|
|
|
collector_names.append(collector_factory->Name());
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.append("; ");
|
|
|
|
}
|
2013-11-20 00:29:42 +00:00
|
|
|
Log(log, " Options.table_properties_collectors: %s",
|
2013-10-16 18:50:50 +00:00
|
|
|
collector_names.c_str());
|
2013-11-08 05:27:21 +00:00
|
|
|
Log(log, " Options.inplace_update_support: %d",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
inplace_update_support);
|
2013-11-08 05:27:21 +00:00
|
|
|
Log(log, " Options.inplace_update_num_locks: %zd",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 21:12:47 +00:00
|
|
|
inplace_update_num_locks);
|
2014-03-25 00:57:13 +00:00
|
|
|
Log(log, " Options.min_partial_merge_operands: %u",
|
|
|
|
min_partial_merge_operands);
|
2013-11-27 22:27:02 +00:00
|
|
|
// TODO: easier config for bloom (maybe based on avg key/value size)
|
|
|
|
Log(log, " Options.memtable_prefix_bloom_bits: %d",
|
|
|
|
memtable_prefix_bloom_bits);
|
|
|
|
Log(log, " Options.memtable_prefix_bloom_probes: %d",
|
|
|
|
memtable_prefix_bloom_probes);
|
2014-05-04 20:55:53 +00:00
|
|
|
Log(log, " Options.memtable_prefix_bloom_huge_page_tlb_size: %zu",
|
|
|
|
memtable_prefix_bloom_huge_page_tlb_size);
|
2014-04-07 23:33:03 +00:00
|
|
|
Log(log, " Options.bloom_locality: %d",
|
|
|
|
bloom_locality);
|
2014-01-11 01:33:56 +00:00
|
|
|
Log(log, " Options.max_successive_merges: %zd",
|
|
|
|
max_successive_merges);
|
2014-02-07 05:39:20 +00:00
|
|
|
} // ColumnFamilyOptions::Dump
|
|
|
|
|
|
|
|
void Options::Dump(Logger* log) const {
|
|
|
|
DBOptions::Dump(log);
|
|
|
|
ColumnFamilyOptions::Dump(log);
|
2012-08-22 18:43:53 +00:00
|
|
|
} // Options::Dump
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
//
|
|
|
|
// The goal of this method is to create a configuration that
|
|
|
|
// allows an application to write all files into L0 and
|
|
|
|
// then do a single compaction to output all files into L1.
|
2013-02-26 06:57:37 +00:00
|
|
|
Options*
|
|
|
|
Options::PrepareForBulkLoad()
|
|
|
|
{
|
2013-03-08 17:19:24 +00:00
|
|
|
// never slowdown ingest.
|
2013-02-26 06:57:37 +00:00
|
|
|
level0_file_num_compaction_trigger = (1<<30);
|
|
|
|
level0_slowdown_writes_trigger = (1<<30);
|
|
|
|
level0_stop_writes_trigger = (1<<30);
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
// no auto compactions please. The application should issue a
|
|
|
|
// manual compaction after all data is loaded into L0.
|
2013-02-26 06:57:37 +00:00
|
|
|
disable_auto_compactions = true;
|
|
|
|
disableDataSync = true;
|
2013-03-08 17:19:24 +00:00
|
|
|
|
|
|
|
// A manual compaction run should pick all files in L0 in
|
|
|
|
// a single compaction run.
|
2013-02-26 06:57:37 +00:00
|
|
|
source_compaction_factor = (1<<30);
|
|
|
|
|
2013-03-08 17:19:24 +00:00
|
|
|
// It is better to have only 2 levels, otherwise a manual
|
|
|
|
// compaction would compact at every possible level, thereby
|
|
|
|
// increasing the total time needed for compactions.
|
|
|
|
num_levels = 2;
|
|
|
|
|
|
|
|
// Prevent a memtable flush to automatically promote files
|
|
|
|
// to L1. This is helpful so that all files that are
|
|
|
|
// input to the manual compaction are all at L0.
|
|
|
|
max_background_compactions = 2;
|
|
|
|
|
|
|
|
// The compaction would create large files in L1.
|
|
|
|
target_file_size_base = 256 * 1024 * 1024;
|
2013-02-26 06:57:37 +00:00
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
2014-05-10 17:49:33 +00:00
|
|
|
// Optimization functions
|
2014-08-26 21:15:00 +00:00
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
|
|
|
|
uint64_t block_cache_size_mb) {
|
2014-05-10 17:49:33 +00:00
|
|
|
prefix_extractor.reset(NewNoopTransform());
|
|
|
|
BlockBasedTableOptions block_based_options;
|
2014-08-26 21:15:00 +00:00
|
|
|
block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
block_based_options.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
|
|
block_based_options.block_cache =
|
|
|
|
NewLRUCache(block_cache_size_mb * 1024 * 1024);
|
2014-05-10 17:49:33 +00:00
|
|
|
table_factory.reset(new BlockBasedTableFactory(block_based_options));
|
2014-06-03 20:58:44 +00:00
|
|
|
#ifndef ROCKSDB_LITE
|
2014-05-10 17:49:33 +00:00
|
|
|
memtable_factory.reset(NewHashLinkListRepFactory());
|
2014-06-03 20:58:44 +00:00
|
|
|
#endif
|
2014-05-10 17:49:33 +00:00
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction(
|
|
|
|
uint64_t memtable_memory_budget) {
|
|
|
|
write_buffer_size = memtable_memory_budget / 4;
|
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
// write stalls.
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
// start flushing L0->L1 as soon as possible. each file on level0 is
|
|
|
|
// (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
|
|
|
|
// memtable_memory_budget.
|
|
|
|
level0_file_num_compaction_trigger = 2;
|
|
|
|
// doesn't really matter much, but we don't want to create too many files
|
|
|
|
target_file_size_base = memtable_memory_budget / 8;
|
|
|
|
// make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
|
|
|
|
max_bytes_for_level_base = memtable_memory_budget;
|
|
|
|
|
|
|
|
// level style compaction
|
|
|
|
compaction_style = kCompactionStyleLevel;
|
|
|
|
|
|
|
|
// only compress levels >= 2
|
|
|
|
compression_per_level.resize(num_levels);
|
|
|
|
for (int i = 0; i < num_levels; ++i) {
|
|
|
|
if (i < 2) {
|
|
|
|
compression_per_level[i] = kNoCompression;
|
|
|
|
} else {
|
|
|
|
compression_per_level[i] = kSnappyCompression;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
|
|
|
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction(
|
|
|
|
uint64_t memtable_memory_budget) {
|
|
|
|
write_buffer_size = memtable_memory_budget / 4;
|
|
|
|
// merge two memtables when flushing to L0
|
|
|
|
min_write_buffer_number_to_merge = 2;
|
|
|
|
// this means we'll use 50% extra memory in the worst case, but will reduce
|
|
|
|
// write stalls.
|
|
|
|
max_write_buffer_number = 6;
|
|
|
|
// universal style compaction
|
|
|
|
compaction_style = kCompactionStyleUniversal;
|
|
|
|
compaction_options_universal.compression_size_percent = 80;
|
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
|
|
|
DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
|
|
|
|
max_background_compactions = total_threads - 1;
|
|
|
|
max_background_flushes = 1;
|
|
|
|
env->SetBackgroundThreads(total_threads, Env::LOW);
|
|
|
|
env->SetBackgroundThreads(1, Env::HIGH);
|
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
2013-10-04 04:49:15 +00:00
|
|
|
} // namespace rocksdb
|