dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-29 09:36:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

memtable_prefix_bloom_bits -> memtable_prefix_bloom_bits_ratio and deprecate memtable_prefix_bloom_probes

Browse source 
Summary:
memtable_prefix_bloom_probes is not a critical option. Remove it to reduce number of options.
It's easier for users to make mistakes with memtable_prefix_bloom_bits, turn it to memtable_prefix_bloom_bits_ratio

Test Plan: Run all existing tests

Reviewers: yhchiang, igor, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: gunnarku, yoshinorim, MarkCallaghan, leveldb, andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D59199
This commit is contained in:
sdong 2016-06-03 17:02:10 -07:00
parent e9c1face6c
commit 20699df843
24 changed files with 128 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
HISTORY.md
View file

@ -1,5 +1,8 @@
# Rocksdb Change Log
## Unreleased
### Public API Change
* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false
* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
## 4.9.0 (6/9/2016)
### Public API changes

11
db/c.cc
View file

@ -1799,14 +1799,9 @@ void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t *opt) {
opt->rep.memtable_factory.reset(factory);
}
void rocksdb_options_set_memtable_prefix_bloom_bits(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.memtable_prefix_bloom_bits = v;
}
void rocksdb_options_set_memtable_prefix_bloom_probes(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.memtable_prefix_bloom_probes = v;
void rocksdb_options_set_memtable_prefix_bloom_size_ratio(
rocksdb_options_t* opt, double v) {
opt->rep.memtable_prefix_bloom_size_ratio = v;
}
void rocksdb_options_set_memtable_prefix_bloom_huge_page_tlb_size(

6
db/column_family.cc
View file

@ -177,6 +177,12 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
if (result.max_write_buffer_number_to_maintain < 0) {
result.max_write_buffer_number_to_maintain = result.max_write_buffer_number;
}
// bloom filter size shouldn't exceed 1/4 of memtable size.
if (result.memtable_prefix_bloom_size_ratio > 0.25) {
result.memtable_prefix_bloom_size_ratio = 0.25;
} else if (result.memtable_prefix_bloom_size_ratio < 0) {
result.memtable_prefix_bloom_size_ratio = 0;
}
XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
xf_transaction_set_memtable_history1,
xf_transaction_set_memtable_history,

3
db/db_bloom_filter_test.cc
View file

@ -615,7 +615,8 @@ class BloomStatsTestWithParam
options_.create_if_missing = true;
options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
options_.memtable_prefix_bloom_bits = 8 * 1024;
options_.memtable_prefix_bloom_size_ratio =
8.0 * 1024.0 / static_cast<double>(options_.write_buffer_size);
if (use_block_table_) {
BlockBasedTableOptions table_options;
table_options.hash_index_allow_collision = false;

45
db/memtable.cc
View file

@ -34,24 +34,25 @@
namespace rocksdb {
MemTableOptions::MemTableOptions(
const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options)
: write_buffer_size(mutable_cf_options.write_buffer_size),
arena_block_size(mutable_cf_options.arena_block_size),
memtable_prefix_bloom_bits(mutable_cf_options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(
mutable_cf_options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_huge_page_tlb_size(
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
inplace_update_support(ioptions.inplace_update_support),
inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
inplace_callback(ioptions.inplace_callback),
max_successive_merges(mutable_cf_options.max_successive_merges),
filter_deletes(mutable_cf_options.filter_deletes),
statistics(ioptions.statistics),
merge_operator(ioptions.merge_operator),
info_log(ioptions.info_log) {}
MemTableOptions::MemTableOptions(const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options)
: write_buffer_size(mutable_cf_options.write_buffer_size),
arena_block_size(mutable_cf_options.arena_block_size),
memtable_prefix_bloom_bits(
static_cast<uint32_t>(
static_cast<double>(mutable_cf_options.write_buffer_size) *
mutable_cf_options.memtable_prefix_bloom_size_ratio) *
8u),
memtable_prefix_bloom_huge_page_tlb_size(
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
inplace_update_support(ioptions.inplace_update_support),
inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
inplace_callback(ioptions.inplace_callback),
max_successive_merges(mutable_cf_options.max_successive_merges),
filter_deletes(mutable_cf_options.filter_deletes),
statistics(ioptions.statistics),
merge_operator(ioptions.merge_operator),
info_log(ioptions.info_log) {}
MemTable::MemTable(const InternalKeyComparator& cmp,
const ImmutableCFOptions& ioptions,
@ -88,11 +89,9 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
prefix_bloom_.reset(new DynamicBloom(
&allocator_,
moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
moptions_.memtable_prefix_bloom_probes, nullptr,
moptions_.memtable_prefix_bloom_huge_page_tlb_size,
ioptions.info_log));
&allocator_, moptions_.memtable_prefix_bloom_bits,
ioptions.bloom_locality, 6 /* hard coded 6 probes */, nullptr,
moptions_.memtable_prefix_bloom_huge_page_tlb_size, ioptions.info_log));
}
}

1
db/memtable.h
View file

@ -42,7 +42,6 @@ struct MemTableOptions {
size_t write_buffer_size;
size_t arena_block_size;
uint32_t memtable_prefix_bloom_bits;
uint32_t memtable_prefix_bloom_probes;
size_t memtable_prefix_bloom_huge_page_tlb_size;
bool inplace_update_support;
size_t inplace_update_num_locks;

7
db/prefix_test.cc
View file

@ -43,8 +43,7 @@ DEFINE_int64(write_buffer_size, 33554432, "");
DEFINE_int32(max_write_buffer_number, 2, "");
DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
DEFINE_int32(skiplist_height, 4, "");
DEFINE_int32(memtable_prefix_bloom_bits, 10000000, "");
DEFINE_int32(memtable_prefix_bloom_probes, 10, "");
DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, "");
DEFINE_int32(memtable_prefix_bloom_huge_page_tlb_size, 2 * 1024 * 1024, "");
DEFINE_int32(value_size, 40, "");
@ -160,8 +159,8 @@ class PrefixTest : public testing::Test {
options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge;
options.memtable_prefix_bloom_bits = FLAGS_memtable_prefix_bloom_bits;
options.memtable_prefix_bloom_probes = FLAGS_memtable_prefix_bloom_probes;
options.memtable_prefix_bloom_size_ratio =
FLAGS_memtable_prefix_bloom_size_ratio;
options.memtable_prefix_bloom_huge_page_tlb_size =
FLAGS_memtable_prefix_bloom_huge_page_tlb_size;

1
db/version_set.h
View file

@ -57,7 +57,6 @@ class Version;
class VersionSet;
class WriteBuffer;
class MergeContext;
class ColumnFamilyData;
class ColumnFamilySet;
class TableCache;
class MergeIteratorBuilder;

15
include/rocksdb/options.h
View file

@ -745,16 +745,15 @@ struct ColumnFamilyOptions {
Slice delta_value,
std::string* merged_value);
// if prefix_extractor is set and bloom_bits is not 0, create prefix bloom
// for memtable
// if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
// create prefix bloom for memtable with the size of
// write_buffer_size * memtable_prefix_bloom_size_ratio.
// If it is larger than 0.25, it is santinized to 0.25.
//
// Default: 0 (disable)
//
// Dynamically changeable through SetOptions() API
uint32_t memtable_prefix_bloom_bits;
// number of hash probes per key
//
// Dynamically changeable through SetOptions() API
uint32_t memtable_prefix_bloom_probes;
double memtable_prefix_bloom_size_ratio;
// Page size for huge page TLB for bloom in memtable. If <=0, not allocate
// from huge page TLB but from malloc.

91
java/rocksjni/options.cc
View file

@ -1732,50 +1732,27 @@ void Java_org_rocksdb_Options_setInplaceUpdateNumLocks(
/*
* Class: org_rocksdb_Options
* Method: memtablePrefixBloomBits
* Method: memtablePrefixBloomSizeRatio
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_memtablePrefixBloomBits(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_bits;
jdouble Java_org_rocksdb_Options_memtablePrefixBloomSizeRatio(JNIEnv* env,
jobject jobj,
jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(jhandle)
->memtable_prefix_bloom_size_ratio;
}
/*
* Class: org_rocksdb_Options
* Method: setMemtablePrefixBloomBits
* Method: setMemtablePrefixBloomSizeRatio
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMemtablePrefixBloomBits(
void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_bits) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_bits =
static_cast<int32_t>(jmemtable_prefix_bloom_bits);
}
/*
* Class: org_rocksdb_Options
* Method: memtablePrefixBloomProbes
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_memtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_probes;
}
/*
* Class: org_rocksdb_Options
* Method: setMemtablePrefixBloomProbes
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMemtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_probes) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_probes =
static_cast<int32_t>(jmemtable_prefix_bloom_probes);
jdouble jmemtable_prefix_bloom_size_ratio) {
reinterpret_cast<rocksdb::Options*>(jhandle)
->memtable_prefix_bloom_size_ratio =
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*
@ -2915,50 +2892,26 @@ void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks(
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtablePrefixBloomBits
* Method: memtablePrefixBloomSizeRatio
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomBits(
jdouble Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_bits;
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
->memtable_prefix_bloom_size_ratio;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtablePrefixBloomBits
* Method: setMemtablePrefixBloomSizeRatio
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomBits(
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_bits) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_bits =
static_cast<int32_t>(jmemtable_prefix_bloom_bits);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtablePrefixBloomProbes
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_probes;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtablePrefixBloomProbes
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_probes) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_probes =
static_cast<int32_t>(jmemtable_prefix_bloom_probes);
jdouble jmemtable_prefix_bloom_size_ratio) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
->memtable_prefix_bloom_size_ratio =
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*

30
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
View file

@ -589,27 +589,15 @@ public class ColumnFamilyOptions extends RocksObject
}
@Override
public ColumnFamilyOptions setMemtablePrefixBloomBits(
final int memtablePrefixBloomBits) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits);
public ColumnFamilyOptions setMemtablePrefixBloomSizeRatio(
final double memtablePrefixBloomSizeRatio) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomSizeRatio);
return this;
}
@Override
public int memtablePrefixBloomBits() {
return memtablePrefixBloomBits(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMemtablePrefixBloomProbes(
final int memtablePrefixBloomProbes) {
setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
return this;
}
@Override
public int memtablePrefixBloomProbes() {
return memtablePrefixBloomProbes(nativeHandle_);
public double memtablePrefixBloomSizeRatio() {
return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
@ -787,12 +775,8 @@ public class ColumnFamilyOptions extends RocksObject
long handle, long inplaceUpdateNumLocks)
throws IllegalArgumentException;
private native long inplaceUpdateNumLocks(long handle);
private native void setMemtablePrefixBloomBits(
long handle, int memtablePrefixBloomBits);
private native int memtablePrefixBloomBits(long handle);
private native void setMemtablePrefixBloomProbes(
long handle, int memtablePrefixBloomProbes);
private native int memtablePrefixBloomProbes(long handle);
private native void setMemtablePrefixBloomBits(long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

25
java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
View file

@ -1011,15 +1011,15 @@ public interface ColumnFamilyOptionsInterface {
long inplaceUpdateNumLocks();
/**
* Sets the number of bits used in the prefix bloom filter.
* Sets the size ratio of the memtable used in the prefix bloom filter.
*
* This value will be used only when a prefix-extractor is specified.
*
* @param memtablePrefixBloomBits the number of bits used in the
* @param memtablePrefixBloomSizeRatio the number of bits used in the
* prefix bloom filter.
* @return the reference to the current option.
*/
Object setMemtablePrefixBloomBits(int memtablePrefixBloomBits);
Object setMemtablePrefixBloomSizeRatio(double memtablePrefixBloomSizeRatio);
/**
* Returns the number of bits used in the prefix bloom filter.
@ -1029,22 +1029,7 @@ public interface ColumnFamilyOptionsInterface {
* @return the number of bloom-bits.
* @see #useFixedLengthPrefixExtractor(int)
*/
int memtablePrefixBloomBits();
/**
* The number of hash probes per key used in the mem-table.
*
* @param memtablePrefixBloomProbes the number of hash probes per key.
* @return the reference to the current option.
*/
Object setMemtablePrefixBloomProbes(int memtablePrefixBloomProbes);
/**
* The number of hash probes per key used in the mem-table.
*
* @return the number of hash probes per key.
*/
int memtablePrefixBloomProbes();
double memtablePrefixBloomSizeRatio();
/**
* Control locality of bloom filter probes to improve cache miss rate.
@ -1074,7 +1059,7 @@ public interface ColumnFamilyOptionsInterface {
* Default: 0
*
* @return the level of locality of bloom-filter probes.
* @see #setMemtablePrefixBloomProbes(int)
* @see #setBloomLocality(int)
*/
int bloomLocality();

30
java/src/main/java/org/rocksdb/Options.java
View file

@ -1005,26 +1005,13 @@ public class Options extends RocksObject
}
@Override
public int memtablePrefixBloomBits() {
return memtablePrefixBloomBits(nativeHandle_);
public double memtablePrefixBloomSizeRatio() {
return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
public Options setMemtablePrefixBloomBits(
final int memtablePrefixBloomBits) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits);
return this;
}
@Override
public int memtablePrefixBloomProbes() {
return memtablePrefixBloomProbes(nativeHandle_);
}
@Override
public Options setMemtablePrefixBloomProbes(
final int memtablePrefixBloomProbes) {
setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
public Options setMemtablePrefixBloomSizeRatio(final double memtablePrefixBloomSizeRatio) {
setMemtablePrefixBloomSizeRatio(nativeHandle_, memtablePrefixBloomSizeRatio);
return this;
}
@ -1293,12 +1280,9 @@ public class Options extends RocksObject
long handle, long inplaceUpdateNumLocks)
throws IllegalArgumentException;
private native long inplaceUpdateNumLocks(long handle);
private native void setMemtablePrefixBloomBits(
long handle, int memtablePrefixBloomBits);
private native int memtablePrefixBloomBits(long handle);
private native void setMemtablePrefixBloomProbes(
long handle, int memtablePrefixBloomProbes);
private native int memtablePrefixBloomProbes(long handle);
private native void setMemtablePrefixBloomSizeRatio(
long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

17
java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
View file

@ -305,20 +305,11 @@ public class ColumnFamilyOptionsTest {
}
@Test
public void memtablePrefixBloomBits() {
public void memtablePrefixBloomSizeRatio() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomBits(intValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
final double doubleValue = rand.nextDouble();
opt.setMemtablePrefixBloomSizeRatio(doubleValue);
assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue);
}
}

17
java/src/test/java/org/rocksdb/OptionsTest.java
View file

@ -268,20 +268,11 @@ public class OptionsTest {
}
@Test
public void memtablePrefixBloomBits() {
public void memtablePrefixBloomSizeRatio() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomBits(intValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
final double doubleValue = rand.nextDouble();
opt.setMemtablePrefixBloomSizeRatio(doubleValue);
assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue);
}
}

7
tools/db_bench_tool.cc
View file

@ -396,8 +396,9 @@ DEFINE_int32(skip_table_builder_flush, false, "Skip flushing block in "
DEFINE_int32(bloom_bits, -1, "Bloom filter bits per key. Negative means"
" use default settings.");
DEFINE_int32(memtable_bloom_bits, 0, "Bloom filter bits per key for memtable. "
"Negative means no bloom filter.");
DEFINE_double(memtable_bloom_size_ratio, 0,
"Ratio of memtable size used for bloom filter. 0 means no bloom "
"filter.");
DEFINE_bool(use_existing_db, false, "If true, do not destroy the existing"
" database. If you set this flag and also specify a benchmark that"
@ -2461,7 +2462,7 @@ class Benchmark {
exit(1);
}
}
options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits;
options.memtable_prefix_bloom_size_ratio = FLAGS_memtable_bloom_size_ratio;
options.bloom_locality = FLAGS_bloom_locality;
options.max_file_opening_threads = FLAGS_file_opening_threads;
options.new_table_reader_for_compaction_inputs =

6
util/mutable_cf_options.cc
View file

@ -70,10 +70,8 @@ void MutableCFOptions::Dump(Logger* log) const {
max_write_buffer_number);
Log(log, " arena_block_size: %" ROCKSDB_PRIszt,
arena_block_size);
Log(log, " memtable_prefix_bloom_bits: %" PRIu32,
memtable_prefix_bloom_bits);
Log(log, " memtable_prefix_bloom_probes: %" PRIu32,
memtable_prefix_bloom_probes);
Log(log, " memtable_prefix_bloom_ratio: %f",
memtable_prefix_bloom_size_ratio);
Log(log, " memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,
memtable_prefix_bloom_huge_page_tlb_size);
Log(log, " max_successive_merges: %" ROCKSDB_PRIszt,

10
util/mutable_cf_options.h
View file

@ -17,8 +17,8 @@ struct MutableCFOptions {
: write_buffer_size(options.write_buffer_size),
max_write_buffer_number(options.max_write_buffer_number),
arena_block_size(options.arena_block_size),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_size_ratio(
options.memtable_prefix_bloom_size_ratio),
memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size),
max_successive_merges(options.max_successive_merges),
@ -58,8 +58,7 @@ struct MutableCFOptions {
: write_buffer_size(0),
max_write_buffer_number(0),
arena_block_size(0),
memtable_prefix_bloom_bits(0),
memtable_prefix_bloom_probes(0),
memtable_prefix_bloom_size_ratio(0),
memtable_prefix_bloom_huge_page_tlb_size(0),
max_successive_merges(0),
filter_deletes(false),
@ -110,8 +109,7 @@ struct MutableCFOptions {
size_t write_buffer_size;
int max_write_buffer_number;
size_t arena_block_size;
uint32_t memtable_prefix_bloom_bits;
uint32_t memtable_prefix_bloom_probes;
double memtable_prefix_bloom_size_ratio;
size_t memtable_prefix_bloom_huge_page_tlb_size;
size_t max_successive_merges;
bool filter_deletes;

13
util/options.cc
View file

@ -121,8 +121,7 @@ ColumnFamilyOptions::ColumnFamilyOptions()
inplace_update_support(false),
inplace_update_num_locks(10000),
inplace_callback(nullptr),
memtable_prefix_bloom_bits(0),
memtable_prefix_bloom_probes(6),
memtable_prefix_bloom_size_ratio(0.0),
memtable_prefix_bloom_huge_page_tlb_size(0),
bloom_locality(0),
max_successive_merges(0),
@ -190,8 +189,8 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
inplace_update_support(options.inplace_update_support),
inplace_update_num_locks(options.inplace_update_num_locks),
inplace_callback(options.inplace_callback),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_size_ratio(
options.memtable_prefix_bloom_size_ratio),
memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size),
bloom_locality(options.bloom_locality),
@ -596,10 +595,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
Header(log, " Options.min_partial_merge_operands: %u",
min_partial_merge_operands);
// TODO: easier config for bloom (maybe based on avg key/value size)
Header(log, " Options.memtable_prefix_bloom_bits: %d",
memtable_prefix_bloom_bits);
Header(log, " Options.memtable_prefix_bloom_probes: %d",
memtable_prefix_bloom_probes);
Header(log, " Options.memtable_prefix_bloom_size_ratio: %f",
memtable_prefix_bloom_size_ratio);
Header(log,
" Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,

12
util/options_helper.cc
View file

@ -537,9 +537,11 @@ bool ParseMemtableOptions(const std::string& name, const std::string& value,
} else if (name == "arena_block_size") {
new_options->arena_block_size = ParseSizeT(value);
} else if (name == "memtable_prefix_bloom_bits") {
new_options->memtable_prefix_bloom_bits = ParseUint32(value);
// deprecated
} else if (name == "memtable_prefix_bloom_size_ratio") {
new_options->memtable_prefix_bloom_size_ratio = ParseDouble(value);
} else if (name == "memtable_prefix_bloom_probes") {
new_options->memtable_prefix_bloom_probes = ParseUint32(value);
// Deprecated
} else if (name == "memtable_prefix_bloom_huge_page_tlb_size") {
new_options->memtable_prefix_bloom_huge_page_tlb_size =
ParseSizeT(value);
@ -1437,10 +1439,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
cf_opts.memtable_prefix_bloom_bits =
mutable_cf_options.memtable_prefix_bloom_bits;
cf_opts.memtable_prefix_bloom_probes =
mutable_cf_options.memtable_prefix_bloom_probes;
cf_opts.memtable_prefix_bloom_size_ratio =
mutable_cf_options.memtable_prefix_bloom_size_ratio;
cf_opts.memtable_prefix_bloom_huge_page_tlb_size =
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size;
cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;

9
util/options_helper.h
View file

@ -437,11 +437,12 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{offsetof(struct ColumnFamilyOptions, bloom_locality),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_bits",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}},
{"memtable_prefix_bloom_size_ratio",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_size_ratio),
OptionType::kDouble, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_probes",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}},
{"min_partial_merge_operands",
{offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
OptionType::kUInt32T, OptionVerificationType::kNormal}},

3
util/options_settable_test.cc
View file

@ -413,14 +413,13 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"max_write_buffer_number_to_maintain=84;"
"verify_checksums_in_compaction=false;"
"merge_operator=aabcxehazrMergeOperator;"
"memtable_prefix_bloom_bits=4642;"
"memtable_prefix_bloom_size_ratio=0.4642;"
"paranoid_file_checks=true;"
"inplace_update_num_locks=7429;"
"optimize_filters_for_hits=false;"
"level_compaction_dynamic_level_bytes=false;"
"inplace_update_support=false;"
"compaction_style=kCompactionStyleFIFO;"
"memtable_prefix_bloom_probes=2511;"
"purge_redundant_kvs_while_flush=true;"
"filter_deletes=false;"
"hard_pending_compaction_bytes_limit=0;"

12
util/options_test.cc
View file

@ -86,8 +86,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"report_bg_io_stats", "true"},
{"compaction_measure_io_stats", "false"},
{"inplace_update_num_locks", "25"},
{"memtable_prefix_bloom_bits", "26"},
{"memtable_prefix_bloom_probes", "27"},
{"memtable_prefix_bloom_size_ratio", "0.26"},
{"memtable_prefix_bloom_huge_page_tlb_size", "28"},
{"bloom_locality", "29"},
{"max_successive_merges", "30"},
@ -187,8 +186,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
static_cast<uint64_t>(24));
ASSERT_EQ(new_cf_opt.inplace_update_support, true);
ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 26U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_probes, 27U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_huge_page_tlb_size, 28U);
ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
@ -300,10 +298,8 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
const int64_t tera = 1024 * giga;
// Units (k)
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
"memtable_prefix_bloom_bits=14k;max_write_buffer_number=-15K",
&new_cf_opt));
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 14UL * kilo);
ASSERT_OK(GetColumnFamilyOptionsFromString(
base_cf_opt, "max_write_buffer_number=-15K", &new_cf_opt));
ASSERT_EQ(new_cf_opt.max_write_buffer_number, -15 * kilo);
// Units (m)
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,

4
util/testutil.cc
View file

@ -307,6 +307,8 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// double options
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->memtable_prefix_bloom_size_ratio =
static_cast<double>(rnd->Uniform(10000)) / 20000.0;
// int options
cf_opt->expanded_compaction_factor = rnd->Uniform(100);
@ -332,8 +334,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// uint32_t options
cf_opt->bloom_locality = rnd->Uniform(10000);
cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000);
cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000);
cf_opt->min_partial_merge_operands = rnd->Uniform(10000);
cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);