dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-25 22:44:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add db_bench options of data block hash index (#4281)

Browse source 
Summary:
Add `--data_block_index_type` and `--data_block_hash_table_util_ratio` option to `db_bench`.

`--data_block_index_type` can be either of `binary` (default) or `binary_and_hash`;
`--data_block_hash_table_util_ratio` will be a double. The default value is `0.75`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4281

Differential Revision: D9361476

Pulled By: fgwu

fbshipit-source-id: dc53e01acef9db81b9eec5e8a96f3bc8ed718c10
This commit is contained in:
Fenggang Wu 2018-08-16 18:29:13 -07:00 committed by Facebook Github Bot
parent 889a0553c8
commit 9d646a6311
10 changed files with 56 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/rocksdb/table.h
View file

@ -102,8 +102,8 @@ struct BlockBasedTableOptions {
// The index type that will be used for the data block.
enum DataBlockIndexType : char {
kDataBlockBinarySearch = 0, // traditional block type
kDataBlockBinaryAndHash = 1, // additional hash index
kDataBlockBinarySearch = 0, // traditional block type
kDataBlockBinaryAndHash = 1, // additional hash index
};
DataBlockIndexType data_block_index_type = kDataBlockBinarySearch;
@ -112,7 +112,6 @@ struct BlockBasedTableOptions {
// kDataBlockBinaryAndHash.
double data_block_hash_table_util_ratio = 0.75;
// This option is now deprecated. No matter what value it is set to,
// it will behave as if hash_index_allow_collision=true.
bool hash_index_allow_collision = true;

10
table/block.cc
View file

@ -262,7 +262,7 @@ bool DataBlockIter::SeekForGetImpl(const Slice& target) {
// Even if the user_key is not found in the hash map, the caller still
// have to conntinue searching the next block. So we invalidate the
// iterator to tell the caller to go on.
current_ = restarts_; // Invalidate the iter
current_ = restarts_; // Invalidate the iter
return true;
}
@ -781,7 +781,7 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
break;
case BlockBasedTableOptions::kDataBlockBinaryAndHash:
if (size_ < sizeof(uint32_t) /* block footer */ +
sizeof(uint16_t) /* NUM_BUCK */) {
sizeof(uint16_t) /* NUM_BUCK */) {
size_ = 0;
break;
}
@ -789,9 +789,9 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
uint16_t map_offset;
data_block_hash_index_.Initialize(
contents.data.data(),
static_cast<uint16_t>(
contents.data.size() - sizeof(uint32_t)), /*chop off
NUM_RESTARTS*/
static_cast<uint16_t>(contents.data.size() -
sizeof(uint32_t)), /*chop off
NUM_RESTARTS*/
&map_offset);
restart_offset_ = map_offset - num_restarts_ * sizeof(uint32_t);

2
table/block_based_table_factory.cc
View file

@ -260,7 +260,7 @@ Status BlockBasedTableFactory::SanitizeOptions(
"Block alignment requested but block size is not a power of 2");
}
if (table_options_.data_block_index_type ==
BlockBasedTableOptions::kDataBlockBinaryAndHash &&
BlockBasedTableOptions::kDataBlockBinaryAndHash &&
table_options_.data_block_hash_table_util_ratio <= 0) {
return Status::InvalidArgument(
"data_block_hash_table_util_ratio should be greater than 0 when "

7
table/block_builder.cc
View file

@ -33,8 +33,8 @@
#include "table/block_builder.h"
#include <algorithm>
#include <assert.h>
#include <algorithm>
#include "db/dbformat.h"
#include "rocksdb/comparator.h"
#include "table/data_block_footer.h"
@ -118,7 +118,7 @@ Slice BlockBuilder::Finish() {
uint32_t num_restarts = static_cast<uint32_t>(restarts_.size());
BlockBasedTableOptions::DataBlockIndexType index_type =
BlockBasedTableOptions::kDataBlockBinarySearch;
BlockBasedTableOptions::kDataBlockBinarySearch;
if (data_block_hash_index_builder_.Valid() &&
CurrentSizeEstimate() <= kMaxBlockSizeSupportedByHashIndex) {
data_block_hash_index_builder_.Finish(buffer_);
@ -126,8 +126,7 @@ Slice BlockBuilder::Finish() {
}
// footer is a packed format of data_block_index_type and num_restarts
uint32_t block_footer = PackIndexTypeAndNumRestarts(
index_type, num_restarts);
uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts);
PutFixed32(&buffer_, block_footer);
finished_ = true;

5
table/data_block_footer.cc
View file

@ -32,13 +32,12 @@ uint32_t PackIndexTypeAndNumRestarts(
if (index_type == BlockBasedTableOptions::kDataBlockBinaryAndHash) {
block_footer |= 1u << kDataBlockIndexTypeBitShift;
} else if (index_type != BlockBasedTableOptions::kDataBlockBinarySearch) {
assert(0);
assert(0);
}
return block_footer;
}
void UnPackIndexTypeAndNumRestarts(
uint32_t block_footer,
BlockBasedTableOptions::DataBlockIndexType* index_type,
@ -57,4 +56,4 @@ void UnPackIndexTypeAndNumRestarts(
}
}
} // namespace rocksdb
} // namespace rocksdb

3
table/data_block_footer.h
View file

@ -17,10 +17,9 @@ uint32_t PackIndexTypeAndNumRestarts(
BlockBasedTableOptions::DataBlockIndexType index_type,
uint32_t num_restarts);
void UnPackIndexTypeAndNumRestarts(
uint32_t block_footer,
BlockBasedTableOptions::DataBlockIndexType* index_type,
uint32_t* num_restarts);
} // namespace rocksdb
} // namespace rocksdb

8
table/data_block_hash_index.cc
View file

@ -54,9 +54,9 @@ void DataBlockHashIndexBuilder::Finish(std::string& buffer) {
}
for (uint8_t restart_index : buckets) {
buffer.append(const_cast<const char*>(
reinterpret_cast<char*>(&restart_index)),
sizeof(restart_index));
buffer.append(
const_cast<const char*>(reinterpret_cast<char*>(&restart_index)),
sizeof(restart_index));
}
// write NUM_BUCK
@ -81,7 +81,7 @@ void DataBlockHashIndex::Initialize(const char* data, uint16_t size,
}
uint8_t DataBlockHashIndex::Lookup(const char* data, uint32_t map_offset,
const Slice& key) const {
const Slice& key) const {
uint32_t hash_value = GetSliceHash(key);
uint16_t idx = static_cast<uint16_t>(hash_value % num_buckets_);
const char* bucket_table = data + map_offset;

26
table/data_block_hash_index_test.cc
View file

@ -9,10 +9,10 @@
#include "rocksdb/slice.h"
#include "table/block.h"
#include "table/block_based_table_reader.h"
#include "table/block_builder.h"
#include "table/data_block_hash_index.h"
#include "table/get_context.h"
#include "table/block_based_table_reader.h"
#include "util/testharness.h"
#include "util/testutil.h"
@ -40,9 +40,9 @@ static std::string RandomString(Random* rnd, int len) {
return r;
}
std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
Random *rnd) {
Random* rnd) {
char buf[50];
char *p = &buf[0];
char* p = &buf[0];
snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key);
std::string k(p);
if (padding_size) {
@ -55,8 +55,8 @@ std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
// Generate random key value pairs.
// The generated key will be sorted. You can tune the parameters to generated
// different kinds of test key/value pairs for different scenario.
void GenerateRandomKVs(std::vector<std::string> *keys,
std::vector<std::string> *values, const int from,
void GenerateRandomKVs(std::vector<std::string>* keys,
std::vector<std::string>* values, const int from,
const int len, const int step = 1,
const int padding_size = 0,
const int keys_share_prefix = 1) {
@ -93,8 +93,7 @@ TEST(DataBlockHashIndex, DataBlockHashTestSmall) {
ASSERT_EQ(buffer.size(), estimated_size);
buffer2 = buffer; // test for the correctness of relative offset
buffer2 = buffer; // test for the correctness of relative offset
Slice s(buffer2);
DataBlockHashIndex index;
@ -290,7 +289,6 @@ TEST(DataBlockHashIndex, BlockRestartIndexExceedMax) {
BlockBasedTableOptions::kDataBlockBinaryAndHash);
}
builder.Reset();
// #restarts > 253. HashIndex is not used
@ -343,7 +341,7 @@ TEST(DataBlockHashIndex, BlockSizeExceedMax) {
Block reader(std::move(contents), kDisableGlobalSequenceNumber);
ASSERT_EQ(reader.IndexType(),
BlockBasedTableOptions::kDataBlockBinaryAndHash);
BlockBasedTableOptions::kDataBlockBinaryAndHash);
}
builder.Reset();
@ -404,8 +402,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
may_exist = iter->SeekForGet(seek_ikey.Encode().ToString());
ASSERT_TRUE(may_exist);
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(options.comparator->Compare(
iter->key(), ikey.Encode().ToString()), 0);
ASSERT_EQ(
options.comparator->Compare(iter->key(), ikey.Encode().ToString()), 0);
ASSERT_EQ(iter->value(), value);
}
@ -419,8 +417,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
ASSERT_TRUE(iter->Valid());
// user key should match
ASSERT_EQ(options.comparator->Compare(
ExtractUserKey(iter->key()), ukey), 0);
ASSERT_EQ(options.comparator->Compare(ExtractUserKey(iter->key()), ukey),
0);
// seek_key seqno number should be greater than that of iter result
ASSERT_GT(GetInternalKeySeqno(seek_ikey.Encode()),
@ -601,7 +599,7 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
ReadOptions ro;
ASSERT_OK(table_reader->Get(ro, seek_ikey.Encode().ToString(), &get_context,
moptions.prefix_extractor.get()));
moptions.prefix_extractor.get()));
}
TEST(DataBlockHashIndex, BlockBoundary) {

16
table/table_test.cc
View file

@ -3660,7 +3660,7 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.data_block_index_type =
BlockBasedTableOptions::kDataBlockBinaryAndHash;
BlockBasedTableOptions::kDataBlockBinaryAndHash;
Options options;
options.comparator = BytewiseComparator();
@ -3685,14 +3685,12 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
c.Finish(options, ioptions, moptions, table_options, internal_comparator,
&keys, &kvmap);
auto reader = c.GetTableReader();
std::unique_ptr<InternalIterator> seek_iter;
seek_iter.reset(reader->NewIterator(ReadOptions(),
moptions.prefix_extractor.get()));
seek_iter.reset(
reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
for (int i = 0; i < 2; ++i) {
ReadOptions ro;
// for every kv, we seek using two method: Get() and Seek()
// Get() will use the SuffixIndexHash in Block. For non-existent key it
@ -3727,17 +3725,17 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
// Search for non-existent keys
for (auto& kv : kvmap) {
std::string user_key = ExtractUserKey(kv.first).ToString();
user_key.back() = '0'; // make it non-existent key
user_key.back() = '0'; // make it non-existent key
InternalKey internal_key(user_key, 0, kTypeValue);
std::string encoded_key = internal_key.Encode().ToString();
if (i == 0) { // Search using Seek()
if (i == 0) { // Search using Seek()
seek_iter->Seek(encoded_key);
ASSERT_OK(seek_iter->status());
if (seek_iter->Valid()){
if (seek_iter->Valid()) {
ASSERT_TRUE(BytewiseComparator()->Compare(
user_key, ExtractUserKey(seek_iter->key())) < 0);
}
} else { // Search using Get()
} else { // Search using Get()
PinnableSlice value;
GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr,

19
tools/db_bench_tool.cc
View file

@ -466,6 +466,16 @@ DEFINE_bool(enable_index_compression,
DEFINE_bool(block_align, rocksdb::BlockBasedTableOptions().block_align,
"Align data blocks on page size");
DEFINE_bool(use_data_block_hash_index, false,
"if use kDataBlockBinaryAndHash "
"instead of kDataBlockBinarySearch. "
"This is valid if only we use BlockTable");
DEFINE_double(data_block_hash_table_util_ratio, 0.75,
"util ratio for data block hash index table. "
"This is only valid if use_data_block_hash_index is "
"set to true");
DEFINE_int64(compressed_cache_size, -1,
"Number of bytes to use as a cache of compressed data.");
@ -3265,6 +3275,15 @@ void VerifyDBFromDB(std::string& truth_db_name) {
block_based_options.enable_index_compression =
FLAGS_enable_index_compression;
block_based_options.block_align = FLAGS_block_align;
if (FLAGS_use_data_block_hash_index) {
block_based_options.data_block_index_type =
rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
} else {
block_based_options.data_block_index_type =
rocksdb::BlockBasedTableOptions::kDataBlockBinarySearch;
}
block_based_options.data_block_hash_table_util_ratio =
FLAGS_data_block_hash_table_util_ratio;
if (FLAGS_read_cache_path != "") {
#ifndef ROCKSDB_LITE
Status rc_status;