dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some refactorings on plain table 

Summary:
Plain table has been working well and this is just a nit-picking patch,
which is generated during my coding reading. No real functional changes.
only some changes regarding:

* Improve some comments from the perspective a "new" code reader.
* Change some magic number to constant, which can help us to parameterize them
  in the future.
* Did some style, naming, C++ convention changes.
* Fix warnings from new "arc lint"

Test Plan: make check

Reviewers: sdong, haobo

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15429
This commit is contained in:
Kai Liu 2014-01-24 21:10:19 -08:00
parent 0ab766132b
commit 4b51dffcf8
3 changed files with 200 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

244
table/plain_table_reader.cc
View File

@ -34,29 +34,66 @@
namespace rocksdb { namespace rocksdb {
extern const uint64_t kPlainTableMagicNumber; namespace {
static uint32_t GetSliceHash(Slice const& s) { inline uint32_t GetSliceHash(Slice const& s) {
return Hash(s.data(), s.size(), 397) ; return Hash(s.data(), s.size(), 397) ;
} }
static uint32_t getBucketIdFromHash(uint32_t hash, uint32_t num_buckets) {
inline uint32_t GetBucketIdFromHash(uint32_t hash, uint32_t num_buckets) {
return hash % num_buckets; return hash % num_buckets;
} }
} // namespace
// Iterator to iterate IndexedTable
class PlainTableIterator : public Iterator {
public:
explicit PlainTableIterator(PlainTableReader* table);
~PlainTableIterator();
bool Valid() const;
void SeekToFirst();
void SeekToLast();
void Seek(const Slice& target);
void Next();
void Prev();
Slice key() const;
Slice value() const;
Status status() const;
private:
PlainTableReader* table_;
uint32_t offset_;
uint32_t next_offset_;
Slice key_;
Slice value_;
Status status_;
// No copying allowed
PlainTableIterator(const PlainTableIterator&) = delete;
void operator=(const Iterator&) = delete;
};
extern const uint64_t kPlainTableMagicNumber;
PlainTableReader::PlainTableReader(const EnvOptions& storage_options, PlainTableReader::PlainTableReader(const EnvOptions& storage_options,
uint64_t file_size, int bloom_bits_per_key, uint64_t file_size, int bloom_bits_per_key,
double hash_table_ratio, double hash_table_ratio,
const TableProperties& table_properties) : const TableProperties& table_properties)
hash_table_size_(0), soptions_(storage_options), file_size_(file_size), : soptions_(storage_options),
hash_table_ratio_(hash_table_ratio), file_size_(file_size),
bloom_bits_per_key_(bloom_bits_per_key), kHashTableRatio(hash_table_ratio),
table_properties_(table_properties), data_start_offset_(0), kBloomBitsPerKey(bloom_bits_per_key),
data_end_offset_(table_properties_.data_size), table_properties_(table_properties),
user_key_len_(table_properties.fixed_key_len) { data_end_offset_(table_properties_.data_size),
hash_table_ = nullptr; user_key_len_(table_properties.fixed_key_len) {}
bloom_ = nullptr;
sub_index_ = nullptr;
}
PlainTableReader::~PlainTableReader() { PlainTableReader::~PlainTableReader() {
delete[] hash_table_; delete[] hash_table_;
@ -73,30 +110,20 @@ Status PlainTableReader::Open(const Options& options,
double hash_table_ratio) { double hash_table_ratio) {
assert(options.allow_mmap_reads); assert(options.allow_mmap_reads);
if (file_size > 2147483646) { if (file_size > kMaxFileSize) {
return Status::NotSupported("File is too large for PlainTableReader!"); return Status::NotSupported("File is too large for PlainTableReader!");
} }
TableProperties table_properties; TableProperties table_properties;
auto s = ReadTableProperties( auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
file.get(), options.env, options.info_log.get(),
file_size, &table_properties);
kPlainTableMagicNumber,
options.env,
options.info_log.get(),
&table_properties
);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
std::unique_ptr<PlainTableReader> new_reader(new PlainTableReader( std::unique_ptr<PlainTableReader> new_reader(new PlainTableReader(
soptions, soptions, file_size, bloom_num_bits, hash_table_ratio, table_properties));
file_size,
bloom_num_bits,
hash_table_ratio,
table_properties
));
new_reader->file_ = std::move(file); new_reader->file_ = std::move(file);
new_reader->options_ = options; new_reader->options_ = options;
@ -129,12 +156,11 @@ struct PlainTableReader::IndexRecord {
// Helper class to track all the index records // Helper class to track all the index records
class PlainTableReader::IndexRecordList { class PlainTableReader::IndexRecordList {
public: public:
explicit IndexRecordList(size_t num_records_per_group) : explicit IndexRecordList(size_t num_records_per_group)
num_records_per_group_(num_records_per_group), : kNumRecordsPerGroup(num_records_per_group),
current_group_(nullptr), current_group_(nullptr),
num_records_in_current_group_(num_records_per_group) { num_records_in_current_group_(num_records_per_group) {}
}
~IndexRecordList() { ~IndexRecordList() {
for (size_t i = 0; i < groups_.size(); i++) { for (size_t i = 0; i < groups_.size(); i++) {
@ -143,65 +169,59 @@ public:
} }
void AddRecord(murmur_t hash, uint32_t offset) { void AddRecord(murmur_t hash, uint32_t offset) {
if (num_records_in_current_group_ == num_records_per_group_) { if (num_records_in_current_group_ == kNumRecordsPerGroup) {
current_group_ = AllocateNewGroup(); current_group_ = AllocateNewGroup();
num_records_in_current_group_ = 0; num_records_in_current_group_ = 0;
} }
auto& new_record = current_group_[num_records_in_current_group_]; auto& new_record = current_group_[num_records_in_current_group_++];
new_record.hash = hash; new_record.hash = hash;
new_record.offset = offset; new_record.offset = offset;
new_record.next = nullptr; new_record.next = nullptr;
num_records_in_current_group_++;
} }
size_t GetNumRecords() { size_t GetNumRecords() const {
return (groups_.size() - 1) * num_records_per_group_ return (groups_.size() - 1) * kNumRecordsPerGroup +
+ num_records_in_current_group_; num_records_in_current_group_;
} }
IndexRecord* At(size_t index) { IndexRecord* At(size_t index) {
return &(groups_[index / num_records_per_group_] return &(groups_[index / kNumRecordsPerGroup][index % kNumRecordsPerGroup]);
[index % num_records_per_group_]);
} }
private:
IndexRecord* AllocateNewGroup() { IndexRecord* AllocateNewGroup() {
IndexRecord* result = new IndexRecord[num_records_per_group_]; IndexRecord* result = new IndexRecord[kNumRecordsPerGroup];
groups_.push_back(result); groups_.push_back(result);
return result; return result;
} }
private:
const size_t num_records_per_group_; const size_t kNumRecordsPerGroup;
IndexRecord* current_group_; IndexRecord* current_group_;
// List of arrays allocated // List of arrays allocated
std::vector<IndexRecord*> groups_; std::vector<IndexRecord*> groups_;
size_t num_records_in_current_group_; size_t num_records_in_current_group_;
}; };
int PlainTableReader::PopulateIndexRecordList( int PlainTableReader::PopulateIndexRecordList(IndexRecordList* record_list) {
IndexRecordList& record_list) {
Slice key_slice;
Slice key_prefix_slice;
Slice key_suffix_slice;
Slice value_slice;
Slice prev_key_prefix_slice; Slice prev_key_prefix_slice;
uint32_t prev_key_prefix_hash = 0; uint32_t prev_key_prefix_hash = 0;
uint32_t pos = data_start_offset_; uint32_t pos = data_start_offset_;
int key_index_within_prefix = 0; int key_index_within_prefix = 0;
bool first = true; bool is_first_record = true;
std::string prefix_sub_index;
HistogramImpl keys_per_prefix_hist; HistogramImpl keys_per_prefix_hist;
// Need map to be ordered to make sure sub indexes generated // Need map to be ordered to make sure sub indexes generated
// are in order. // are in order.
int num_prefixes = 0; int num_prefixes = 0;
while (pos < data_end_offset_) { while (pos < data_end_offset_) {
uint32_t key_offset = pos; uint32_t key_offset = pos;
Slice key_slice;
Slice value_slice;
status_ = Next(pos, &key_slice, &value_slice, pos); status_ = Next(pos, &key_slice, &value_slice, pos);
key_prefix_slice = GetPrefix(key_slice); Slice key_prefix_slice = GetPrefix(key_slice);
if (first || prev_key_prefix_slice != key_prefix_slice) { if (is_first_record || prev_key_prefix_slice != key_prefix_slice) {
num_prefixes++; ++num_prefixes;
if (!first) { if (!is_first_record) {
keys_per_prefix_hist.Add(key_index_within_prefix); keys_per_prefix_hist.Add(key_index_within_prefix);
} }
key_index_within_prefix = 0; key_index_within_prefix = 0;
@ -209,12 +229,13 @@ int PlainTableReader::PopulateIndexRecordList(
prev_key_prefix_hash = GetSliceHash(key_prefix_slice); prev_key_prefix_hash = GetSliceHash(key_prefix_slice);
} }
if (key_index_within_prefix++ % 16 == 0) { if (key_index_within_prefix++ % kIndexIntervalForSamePrefixKeys == 0) {
// Add an index key for every 16 keys // Add an index key for every kIndexIntervalForSamePrefixKeys keys
record_list.AddRecord(prev_key_prefix_hash, key_offset); record_list->AddRecord(prev_key_prefix_hash, key_offset);
} }
first = false; is_first_record = false;
} }
keys_per_prefix_hist.Add(key_index_within_prefix); keys_per_prefix_hist.Add(key_index_within_prefix);
Log(options_.info_log, "Number of Keys per prefix Histogram: %s", Log(options_.info_log, "Number of Keys per prefix Histogram: %s",
keys_per_prefix_hist.ToString().c_str()); keys_per_prefix_hist.ToString().c_str());
@ -222,23 +243,22 @@ int PlainTableReader::PopulateIndexRecordList(
return num_prefixes; return num_prefixes;
} }
void PlainTableReader::Allocate(int num_prefixes) { void PlainTableReader::AllocateIndexAndBloom(int num_prefixes) {
if (hash_table_ != nullptr) { delete[] hash_table_;
delete[] hash_table_;
} if (kBloomBitsPerKey > 0) {
if (bloom_bits_per_key_ > 0) { bloom_ = new DynamicBloom(num_prefixes * kBloomBitsPerKey);
bloom_ = new DynamicBloom(num_prefixes * bloom_bits_per_key_);
} }
double hash_table_size_multipier = double hash_table_size_multipier =
(hash_table_ratio_ > 1.0) ? 1.0 : 1.0 / hash_table_ratio_; (kHashTableRatio > 1.0) ? 1.0 : 1.0 / kHashTableRatio;
hash_table_size_ = num_prefixes * hash_table_size_multipier + 1; hash_table_size_ = num_prefixes * hash_table_size_multipier + 1;
hash_table_ = new uint32_t[hash_table_size_]; hash_table_ = new uint32_t[hash_table_size_];
} }
size_t PlainTableReader::BucketizeIndexesAndFillBloom( size_t PlainTableReader::BucketizeIndexesAndFillBloom(
IndexRecordList& record_list, int num_prefixes, IndexRecordList& record_list, int num_prefixes,
std::vector<IndexRecord*>& hash2offsets, std::vector<IndexRecord*>* hash_to_offsets,
std::vector<uint32_t>& bucket_count) { std::vector<uint32_t>* bucket_count) {
size_t sub_index_size_needed = 0; size_t sub_index_size_needed = 0;
bool first = true; bool first = true;
uint32_t prev_hash = 0; uint32_t prev_hash = 0;
@ -253,32 +273,34 @@ size_t PlainTableReader::BucketizeIndexesAndFillBloom(
bloom_->AddHash(cur_hash); bloom_->AddHash(cur_hash);
} }
} }
uint32_t bucket = getBucketIdFromHash(cur_hash, hash_table_size_); uint32_t bucket = GetBucketIdFromHash(cur_hash, hash_table_size_);
IndexRecord* prev_bucket_head = hash2offsets[bucket]; IndexRecord* prev_bucket_head = (*hash_to_offsets)[bucket];
index_record->next = prev_bucket_head; index_record->next = prev_bucket_head;
hash2offsets[bucket] = index_record; (*hash_to_offsets)[bucket] = index_record;
if (bucket_count[bucket] > 0) { auto& item_count = (*bucket_count)[bucket];
if (bucket_count[bucket] == 1) { if (item_count > 0) {
if (item_count == 1) {
sub_index_size_needed += kOffsetLen + 1; sub_index_size_needed += kOffsetLen + 1;
} }
if (bucket_count[bucket] == 127) { if (item_count == 127) {
// Need more than one byte for length // Need more than one byte for length
sub_index_size_needed++; sub_index_size_needed++;
} }
sub_index_size_needed += kOffsetLen; sub_index_size_needed += kOffsetLen;
} }
bucket_count[bucket]++; item_count++;
} }
return sub_index_size_needed; return sub_index_size_needed;
} }
void PlainTableReader::FillIndexes(size_t sub_index_size_needed, void PlainTableReader::FillIndexes(
std::vector<IndexRecord*>& hash2offsets, size_t sub_index_size_needed,
std::vector<uint32_t>& bucket_count) { const std::vector<IndexRecord*>& hash_to_offsets,
const std::vector<uint32_t>& bucket_count) {
Log(options_.info_log, "Reserving %zu bytes for sub index", Log(options_.info_log, "Reserving %zu bytes for sub index",
sub_index_size_needed); sub_index_size_needed);
// 4 bytes buffer for variable length size // 8 bytes buffer for variable length size
size_t buffer_size = 64; size_t buffer_size = 8 * 8;
size_t buffer_used = 0; size_t buffer_used = 0;
sub_index_size_needed += buffer_size; sub_index_size_needed += buffer_size;
sub_index_ = new char[sub_index_size_needed]; sub_index_ = new char[sub_index_size_needed];
@ -286,7 +308,6 @@ void PlainTableReader::FillIndexes(size_t sub_index_size_needed,
char* prev_ptr; char* prev_ptr;
char* cur_ptr; char* cur_ptr;
uint32_t* sub_index_ptr; uint32_t* sub_index_ptr;
IndexRecord* record;
for (int i = 0; i < hash_table_size_; i++) { for (int i = 0; i < hash_table_size_; i++) {
uint32_t num_keys_for_bucket = bucket_count[i]; uint32_t num_keys_for_bucket = bucket_count[i];
switch (num_keys_for_bucket) { switch (num_keys_for_bucket) {
@ -296,14 +317,14 @@ void PlainTableReader::FillIndexes(size_t sub_index_size_needed,
break; break;
case 1: case 1:
// point directly to the file offset // point directly to the file offset
hash_table_[i] = hash2offsets[i]->offset; hash_table_[i] = hash_to_offsets[i]->offset;
break; break;
default: default:
// point to second level indexes. // point to second level indexes.
hash_table_[i] = sub_index_offset | kSubIndexMask; hash_table_[i] = sub_index_offset | kSubIndexMask;
prev_ptr = sub_index_ + sub_index_offset; prev_ptr = sub_index_ + sub_index_offset;
cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket); cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket);
sub_index_offset += cur_ptr - prev_ptr; sub_index_offset += (cur_ptr - prev_ptr);
if (cur_ptr - prev_ptr > 2 if (cur_ptr - prev_ptr > 2
|| (cur_ptr - prev_ptr == 2 && num_keys_for_bucket <= 127)) { || (cur_ptr - prev_ptr == 2 && num_keys_for_bucket <= 127)) {
// Need to resize sub_index. Exponentially grow buffer. // Need to resize sub_index. Exponentially grow buffer.
@ -321,10 +342,10 @@ void PlainTableReader::FillIndexes(size_t sub_index_size_needed,
} }
} }
sub_index_ptr = (uint32_t*) (sub_index_ + sub_index_offset); sub_index_ptr = (uint32_t*) (sub_index_ + sub_index_offset);
record = hash2offsets[i]; IndexRecord* record = hash_to_offsets[i];
int j; int j;
for (j = num_keys_for_bucket - 1; for (j = num_keys_for_bucket - 1; j >= 0 && record;
j >= 0 && record; j--, record = record->next) { j--, record = record->next) {
sub_index_ptr[j] = record->offset; sub_index_ptr[j] = record->offset;
} }
assert(j == -1 && record == nullptr); assert(j == -1 && record == nullptr);
@ -337,24 +358,6 @@ void PlainTableReader::FillIndexes(size_t sub_index_size_needed,
hash_table_size_, sub_index_size_needed); hash_table_size_, sub_index_size_needed);
} }
// PopulateIndex() builds index of keys.
// hash_table_ contains buckets size of hash_table_size_, each is a 32-bit
// integer. The lower 31 bits contain an offset value (explained below) and
// the first bit of the integer indicates type of the offset:
//
// 0 indicates that the bucket contains only one prefix (no conflict when
// hashing this prefix), whose first row starts from this offset of the file.
// 1 indicates that the bucket contains more than one prefixes, or there
// are too many rows for one prefix so we need a binary search for it. In
// this case, the offset indicates the offset of sub_index_ holding the
// binary search indexes of keys for those rows. Those binary search indexes
// are organized in this way:
//
// The first 4 bytes, indicates how many indexes (N) are stored after it. After
// it, there are N 32-bit integers, each points of an offset of the file, which
// points to starting of a row. Those offsets need to be guaranteed to be in
// ascending order so the keys they are pointing to are also in ascending order
// to make sure we can use them to do binary searches.
Status PlainTableReader::PopulateIndex() { Status PlainTableReader::PopulateIndex() {
// Get mmapped memory to file_data_. // Get mmapped memory to file_data_.
Status s = file_->Read(0, file_size_, &file_data_, nullptr); Status s = file_->Read(0, file_size_, &file_data_, nullptr);
@ -362,25 +365,24 @@ Status PlainTableReader::PopulateIndex() {
return s; return s;
} }
IndexRecordList record_list(256); IndexRecordList record_list(kRecordsPerGroup);
// First, read the whole file, for every 16 rows for a prefix (starting from // First, read the whole file, for every kIndexIntervalForSamePrefixKeys rows
// the first one), generate a record of (hash, offset) and append it to // for a prefix (starting from the first one), generate a record of (hash,
// IndexRecordList, which is a data structure created to store them. // offset) and append it to IndexRecordList, which is a data structure created
int num_prefixes = PopulateIndexRecordList(record_list); // to store them.
int num_prefixes = PopulateIndexRecordList(&record_list);
// Calculated hash table and bloom filter size and allocate memory for indexes // Calculated hash table and bloom filter size and allocate memory for indexes
// and bloom filter based on the number of prefixes. // and bloom filter based on the number of prefixes.
Allocate(num_prefixes); AllocateIndexAndBloom(num_prefixes);
// Bucketize all the index records to a temp data structure, in which for // Bucketize all the index records to a temp data structure, in which for
// each bucket, we generate a linked list of IndexRecord, in reversed order. // each bucket, we generate a linked list of IndexRecord, in reversed order.
std::vector<IndexRecord*> hash2offsets(hash_table_size_, nullptr); std::vector<IndexRecord*> hash_to_offsets(hash_table_size_, nullptr);
std::vector<uint32_t> bucket_count(hash_table_size_, 0); std::vector<uint32_t> bucket_count(hash_table_size_, 0);
size_t sub_index_size_needed = BucketizeIndexesAndFillBloom(record_list, size_t sub_index_size_needed = BucketizeIndexesAndFillBloom(
num_prefixes, record_list, num_prefixes, &hash_to_offsets, &bucket_count);
hash2offsets,
bucket_count);
// From the temp data structure, populate indexes. // From the temp data structure, populate indexes.
FillIndexes(sub_index_size_needed, hash2offsets, bucket_count); FillIndexes(sub_index_size_needed, hash_to_offsets, bucket_count);
return Status::OK(); return Status::OK();
} }
@ -389,7 +391,7 @@ Status PlainTableReader::GetOffset(const Slice& target, const Slice& prefix,
uint32_t prefix_hash, bool& prefix_matched, uint32_t prefix_hash, bool& prefix_matched,
uint32_t& ret_offset) { uint32_t& ret_offset) {
prefix_matched = false; prefix_matched = false;
int bucket = getBucketIdFromHash(prefix_hash, hash_table_size_); int bucket = GetBucketIdFromHash(prefix_hash, hash_table_size_);
uint32_t bucket_value = hash_table_[bucket]; uint32_t bucket_value = hash_table_[bucket];
if (bucket_value == data_end_offset_) { if (bucket_value == data_end_offset_) {
ret_offset = data_end_offset_; ret_offset = data_end_offset_;

125
table/plain_table_reader.h
View File

@ -5,6 +5,7 @@
#pragma once #pragma once
#include <unordered_map> #include <unordered_map>
#include <memory> #include <memory>
#include <vector>
#include <stdint.h> #include <stdint.h>
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/iterator.h" #include "rocksdb/iterator.h"
@ -35,7 +36,7 @@ using std::unordered_map;
// //
// The implementation of IndexedTableReader requires output file is mmaped // The implementation of IndexedTableReader requires output file is mmaped
class PlainTableReader: public TableReader { class PlainTableReader: public TableReader {
public: public:
static Status Open(const Options& options, const EnvOptions& soptions, static Status Open(const Options& options, const EnvOptions& soptions,
unique_ptr<RandomAccessFile> && file, uint64_t file_size, unique_ptr<RandomAccessFile> && file, uint64_t file_size,
unique_ptr<TableReader>* table, const int bloom_num_bits, unique_ptr<TableReader>* table, const int bloom_num_bits,
@ -65,12 +66,12 @@ public:
const TableProperties& table_properties); const TableProperties& table_properties);
~PlainTableReader(); ~PlainTableReader();
private: private:
struct IndexRecord; struct IndexRecord;
class IndexRecordList; class IndexRecordList;
uint32_t* hash_table_ = nullptr; uint32_t* hash_table_ = nullptr;
int hash_table_size_; int hash_table_size_ = 0;
char* sub_index_ = nullptr; char* sub_index_ = nullptr;
Options options_; Options options_;
@ -82,24 +83,30 @@ private:
uint32_t version_; uint32_t version_;
uint32_t file_size_; uint32_t file_size_;
const double hash_table_ratio_; const double kHashTableRatio;
const int bloom_bits_per_key_; const int kBloomBitsPerKey;
DynamicBloom* bloom_; DynamicBloom* bloom_ = nullptr;
TableProperties table_properties_; TableProperties table_properties_;
const uint32_t data_start_offset_; const uint32_t data_start_offset_ = 0;
const uint32_t data_end_offset_; const uint32_t data_end_offset_;
const size_t user_key_len_; const size_t user_key_len_;
static const size_t kNumInternalBytes = 8; static const size_t kNumInternalBytes = 8;
static const uint32_t kSubIndexMask = 0x80000000; static const uint32_t kSubIndexMask = 0x80000000;
static const size_t kOffsetLen = sizeof(uint32_t); static const size_t kOffsetLen = sizeof(uint32_t);
static const uint64_t kMaxFileSize = 1u << 31;
static const size_t kRecordsPerGroup = 256;
// To speed up the search for keys with same prefix, we'll add index key for
// every N keys, where the "N" is determined by
// kIndexIntervalForSamePrefixKeys
static const size_t kIndexIntervalForSamePrefixKeys = 16;
bool IsFixedLength() { bool IsFixedLength() const {
return user_key_len_ != PlainTableFactory::kVariableLength; return user_key_len_ != PlainTableFactory::kVariableLength;
} }
size_t GetFixedInternalKeyLength() { size_t GetFixedInternalKeyLength() const {
return user_key_len_ + kNumInternalBytes; return user_key_len_ + kNumInternalBytes;
} }
@ -108,32 +115,67 @@ private:
// Internal helper function to generate an IndexRecordList object from all // Internal helper function to generate an IndexRecordList object from all
// the rows, which contains index records as a list. // the rows, which contains index records as a list.
int PopulateIndexRecordList(IndexRecordList& record_list); int PopulateIndexRecordList(IndexRecordList* record_list);
// Internal helper function to allocate memory for indexes and bloom filters // Internal helper function to allocate memory for indexes and bloom filters
void Allocate(int num_prefixes); void AllocateIndexAndBloom(int num_prefixes);
// Internal helper function to bucket index record list to hash buckets. // Internal helper function to bucket index record list to hash buckets.
// hash2offsets is sized of of hash_table_size_, each contains a linked list // hash_to_offsets is sized of of hash_table_size_, each contains a linked
// list
// of offsets for the hash, in reversed order. // of offsets for the hash, in reversed order.
// bucket_count is sized of hash_table_size_. The value is how many index // bucket_count is sized of hash_table_size_. The value is how many index
// records are there in hash2offsets for the same bucket. // records are there in hash_to_offsets for the same bucket.
size_t BucketizeIndexesAndFillBloom( size_t BucketizeIndexesAndFillBloom(
IndexRecordList& record_list, int num_prefixes, IndexRecordList& record_list, int num_prefixes,
std::vector<IndexRecord*>& hash2offsets, std::vector<IndexRecord*>* hash_to_offsets,
std::vector<uint32_t>& bucket_count); std::vector<uint32_t>* bucket_count);
// Internal helper class to fill the indexes and bloom filters to internal // Internal helper class to fill the indexes and bloom filters to internal
// data structures. hash2offsets and bucket_count are bucketized indexes and // data structures. hash_to_offsets and bucket_count are bucketized indexes
// counts generated by BucketizeIndexesAndFillBloom(). // and counts generated by BucketizeIndexesAndFillBloom().
void FillIndexes(size_t sub_index_size_needed, void FillIndexes(size_t sub_index_size_needed,
std::vector<IndexRecord*>& hash2offsets, const std::vector<IndexRecord*>& hash_to_offsets,
std::vector<uint32_t>& bucket_count); const std::vector<uint32_t>& bucket_count);
// Populate the internal indexes. It must be called before // PopulateIndex() builds index of keys. It must be called before any query
// any query to the table. // to the table.
// This query will populate the hash table hash_table_, the second //
// level of indexes sub_index_ and bloom filter filter_slice_ if enabled. // hash_table_ contains buckets size of hash_table_size_, each is a 32-bit
// integer. The lower 31 bits contain an offset value (explained below) and
// the first bit of the integer indicates type of the offset.
//
// +--------------+------------------------------------------------------+
// | Flag (1 bit) | Offset to binary search buffer or file (31 bits) +
// +--------------+------------------------------------------------------+
//
// Explanation for the "flag bit":
//
// 0 indicates that the bucket contains only one prefix (no conflict when
// hashing this prefix), whose first row starts from this offset of the
// file.
// 1 indicates that the bucket contains more than one prefixes, or there
// are too many rows for one prefix so we need a binary search for it. In
// this case, the offset indicates the offset of sub_index_ holding the
// binary search indexes of keys for those rows. Those binary search indexes
// are organized in this way:
//
// The first 4 bytes, indicate how many indexes (N) are stored after it. After
// it, there are N 32-bit integers, each points of an offset of the file,
// which
// points to starting of a row. Those offsets need to be guaranteed to be in
// ascending order so the keys they are pointing to are also in ascending
// order
// to make sure we can use them to do binary searches. Below is visual
// presentation of a bucket.
//
// <begin>
// number_of_records: varint32
// record 1 file offset: fixedint32
// record 2 file offset: fixedint32
// ....
// record N file offset: fixedint32
// <end>
Status PopulateIndex(); Status PopulateIndex();
// Check bloom filter to see whether it might contain this prefix. // Check bloom filter to see whether it might contain this prefix.
@ -163,41 +205,4 @@ private:
explicit PlainTableReader(const TableReader&) = delete; explicit PlainTableReader(const TableReader&) = delete;
void operator=(const TableReader&) = delete; void operator=(const TableReader&) = delete;
}; };
// Iterator to iterate IndexedTable
class PlainTableIterator: public Iterator {
public:
explicit PlainTableIterator(PlainTableReader* table);
~PlainTableIterator();
bool Valid() const;
void SeekToFirst();
void SeekToLast();
void Seek(const Slice& target);
void Next();
void Prev();
Slice key() const;
Slice value() const;
Status status() const;
private:
PlainTableReader* table_;
uint32_t offset_;
uint32_t next_offset_;
Slice key_;
Slice value_;
Status status_;
// No copying allowed
PlainTableIterator(const PlainTableIterator&) = delete;
void operator=(const Iterator&) = delete;
};
} // namespace rocksdb } // namespace rocksdb

33
table/table_test.cc
View File

@ -874,9 +874,9 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
} }
// Tests against all kinds of tables // Tests against all kinds of tables
class GeneralTableTest { }; class GeneralTableTest {};
class BlockBasedTableTest { }; class BlockBasedTableTest {};
class PlainTableTest { }; class PlainTableTest {};
// This test include all the basic checks except those for index size and block // This test include all the basic checks except those for index size and block
// size, which will be conducted in separated unit tests. // size, which will be conducted in separated unit tests.
@ -1184,8 +1184,9 @@ TEST(BlockBasedTableTest, BlockCacheLeak) {
Options opt; Options opt;
opt.block_size = 1024; opt.block_size = 1024;
opt.compression = kNoCompression; opt.compression = kNoCompression;
opt.block_cache = NewLRUCache(16*1024*1024); // big enough so we don't ever opt.block_cache =
// lose cached values. NewLRUCache(16 * 1024 * 1024); // big enough so we don't ever
// lose cached values.
TableConstructor c(BytewiseComparator()); TableConstructor c(BytewiseComparator());
c.Add("k01", "hello"); c.Add("k01", "hello");
@ -1209,21 +1210,17 @@ TEST(BlockBasedTableTest, BlockCacheLeak) {
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_OK(c.Reopen(opt)); ASSERT_OK(c.Reopen(opt));
for (const std::string& key: keys) { for (const std::string& key : keys) {
ASSERT_TRUE(c.table_reader()->TEST_KeyInCache(ReadOptions(), key)); ASSERT_TRUE(c.table_reader()->TEST_KeyInCache(ReadOptions(), key));
} }
} }
extern const uint64_t kPlainTableMagicNumber; extern const uint64_t kPlainTableMagicNumber;
TEST(PlainTableTest, BasicPlainTableProperties) { TEST(PlainTableTest, BasicPlainTableProperties) {
PlainTableFactory factory(8, 8, 0); PlainTableFactory factory(8, 8, 0);
StringSink sink; StringSink sink;
std::unique_ptr<TableBuilder> builder(factory.GetTableBuilder( std::unique_ptr<TableBuilder> builder(
Options(), factory.GetTableBuilder(Options(), &sink, kNoCompression));
&sink,
kNoCompression
));
for (char c = 'a'; c <= 'z'; ++c) { for (char c = 'a'; c <= 'z'; ++c) {
std::string key(16, c); std::string key(16, c);
@ -1235,14 +1232,9 @@ TEST(PlainTableTest, BasicPlainTableProperties) {
StringSource source(sink.contents(), 72242, true); StringSource source(sink.contents(), 72242, true);
TableProperties props; TableProperties props;
auto s = ReadTableProperties( auto s = ReadTableProperties(&source, sink.contents().size(),
&source, kPlainTableMagicNumber, Env::Default(), nullptr,
sink.contents().size(), &props);
kPlainTableMagicNumber,
Env::Default(),
nullptr,
&props
);
ASSERT_OK(s); ASSERT_OK(s);
ASSERT_EQ(0ul, props.index_size); ASSERT_EQ(0ul, props.index_size);
@ -1253,7 +1245,6 @@ TEST(PlainTableTest, BasicPlainTableProperties) {
ASSERT_EQ(1ul, props.num_data_blocks); ASSERT_EQ(1ul, props.num_data_blocks);
} }
TEST(GeneralTableTest, ApproximateOffsetOfPlain) { TEST(GeneralTableTest, ApproximateOffsetOfPlain) {
TableConstructor c(BytewiseComparator()); TableConstructor c(BytewiseComparator());
c.Add("k01", "hello"); c.Add("k01", "hello");