Add Prev() for merge operator

Summary: Implement Prev() with merge operator for DBIterator. Request from mongoDB. Task 4673663.

Test Plan: make all check

Reviewers: sdong

Reviewed By: sdong

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D19743
This commit is contained in:
Stanislau Hlebik 2014-07-15 16:10:18 -07:00
parent 0abaed2e08
commit d916593ead
2 changed files with 1175 additions and 110 deletions

View file

@ -10,6 +10,7 @@
#include "db/db_iter.h" #include "db/db_iter.h"
#include <stdexcept> #include <stdexcept>
#include <deque> #include <deque>
#include <string>
#include "db/filename.h" #include "db/filename.h"
#include "db/dbformat.h" #include "db/dbformat.h"
@ -109,9 +110,14 @@ class DBIter: public Iterator {
virtual void SeekToLast(); virtual void SeekToLast();
private: private:
void PrevInternal();
void FindParseableKey(ParsedInternalKey* ikey, Direction direction);
bool FindValueForCurrentKey();
bool FindValueForCurrentKeyUsingSeek();
void FindPrevUserKey();
void FindNextUserKey();
inline void FindNextUserEntry(bool skipping); inline void FindNextUserEntry(bool skipping);
void FindNextUserEntryInternal(bool skipping); void FindNextUserEntryInternal(bool skipping);
void FindPrevUserEntry();
bool ParseKey(ParsedInternalKey* key); bool ParseKey(ParsedInternalKey* key);
void MergeValuesNewToOld(); void MergeValuesNewToOld();
@ -133,8 +139,8 @@ class DBIter: public Iterator {
SequenceNumber const sequence_; SequenceNumber const sequence_;
Status status_; Status status_;
IterKey saved_key_; // == current key when direction_==kReverse IterKey saved_key_;
std::string saved_value_; // == current raw value when direction_==kReverse std::string saved_value_;
Direction direction_; Direction direction_;
bool valid_; bool valid_;
bool current_entry_is_merged_; bool current_entry_is_merged_;
@ -160,20 +166,11 @@ inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
void DBIter::Next() { void DBIter::Next() {
assert(valid_); assert(valid_);
if (direction_ == kReverse) { // Switch directions? if (direction_ == kReverse) {
FindNextUserKey();
direction_ = kForward; direction_ = kForward;
// iter_ is pointing just before the entries for this->key(),
// so advance into the range of entries for this->key() and then
// use the normal skipping code below.
if (!iter_->Valid()) { if (!iter_->Valid()) {
iter_->SeekToFirst(); iter_->SeekToFirst();
} else {
iter_->Next();
}
if (!iter_->Valid()) {
valid_ = false;
saved_key_.Clear();
return;
} }
} }
@ -185,7 +182,6 @@ void DBIter::Next() {
FindNextUserEntry(true /* skipping the current user key */); FindNextUserEntry(true /* skipping the current user key */);
} }
// PRE: saved_key_ has the current user key if skipping // PRE: saved_key_ has the current user key if skipping
// POST: saved_key_ should have the next user key if valid_, // POST: saved_key_ should have the next user key if valid_,
// if the current entry is a result of merge // if the current entry is a result of merge
@ -327,100 +323,228 @@ void DBIter::MergeValuesNewToOld() {
void DBIter::Prev() { void DBIter::Prev() {
assert(valid_); assert(valid_);
if (direction_ == kForward) {
// Throw an exception now if merge_operator is provided FindPrevUserKey();
// TODO: support backward iteration
if (user_merge_operator_) {
Log(logger_, "Prev not supported yet if merge_operator is provided");
throw std::logic_error("DBIter::Prev backward iteration not supported"
" if merge_operator is provided");
}
if (direction_ == kForward) { // Switch directions?
// iter_ is pointing at the current entry. Scan backwards until
// the key changes so we can use the normal reverse scanning code.
assert(iter_->Valid()); // Otherwise valid_ would have been false
saved_key_.SetKey(ExtractUserKey(iter_->key()));
while (true) {
iter_->Prev();
if (!iter_->Valid()) {
valid_ = false;
saved_key_.Clear();
ClearSavedValue();
return;
}
if (user_comparator_->Compare(ExtractUserKey(iter_->key()),
saved_key_.GetKey()) < 0) {
break;
}
}
direction_ = kReverse; direction_ = kReverse;
} }
PrevInternal();
FindPrevUserEntry();
} }
void DBIter::FindPrevUserEntry() { void DBIter::PrevInternal() {
assert(direction_ == kReverse); if (!iter_->Valid()) {
uint64_t num_skipped = 0; valid_ = false;
return;
ValueType value_type = kTypeDeletion;
bool saved_key_valid = true;
if (iter_->Valid()) {
do {
ParsedInternalKey ikey;
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
if ((value_type != kTypeDeletion) &&
user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) < 0) {
// We encountered a non-deleted value in entries for previous keys,
break;
}
value_type = ikey.type;
if (value_type == kTypeDeletion) {
saved_key_.Clear();
ClearSavedValue();
saved_key_valid = false;
} else {
Slice raw_value = iter_->value();
if (saved_value_.capacity() > raw_value.size() + 1048576) {
std::string empty;
swap(empty, saved_value_);
}
saved_key_.SetKey(ExtractUserKey(iter_->key()));
saved_value_.assign(raw_value.data(), raw_value.size());
}
} else {
// In the case of ikey.sequence > sequence_, we might have already
// iterated to a different user key.
saved_key_valid = false;
}
num_skipped++;
// If we have sequentially iterated via numerous keys and still not
// found the prev user-key, then it is better to seek so that we can
// avoid too many key comparisons. We seek to the first occurence of
// our current key by looking for max sequence number.
if (saved_key_valid && num_skipped > max_skip_) {
num_skipped = 0;
std::string last_key;
AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetKey(),
kMaxSequenceNumber,
kValueTypeForSeek));
iter_->Seek(last_key);
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
} else {
iter_->Prev();
}
} while (iter_->Valid());
} }
if (value_type == kTypeDeletion) { ParsedInternalKey ikey;
// End
while (iter_->Valid()) {
saved_key_.SetKey(ExtractUserKey(iter_->key()));
if (FindValueForCurrentKey()) {
valid_ = true;
if (!iter_->Valid()) {
return;
}
FindParseableKey(&ikey, kReverse);
if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) {
FindPrevUserKey();
}
return;
}
if (!iter_->Valid()) {
break;
}
FindParseableKey(&ikey, kReverse);
if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) {
FindPrevUserKey();
}
}
// We haven't found any key - iterator is not valid
assert(!iter_->Valid());
valid_ = false;
}
// This function checks, if the entry with biggest sequence_number <= sequence_
// is non kTypeDeletion. If it's not, we save value in saved_value_
bool DBIter::FindValueForCurrentKey() {
assert(iter_->Valid());
// Contains operands for merge operator.
std::deque<std::string> operands;
// last entry before merge (could be kTypeDeletion or kTypeValue)
ValueType last_not_merge_type = kTypeDeletion;
ValueType last_key_entry_type = kTypeDeletion;
ParsedInternalKey ikey;
FindParseableKey(&ikey, kReverse);
size_t num_skipped = 0;
while (iter_->Valid() && ikey.sequence <= sequence_ &&
(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0)) {
// We iterate too much: let's use Seek() to avoid too much key comparisons
if (num_skipped >= max_skip_) {
return FindValueForCurrentKeyUsingSeek();
}
last_key_entry_type = ikey.type;
switch (last_key_entry_type) {
case kTypeValue:
operands.clear();
saved_value_ = iter_->value().ToString();
last_not_merge_type = kTypeValue;
break;
case kTypeDeletion:
operands.clear();
last_not_merge_type = kTypeDeletion;
break;
case kTypeMerge:
assert(user_merge_operator_ != nullptr);
operands.push_back(iter_->value().ToString());
break;
default:
assert(false);
}
assert(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0);
iter_->Prev();
++num_skipped;
FindParseableKey(&ikey, kReverse);
}
switch (last_key_entry_type) {
case kTypeDeletion:
valid_ = false;
return false;
case kTypeMerge:
if (last_not_merge_type == kTypeDeletion) {
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
&saved_value_, logger_);
} else {
assert(last_not_merge_type == kTypeValue);
std::string last_put_value = saved_value_;
Slice temp_slice(last_put_value);
user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
operands, &saved_value_, logger_);
}
break;
case kTypeValue:
// do nothing - we've already has value in saved_value_
break;
default:
assert(false);
break;
}
valid_ = true;
return true;
}
// This function is used in FindValueForCurrentKey.
// We use Seek() function instead of Prev() to find necessary value
bool DBIter::FindValueForCurrentKeyUsingSeek() {
std::string last_key;
AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetKey(), sequence_,
kValueTypeForSeek));
iter_->Seek(last_key);
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
// assume there is at least one parseable key for this user key
ParsedInternalKey ikey;
FindParseableKey(&ikey, kForward);
if (ikey.type == kTypeValue || ikey.type == kTypeDeletion) {
if (ikey.type == kTypeValue) {
saved_value_ = iter_->value().ToString();
valid_ = true;
return true;
}
valid_ = false; valid_ = false;
saved_key_.Clear(); return false;
ClearSavedValue(); }
direction_ = kForward;
} else { // kTypeMerge. We need to collect all kTypeMerge values and save them
// in operands
std::deque<std::string> operands;
while (iter_->Valid() &&
(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) &&
ikey.type == kTypeMerge) {
operands.push_front(iter_->value().ToString());
iter_->Next();
FindParseableKey(&ikey, kForward);
}
if (!iter_->Valid() ||
(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) ||
ikey.type == kTypeDeletion) {
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
&saved_value_, logger_);
// Make iter_ valid and point to saved_key_
if (!iter_->Valid() ||
(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0)) {
iter_->Seek(last_key);
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
}
valid_ = true; valid_ = true;
return true;
}
const Slice& value = iter_->value();
user_merge_operator_->FullMerge(saved_key_.GetKey(), &value, operands,
&saved_value_, logger_);
valid_ = true;
return true;
}
// Used in Next to change directions
// Go to next user key
// Don't use Seek(),
// because next user key will be very close
void DBIter::FindNextUserKey() {
if (!iter_->Valid()) {
return;
}
ParsedInternalKey ikey;
FindParseableKey(&ikey, kForward);
while (iter_->Valid() &&
user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) {
iter_->Next();
FindParseableKey(&ikey, kForward);
}
}
// Go to previous user_key
void DBIter::FindPrevUserKey() {
if (!iter_->Valid()) {
return;
}
size_t num_skipped = 0;
ParsedInternalKey ikey;
FindParseableKey(&ikey, kReverse);
while (iter_->Valid() &&
user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) {
if (num_skipped >= max_skip_) {
num_skipped = 0;
IterKey last_key;
last_key.SetInternalKey(ParsedInternalKey(
saved_key_.GetKey(), kMaxSequenceNumber, kValueTypeForSeek));
iter_->Seek(last_key.GetKey());
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
}
iter_->Prev();
++num_skipped;
FindParseableKey(&ikey, kReverse);
}
}
// Skip all unparseable keys
void DBIter::FindParseableKey(ParsedInternalKey* ikey, Direction direction) {
while (iter_->Valid() && !ParseKey(ikey)) {
if (direction == kReverse) {
iter_->Prev();
} else {
iter_->Next();
}
} }
} }
@ -454,20 +578,13 @@ void DBIter::SeekToFirst() {
} }
void DBIter::SeekToLast() { void DBIter::SeekToLast() {
// Throw an exception for now if merge_operator is provided
// TODO: support backward iteration
if (user_merge_operator_) {
Log(logger_, "SeekToLast not supported yet if merge_operator is provided");
throw std::logic_error("DBIter::SeekToLast: backward iteration not"
" supported if merge_operator is provided");
}
direction_ = kReverse; direction_ = kReverse;
ClearSavedValue(); ClearSavedValue();
PERF_TIMER_AUTO(seek_internal_seek_time); PERF_TIMER_AUTO(seek_internal_seek_time);
iter_->SeekToLast(); iter_->SeekToLast();
PERF_TIMER_STOP(seek_internal_seek_time); PERF_TIMER_STOP(seek_internal_seek_time);
FindPrevUserEntry();
PrevInternal();
} }
Iterator* NewDBIterator(Env* env, const Options& options, Iterator* NewDBIterator(Env* env, const Options& options,

View file

@ -12,8 +12,10 @@
#include <set> #include <set>
#include <unistd.h> #include <unistd.h>
#include <unordered_set> #include <unordered_set>
#include <utility>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/db_iter.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/filename.h" #include "db/filename.h"
#include "db/version_set.h" #include "db/version_set.h"
@ -1162,6 +1164,952 @@ TEST(DBTest, ReadOnlyDB) {
delete iter; delete iter;
} }
class TestIterator : public Iterator {
public:
explicit TestIterator(const Comparator* comparator)
: initialized_(false),
valid_(false),
sequence_number_(0),
iter_(0),
cmp(comparator) {}
void AddMerge(std::string key, std::string value) {
Add(key, kTypeMerge, value);
}
void AddDeletion(std::string key) { Add(key, kTypeDeletion, std::string()); }
void AddPut(std::string key, std::string value) {
Add(key, kTypeValue, value);
}
void Add(std::string key, ValueType type, std::string value) {
valid_ = true;
ParsedInternalKey internal_key(key, sequence_number_++, type);
data_.push_back(std::pair<std::string, std::string>(std::string(), value));
AppendInternalKey(&data_.back().first, internal_key);
}
// should be called before operations with iterator
void Finish() {
initialized_ = true;
std::sort(data_.begin(), data_.end(),
[this](std::pair<std::string, std::string> a,
std::pair<std::string, std::string> b) {
return (cmp.Compare(a.first, b.first) < 0);
});
}
virtual bool Valid() const override {
assert(initialized_);
return valid_;
}
virtual void SeekToFirst() override {
assert(initialized_);
valid_ = (data_.size() > 0);
iter_ = 0;
}
virtual void SeekToLast() override {
assert(initialized_);
valid_ = (data_.size() > 0);
iter_ = data_.size() - 1;
}
virtual void Seek(const Slice& target) override {
assert(initialized_);
SeekToFirst();
if (!valid_) {
return;
}
while (iter_ < data_.size() &&
(cmp.Compare(data_[iter_].first, target) < 0)) {
++iter_;
}
if (iter_ == data_.size()) {
valid_ = false;
}
}
virtual void Next() override {
assert(initialized_);
if (data_.empty() || (iter_ == data_.size() - 1)) {
valid_ = false;
} else {
++iter_;
}
}
virtual void Prev() override {
assert(initialized_);
if (iter_ == 0) {
valid_ = false;
} else {
--iter_;
}
}
virtual Slice key() const override {
assert(initialized_);
return data_[iter_].first;
}
virtual Slice value() const override {
assert(initialized_);
return data_[iter_].second;
}
virtual Status status() const override {
assert(initialized_);
return Status::OK();
}
private:
bool initialized_;
bool valid_;
size_t sequence_number_;
size_t iter_;
InternalKeyComparator cmp;
std::vector<std::pair<std::string, std::string>> data_;
};
TEST(DBTest, DBIteratorPrevNext) {
Options options;
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddDeletion("a");
internal_iter.AddDeletion("a");
internal_iter.AddDeletion("a");
internal_iter.AddDeletion("a");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.Finish();
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val_b");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "val_a");
db_iter->Next();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val_b");
db_iter->Next();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10);
db_iter->SeekToFirst();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "val_a");
db_iter->Next();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val_b");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "val_a");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
Options options;
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.Finish();
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val_b");
db_iter->Next();
ASSERT_TRUE(!db_iter->Valid());
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val_b");
}
{
Options options;
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("a", "val_a");
internal_iter.AddPut("b", "val_b");
internal_iter.AddPut("c", "val_c");
internal_iter.Finish();
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "val_c");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val_b");
db_iter->Next();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "val_c");
}
}
TEST(DBTest, DBIteratorEmpty) {
Options options = CurrentOptions();
TestIterator internal_iter(BytewiseComparator());
internal_iter.Finish();
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0);
db_iter->SeekToLast();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0);
db_iter->SeekToFirst();
ASSERT_TRUE(!db_iter->Valid());
}
}
TEST(DBTest, DBIteratorUseSkipCountSkips) {
Options options = CurrentOptions();
options.statistics = rocksdb::CreateDBStatistics();
options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
TestIterator internal_iter(BytewiseComparator());
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddPut("a", "a");
internal_iter.AddPut("b", "b");
internal_iter.AddPut("c", "c");
}
internal_iter.Finish();
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "c");
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1);
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "b");
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2);
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "a");
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3);
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3);
}
TEST(DBTest, DBIteratorUseSkip) {
Options options = CurrentOptions();
options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("b", "merge_1");
internal_iter.AddMerge("a", "merge_2");
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddPut("c", std::to_string(i));
}
internal_iter.Finish();
for (size_t i = 0; i < 200; ++i) {
options.statistics = rocksdb::CreateDBStatistics();
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, i + 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_2");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("b", "merge_1");
internal_iter.AddMerge("a", "merge_2");
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddDeletion("c");
}
internal_iter.AddPut("c", "200");
internal_iter.Finish();
for (size_t i = 0; i < 200; ++i) {
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, i + 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_2");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, 202);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "200");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_2");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
{
TestIterator internal_iter(BytewiseComparator());
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddDeletion("c");
}
internal_iter.AddPut("c", "200");
internal_iter.Finish();
for (size_t i = 0; i < 200; ++i) {
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, i);
db_iter->SeekToLast();
ASSERT_TRUE(!db_iter->Valid());
db_iter->SeekToFirst();
ASSERT_TRUE(!db_iter->Valid());
}
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 200);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "200");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
db_iter->SeekToFirst();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "200");
db_iter->Next();
ASSERT_TRUE(!db_iter->Valid());
}
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("b", "merge_1");
internal_iter.AddMerge("a", "merge_2");
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddPut("d", std::to_string(i));
}
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddPut("c", std::to_string(i));
}
internal_iter.Finish();
for (size_t i = 0; i < 200; ++i) {
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, i + 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "d");
ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_2");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("b", "b");
internal_iter.AddMerge("a", "a");
for (size_t i = 0; i < 200; ++i) {
internal_iter.AddMerge("c", std::to_string(i));
}
internal_iter.Finish();
for (size_t i = 0; i < 200; ++i) {
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, i + 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
std::string merge_result = "0";
for (size_t j = 1; j <= i; ++j) {
merge_result += "," + std::to_string(j);
}
ASSERT_EQ(db_iter->value().ToString(), merge_result);
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "b");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "a");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
}
TEST(DBTest, DBIterator) {
Options options = CurrentOptions();
options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddPut("a", "0");
internal_iter.AddPut("b", "0");
internal_iter.AddDeletion("b");
internal_iter.AddMerge("a", "1");
internal_iter.AddMerge("b", "2");
internal_iter.Finish();
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 1);
db_iter->SeekToFirst();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "0");
db_iter->Next();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0);
db_iter->SeekToFirst();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "0");
db_iter->Next();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2);
db_iter->SeekToFirst();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "0");
db_iter->Next();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4);
db_iter->SeekToFirst();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "0,1");
db_iter->Next();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "2");
db_iter->Next();
ASSERT_TRUE(!db_iter->Valid());
}
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("a", "merge_1");
internal_iter.AddMerge("a", "merge_2");
internal_iter.AddMerge("a", "merge_3");
internal_iter.AddPut("a", "put_1");
internal_iter.AddMerge("a", "merge_4");
internal_iter.AddMerge("a", "merge_5");
internal_iter.AddMerge("a", "merge_6");
internal_iter.Finish();
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 1);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 3);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "put_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 5);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 6);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5,merge_6");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("a", "merge_1");
internal_iter.AddMerge("a", "merge_2");
internal_iter.AddMerge("a", "merge_3");
internal_iter.AddDeletion("a");
internal_iter.AddMerge("a", "merge_4");
internal_iter.AddMerge("a", "merge_5");
internal_iter.AddMerge("a", "merge_6");
internal_iter.Finish();
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 1);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 3);
db_iter->SeekToLast();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_4");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 5);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 6);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5,merge_6");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
{
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddMerge("a", "merge_1");
internal_iter.AddPut("b", "val");
internal_iter.AddMerge("b", "merge_2");
internal_iter.AddDeletion("b");
internal_iter.AddMerge("b", "merge_3");
internal_iter.AddMerge("c", "merge_4");
internal_iter.AddMerge("c", "merge_5");
internal_iter.AddDeletion("b");
internal_iter.AddMerge("b", "merge_6");
internal_iter.AddMerge("b", "merge_7");
internal_iter.AddMerge("b", "merge_8");
internal_iter.AddMerge("b", "merge_9");
internal_iter.AddMerge("b", "merge_10");
internal_iter.AddMerge("b", "merge_11");
internal_iter.AddDeletion("c");
internal_iter.Finish();
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "val,merge_2");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_3");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 5);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "merge_4");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_3");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 6);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_3");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 7);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 9);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "merge_6,merge_7");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, 13);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(),
"merge_6,merge_7,merge_8,merge_9,merge_10,merge_11");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
{
auto db_iter = NewDBIterator(env_, options, BytewiseComparator(),
&internal_iter, 14);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(),
"merge_6,merge_7,merge_8,merge_9,merge_10,merge_11");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "merge_1");
db_iter->Prev();
ASSERT_TRUE(!db_iter->Valid());
}
}
{
Options options = CurrentOptions();
TestIterator internal_iter(BytewiseComparator());
internal_iter.AddDeletion("a");
internal_iter.AddPut("a", "0");
internal_iter.AddPut("b", "0");
internal_iter.Finish();
auto db_iter =
NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10);
db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "b");
ASSERT_EQ(db_iter->value().ToString(), "0");
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "a");
ASSERT_EQ(db_iter->value().ToString(), "0");
}
}
// Make sure that when options.block_cache is set, after a new table is // Make sure that when options.block_cache is set, after a new table is
// created its index/filter blocks are added to block cache. // created its index/filter blocks are added to block cache.
TEST(DBTest, IndexAndFilterBlocksOfNewTableAddedToCache) { TEST(DBTest, IndexAndFilterBlocksOfNewTableAddedToCache) {