mirror of https://github.com/facebook/rocksdb.git
Support allow_unprepared_value for multi-CF iterators (#13079)
Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/13079 The patch adds support for the new read option `allow_unprepared_value` to the multi-column-family iterators `CoalescingIterator` and `AttributeGroupIterator`. When this option is set, these iterators populate their value (`value()` + `columns()` or `attribute_groups()`) in an on-demand fashion when `PrepareValue()` is called. Calling `PrepareValue()` on the child iterators is similarly deferred until `PrepareValue()` is called on the main iterator. Reviewed By: jowlyzhang Differential Revision: D64570587 fbshipit-source-id: 783c8d408ad10074417dabca7b82c5e1fe5cab36
This commit is contained in:
parent
0ca691654f
commit
c0be6a4b90
|
@ -13,11 +13,11 @@ namespace ROCKSDB_NAMESPACE {
|
||||||
class AttributeGroupIteratorImpl : public AttributeGroupIterator {
|
class AttributeGroupIteratorImpl : public AttributeGroupIterator {
|
||||||
public:
|
public:
|
||||||
AttributeGroupIteratorImpl(
|
AttributeGroupIteratorImpl(
|
||||||
const Comparator* comparator,
|
const Comparator* comparator, bool allow_unprepared_value,
|
||||||
const std::vector<ColumnFamilyHandle*>& column_families,
|
const std::vector<ColumnFamilyHandle*>& column_families,
|
||||||
const std::vector<Iterator*>& child_iterators)
|
const std::vector<Iterator*>& child_iterators)
|
||||||
: impl_(comparator, column_families, child_iterators, ResetFunc(this),
|
: impl_(comparator, allow_unprepared_value, column_families,
|
||||||
PopulateFunc(this)) {}
|
child_iterators, ResetFunc(this), PopulateFunc(this)) {}
|
||||||
~AttributeGroupIteratorImpl() override {}
|
~AttributeGroupIteratorImpl() override {}
|
||||||
|
|
||||||
// No copy allowed
|
// No copy allowed
|
||||||
|
@ -42,6 +42,8 @@ class AttributeGroupIteratorImpl : public AttributeGroupIterator {
|
||||||
|
|
||||||
void Reset() { attribute_groups_.clear(); }
|
void Reset() { attribute_groups_.clear(); }
|
||||||
|
|
||||||
|
bool PrepareValue() override { return impl_.PrepareValue(); }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
class ResetFunc {
|
class ResetFunc {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -12,11 +12,11 @@ namespace ROCKSDB_NAMESPACE {
|
||||||
// EXPERIMENTAL
|
// EXPERIMENTAL
|
||||||
class CoalescingIterator : public Iterator {
|
class CoalescingIterator : public Iterator {
|
||||||
public:
|
public:
|
||||||
CoalescingIterator(const Comparator* comparator,
|
CoalescingIterator(const Comparator* comparator, bool allow_unprepared_value,
|
||||||
const std::vector<ColumnFamilyHandle*>& column_families,
|
const std::vector<ColumnFamilyHandle*>& column_families,
|
||||||
const std::vector<Iterator*>& child_iterators)
|
const std::vector<Iterator*>& child_iterators)
|
||||||
: impl_(comparator, column_families, child_iterators, ResetFunc(this),
|
: impl_(comparator, allow_unprepared_value, column_families,
|
||||||
PopulateFunc(this)) {}
|
child_iterators, ResetFunc(this), PopulateFunc(this)) {}
|
||||||
~CoalescingIterator() override {}
|
~CoalescingIterator() override {}
|
||||||
|
|
||||||
// No copy allowed
|
// No copy allowed
|
||||||
|
@ -47,6 +47,8 @@ class CoalescingIterator : public Iterator {
|
||||||
wide_columns_.clear();
|
wide_columns_.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool PrepareValue() override { return impl_.PrepareValue(); }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
class ResetFunc {
|
class ResetFunc {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -3992,9 +3992,9 @@ std::unique_ptr<IterType> DBImpl::NewMultiCfIterator(
|
||||||
if (!s.ok()) {
|
if (!s.ok()) {
|
||||||
return error_iterator_func(s);
|
return error_iterator_func(s);
|
||||||
}
|
}
|
||||||
return std::make_unique<ImplType>(column_families[0]->GetComparator(),
|
return std::make_unique<ImplType>(
|
||||||
column_families,
|
column_families[0]->GetComparator(), _read_options.allow_unprepared_value,
|
||||||
std::move(child_iterators));
|
column_families, std::move(child_iterators));
|
||||||
}
|
}
|
||||||
|
|
||||||
Status DBImpl::NewIterators(
|
Status DBImpl::NewIterators(
|
||||||
|
|
|
@ -24,11 +24,12 @@ struct MultiCfIteratorInfo {
|
||||||
template <typename ResetFunc, typename PopulateFunc>
|
template <typename ResetFunc, typename PopulateFunc>
|
||||||
class MultiCfIteratorImpl {
|
class MultiCfIteratorImpl {
|
||||||
public:
|
public:
|
||||||
MultiCfIteratorImpl(const Comparator* comparator,
|
MultiCfIteratorImpl(const Comparator* comparator, bool allow_unprepared_value,
|
||||||
const std::vector<ColumnFamilyHandle*>& column_families,
|
const std::vector<ColumnFamilyHandle*>& column_families,
|
||||||
const std::vector<Iterator*>& child_iterators,
|
const std::vector<Iterator*>& child_iterators,
|
||||||
ResetFunc reset_func, PopulateFunc populate_func)
|
ResetFunc reset_func, PopulateFunc populate_func)
|
||||||
: comparator_(comparator),
|
: comparator_(comparator),
|
||||||
|
allow_unprepared_value_(allow_unprepared_value),
|
||||||
heap_(MultiCfMinHeap(
|
heap_(MultiCfMinHeap(
|
||||||
MultiCfHeapItemComparator<std::greater<int>>(comparator_))),
|
MultiCfHeapItemComparator<std::greater<int>>(comparator_))),
|
||||||
reset_func_(std::move(reset_func)),
|
reset_func_(std::move(reset_func)),
|
||||||
|
@ -100,6 +101,41 @@ class MultiCfIteratorImpl {
|
||||||
AdvanceIterator(max_heap, [](Iterator* iter) { iter->Prev(); });
|
AdvanceIterator(max_heap, [](Iterator* iter) { iter->Prev(); });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool PrepareValue() {
|
||||||
|
assert(Valid());
|
||||||
|
|
||||||
|
if (!allow_unprepared_value_) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto prepare_value_func = [this](auto& heap, Iterator* iterator) {
|
||||||
|
assert(iterator);
|
||||||
|
assert(iterator->Valid());
|
||||||
|
assert(iterator->status().ok());
|
||||||
|
|
||||||
|
if (!iterator->PrepareValue()) {
|
||||||
|
assert(!iterator->Valid());
|
||||||
|
assert(!iterator->status().ok());
|
||||||
|
|
||||||
|
considerStatus(iterator->status());
|
||||||
|
assert(!status_.ok());
|
||||||
|
|
||||||
|
heap.clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (std::holds_alternative<MultiCfMaxHeap>(heap_)) {
|
||||||
|
return PopulateIterator(std::get<MultiCfMaxHeap>(heap_),
|
||||||
|
prepare_value_func);
|
||||||
|
}
|
||||||
|
|
||||||
|
return PopulateIterator(std::get<MultiCfMinHeap>(heap_),
|
||||||
|
prepare_value_func);
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>
|
std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>
|
||||||
cfh_iter_pairs_;
|
cfh_iter_pairs_;
|
||||||
|
@ -124,7 +160,10 @@ class MultiCfIteratorImpl {
|
||||||
private:
|
private:
|
||||||
const Comparator* comparator_;
|
const Comparator* comparator_;
|
||||||
};
|
};
|
||||||
|
|
||||||
const Comparator* comparator_;
|
const Comparator* comparator_;
|
||||||
|
bool allow_unprepared_value_;
|
||||||
|
|
||||||
using MultiCfMinHeap =
|
using MultiCfMinHeap =
|
||||||
BinaryHeap<MultiCfIteratorInfo,
|
BinaryHeap<MultiCfIteratorInfo,
|
||||||
MultiCfHeapItemComparator<std::greater<int>>>;
|
MultiCfHeapItemComparator<std::greater<int>>>;
|
||||||
|
@ -185,13 +224,16 @@ class MultiCfIteratorImpl {
|
||||||
if (!status_.ok()) {
|
if (!status_.ok()) {
|
||||||
// Non-OK status from the iterator. Bail out early
|
// Non-OK status from the iterator. Bail out early
|
||||||
heap.clear();
|
heap.clear();
|
||||||
break;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
++i;
|
++i;
|
||||||
}
|
}
|
||||||
if (!heap.empty()) {
|
if (!allow_unprepared_value_ && !heap.empty()) {
|
||||||
PopulateIterator(heap);
|
[[maybe_unused]] const bool result = PopulateIterator(
|
||||||
|
heap,
|
||||||
|
[](auto& /* heap */, Iterator* /* iterator */) { return true; });
|
||||||
|
assert(result);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -257,13 +299,16 @@ class MultiCfIteratorImpl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!heap.empty()) {
|
if (!allow_unprepared_value_ && !heap.empty()) {
|
||||||
PopulateIterator(heap);
|
[[maybe_unused]] const bool result = PopulateIterator(
|
||||||
|
heap,
|
||||||
|
[](auto& /* heap */, Iterator* /* iterator */) { return true; });
|
||||||
|
assert(result);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename BinaryHeap>
|
template <typename BinaryHeap, typename PrepareValueFunc>
|
||||||
void PopulateIterator(BinaryHeap& heap) {
|
bool PopulateIterator(BinaryHeap& heap, PrepareValueFunc prepare_value_func) {
|
||||||
// 1. Keep the top iterator (by popping it from the heap) and add it to list
|
// 1. Keep the top iterator (by popping it from the heap) and add it to list
|
||||||
// to populate
|
// to populate
|
||||||
// 2. For all non-top iterators having the same key as top iter popped
|
// 2. For all non-top iterators having the same key as top iter popped
|
||||||
|
@ -279,10 +324,14 @@ class MultiCfIteratorImpl {
|
||||||
assert(top.iterator->Valid());
|
assert(top.iterator->Valid());
|
||||||
assert(top.iterator->status().ok());
|
assert(top.iterator->status().ok());
|
||||||
|
|
||||||
heap.pop();
|
if (!prepare_value_func(heap, top.iterator)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
autovector<MultiCfIteratorInfo> to_populate;
|
autovector<MultiCfIteratorInfo> to_populate;
|
||||||
|
|
||||||
to_populate.push_back(top);
|
to_populate.push_back(top);
|
||||||
|
heap.pop();
|
||||||
|
|
||||||
while (!heap.empty()) {
|
while (!heap.empty()) {
|
||||||
auto current = heap.top();
|
auto current = heap.top();
|
||||||
|
@ -295,6 +344,10 @@ class MultiCfIteratorImpl {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!prepare_value_func(heap, current.iterator)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
to_populate.push_back(current);
|
to_populate.push_back(current);
|
||||||
heap.pop();
|
heap.pop();
|
||||||
}
|
}
|
||||||
|
@ -305,6 +358,8 @@ class MultiCfIteratorImpl {
|
||||||
}
|
}
|
||||||
|
|
||||||
populate_func_(to_populate);
|
populate_func_(to_populate);
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -21,18 +21,27 @@ class CoalescingIteratorTest : public DBTestBase {
|
||||||
const std::optional<std::vector<WideColumns>>&
|
const std::optional<std::vector<WideColumns>>&
|
||||||
expected_wide_columns = std::nullopt,
|
expected_wide_columns = std::nullopt,
|
||||||
const Slice* lower_bound = nullptr,
|
const Slice* lower_bound = nullptr,
|
||||||
const Slice* upper_bound = nullptr) {
|
const Slice* upper_bound = nullptr,
|
||||||
|
bool allow_unprepared_value = false) {
|
||||||
const size_t num_keys = expected_keys.size();
|
const size_t num_keys = expected_keys.size();
|
||||||
|
|
||||||
ReadOptions read_options;
|
ReadOptions read_options;
|
||||||
read_options.iterate_lower_bound = lower_bound;
|
read_options.iterate_lower_bound = lower_bound;
|
||||||
read_options.iterate_upper_bound = upper_bound;
|
read_options.iterate_upper_bound = upper_bound;
|
||||||
|
read_options.allow_unprepared_value = allow_unprepared_value;
|
||||||
|
|
||||||
std::unique_ptr<Iterator> iter =
|
std::unique_ptr<Iterator> iter =
|
||||||
db_->NewCoalescingIterator(read_options, cfhs);
|
db_->NewCoalescingIterator(read_options, cfhs);
|
||||||
|
|
||||||
auto check_iter_entry = [&](size_t idx) {
|
auto check_iter_entry = [&](size_t idx) {
|
||||||
ASSERT_EQ(iter->key(), expected_keys[idx]);
|
ASSERT_EQ(iter->key(), expected_keys[idx]);
|
||||||
|
|
||||||
|
if (allow_unprepared_value) {
|
||||||
|
ASSERT_TRUE(iter->value().empty());
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
}
|
||||||
|
|
||||||
ASSERT_EQ(iter->value(), expected_values[idx]);
|
ASSERT_EQ(iter->value(), expected_values[idx]);
|
||||||
if (expected_wide_columns.has_value()) {
|
if (expected_wide_columns.has_value()) {
|
||||||
ASSERT_EQ(iter->columns(), expected_wide_columns.value()[idx]);
|
ASSERT_EQ(iter->columns(), expected_wide_columns.value()[idx]);
|
||||||
|
@ -781,6 +790,137 @@ TEST_F(CoalescingIteratorTest, CustomComparatorsInMultiCFs) {
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(CoalescingIteratorTest, AllowUnpreparedValue) {
|
||||||
|
Options options = GetDefaultOptions();
|
||||||
|
options.enable_blob_files = true;
|
||||||
|
|
||||||
|
CreateAndReopenWithCF({"cf_1", "cf_2", "cf_3"}, options);
|
||||||
|
|
||||||
|
ASSERT_OK(Put(0, "key_1", "key_1_cf_0_val"));
|
||||||
|
ASSERT_OK(Put(3, "key_1", "key_1_cf_3_val"));
|
||||||
|
ASSERT_OK(Put(1, "key_2", "key_2_cf_1_val"));
|
||||||
|
ASSERT_OK(Put(2, "key_2", "key_2_cf_2_val"));
|
||||||
|
ASSERT_OK(Put(0, "key_3", "key_3_cf_0_val"));
|
||||||
|
ASSERT_OK(Put(1, "key_3", "key_3_cf_1_val"));
|
||||||
|
ASSERT_OK(Put(3, "key_3", "key_3_cf_3_val"));
|
||||||
|
|
||||||
|
ASSERT_OK(Flush());
|
||||||
|
|
||||||
|
std::vector<ColumnFamilyHandle*> cfhs_order_3_2_0_1{handles_[3], handles_[2],
|
||||||
|
handles_[0], handles_[1]};
|
||||||
|
std::vector<Slice> expected_keys{"key_1", "key_2", "key_3"};
|
||||||
|
std::vector<Slice> expected_values{"key_1_cf_0_val", "key_2_cf_1_val",
|
||||||
|
"key_3_cf_1_val"};
|
||||||
|
|
||||||
|
VerifyCoalescingIterator(cfhs_order_3_2_0_1, expected_keys, expected_values,
|
||||||
|
/* expected_wide_columns */ std::nullopt,
|
||||||
|
/* lower_bound */ nullptr, /* upper_bound */ nullptr,
|
||||||
|
/* allow_unprepared_value */ true);
|
||||||
|
|
||||||
|
ReadOptions read_options;
|
||||||
|
read_options.allow_unprepared_value = true;
|
||||||
|
|
||||||
|
{
|
||||||
|
std::unique_ptr<Iterator> iter =
|
||||||
|
db_->NewCoalescingIterator(read_options, cfhs_order_3_2_0_1);
|
||||||
|
iter->Seek("");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_1->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_1->key_1_cf_0_val");
|
||||||
|
|
||||||
|
iter->Seek("key_1");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_1->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_1->key_1_cf_0_val");
|
||||||
|
|
||||||
|
iter->Seek("key_2");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_2->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_2->key_2_cf_1_val");
|
||||||
|
|
||||||
|
iter->Next();
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_3->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_3->key_3_cf_1_val");
|
||||||
|
|
||||||
|
iter->Seek("key_x");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "(invalid)");
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
std::unique_ptr<Iterator> iter =
|
||||||
|
db_->NewCoalescingIterator(read_options, cfhs_order_3_2_0_1);
|
||||||
|
iter->SeekForPrev("");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "(invalid)");
|
||||||
|
|
||||||
|
iter->SeekForPrev("key_1");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_1->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_1->key_1_cf_0_val");
|
||||||
|
|
||||||
|
iter->Next();
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_2->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_2->key_2_cf_1_val");
|
||||||
|
|
||||||
|
iter->SeekForPrev("key_x");
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_3->");
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "key_3->key_3_cf_1_val");
|
||||||
|
|
||||||
|
iter->Next();
|
||||||
|
ASSERT_EQ(IterStatus(iter.get()), "(invalid)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(CoalescingIteratorTest, AllowUnpreparedValue_Corruption) {
|
||||||
|
Options options = GetDefaultOptions();
|
||||||
|
options.enable_blob_files = true;
|
||||||
|
|
||||||
|
CreateAndReopenWithCF({"cf_1", "cf_2", "cf_3"}, options);
|
||||||
|
|
||||||
|
ASSERT_OK(Put(0, "key_1", "key_1_cf_0_val"));
|
||||||
|
ASSERT_OK(Put(3, "key_1", "key_1_cf_3_val"));
|
||||||
|
ASSERT_OK(Put(1, "key_2", "key_2_cf_1_val"));
|
||||||
|
ASSERT_OK(Put(2, "key_2", "key_2_cf_2_val"));
|
||||||
|
ASSERT_OK(Put(0, "key_3", "key_3_cf_0_val"));
|
||||||
|
ASSERT_OK(Put(1, "key_3", "key_3_cf_1_val"));
|
||||||
|
ASSERT_OK(Put(3, "key_3", "key_3_cf_3_val"));
|
||||||
|
|
||||||
|
ASSERT_OK(Flush());
|
||||||
|
|
||||||
|
ReadOptions read_options;
|
||||||
|
read_options.allow_unprepared_value = true;
|
||||||
|
|
||||||
|
std::vector<ColumnFamilyHandle*> cfhs_order_3_2_0_1{handles_[3], handles_[2],
|
||||||
|
handles_[0], handles_[1]};
|
||||||
|
|
||||||
|
std::unique_ptr<Iterator> iter =
|
||||||
|
db_->NewCoalescingIterator(read_options, cfhs_order_3_2_0_1);
|
||||||
|
iter->SeekToFirst();
|
||||||
|
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
ASSERT_OK(iter->status());
|
||||||
|
ASSERT_EQ(iter->key(), "key_1");
|
||||||
|
ASSERT_TRUE(iter->value().empty());
|
||||||
|
|
||||||
|
SyncPoint::GetInstance()->SetCallBack(
|
||||||
|
"BlobFileReader::GetBlob:TamperWithResult", [](void* arg) {
|
||||||
|
Slice* const blob_index = static_cast<Slice*>(arg);
|
||||||
|
assert(blob_index);
|
||||||
|
assert(!blob_index->empty());
|
||||||
|
blob_index->remove_prefix(1);
|
||||||
|
});
|
||||||
|
SyncPoint::GetInstance()->EnableProcessing();
|
||||||
|
|
||||||
|
ASSERT_FALSE(iter->PrepareValue());
|
||||||
|
ASSERT_FALSE(iter->Valid());
|
||||||
|
ASSERT_TRUE(iter->status().IsCorruption());
|
||||||
|
|
||||||
|
SyncPoint::GetInstance()->DisableProcessing();
|
||||||
|
SyncPoint::GetInstance()->ClearAllCallBacks();
|
||||||
|
}
|
||||||
|
|
||||||
class AttributeGroupIteratorTest : public DBTestBase {
|
class AttributeGroupIteratorTest : public DBTestBase {
|
||||||
public:
|
public:
|
||||||
AttributeGroupIteratorTest()
|
AttributeGroupIteratorTest()
|
||||||
|
@ -790,17 +930,27 @@ class AttributeGroupIteratorTest : public DBTestBase {
|
||||||
const std::vector<ColumnFamilyHandle*>& cfhs,
|
const std::vector<ColumnFamilyHandle*>& cfhs,
|
||||||
const std::vector<Slice>& expected_keys,
|
const std::vector<Slice>& expected_keys,
|
||||||
const std::vector<IteratorAttributeGroups>& expected_attribute_groups,
|
const std::vector<IteratorAttributeGroups>& expected_attribute_groups,
|
||||||
const Slice* lower_bound = nullptr, const Slice* upper_bound = nullptr) {
|
const Slice* lower_bound = nullptr, const Slice* upper_bound = nullptr,
|
||||||
|
bool allow_unprepared_value = false) {
|
||||||
const size_t num_keys = expected_keys.size();
|
const size_t num_keys = expected_keys.size();
|
||||||
|
|
||||||
ReadOptions read_options;
|
ReadOptions read_options;
|
||||||
read_options.iterate_lower_bound = lower_bound;
|
read_options.iterate_lower_bound = lower_bound;
|
||||||
read_options.iterate_upper_bound = upper_bound;
|
read_options.iterate_upper_bound = upper_bound;
|
||||||
|
read_options.allow_unprepared_value = allow_unprepared_value;
|
||||||
|
|
||||||
std::unique_ptr<AttributeGroupIterator> iter =
|
std::unique_ptr<AttributeGroupIterator> iter =
|
||||||
db_->NewAttributeGroupIterator(read_options, cfhs);
|
db_->NewAttributeGroupIterator(read_options, cfhs);
|
||||||
|
|
||||||
auto check_iter_entry = [&](size_t idx) {
|
auto check_iter_entry = [&](size_t idx) {
|
||||||
ASSERT_EQ(iter->key(), expected_keys[idx]);
|
ASSERT_EQ(iter->key(), expected_keys[idx]);
|
||||||
|
|
||||||
|
if (allow_unprepared_value) {
|
||||||
|
ASSERT_TRUE(iter->attribute_groups().empty());
|
||||||
|
ASSERT_TRUE(iter->PrepareValue());
|
||||||
|
ASSERT_TRUE(iter->Valid());
|
||||||
|
}
|
||||||
|
|
||||||
ASSERT_EQ(iter->attribute_groups(), expected_attribute_groups[idx]);
|
ASSERT_EQ(iter->attribute_groups(), expected_attribute_groups[idx]);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -938,6 +1088,82 @@ TEST_F(AttributeGroupIteratorTest, IterateAttributeGroups) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(AttributeGroupIteratorTest, AllowUnpreparedValue) {
|
||||||
|
Options options = GetDefaultOptions();
|
||||||
|
CreateAndReopenWithCF({"cf_1", "cf_2", "cf_3"}, options);
|
||||||
|
|
||||||
|
constexpr char key_1[] = "key_1";
|
||||||
|
WideColumns key_1_columns_in_cf_2{
|
||||||
|
{kDefaultWideColumnName, "cf_2_col_val_0_key_1"},
|
||||||
|
{"cf_2_col_name_1", "cf_2_col_val_1_key_1"},
|
||||||
|
{"cf_2_col_name_2", "cf_2_col_val_2_key_1"}};
|
||||||
|
WideColumns key_1_columns_in_cf_3{
|
||||||
|
{"cf_3_col_name_1", "cf_3_col_val_1_key_1"},
|
||||||
|
{"cf_3_col_name_2", "cf_3_col_val_2_key_1"},
|
||||||
|
{"cf_3_col_name_3", "cf_3_col_val_3_key_1"}};
|
||||||
|
|
||||||
|
constexpr char key_2[] = "key_2";
|
||||||
|
WideColumns key_2_columns_in_cf_1{
|
||||||
|
{"cf_1_col_name_1", "cf_1_col_val_1_key_2"}};
|
||||||
|
WideColumns key_2_columns_in_cf_2{
|
||||||
|
{"cf_2_col_name_1", "cf_2_col_val_1_key_2"},
|
||||||
|
{"cf_2_col_name_2", "cf_2_col_val_2_key_2"}};
|
||||||
|
|
||||||
|
constexpr char key_3[] = "key_3";
|
||||||
|
WideColumns key_3_columns_in_cf_1{
|
||||||
|
{"cf_1_col_name_1", "cf_1_col_val_1_key_3"}};
|
||||||
|
WideColumns key_3_columns_in_cf_3{
|
||||||
|
{"cf_3_col_name_1", "cf_3_col_val_1_key_3"}};
|
||||||
|
|
||||||
|
constexpr char key_4[] = "key_4";
|
||||||
|
WideColumns key_4_columns_in_cf_0{
|
||||||
|
{"cf_0_col_name_1", "cf_0_col_val_1_key_4"}};
|
||||||
|
WideColumns key_4_columns_in_cf_2{
|
||||||
|
{"cf_2_col_name_1", "cf_2_col_val_1_key_4"}};
|
||||||
|
|
||||||
|
AttributeGroups key_1_attribute_groups{
|
||||||
|
AttributeGroup(handles_[2], key_1_columns_in_cf_2),
|
||||||
|
AttributeGroup(handles_[3], key_1_columns_in_cf_3)};
|
||||||
|
AttributeGroups key_2_attribute_groups{
|
||||||
|
AttributeGroup(handles_[1], key_2_columns_in_cf_1),
|
||||||
|
AttributeGroup(handles_[2], key_2_columns_in_cf_2)};
|
||||||
|
AttributeGroups key_3_attribute_groups{
|
||||||
|
AttributeGroup(handles_[1], key_3_columns_in_cf_1),
|
||||||
|
AttributeGroup(handles_[3], key_3_columns_in_cf_3)};
|
||||||
|
AttributeGroups key_4_attribute_groups{
|
||||||
|
AttributeGroup(handles_[0], key_4_columns_in_cf_0),
|
||||||
|
AttributeGroup(handles_[2], key_4_columns_in_cf_2)};
|
||||||
|
|
||||||
|
ASSERT_OK(db_->PutEntity(WriteOptions(), key_1, key_1_attribute_groups));
|
||||||
|
ASSERT_OK(db_->PutEntity(WriteOptions(), key_2, key_2_attribute_groups));
|
||||||
|
ASSERT_OK(db_->PutEntity(WriteOptions(), key_3, key_3_attribute_groups));
|
||||||
|
ASSERT_OK(db_->PutEntity(WriteOptions(), key_4, key_4_attribute_groups));
|
||||||
|
|
||||||
|
IteratorAttributeGroups key_1_expected_attribute_groups{
|
||||||
|
IteratorAttributeGroup(key_1_attribute_groups[0]),
|
||||||
|
IteratorAttributeGroup(key_1_attribute_groups[1])};
|
||||||
|
IteratorAttributeGroups key_2_expected_attribute_groups{
|
||||||
|
IteratorAttributeGroup(key_2_attribute_groups[0]),
|
||||||
|
IteratorAttributeGroup(key_2_attribute_groups[1])};
|
||||||
|
IteratorAttributeGroups key_3_expected_attribute_groups{
|
||||||
|
IteratorAttributeGroup(key_3_attribute_groups[0]),
|
||||||
|
IteratorAttributeGroup(key_3_attribute_groups[1])};
|
||||||
|
IteratorAttributeGroups key_4_expected_attribute_groups{
|
||||||
|
IteratorAttributeGroup(key_4_attribute_groups[0]),
|
||||||
|
IteratorAttributeGroup(key_4_attribute_groups[1])};
|
||||||
|
|
||||||
|
std::vector<ColumnFamilyHandle*> cfhs_order_0_1_2_3{handles_[0], handles_[1],
|
||||||
|
handles_[2], handles_[3]};
|
||||||
|
std::vector<Slice> expected_keys{key_1, key_2, key_3, key_4};
|
||||||
|
std::vector<IteratorAttributeGroups> expected_attribute_groups{
|
||||||
|
key_1_expected_attribute_groups, key_2_expected_attribute_groups,
|
||||||
|
key_3_expected_attribute_groups, key_4_expected_attribute_groups};
|
||||||
|
VerifyAttributeGroupIterator(
|
||||||
|
cfhs_order_0_1_2_3, expected_keys, expected_attribute_groups,
|
||||||
|
/* lower_bound */ nullptr, /* upper_bound */ nullptr,
|
||||||
|
/* allow_unprepared_value */ true);
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
int main(int argc, char** argv) {
|
int main(int argc, char** argv) {
|
||||||
|
|
|
@ -75,20 +75,22 @@ class IteratorBase : public Cleanable {
|
||||||
}
|
}
|
||||||
|
|
||||||
// When ReadOptions::allow_unprepared_value is set, the iterator may defer
|
// When ReadOptions::allow_unprepared_value is set, the iterator may defer
|
||||||
// loading the value when moving to a different entry (i.e. during
|
// loading and/or preparing the value when moving to a different entry (i.e.
|
||||||
// SeekToFirst/SeekToLast/Seek/SeekForPrev/Next/Prev operations). This can be
|
// during SeekToFirst/SeekToLast/Seek/SeekForPrev/Next/Prev operations). This
|
||||||
// used to save on I/O when the values associated with certain keys may not be
|
// can be used to save on I/O and/or CPU when the values associated with
|
||||||
// used by the application. When allow_unprepared_value is true, the
|
// certain keys may not be used by the application. When
|
||||||
// application is expected to call this method before accessing the value to
|
// allow_unprepared_value is true, the application is expected to call this
|
||||||
// ensure it is loaded (for all entries whose values are actually needed).
|
// method before accessing the value to ensure it is prepared (for all entries
|
||||||
// Note: it is safe to call this method for entries whose values are already
|
// whose values are actually needed). Note: it is safe to call this method for
|
||||||
// loaded.
|
// entries whose values are already prepared.
|
||||||
//
|
//
|
||||||
// Returns true on success. Returns false and sets Valid() to false and
|
// Returns true on success. Returns false and sets Valid() to false and
|
||||||
// status() to non-OK if there is an error while loading the value.
|
// status() to non-OK if there is an error while loading or preparing the
|
||||||
|
// value.
|
||||||
//
|
//
|
||||||
// Note: this method is currently only applicable to large values stored in
|
// Note: this option currently only applies to 1) large values stored in blob
|
||||||
// blob files using BlobDB, and has no effect otherwise.
|
// files using BlobDB and 2) multi-column-family iterators (CoalescingIterator
|
||||||
|
// and AttributeGroupIterator). Otherwise, it has no effect.
|
||||||
//
|
//
|
||||||
// REQUIRES: Valid()
|
// REQUIRES: Valid()
|
||||||
virtual bool PrepareValue() { return true; }
|
virtual bool PrepareValue() { return true; }
|
||||||
|
|
|
@ -1938,14 +1938,15 @@ struct ReadOptions {
|
||||||
// Default: true
|
// Default: true
|
||||||
bool auto_readahead_size = true;
|
bool auto_readahead_size = true;
|
||||||
|
|
||||||
// When set, the iterator may defer loading the value when moving to a
|
// When set, the iterator may defer loading and/or preparing the value when
|
||||||
// different entry (i.e. during SeekToFirst/SeekToLast/Seek/SeekForPrev/
|
// moving to a different entry (i.e. during SeekToFirst/SeekToLast/Seek/
|
||||||
// Next/Prev operations). This can be used to save on I/O when the values
|
// SeekForPrev/Next/Prev operations). This can be used to save on I/O and/or
|
||||||
// associated with certain keys may not be used by the application. See also
|
// CPU when the values associated with certain keys may not be used by the
|
||||||
// IteratorBase::PrepareValue().
|
// application. See also IteratorBase::PrepareValue().
|
||||||
//
|
//
|
||||||
// Note: this option currently only applies to large values stored in blob
|
// Note: this option currently only applies to 1) large values stored in blob
|
||||||
// files using BlobDB, and has no effect otherwise.
|
// files using BlobDB and 2) multi-column-family iterators (CoalescingIterator
|
||||||
|
// and AttributeGroupIterator). Otherwise, it has no effect.
|
||||||
//
|
//
|
||||||
// Default: false
|
// Default: false
|
||||||
bool allow_unprepared_value = false;
|
bool allow_unprepared_value = false;
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
When using iterators with BlobDB, it is now possible to load large values on an on-demand basis, i.e. only if they are actually needed by the application. This can save I/O in use cases where the values associated with certain keys are not needed. For more details, see the new read option `allow_unprepared_value` and the iterator API `PrepareValue`. Currently, this functionality is only supported for regular single-column-family iterators.
|
When using iterators with BlobDB, it is now possible to load large values on an on-demand basis, i.e. only if they are actually needed by the application. This can save I/O in use cases where the values associated with certain keys are not needed. For more details, see the new read option `allow_unprepared_value` and the iterator API `PrepareValue`.
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
The option `allow_unprepared_value` is now also supported for multi-column-family iterators (i.e. `CoalescingIterator` and `AttributeGroupIterator`).
|
Loading…
Reference in New Issue