dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-30 04:41:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
rocksdb/db/range_del_aggregator_test.cc
Abhishek Madan 3a4bd36fed Truncate range tombstones by leveraging InternalKeys (#4432) 
Summary:
To more accurately truncate range tombstones at SST boundaries,
we now represent them in RangeDelAggregator using InternalKeys, which
are end-key-exclusive as they were before this change.

During compaction, "atomic compaction unit boundaries" (the range of
keys contained in neighbouring and overlaping SSTs) are propagated down
to RangeDelAggregator to truncate range tombstones at those boundariies
instead. See https://github.com/facebook/rocksdb/pull/4432#discussion_r221072219 and https://github.com/facebook/rocksdb/pull/4432#discussion_r221138683
for motivating examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4432

Differential Revision: D10263952

Pulled By: abhimadan

fbshipit-source-id: 2fe85ff8a02b3a6a2de2edfe708012797a7bd579
2018-10-09 15:19:38 -07:00

473 lines
18 KiB
C++
Raw Blame History

// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include <algorithm>
#include "db/db_test_util.h"
#include "db/range_del_aggregator.h"
#include "rocksdb/comparator.h"
#include "util/testutil.h"
namespace rocksdb {
class RangeDelAggregatorTest : public testing::Test {};
namespace {
struct ExpectedPoint {
Slice begin;
SequenceNumber seq;
bool expectAlive;
};
enum Direction {
kForward,
kReverse,
};
struct AddTombstonesArgs {
const std::vector<RangeTombstone> tombstones;
const InternalKey* smallest;
const InternalKey* largest;
};
static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator());
void AddTombstones(RangeDelAggregator* range_del_agg,
const std::vector<RangeTombstone>& range_dels,
const InternalKey* smallest = nullptr,
const InternalKey* largest = nullptr) {
std::vector<std::string> keys, values;
for (const auto& range_del : range_dels) {
auto key_and_value = range_del.Serialize();
keys.push_back(key_and_value.first.Encode().ToString());
values.push_back(key_and_value.second.ToString());
}
std::unique_ptr<test::VectorIterator> range_del_iter(
new test::VectorIterator(keys, values));
range_del_agg->AddTombstones(std::move(range_del_iter), smallest, largest);
}
void VerifyTombstonesEq(const RangeTombstone& a, const RangeTombstone& b) {
ASSERT_EQ(a.seq_, b.seq_);
ASSERT_EQ(a.start_key_, b.start_key_);
ASSERT_EQ(a.end_key_, b.end_key_);
}
void VerifyRangeDelIter(
RangeDelIterator* range_del_iter,
const std::vector<RangeTombstone>& expected_range_dels) {
size_t i = 0;
for (; range_del_iter->Valid(); range_del_iter->Next(), i++) {
VerifyTombstonesEq(expected_range_dels[i], range_del_iter->Tombstone());
}
ASSERT_EQ(expected_range_dels.size(), i);
ASSERT_FALSE(range_del_iter->Valid());
}
void VerifyRangeDels(
const std::vector<AddTombstonesArgs>& all_args,
const std::vector<ExpectedPoint>& expected_points,
const std::vector<RangeTombstone>& expected_collapsed_range_dels,
const InternalKeyComparator& icmp = bytewise_icmp) {
// Test same result regardless of which order the range deletions are added
// and regardless of collapsed mode.
for (bool collapsed : {false, true}) {
for (Direction dir : {kForward, kReverse}) {
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */, collapsed);
std::vector<RangeTombstone> all_range_dels;
for (const auto& args : all_args) {
std::vector<RangeTombstone> range_dels = args.tombstones;
if (dir == kReverse) {
std::reverse(range_dels.begin(), range_dels.end());
}
all_range_dels.insert(all_range_dels.end(), range_dels.begin(),
range_dels.end());
AddTombstones(&range_del_agg, range_dels, args.smallest, args.largest);
}
auto mode = RangeDelPositioningMode::kFullScan;
if (collapsed) {
mode = RangeDelPositioningMode::kForwardTraversal;
}
for (const auto expected_point : expected_points) {
ParsedInternalKey parsed_key;
parsed_key.user_key = expected_point.begin;
parsed_key.sequence = expected_point.seq;
parsed_key.type = kTypeValue;
std::string ikey;
AppendInternalKey(&ikey, parsed_key);
ASSERT_FALSE(range_del_agg.ShouldDelete(ikey, mode));
if (parsed_key.sequence > 0) {
--parsed_key.sequence;
ikey.clear();
AppendInternalKey(&ikey, parsed_key);
if (expected_point.expectAlive) {
ASSERT_FALSE(range_del_agg.ShouldDelete(ikey, mode));
} else {
ASSERT_TRUE(range_del_agg.ShouldDelete(ikey, mode));
}
}
}
if (collapsed) {
all_range_dels = expected_collapsed_range_dels;
VerifyRangeDelIter(range_del_agg.NewIterator().get(), all_range_dels);
} else if (all_args.size() == 1 && all_args[0].smallest == nullptr &&
all_args[0].largest == nullptr) {
// Tombstones in an uncollapsed map are presented in start key
// order. Tombstones with the same start key are presented in
// insertion order. We don't handle tombstone truncation here, so the
// verification is only performed if no truncation was requested.
std::stable_sort(all_range_dels.begin(), all_range_dels.end(),
[&](const RangeTombstone& a, const RangeTombstone& b) {
return icmp.user_comparator()->Compare(
a.start_key_, b.start_key_) < 0;
});
VerifyRangeDelIter(range_del_agg.NewIterator().get(), all_range_dels);
}
}
}
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */,
false /* collapse_deletions */);
for (const auto& args : all_args) {
AddTombstones(&range_del_agg, args.tombstones, args.smallest, args.largest);
}
for (size_t i = 1; i < expected_points.size(); ++i) {
bool overlapped = range_del_agg.IsRangeOverlapped(
expected_points[i - 1].begin, expected_points[i].begin);
if (expected_points[i - 1].seq > 0 || expected_points[i].seq > 0) {
ASSERT_TRUE(overlapped);
} else {
ASSERT_FALSE(overlapped);
}
}
}
} // anonymous namespace
TEST_F(RangeDelAggregatorTest, Empty) { VerifyRangeDels({}, {{"a", 0}}, {}); }
TEST_F(RangeDelAggregatorTest, SameStartAndEnd) {
VerifyRangeDels({{{{"a", "a", 5}}}}, {{" ", 0}, {"a", 0}, {"b", 0}}, {});
}
TEST_F(RangeDelAggregatorTest, Single) {
VerifyRangeDels({{{{"a", "b", 10}}}}, {{" ", 0}, {"a", 10}, {"b", 0}},
{{"a", "b", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlapAboveLeft) {
VerifyRangeDels({{{{"a", "c", 10}, {"b", "d", 5}}}},
{{" ", 0}, {"a", 10}, {"c", 5}, {"d", 0}},
{{"a", "c", 10}, {"c", "d", 5}});
}
TEST_F(RangeDelAggregatorTest, OverlapAboveRight) {
VerifyRangeDels({{{{"a", "c", 5}, {"b", "d", 10}}}},
{{" ", 0}, {"a", 5}, {"b", 10}, {"d", 0}},
{{"a", "b", 5}, {"b", "d", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlapAboveMiddle) {
VerifyRangeDels({{{{"a", "d", 5}, {"b", "c", 10}}}},
{{" ", 0}, {"a", 5}, {"b", 10}, {"c", 5}, {"d", 0}},
{{"a", "b", 5}, {"b", "c", 10}, {"c", "d", 5}});
}
TEST_F(RangeDelAggregatorTest, OverlapAboveMiddleReverse) {
VerifyRangeDels({{{{"d", "a", 5}, {"c", "b", 10}}}},
{{"z", 0}, {"d", 5}, {"c", 10}, {"b", 5}, {"a", 0}},
{{"d", "c", 5}, {"c", "b", 10}, {"b", "a", 5}},
InternalKeyComparator(ReverseBytewiseComparator()));
}
TEST_F(RangeDelAggregatorTest, OverlapFully) {
VerifyRangeDels({{{{"a", "d", 10}, {"b", "c", 5}}}},
{{" ", 0}, {"a", 10}, {"d", 0}}, {{"a", "d", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlapPoint) {
VerifyRangeDels({{{{"a", "b", 5}, {"b", "c", 10}}}},
{{" ", 0}, {"a", 5}, {"b", 10}, {"c", 0}},
{{"a", "b", 5}, {"b", "c", 10}});
}
TEST_F(RangeDelAggregatorTest, SameStartKey) {
VerifyRangeDels({{{{"a", "c", 5}, {"a", "b", 10}}}},
{{" ", 0}, {"a", 10}, {"b", 5}, {"c", 0}},
{{"a", "b", 10}, {"b", "c", 5}});
}
TEST_F(RangeDelAggregatorTest, SameEndKey) {
VerifyRangeDels({{{{"a", "d", 5}, {"b", "d", 10}}}},
{{" ", 0}, {"a", 5}, {"b", 10}, {"d", 0}},
{{"a", "b", 5}, {"b", "d", 10}});
}
TEST_F(RangeDelAggregatorTest, GapsBetweenRanges) {
VerifyRangeDels({{{{"a", "b", 5}, {"c", "d", 10}, {"e", "f", 15}}}},
{{" ", 0},
{"a", 5},
{"b", 0},
{"c", 10},
{"d", 0},
{"da", 0},
{"e", 15},
{"f", 0}},
{{"a", "b", 5}, {"c", "d", 10}, {"e", "f", 15}});
}
TEST_F(RangeDelAggregatorTest, IdenticalSameSeqNo) {
VerifyRangeDels({{{{"a", "b", 5}, {"a", "b", 5}}}},
{{" ", 0}, {"a", 5}, {"b", 0}},
{{"a", "b", 5}});
}
TEST_F(RangeDelAggregatorTest, ContiguousSameSeqNo) {
VerifyRangeDels({{{{"a", "b", 5}, {"b", "c", 5}}}},
{{" ", 0}, {"a", 5}, {"b", 5}, {"c", 0}},
{{"a", "c", 5}});
}
TEST_F(RangeDelAggregatorTest, OverlappingSameSeqNo) {
VerifyRangeDels({{{{"a", "c", 5}, {"b", "d", 5}}}},
{{" ", 0}, {"a", 5}, {"b", 5}, {"c", 5}, {"d", 0}},
{{"a", "d", 5}});
}
TEST_F(RangeDelAggregatorTest, CoverSameSeqNo) {
VerifyRangeDels({{{{"a", "d", 5}, {"b", "c", 5}}}},
{{" ", 0}, {"a", 5}, {"b", 5}, {"c", 5}, {"d", 0}},
{{"a", "d", 5}});
}
// Note the Cover* tests also test cases where tombstones are inserted under a
// larger one when VerifyRangeDels() runs them in reverse
TEST_F(RangeDelAggregatorTest, CoverMultipleFromLeft) {
VerifyRangeDels(
{{{{"b", "d", 5}, {"c", "f", 10}, {"e", "g", 15}, {"a", "f", 20}}}},
{{" ", 0}, {"a", 20}, {"f", 15}, {"g", 0}},
{{"a", "f", 20}, {"f", "g", 15}});
}
TEST_F(RangeDelAggregatorTest, CoverMultipleFromRight) {
VerifyRangeDels(
{{{{"b", "d", 5}, {"c", "f", 10}, {"e", "g", 15}, {"c", "h", 20}}}},
{{" ", 0}, {"b", 5}, {"c", 20}, {"h", 0}},
{{"b", "c", 5}, {"c", "h", 20}});
}
TEST_F(RangeDelAggregatorTest, CoverMultipleFully) {
VerifyRangeDels(
{{{{"b", "d", 5}, {"c", "f", 10}, {"e", "g", 15}, {"a", "h", 20}}}},
{{" ", 0}, {"a", 20}, {"h", 0}}, {{"a", "h", 20}});
}
TEST_F(RangeDelAggregatorTest, AlternateMultipleAboveBelow) {
VerifyRangeDels(
{{{{"b", "d", 15}, {"c", "f", 10}, {"e", "g", 20}, {"a", "h", 5}}}},
{{" ", 0}, {"a", 5}, {"b", 15}, {"d", 10}, {"e", 20}, {"g", 5}, {"h", 0}},
{{"a", "b", 5},
{"b", "d", 15},
{"d", "e", 10},
{"e", "g", 20},
{"g", "h", 5}});
}
TEST_F(RangeDelAggregatorTest, MergingIteratorAllEmptyStripes) {
for (bool collapsed : {true, false}) {
RangeDelAggregator range_del_agg(bytewise_icmp, {1, 2}, collapsed);
VerifyRangeDelIter(range_del_agg.NewIterator().get(), {});
}
}
TEST_F(RangeDelAggregatorTest, MergingIteratorOverlappingStripes) {
for (bool collapsed : {true, false}) {
RangeDelAggregator range_del_agg(bytewise_icmp, {5, 15, 25, 35}, collapsed);
AddTombstones(
&range_del_agg,
{{"d", "e", 10}, {"aa", "b", 20}, {"c", "d", 30}, {"a", "b", 10}});
VerifyRangeDelIter(
range_del_agg.NewIterator().get(),
{{"a", "b", 10}, {"aa", "b", 20}, {"c", "d", 30}, {"d", "e", 10}});
}
}
TEST_F(RangeDelAggregatorTest, MergingIteratorSeek) {
RangeDelAggregator range_del_agg(bytewise_icmp, {5, 15},
true /* collapsed */);
AddTombstones(&range_del_agg, {{"a", "c", 10},
{"b", "c", 11},
{"f", "g", 10},
{"c", "d", 20},
{"e", "f", 20}});
auto it = range_del_agg.NewIterator();
// Verify seek positioning.
it->Seek("");
VerifyTombstonesEq(it->Tombstone(), {"a", "b", 10});
it->Seek("a");
VerifyTombstonesEq(it->Tombstone(), {"a", "b", 10});
it->Seek("aa");
VerifyTombstonesEq(it->Tombstone(), {"a", "b", 10});
it->Seek("b");
VerifyTombstonesEq(it->Tombstone(), {"b", "c", 11});
it->Seek("c");
VerifyTombstonesEq(it->Tombstone(), {"c", "d", 20});
it->Seek("dd");
VerifyTombstonesEq(it->Tombstone(), {"e", "f", 20});
it->Seek("f");
VerifyTombstonesEq(it->Tombstone(), {"f", "g", 10});
it->Seek("g");
ASSERT_EQ(it->Valid(), false);
it->Seek("h");
ASSERT_EQ(it->Valid(), false);
// Verify iteration after seek.
it->Seek("c");
VerifyRangeDelIter(it.get(),
{{"c", "d", 20}, {"e", "f", 20}, {"f", "g", 10}});
}
TEST_F(RangeDelAggregatorTest, TruncateTombstones) {
const InternalKey smallest("b", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest("e", kMaxSequenceNumber, kTypeRangeDeletion);
VerifyRangeDels(
{{{{"a", "c", 10}, {"d", "f", 10}}, &smallest, &largest}},
{{"a", 10, true}, // truncated
{"b", 10, false}, // not truncated
{"d", 10, false}, // not truncated
{"e", 10, true}}, // truncated
{{"b", "c", 10}, {"d", "e", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlappingLargestKeyTruncateBelowTombstone) {
const InternalKey smallest("b", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest(
"e", 3, // could happen if "e" is in consecutive sstables
kTypeValue);
VerifyRangeDels(
{{{{"a", "c", 10}, {"d", "f", 10}}, &smallest, &largest}},
{{"a", 10, true}, // truncated
{"b", 10, false}, // not truncated
{"d", 10, false}, // not truncated
{"e", 10, false}, // not truncated
{"e", 2, true}}, // truncated here
{{"b", "c", 10}, {"d", "e", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlappingLargestKeyTruncateAboveTombstone) {
const InternalKey smallest("b", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest(
"e", 15, // could happen if "e" is in consecutive sstables
kTypeValue);
VerifyRangeDels(
{{{{"a", "c", 10}, {"d", "f", 10}}, &smallest, &largest}},
{{"a", 10, true}, // truncated
{"b", 10, false}, // not truncated
{"d", 10, false}, // not truncated
{"e", kMaxSequenceNumber, true}}, // truncated
{{"b", "c", 10}, {"d", "e", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlappingSmallestKeyTruncateBelowTombstone) {
const InternalKey smallest("b", 5, kTypeValue);
const InternalKey largest("e", kMaxSequenceNumber, kTypeRangeDeletion);
VerifyRangeDels(
{{{{"a", "c", 10}, {"d", "f", 10}}, &smallest, &largest}},
{{"a", 10, true}, // truncated
{"b", 10, true}, // truncated
{"b", 6, false}, // not truncated; start boundary moved
{"d", 10, false}, // not truncated
{"e", kMaxSequenceNumber, true}}, // truncated
{{"b", "c", 10}, {"d", "e", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlappingSmallestKeyTruncateAboveTombstone) {
const InternalKey smallest("b", 15, kTypeValue);
const InternalKey largest("e", kMaxSequenceNumber, kTypeRangeDeletion);
VerifyRangeDels(
{{{{"a", "c", 10}, {"d", "f", 10}}, &smallest, &largest}},
{{"a", 10, true}, // truncated
{"b", 15, true}, // truncated
{"b", 10, false}, // not truncated
{"d", 10, false}, // not truncated
{"e", kMaxSequenceNumber, true}}, // truncated
{{"b", "c", 10}, {"d", "e", 10}});
}
TEST_F(RangeDelAggregatorTest, OverlappingBoundaryGapAboveTombstone) {
const InternalKey smallest1("b", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest1("c", 20, kTypeValue);
const InternalKey smallest2("c", 10, kTypeValue);
const InternalKey largest2("e", kMaxSequenceNumber, kTypeRangeDeletion);
VerifyRangeDels(
{{{{"b", "d", 5}}, &smallest1, &largest1},
{{{"b", "d", 5}}, &smallest2, &largest2}},
{{"b", 5, false}, // not truncated
{"c", 5, false}}, // not truncated
{{"b", "c", 5}, {"c", "d", 5}}); // not collapsed due to boundaries
}
TEST_F(RangeDelAggregatorTest, OverlappingBoundaryGapBelowTombstone) {
const InternalKey smallest1("b", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest1("c", 20, kTypeValue);
const InternalKey smallest2("c", 10, kTypeValue);
const InternalKey largest2("e", kMaxSequenceNumber, kTypeRangeDeletion);
VerifyRangeDels(
{{{{"b", "d", 30}}, &smallest1, &largest1},
{{{"b", "d", 30}}, &smallest2, &largest2}},
{{"b", 30, false}, // not truncated
{"c", 30, false}, // not truncated
{"c", 19, true}, // truncated here (keys in this range should not exist)
{"c", 11, false}}, // not truncated again
{{"b", "c", 30}, {"c", "d", 30}}); // not collapsed due to boundaries
}
TEST_F(RangeDelAggregatorTest, OverlappingBoundaryGapContainsTombstone) {
const InternalKey smallest1("b", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest1("c", 20, kTypeValue);
const InternalKey smallest2("c", 10, kTypeValue);
const InternalKey largest2("e", kMaxSequenceNumber, kTypeRangeDeletion);
VerifyRangeDels(
{{{{"b", "d", 15}}, &smallest1, &largest1},
{{{"b", "d", 15}}, &smallest2, &largest2}},
{{"b", 15, false}, // not truncated
{"c", 15, true}, // truncated (keys in this range should not exist)
{"c", 11, false}}, // not truncated here
{{"b", "c", 15}, {"c", "d", 15}}); // not collapsed due to boundaries
}
TEST_F(RangeDelAggregatorTest, FileCoversOneKeyAndTombstoneAbove) {
const InternalKey smallest("a", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest("a", 20, kTypeValue);
VerifyRangeDels(
{{{{"a", "b", 35}}, &smallest, &largest}},
{{"a", 40, true}, // not truncated
{"a", 35, false}}, // not truncated
{{"a", "a", 35}}); // empty tombstone but can't occur during a compaction
}
TEST_F(RangeDelAggregatorTest, FileCoversOneKeyAndTombstoneBelow) {
const InternalKey smallest("a", kMaxSequenceNumber, kTypeRangeDeletion);
const InternalKey largest("a", 20, kTypeValue);
VerifyRangeDels(
{{{{"a", "b", 15}}, &smallest, &largest}},
{{"a", 20, true}, // truncated here
{"a", 15, true}}, // truncated
{{"a", "a", 15}}); // empty tombstone but can't occur during a compaction
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
Reference in a new issue View git blame Copy permalink