rocksdb/util/udt_util_test.cc
Yu Zhang 282f5a463b Fix write committed transactions replay when UDT setting toggles (#13121)
Summary:
This PR adds some missing pieces in order to handle UDT setting toggles while replay WALs for WriteCommitted transactions DB. Specifically, all the transaction markers for no op, prepare, commit, rollback are currently not carried over from the original WriteBatch to the new WriteBatch when there is a timestamp setting difference detected. This PR fills that gap.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/13121

Test Plan: Added unit tests

Reviewed By: ltamasi

Differential Revision: D65558801

Pulled By: jowlyzhang

fbshipit-source-id: 8176882637b95f6dc0dad10d7fe21056fa5173d1
2024-11-06 17:32:03 -08:00

508 lines
21 KiB
C++

// Copyright (c) Meta Platforms, Inc. and affiliates.
//
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "util/udt_util.h"
#include <gtest/gtest.h>
#include "db/dbformat.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
namespace ROCKSDB_NAMESPACE {
namespace {
static const std::string kTestKeyWithoutTs = "key";
static const std::string kValuePlaceHolder = "value";
static const uint64_t kWriteUnixTime = 100;
} // namespace
class HandleTimestampSizeDifferenceTest : public testing::Test {
public:
HandleTimestampSizeDifferenceTest() = default;
// Test handler used to collect the column family id and user keys contained
// in a WriteBatch for test verification. And verifies the value part stays
// the same if it's available.
class KeyCollector : public WriteBatch::Handler {
public:
explicit KeyCollector() = default;
~KeyCollector() override = default;
Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
if (value.compare(kValuePlaceHolder) != 0) {
return Status::InvalidArgument();
}
return AddKey(cf, key);
}
Status TimedPutCF(uint32_t cf, const Slice& key, const Slice& value,
uint64_t write_unix_time) override {
if (value.compare(kValuePlaceHolder) != 0) {
return Status::InvalidArgument();
}
if (write_unix_time != kWriteUnixTime) {
return Status::InvalidArgument();
}
return AddKey(cf, key);
}
Status PutEntityCF(uint32_t cf, const Slice& key,
const Slice& entity) override {
Slice entity_copy = entity;
WideColumns columns;
Status s = WideColumnSerialization::Deserialize(entity_copy, columns);
if (!s.ok()) {
return s;
}
if (columns.size() != 1) {
return Status::InvalidArgument();
}
if (columns[0].value().compare(kValuePlaceHolder) != 0) {
return Status::InvalidArgument();
}
return AddKey(cf, key);
}
Status DeleteCF(uint32_t cf, const Slice& key) override {
return AddKey(cf, key);
}
Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
return AddKey(cf, key);
}
Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
const Slice& end_key) override {
Status status = AddKey(cf, begin_key);
if (!status.ok()) {
return status;
}
return AddKey(cf, end_key);
}
Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
if (value.compare(kValuePlaceHolder) != 0) {
return Status::InvalidArgument();
}
return AddKey(cf, key);
}
Status PutBlobIndexCF(uint32_t cf, const Slice& key,
const Slice& value) override {
if (value.compare(kValuePlaceHolder) != 0) {
return Status::InvalidArgument();
}
return AddKey(cf, key);
}
Status MarkBeginPrepare(bool) override { return Status::OK(); }
Status MarkEndPrepare(const Slice&) override { return Status::OK(); }
Status MarkRollback(const Slice&) override { return Status::OK(); }
Status MarkCommit(const Slice&) override { return Status::OK(); }
Status MarkCommitWithTimestamp(const Slice&, const Slice&) override {
return Status::OK();
}
Status MarkNoop(bool) override { return Status::OK(); }
const std::vector<std::pair<uint32_t, const Slice>>& GetKeys() const {
return keys_;
}
private:
Status AddKey(uint32_t cf, const Slice& key) {
keys_.emplace_back(cf, key);
return Status::OK();
}
std::vector<std::pair<uint32_t, const Slice>> keys_;
};
void CreateKey(std::string* key_buf, size_t ts_sz) {
if (ts_sz > 0) {
AppendKeyWithMinTimestamp(key_buf, kTestKeyWithoutTs, ts_sz);
} else {
key_buf->assign(kTestKeyWithoutTs);
}
}
void CreateWriteBatch(const UnorderedMap<uint32_t, size_t>& ts_sz_for_batch,
WriteBatch* batch) {
for (const auto& [cf_id, ts_sz] : ts_sz_for_batch) {
std::string key;
CreateKey(&key, ts_sz);
ASSERT_OK(WriteBatchInternal::Put(batch, cf_id, key, kValuePlaceHolder));
ASSERT_OK(WriteBatchInternal::Delete(batch, cf_id, key));
ASSERT_OK(WriteBatchInternal::SingleDelete(batch, cf_id, key));
ASSERT_OK(WriteBatchInternal::DeleteRange(batch, cf_id, key, key));
ASSERT_OK(
WriteBatchInternal::Merge(batch, cf_id, key, kValuePlaceHolder));
ASSERT_OK(WriteBatchInternal::PutBlobIndex(batch, cf_id, key,
kValuePlaceHolder));
ASSERT_OK(WriteBatchInternal::TimedPut(
batch, cf_id, key, kValuePlaceHolder, kWriteUnixTime));
WideColumns columns{{kDefaultWideColumnName, kValuePlaceHolder}};
ASSERT_OK(WriteBatchInternal::PutEntity(batch, cf_id, key, columns));
}
}
void CheckSequenceEqual(const WriteBatch& orig_batch,
const WriteBatch& new_batch) {
ASSERT_EQ(WriteBatchInternal::Sequence(&orig_batch),
WriteBatchInternal::Sequence(&new_batch));
}
void CheckCountEqual(const WriteBatch& orig_batch,
const WriteBatch& new_batch) {
ASSERT_EQ(WriteBatchInternal::Count(&orig_batch),
WriteBatchInternal::Count(&new_batch));
}
void VerifyKeys(
const std::vector<std::pair<uint32_t, const Slice>>& keys_with_ts,
const std::vector<std::pair<uint32_t, const Slice>>& keys_without_ts,
size_t ts_sz, std::optional<uint32_t> dropped_cf) {
ASSERT_EQ(keys_with_ts.size(), keys_without_ts.size());
const std::string kTsMin(ts_sz, static_cast<unsigned char>(0));
for (size_t i = 0; i < keys_with_ts.size(); i++) {
// TimestampRecoveryHandler ignores dropped column family and copy it over
// as is. Check the keys stay the same.
if (dropped_cf.has_value() &&
keys_with_ts[i].first == dropped_cf.value()) {
ASSERT_EQ(keys_with_ts[i].first, keys_without_ts[i].first);
ASSERT_EQ(keys_with_ts[i].second, keys_without_ts[i].second);
continue;
}
const Slice& key_with_ts = keys_with_ts[i].second;
const Slice& key_without_ts = keys_without_ts[i].second;
ASSERT_TRUE(key_with_ts.starts_with(key_without_ts));
ASSERT_EQ(key_with_ts.size() - key_without_ts.size(), ts_sz);
ASSERT_TRUE(key_with_ts.ends_with(kTsMin));
}
}
void CheckContentsWithTimestampStripping(const WriteBatch& orig_batch,
const WriteBatch& new_batch,
size_t ts_sz,
std::optional<uint32_t> dropped_cf) {
CheckSequenceEqual(orig_batch, new_batch);
CheckCountEqual(orig_batch, new_batch);
KeyCollector collector_for_orig_batch;
ASSERT_OK(orig_batch.Iterate(&collector_for_orig_batch));
KeyCollector collector_for_new_batch;
ASSERT_OK(new_batch.Iterate(&collector_for_new_batch));
VerifyKeys(collector_for_orig_batch.GetKeys(),
collector_for_new_batch.GetKeys(), ts_sz, dropped_cf);
}
void CheckContentsWithTimestampPadding(const WriteBatch& orig_batch,
const WriteBatch& new_batch,
size_t ts_sz) {
CheckSequenceEqual(orig_batch, new_batch);
CheckCountEqual(orig_batch, new_batch);
KeyCollector collector_for_orig_batch;
ASSERT_OK(orig_batch.Iterate(&collector_for_orig_batch));
KeyCollector collector_for_new_batch;
ASSERT_OK(new_batch.Iterate(&collector_for_new_batch));
VerifyKeys(collector_for_new_batch.GetKeys(),
collector_for_orig_batch.GetKeys(), ts_sz,
std::nullopt /* dropped_cf */);
}
};
TEST_F(HandleTimestampSizeDifferenceTest, AllColumnFamiliesConsistent) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{1, sizeof(uint64_t)},
{2, 0}};
UnorderedMap<uint32_t, size_t> record_ts_sz = {{1, sizeof(uint64_t)}};
WriteBatch batch;
CreateWriteBatch(running_ts_sz, &batch);
// All `check_mode` pass with OK status and `batch` not checked or updated.
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kVerifyConsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true));
std::unique_ptr<WriteBatch> new_batch(nullptr);
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true, &new_batch));
ASSERT_TRUE(new_batch.get() == nullptr);
}
TEST_F(HandleTimestampSizeDifferenceTest,
AllInconsistentColumnFamiliesDropped) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{2, 0}};
UnorderedMap<uint32_t, size_t> record_ts_sz = {{1, sizeof(uint64_t)},
{3, sizeof(char)}};
WriteBatch batch;
CreateWriteBatch(record_ts_sz, &batch);
// All `check_mode` pass with OK status and `batch` not checked or updated.
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kVerifyConsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true));
std::unique_ptr<WriteBatch> new_batch(nullptr);
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true, &new_batch));
ASSERT_TRUE(new_batch.get() == nullptr);
}
TEST_F(HandleTimestampSizeDifferenceTest, InvolvedColumnFamiliesConsistent) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{1, sizeof(uint64_t)},
{2, sizeof(char)}};
UnorderedMap<uint32_t, size_t> record_ts_sz = {{1, sizeof(uint64_t)}};
WriteBatch batch;
CreateWriteBatch(record_ts_sz, &batch);
// All `check_mode` pass with OK status and `batch` not updated.
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kVerifyConsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true));
std::unique_ptr<WriteBatch> new_batch(nullptr);
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true, &new_batch));
ASSERT_TRUE(new_batch.get() == nullptr);
}
TEST_F(HandleTimestampSizeDifferenceTest,
InconsistentColumnFamilyNeedsTimestampStripping) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{1, 0}, {2, sizeof(char)}};
UnorderedMap<uint32_t, size_t> record_ts_sz = {{1, sizeof(uint64_t)}};
WriteBatch batch;
CreateWriteBatch(record_ts_sz, &batch);
// kVerifyConsistency doesn't tolerate inconsistency for running column
// families.
ASSERT_TRUE(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kVerifyConsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true)
.IsInvalidArgument());
std::unique_ptr<WriteBatch> new_batch(nullptr);
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true, &new_batch));
ASSERT_TRUE(new_batch.get() != nullptr);
CheckContentsWithTimestampStripping(batch, *new_batch, sizeof(uint64_t),
std::nullopt /* dropped_cf */);
}
TEST_F(HandleTimestampSizeDifferenceTest,
InconsistentColumnFamilyNeedsTimestampPadding) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{1, sizeof(uint64_t)}};
// Make `record_ts_sz` not contain zero timestamp size entries to follow the
// behavior of actual WAL log timestamp size record.
UnorderedMap<uint32_t, size_t> record_ts_sz;
WriteBatch batch;
CreateWriteBatch({{1, 0}}, &batch);
// kVerifyConsistency doesn't tolerate inconsistency for running column
// families.
ASSERT_TRUE(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kVerifyConsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true)
.IsInvalidArgument());
std::unique_ptr<WriteBatch> new_batch(nullptr);
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true, &new_batch));
ASSERT_TRUE(new_batch.get() != nullptr);
CheckContentsWithTimestampPadding(batch, *new_batch, sizeof(uint64_t));
}
TEST_F(HandleTimestampSizeDifferenceTest,
InconsistencyReconcileCopyOverDroppedColumnFamily) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{1, 0}};
UnorderedMap<uint32_t, size_t> record_ts_sz = {{1, sizeof(uint64_t)},
{2, sizeof(char)}};
WriteBatch batch;
CreateWriteBatch(record_ts_sz, &batch);
std::unique_ptr<WriteBatch> new_batch(nullptr);
// kReconcileInconsistency tolerate inconsistency for dropped column family
// and all related entries copied over to the new WriteBatch.
ASSERT_OK(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true, &new_batch));
ASSERT_TRUE(new_batch.get() != nullptr);
CheckContentsWithTimestampStripping(batch, *new_batch, sizeof(uint64_t),
std::optional<uint32_t>(2));
}
TEST_F(HandleTimestampSizeDifferenceTest, UnrecoverableInconsistency) {
UnorderedMap<uint32_t, size_t> running_ts_sz = {{1, sizeof(char)}};
UnorderedMap<uint32_t, size_t> record_ts_sz = {{1, sizeof(uint64_t)}};
WriteBatch batch;
CreateWriteBatch(record_ts_sz, &batch);
ASSERT_TRUE(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kVerifyConsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true)
.IsInvalidArgument());
ASSERT_TRUE(HandleWriteBatchTimestampSizeDifference(
&batch, running_ts_sz, record_ts_sz,
TimestampSizeConsistencyMode::kReconcileInconsistency,
/* seq_per_batch */ false, /* batch_per_txn */ true)
.IsInvalidArgument());
}
TEST(ValidateUserDefinedTimestampsOptionsTest, EnableUserDefinedTimestamps) {
bool mark_sst_files = false;
const Comparator* new_comparator = test::BytewiseComparatorWithU64TsWrapper();
const Comparator* old_comparator = BytewiseComparator();
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
false /*new_persist_udt*/, true /*old_persist_udt*/, &mark_sst_files));
ASSERT_TRUE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files));
ASSERT_TRUE(mark_sst_files);
}
TEST(ValidateUserDefinedTimestampsOptionsTest,
EnableUserDefinedTimestampsNewPersistUDTFlagIncorrect) {
bool mark_sst_files = false;
const Comparator* new_comparator = test::BytewiseComparatorWithU64TsWrapper();
const Comparator* old_comparator = BytewiseComparator();
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
true /*new_persist_udt*/, true /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
true /*new_persist_udt*/, false /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
}
TEST(ValidateUserDefinedTimestampsOptionsTest, DisableUserDefinedTimestamps) {
bool mark_sst_files = false;
const Comparator* new_comparator = ReverseBytewiseComparator();
const Comparator* old_comparator =
test::ReverseBytewiseComparatorWithU64TsWrapper();
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
true /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
}
TEST(ValidateUserDefinedTimestampsOptionsTest,
DisableUserDefinedTimestampsOldPersistUDTFlagIncorrect) {
bool mark_sst_files = false;
const Comparator* new_comparator = BytewiseComparator();
const Comparator* old_comparator = test::BytewiseComparatorWithU64TsWrapper();
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
false /*new_persist_udt*/, true /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
true /*new_persist_udt*/, true /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
}
TEST(ValidateUserDefinedTimestampsOptionsTest, UserComparatorUnchanged) {
bool mark_sst_files = false;
const Comparator* ucmp_without_ts = BytewiseComparator();
const Comparator* ucmp_with_ts = test::BytewiseComparatorWithU64TsWrapper();
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
ucmp_without_ts, std::string(ucmp_without_ts->Name()),
false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
ucmp_without_ts, std::string(ucmp_without_ts->Name()),
true /*new_persist_udt*/, true /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
ucmp_without_ts, std::string(ucmp_without_ts->Name()),
true /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
ucmp_without_ts, std::string(ucmp_without_ts->Name()),
false /*new_persist_udt*/, true /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
ucmp_with_ts, std::string(ucmp_with_ts->Name()), true /*new_persist_udt*/,
true /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_OK(ValidateUserDefinedTimestampsOptions(
ucmp_with_ts, std::string(ucmp_with_ts->Name()),
false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files));
ASSERT_FALSE(mark_sst_files);
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
ucmp_with_ts, std::string(ucmp_with_ts->Name()),
true /*new_persist_udt*/, false /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
ucmp_with_ts, std::string(ucmp_with_ts->Name()),
false /*new_persist_udt*/, true /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
}
TEST(ValidateUserDefinedTimestampsOptionsTest, InvalidUserComparatorChange) {
bool mark_sst_files = false;
const Comparator* new_comparator = BytewiseComparator();
const Comparator* old_comparator = ReverseBytewiseComparator();
ASSERT_TRUE(ValidateUserDefinedTimestampsOptions(
new_comparator, std::string(old_comparator->Name()),
false /*new_persist_udt*/, true /*old_persist_udt*/,
&mark_sst_files)
.IsInvalidArgument());
}
TEST(GetFullHistoryTsLowFromU64CutoffTsTest, Success) {
std::string cutoff_ts;
uint64_t cutoff_ts_int = 3;
PutFixed64(&cutoff_ts, 3);
Slice cutoff_ts_slice = cutoff_ts;
std::string actual_full_history_ts_low;
GetFullHistoryTsLowFromU64CutoffTs(&cutoff_ts_slice,
&actual_full_history_ts_low);
std::string expected_ts_low;
PutFixed64(&expected_ts_low, cutoff_ts_int + 1);
ASSERT_EQ(expected_ts_low, actual_full_history_ts_low);
}
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}