From 16bdb1f999dab165924cb204be9aa8663454c802 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Thu, 19 May 2022 18:39:41 -0700 Subject: [PATCH] Add timestamp support to DBImplReadOnly (#10004) Summary: This PR adds timestamp support to a read only DB instance opened as `DBImplReadOnly`. A follow up PR will add the same support to `CompactedDBImpl`. With this, read only database has these timestamp related APIs: `ReadOptions.timestamp` : read should return the latest data visible to this specified timestamp `Iterator::timestamp()` : returns the timestamp associated with the key, value `DB:Get(..., std::string* timestamp)` : returns the timestamp associated with the key, value in `timestamp` Test plan (on devserver): ``` $COMPILE_WITH_ASAN=1 make -j24 all $./db_with_timestamp_basic_test --gtest_filter=DBBasicTestWithTimestamp.ReadOnlyDB* ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/10004 Reviewed By: riversand963 Differential Revision: D36434422 Pulled By: jowlyzhang fbshipit-source-id: 5d949e65b1ffb845758000e2b310fdd4aae71cfb --- .gitignore | 1 + CMakeLists.txt | 2 + Makefile | 3 + TARGETS | 7 + db/db_impl/db_impl.cc | 11 - db/db_impl/db_impl.h | 9 + db/db_impl/db_impl_readonly.cc | 77 ++++-- db/db_impl/db_impl_readonly.h | 3 + db/db_readonly_with_timestamp_test.cc | 331 ++++++++++++++++++++++++++ db/db_with_timestamp_basic_test.cc | 172 +------------ db/db_with_timestamp_test_util.cc | 96 ++++++++ db/db_with_timestamp_test_util.h | 126 ++++++++++ src.mk | 4 +- 13 files changed, 636 insertions(+), 206 deletions(-) create mode 100644 db/db_readonly_with_timestamp_test.cc create mode 100644 db/db_with_timestamp_test_util.cc create mode 100644 db/db_with_timestamp_test_util.h diff --git a/.gitignore b/.gitignore index 7d6f61affa..5bdb34212a 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ manifest_dump sst_dump blob_dump block_cache_trace_analyzer +db_readonly_with_timestamp_test db_with_timestamp_basic_test tools/block_cache_analyzer/*.pyc column_aware_encoding_exp diff --git a/CMakeLists.txt b/CMakeLists.txt index 847e3f4adb..040fb526d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1217,6 +1217,7 @@ if(WITH_TESTS) db/comparator_db_test.cc db/corruption_test.cc db/cuckoo_table_db_test.cc + db/db_readonly_with_timestamp_test.cc db/db_with_timestamp_basic_test.cc db/db_block_cache_test.cc db/db_bloom_filter_test.cc @@ -1385,6 +1386,7 @@ if(WITH_TESTS) set(TESTUTIL_SOURCE db/db_test_util.cc + db/db_with_timestamp_test_util.cc monitoring/thread_status_updater_debug.cc table/mock_table.cc utilities/agg_merge/test_agg_merge.cc diff --git a/Makefile b/Makefile index 8e8c7ed5b8..9cd3b70a39 100644 --- a/Makefile +++ b/Makefile @@ -1378,6 +1378,9 @@ db_blob_basic_test: $(OBJ_DIR)/db/blob/db_blob_basic_test.o $(TEST_LIBRARY) $(LI db_blob_compaction_test: $(OBJ_DIR)/db/blob/db_blob_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +db_readonly_with_timestamp_test: $(OBJ_DIR)/db/db_readonly_with_timestamp_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + db_with_timestamp_basic_test: $(OBJ_DIR)/db/db_with_timestamp_basic_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) diff --git a/TARGETS b/TARGETS index 5d0e2836ed..24a88eca98 100644 --- a/TARGETS +++ b/TARGETS @@ -666,6 +666,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[ cpp_library_wrapper(name="rocksdb_test_lib", srcs=[ "db/db_test_util.cc", + "db/db_with_timestamp_test_util.cc", "table/mock_table.cc", "test_util/mock_time_env.cc", "test_util/testharness.cc", @@ -5175,6 +5176,12 @@ cpp_unittest_wrapper(name="db_rate_limiter_test", extra_compiler_flags=[]) +cpp_unittest_wrapper(name="db_readonly_with_timestamp_test", + srcs=["db/db_readonly_with_timestamp_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + cpp_unittest_wrapper(name="db_secondary_test", srcs=["db/db_secondary_test.cc"], deps=[":rocksdb_test_lib"], diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 62853380fa..e0140707d9 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -1723,17 +1723,6 @@ Status DBImpl::Get(const ReadOptions& read_options, return s; } -namespace { -class GetWithTimestampReadCallback : public ReadCallback { - public: - explicit GetWithTimestampReadCallback(SequenceNumber seq) - : ReadCallback(seq) {} - bool IsVisibleFullCheck(SequenceNumber seq) override { - return seq <= max_visible_seq_; - } -}; -} // namespace - Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, GetImplOptions& get_impl_options) { assert(get_impl_options.value != nullptr || diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 00d9810577..6ebb3207e2 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -2400,6 +2400,15 @@ class DBImpl : public DB { std::unique_ptr wbm_stall_; }; +class GetWithTimestampReadCallback : public ReadCallback { + public: + explicit GetWithTimestampReadCallback(SequenceNumber seq) + : ReadCallback(seq) {} + bool IsVisibleFullCheck(SequenceNumber seq) override { + return seq <= max_visible_seq_; + } +}; + extern Options SanitizeOptions(const std::string& db, const Options& src, bool read_only = false); diff --git a/db/db_impl/db_impl_readonly.cc b/db/db_impl/db_impl_readonly.cc index 96d409e8ab..b014f75ce6 100644 --- a/db/db_impl/db_impl_readonly.cc +++ b/db/db_impl/db_impl_readonly.cc @@ -33,20 +33,38 @@ DBImplReadOnly::~DBImplReadOnly() {} Status DBImplReadOnly::Get(const ReadOptions& read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* pinnable_val) { + return Get(read_options, column_family, key, pinnable_val, + /*timestamp*/ nullptr); +} + +Status DBImplReadOnly::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* pinnable_val, + std::string* timestamp) { assert(pinnable_val != nullptr); // TODO: stopwatch DB_GET needed?, perf timer needed? PERF_TIMER_GUARD(get_snapshot_time); assert(column_family); + if (read_options.timestamp) { + const Status s = + FailIfTsSizesMismatch(column_family, *(read_options.timestamp)); + if (!s.ok()) { + return s; + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + } const Comparator* ucmp = column_family->GetComparator(); assert(ucmp); - if (ucmp->timestamp_size() || read_options.timestamp) { - // TODO: support timestamp - return Status::NotSupported(); - } + std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr; Status s; SequenceNumber snapshot = versions_->LastSequence(); + GetWithTimestampReadCallback read_cb(snapshot); auto cfh = static_cast_with_check(column_family); auto cfd = cfh->cfd(); if (tracer_) { @@ -58,19 +76,23 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options, SuperVersion* super_version = cfd->GetSuperVersion(); MergeContext merge_context; SequenceNumber max_covering_tombstone_seq = 0; - LookupKey lkey(key, snapshot); + LookupKey lkey(key, snapshot, read_options.timestamp); PERF_TIMER_STOP(get_snapshot_time); - if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), - /*timestamp=*/nullptr, &s, &merge_context, - &max_covering_tombstone_seq, read_options)) { + if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), ts, &s, + &merge_context, &max_covering_tombstone_seq, + read_options, &read_cb)) { pinnable_val->PinSelf(); RecordTick(stats_, MEMTABLE_HIT); } else { PERF_TIMER_GUARD(get_from_output_files_time); PinnedIteratorsManager pinned_iters_mgr; - super_version->current->Get(read_options, lkey, pinnable_val, - /*timestamp=*/nullptr, &s, &merge_context, - &max_covering_tombstone_seq, &pinned_iters_mgr); + super_version->current->Get( + read_options, lkey, pinnable_val, ts, &s, &merge_context, + &max_covering_tombstone_seq, &pinned_iters_mgr, + /*value_found*/ nullptr, + /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, + /*is_blob*/ nullptr, + /*do_merge*/ true); RecordTick(stats_, MEMTABLE_MISS); } RecordTick(stats_, NUMBER_KEYS_READ); @@ -84,11 +106,17 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options, Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options, ColumnFamilyHandle* column_family) { assert(column_family); - const Comparator* ucmp = column_family->GetComparator(); - assert(ucmp); - if (ucmp->timestamp_size() || read_options.timestamp) { - // TODO: support timestamp - return NewErrorIterator(Status::NotSupported()); + if (read_options.timestamp) { + const Status s = + FailIfTsSizesMismatch(column_family, *(read_options.timestamp)); + if (!s.ok()) { + return NewErrorIterator(s); + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return NewErrorIterator(s); + } } auto cfh = static_cast_with_check(column_family); auto cfd = cfh->cfd(); @@ -118,16 +146,19 @@ Status DBImplReadOnly::NewIterators( const std::vector& column_families, std::vector* iterators) { if (read_options.timestamp) { - // TODO: support timestamp - return Status::NotSupported(); + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfTsSizesMismatch(cf, *(read_options.timestamp)); + if (!s.ok()) { + return s; + } + } } else { for (auto* cf : column_families) { assert(cf); - const Comparator* ucmp = cf->GetComparator(); - assert(ucmp); - if (ucmp->timestamp_size()) { - // TODO: support timestamp - return Status::NotSupported(); + const Status s = FailIfCfHasTs(cf); + if (!s.ok()) { + return s; } } } diff --git a/db/db_impl/db_impl_readonly.h b/db/db_impl/db_impl_readonly.h index 090d67d0f8..3f1ea95a54 100644 --- a/db/db_impl/db_impl_readonly.h +++ b/db/db_impl/db_impl_readonly.h @@ -27,6 +27,9 @@ class DBImplReadOnly : public DBImpl { virtual Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; + Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value, + std::string* timestamp) override; // TODO: Implement ReadOnly MultiGet? diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc new file mode 100644 index 0000000000..908e791eed --- /dev/null +++ b/db/db_readonly_with_timestamp_test.cc @@ -0,0 +1,331 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_with_timestamp_test_util.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { +class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { + public: + DBReadOnlyTestWithTimestamp() + : DBBasicTestWithTimestampBase("db_readonly_test_with_timestamp") {} +}; + +#ifndef ROCKSDB_LITE +TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorAndGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + + auto get_value_and_check = [](DB* db, ReadOptions read_opts, Slice key, + Slice expected_value, std::string expected_ts) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db->Get(read_opts, key.ToString(), &value_from_get, ×tamp)); + ASSERT_EQ(expected_value, value_from_get); + ASSERT_EQ(expected_ts, timestamp); + }; + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (it->Seek(Key1(0)), key = start_keys[i]; it->Valid(); + it->Next(), ++count, ++key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + + // Backward iterate. + count = 0; + for (it->SeekForPrev(Key1(kMaxKey)), key = kMaxKey; it->Valid(); + it->Prev(), ++count, --key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); + + // SeekToFirst()/SeekToLast() with lower/upper bounds. + // Then iter with lower and upper bounds. + uint64_t l = 0; + uint64_t r = kMaxKey + 1; + while (l < r) { + std::string lb_str = Key1(l); + Slice lb = lb_str; + std::string ub_str = Key1(r); + Slice ub = ub_str; + read_opts.iterate_lower_bound = &lb; + read_opts.iterate_upper_bound = &ub; + it.reset(db_->NewIterator(read_opts)); + for (it->SeekToFirst(), key = std::max(l, start_keys[i]), count = 0; + it->Valid(); it->Next(), ++key, ++count) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + ASSERT_EQ(r - std::max(l, start_keys[i]), count); + + for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; + it->Valid(); it->Prev(), --key, ++count) { + CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + l += (kMaxKey / 100); + r -= (kMaxKey / 100); + } + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, Iterators) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + const std::string read_timestamp = Timestamp(2, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + std::vector iters; + ASSERT_OK(db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters)); + ASSERT_EQ(static_cast(1), iters.size()); + + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (iters[0]->Seek(Key1(0)), key = 0; iters[0]->Valid(); + iters[0]->Next(), ++count, ++key) { + CheckIterUserEntry(iters[0], Key1(key), kTypeValue, + "value" + std::to_string(key), write_timestamp); + } + + size_t expected_count = kMaxKey - 0 + 1; + ASSERT_EQ(expected_count, count); + delete iters[0]; + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, IteratorsReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorsReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} +#endif // !ROCKSDB_LITE +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/db/db_with_timestamp_basic_test.cc b/db/db_with_timestamp_basic_test.cc index cd7465b59f..009dbbc108 100644 --- a/db/db_with_timestamp_basic_test.cc +++ b/db/db_with_timestamp_basic_test.cc @@ -7,7 +7,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "db/db_test_util.h" +#include "db/db_with_timestamp_test_util.h" #include "port/stack_trace.h" #include "rocksdb/perf_context.h" #include "rocksdb/utilities/debug.h" @@ -20,176 +20,6 @@ #include "utilities/fault_injection_env.h" namespace ROCKSDB_NAMESPACE { -class DBBasicTestWithTimestampBase : public DBTestBase { - public: - explicit DBBasicTestWithTimestampBase(const std::string& dbname) - : DBTestBase(dbname, /*env_do_fsync=*/true) {} - - protected: - static std::string Key1(uint64_t k) { - std::string ret; - PutFixed64(&ret, k); - std::reverse(ret.begin(), ret.end()); - return ret; - } - - static std::string KeyWithPrefix(std::string prefix, uint64_t k) { - std::string ret; - PutFixed64(&ret, k); - std::reverse(ret.begin(), ret.end()); - return prefix + ret; - } - - static std::vector ConvertStrToSlice( - std::vector& strings) { - std::vector ret; - for (const auto& s : strings) { - ret.emplace_back(s); - } - return ret; - } - - class TestComparator : public Comparator { - private: - const Comparator* cmp_without_ts_; - - public: - explicit TestComparator(size_t ts_sz) - : Comparator(ts_sz), cmp_without_ts_(nullptr) { - cmp_without_ts_ = BytewiseComparator(); - } - - const char* Name() const override { return "TestComparator"; } - - void FindShortSuccessor(std::string*) const override {} - - void FindShortestSeparator(std::string*, const Slice&) const override {} - - int Compare(const Slice& a, const Slice& b) const override { - int r = CompareWithoutTimestamp(a, b); - if (r != 0 || 0 == timestamp_size()) { - return r; - } - return -CompareTimestamp( - Slice(a.data() + a.size() - timestamp_size(), timestamp_size()), - Slice(b.data() + b.size() - timestamp_size(), timestamp_size())); - } - - using Comparator::CompareWithoutTimestamp; - int CompareWithoutTimestamp(const Slice& a, bool a_has_ts, const Slice& b, - bool b_has_ts) const override { - if (a_has_ts) { - assert(a.size() >= timestamp_size()); - } - if (b_has_ts) { - assert(b.size() >= timestamp_size()); - } - Slice lhs = a_has_ts ? StripTimestampFromUserKey(a, timestamp_size()) : a; - Slice rhs = b_has_ts ? StripTimestampFromUserKey(b, timestamp_size()) : b; - return cmp_without_ts_->Compare(lhs, rhs); - } - - int CompareTimestamp(const Slice& ts1, const Slice& ts2) const override { - if (!ts1.data() && !ts2.data()) { - return 0; - } else if (ts1.data() && !ts2.data()) { - return 1; - } else if (!ts1.data() && ts2.data()) { - return -1; - } - assert(ts1.size() == ts2.size()); - uint64_t low1 = 0; - uint64_t low2 = 0; - uint64_t high1 = 0; - uint64_t high2 = 0; - const size_t kSize = ts1.size(); - std::unique_ptr ts1_buf(new char[kSize]); - memcpy(ts1_buf.get(), ts1.data(), ts1.size()); - std::unique_ptr ts2_buf(new char[kSize]); - memcpy(ts2_buf.get(), ts2.data(), ts2.size()); - Slice ts1_copy = Slice(ts1_buf.get(), kSize); - Slice ts2_copy = Slice(ts2_buf.get(), kSize); - auto* ptr1 = const_cast(&ts1_copy); - auto* ptr2 = const_cast(&ts2_copy); - if (!GetFixed64(ptr1, &low1) || !GetFixed64(ptr1, &high1) || - !GetFixed64(ptr2, &low2) || !GetFixed64(ptr2, &high2)) { - assert(false); - } - if (high1 < high2) { - return -1; - } else if (high1 > high2) { - return 1; - } - if (low1 < low2) { - return -1; - } else if (low1 > low2) { - return 1; - } - return 0; - } - }; - - std::string Timestamp(uint64_t low, uint64_t high) { - std::string ts; - PutFixed64(&ts, low); - PutFixed64(&ts, high); - return ts; - } - - void CheckIterUserEntry(const Iterator* it, const Slice& expected_key, - ValueType expected_value_type, - const Slice& expected_value, - const Slice& expected_ts) const { - ASSERT_TRUE(it->Valid()); - ASSERT_OK(it->status()); - ASSERT_EQ(expected_key, it->key()); - if (kTypeValue == expected_value_type) { - ASSERT_EQ(expected_value, it->value()); - } - ASSERT_EQ(expected_ts, it->timestamp()); - } - - void CheckIterEntry(const Iterator* it, const Slice& expected_ukey, - SequenceNumber expected_seq, ValueType expected_val_type, - const Slice& expected_value, const Slice& expected_ts) { - ASSERT_TRUE(it->Valid()); - ASSERT_OK(it->status()); - std::string ukey_and_ts; - ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size()); - ukey_and_ts.append(expected_ts.data(), expected_ts.size()); - ParsedInternalKey parsed_ikey; - ASSERT_OK( - ParseInternalKey(it->key(), &parsed_ikey, true /* log_err_key */)); - ASSERT_EQ(ukey_and_ts, parsed_ikey.user_key); - ASSERT_EQ(expected_val_type, parsed_ikey.type); - ASSERT_EQ(expected_seq, parsed_ikey.sequence); - if (expected_val_type == kTypeValue) { - ASSERT_EQ(expected_value, it->value()); - } - ASSERT_EQ(expected_ts, it->timestamp()); - } - - void CheckIterEntry(const Iterator* it, const Slice& expected_ukey, - ValueType expected_val_type, const Slice& expected_value, - const Slice& expected_ts) { - ASSERT_TRUE(it->Valid()); - ASSERT_OK(it->status()); - std::string ukey_and_ts; - ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size()); - ukey_and_ts.append(expected_ts.data(), expected_ts.size()); - - ParsedInternalKey parsed_ikey; - ASSERT_OK( - ParseInternalKey(it->key(), &parsed_ikey, true /* log_err_key */)); - ASSERT_EQ(expected_val_type, parsed_ikey.type); - ASSERT_EQ(Slice(ukey_and_ts), parsed_ikey.user_key); - if (expected_val_type == kTypeValue) { - ASSERT_EQ(expected_value, it->value()); - } - ASSERT_EQ(expected_ts, it->timestamp()); - } -}; - class DBBasicTestWithTimestamp : public DBBasicTestWithTimestampBase { public: DBBasicTestWithTimestamp() diff --git a/db/db_with_timestamp_test_util.cc b/db/db_with_timestamp_test_util.cc new file mode 100644 index 0000000000..f562bcb489 --- /dev/null +++ b/db/db_with_timestamp_test_util.cc @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_with_timestamp_test_util.h" + +namespace ROCKSDB_NAMESPACE { +std::string DBBasicTestWithTimestampBase::Key1(uint64_t k) { + std::string ret; + PutFixed64(&ret, k); + std::reverse(ret.begin(), ret.end()); + return ret; +} + +std::string DBBasicTestWithTimestampBase::KeyWithPrefix(std::string prefix, + uint64_t k) { + std::string ret; + PutFixed64(&ret, k); + std::reverse(ret.begin(), ret.end()); + return prefix + ret; +} + +std::vector DBBasicTestWithTimestampBase::ConvertStrToSlice( + std::vector& strings) { + std::vector ret; + for (const auto& s : strings) { + ret.emplace_back(s); + } + return ret; +} + +std::string DBBasicTestWithTimestampBase::Timestamp(uint64_t low, + uint64_t high) { + std::string ts; + PutFixed64(&ts, low); + PutFixed64(&ts, high); + return ts; +} + +void DBBasicTestWithTimestampBase::CheckIterUserEntry( + const Iterator* it, const Slice& expected_key, + ValueType expected_value_type, const Slice& expected_value, + const Slice& expected_ts) const { + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_EQ(expected_key, it->key()); + if (kTypeValue == expected_value_type) { + ASSERT_EQ(expected_value, it->value()); + } + ASSERT_EQ(expected_ts, it->timestamp()); +} + +void DBBasicTestWithTimestampBase::CheckIterEntry( + const Iterator* it, const Slice& expected_ukey, SequenceNumber expected_seq, + ValueType expected_val_type, const Slice& expected_value, + const Slice& expected_ts) const { + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + std::string ukey_and_ts; + ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size()); + ukey_and_ts.append(expected_ts.data(), expected_ts.size()); + ParsedInternalKey parsed_ikey; + ASSERT_OK(ParseInternalKey(it->key(), &parsed_ikey, true /* log_err_key */)); + ASSERT_EQ(ukey_and_ts, parsed_ikey.user_key); + ASSERT_EQ(expected_val_type, parsed_ikey.type); + ASSERT_EQ(expected_seq, parsed_ikey.sequence); + if (expected_val_type == kTypeValue) { + ASSERT_EQ(expected_value, it->value()); + } + ASSERT_EQ(expected_ts, it->timestamp()); +} + +void DBBasicTestWithTimestampBase::CheckIterEntry( + const Iterator* it, const Slice& expected_ukey, ValueType expected_val_type, + const Slice& expected_value, const Slice& expected_ts) const { + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + std::string ukey_and_ts; + ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size()); + ukey_and_ts.append(expected_ts.data(), expected_ts.size()); + + ParsedInternalKey parsed_ikey; + ASSERT_OK(ParseInternalKey(it->key(), &parsed_ikey, true /* log_err_key */)); + ASSERT_EQ(expected_val_type, parsed_ikey.type); + ASSERT_EQ(Slice(ukey_and_ts), parsed_ikey.user_key); + if (expected_val_type == kTypeValue) { + ASSERT_EQ(expected_value, it->value()); + } + ASSERT_EQ(expected_ts, it->timestamp()); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/db/db_with_timestamp_test_util.h b/db/db_with_timestamp_test_util.h new file mode 100644 index 0000000000..8a0d8e4e31 --- /dev/null +++ b/db/db_with_timestamp_test_util.h @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { +class DBBasicTestWithTimestampBase : public DBTestBase { + public: + explicit DBBasicTestWithTimestampBase(const std::string& dbname) + : DBTestBase(dbname, /*env_do_fsync=*/true) {} + + protected: + static std::string Key1(uint64_t k); + + static std::string KeyWithPrefix(std::string prefix, uint64_t k); + + static std::vector ConvertStrToSlice( + std::vector& strings); + + class TestComparator : public Comparator { + private: + const Comparator* cmp_without_ts_; + + public: + explicit TestComparator(size_t ts_sz) + : Comparator(ts_sz), cmp_without_ts_(nullptr) { + cmp_without_ts_ = BytewiseComparator(); + } + + const char* Name() const override { return "TestComparator"; } + + void FindShortSuccessor(std::string*) const override {} + + void FindShortestSeparator(std::string*, const Slice&) const override {} + + int Compare(const Slice& a, const Slice& b) const override { + int r = CompareWithoutTimestamp(a, b); + if (r != 0 || 0 == timestamp_size()) { + return r; + } + return -CompareTimestamp( + Slice(a.data() + a.size() - timestamp_size(), timestamp_size()), + Slice(b.data() + b.size() - timestamp_size(), timestamp_size())); + } + + using Comparator::CompareWithoutTimestamp; + int CompareWithoutTimestamp(const Slice& a, bool a_has_ts, const Slice& b, + bool b_has_ts) const override { + if (a_has_ts) { + assert(a.size() >= timestamp_size()); + } + if (b_has_ts) { + assert(b.size() >= timestamp_size()); + } + Slice lhs = a_has_ts ? StripTimestampFromUserKey(a, timestamp_size()) : a; + Slice rhs = b_has_ts ? StripTimestampFromUserKey(b, timestamp_size()) : b; + return cmp_without_ts_->Compare(lhs, rhs); + } + + int CompareTimestamp(const Slice& ts1, const Slice& ts2) const override { + if (!ts1.data() && !ts2.data()) { + return 0; + } else if (ts1.data() && !ts2.data()) { + return 1; + } else if (!ts1.data() && ts2.data()) { + return -1; + } + assert(ts1.size() == ts2.size()); + uint64_t low1 = 0; + uint64_t low2 = 0; + uint64_t high1 = 0; + uint64_t high2 = 0; + const size_t kSize = ts1.size(); + std::unique_ptr ts1_buf(new char[kSize]); + memcpy(ts1_buf.get(), ts1.data(), ts1.size()); + std::unique_ptr ts2_buf(new char[kSize]); + memcpy(ts2_buf.get(), ts2.data(), ts2.size()); + Slice ts1_copy = Slice(ts1_buf.get(), kSize); + Slice ts2_copy = Slice(ts2_buf.get(), kSize); + auto* ptr1 = const_cast(&ts1_copy); + auto* ptr2 = const_cast(&ts2_copy); + if (!GetFixed64(ptr1, &low1) || !GetFixed64(ptr1, &high1) || + !GetFixed64(ptr2, &low2) || !GetFixed64(ptr2, &high2)) { + assert(false); + } + if (high1 < high2) { + return -1; + } else if (high1 > high2) { + return 1; + } + if (low1 < low2) { + return -1; + } else if (low1 > low2) { + return 1; + } + return 0; + } + }; + + std::string Timestamp(uint64_t low, uint64_t high); + + void CheckIterUserEntry(const Iterator* it, const Slice& expected_key, + ValueType expected_value_type, + const Slice& expected_value, + const Slice& expected_ts) const; + + void CheckIterEntry(const Iterator* it, const Slice& expected_ukey, + SequenceNumber expected_seq, ValueType expected_val_type, + const Slice& expected_value, + const Slice& expected_ts) const; + + void CheckIterEntry(const Iterator* it, const Slice& expected_ukey, + ValueType expected_val_type, const Slice& expected_value, + const Slice& expected_ts) const; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/src.mk b/src.mk index 6ed13512ab..08b911475a 100644 --- a/src.mk +++ b/src.mk @@ -365,6 +365,7 @@ STRESS_LIB_SOURCES = \ TEST_LIB_SOURCES = \ db/db_test_util.cc \ + db/db_with_timestamp_test_util.cc \ test_util/mock_time_env.cc \ test_util/testharness.cc \ test_util/testutil.cc \ @@ -430,7 +431,6 @@ TEST_MAIN_SOURCES = \ db/corruption_test.cc \ db/cuckoo_table_db_test.cc \ db/db_basic_test.cc \ - db/db_with_timestamp_basic_test.cc \ db/db_block_cache_test.cc \ db/db_bloom_filter_test.cc \ db/db_compaction_filter_test.cc \ @@ -438,6 +438,8 @@ TEST_MAIN_SOURCES = \ db/db_dynamic_level_test.cc \ db/db_encryption_test.cc \ db/db_flush_test.cc \ + db/db_readonly_with_timestamp_test.cc \ + db/db_with_timestamp_basic_test.cc \ db/import_column_family_test.cc \ db/db_inplace_update_test.cc \ db/db_io_failure_test.cc \