From 7bebd3036d6947fa38db0f8936d6e7706ca22a5a Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Mon, 2 Oct 2023 16:19:05 -0700 Subject: [PATCH] Update tiered storage tests (ahead of next change) (#11917) Summary: After https://github.com/facebook/rocksdb/issues/11905, I am preparing a DBImpl change to ensure all sufficiently recent sequence numbers since Open are covered by SeqnoToTimeMapping. **Intended follow-up** However, there are a number of test changes I want to make prior to that to make it clear that I am not regressing the tests and production behavior at the same time. * Start mock time in the tests well beyond epoch (time 0) so that we aren't normally reaching into pre-history for current time minus the preserve/preclude duration. * Majorly clean up BasicSeqnoToTimeMapping to avoid confusing hard-coded bounds on GetProximalTimeBeforeSeqno() results. * There is an unresolved/unexplained issue marked with FIXME that should be investigated when GetProximalTimeBeforeSeqno() is put into production. * MultiCFs test was strangely generating 5 L0 files, four of which would be compacted into an L1, and then letting TTL compaction compact 1@L0+1@L1. Changing the starting time of the tests seemed to mess up the TTL compaction. But I suspect the TTL compaction was unintentional, so I've cut it down to just 4 L0 files, which compacts predictably. * Unrelated: allow ROCKSDB_NO_STACK=1 to skip printing a stack trace on assertion failures. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11917 Test Plan: no changes to production code Reviewed By: jowlyzhang Differential Revision: D49841436 Pulled By: pdillinger fbshipit-source-id: 753348ace9c548e82bcb77fcc8b2ffb7a6beeb0a --- db/compaction/tiered_compaction_test.cc | 7 ++- db/seqno_time_test.cc | 71 ++++++++++++++----------- port/stack_trace.cc | 5 ++ 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/db/compaction/tiered_compaction_test.cc b/db/compaction/tiered_compaction_test.cc index 51f931603a..3341bddc81 100644 --- a/db/compaction/tiered_compaction_test.cc +++ b/db/compaction/tiered_compaction_test.cc @@ -1216,6 +1216,7 @@ class PrecludeLastLevelTest : public DBTestBase { PrecludeLastLevelTest() : DBTestBase("preclude_last_level_test", /*env_do_fsync=*/false) { mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_clock_->SetCurrentTime(kMockStartTime); mock_env_ = std::make_unique(env_, mock_clock_); } @@ -1223,6 +1224,10 @@ class PrecludeLastLevelTest : public DBTestBase { std::unique_ptr mock_env_; std::shared_ptr mock_clock_; + // Sufficient starting time that preserve time doesn't under-flow into + // pre-history + static constexpr uint32_t kMockStartTime = 10000000; + void SetUp() override { mock_clock_->InstallTimedWaitFixCallback(); SyncPoint::GetInstance()->SetCallBack( @@ -1231,7 +1236,7 @@ class PrecludeLastLevelTest : public DBTestBase { reinterpret_cast(arg); periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); }); - mock_clock_->SetCurrentTime(0); + mock_clock_->SetCurrentTime(kMockStartTime); } }; diff --git a/db/seqno_time_test.cc b/db/seqno_time_test.cc index e4c5751676..3a64499be3 100644 --- a/db/seqno_time_test.cc +++ b/db/seqno_time_test.cc @@ -18,6 +18,7 @@ class SeqnoTimeTest : public DBTestBase { public: SeqnoTimeTest() : DBTestBase("seqno_time_test", /*env_do_fsync=*/false) { mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_clock_->SetCurrentTime(kMockStartTime); mock_env_ = std::make_unique(env_, mock_clock_); } @@ -25,6 +26,10 @@ class SeqnoTimeTest : public DBTestBase { std::unique_ptr mock_env_; std::shared_ptr mock_clock_; + // Sufficient starting time that preserve time doesn't under-flow into + // pre-history + static constexpr uint32_t kMockStartTime = 10000000; + void SetUp() override { mock_clock_->InstallTimedWaitFixCallback(); SyncPoint::GetInstance()->SetCallBack( @@ -33,6 +38,7 @@ class SeqnoTimeTest : public DBTestBase { reinterpret_cast(arg); periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); }); + mock_clock_->SetCurrentTime(kMockStartTime); } // make sure the file is not in cache, otherwise it won't have IO info @@ -324,8 +330,18 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { options.disable_auto_compactions = true; DestroyAndReopen(options); + // bootstrap DB sequence numbers (FIXME: make these steps unnecessary) + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(SingleDelete("foo")); + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownTimeBeforeAll + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + std::set checked_file_nums; SequenceNumber start_seq = dbfull()->GetLatestSequenceNumber() + 1; + uint64_t start_time = mock_clock_->NowSeconds(); + // Write a key every 10 seconds for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i), "value")); @@ -347,17 +363,16 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { ASSERT_GE(seqs.size(), 19); ASSERT_LE(seqs.size(), 21); SequenceNumber seq_end = dbfull()->GetLatestSequenceNumber() + 1; - for (auto i = start_seq; i < start_seq + 10; i++) { - ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), (i + 1) * 10); - } - start_seq += 10; for (auto i = start_seq; i < seq_end; i++) { // The result is within the range - ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), (i - 10) * 10); - ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), (i + 10) * 10); + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 10 - 100); + ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 10); } checked_file_nums.insert(it->second->orig_file_number); start_seq = seq_end; + start_time = mock_clock_->NowSeconds(); // Write a key every 1 seconds for (int i = 0; i < 200; i++) { @@ -387,13 +402,14 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { ASSERT_GE(seqs.size(), 1); ASSERT_LE(seqs.size(), 3); for (auto i = start_seq; i < seq_end; i++) { - // The result is not very accurate, as there is more data write within small - // range of time - ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), (i - start_seq) + 1000); - ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), (i - start_seq) + 3000); + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) - 100); + ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq)); } checked_file_nums.insert(it->second->orig_file_number); start_seq = seq_end; + start_time = mock_clock_->NowSeconds(); // Write a key every 200 seconds for (int i = 0; i < 200; i++) { @@ -422,20 +438,18 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { // The sequence number -> time entries should be maxed ASSERT_GE(seqs.size(), 99); ASSERT_LE(seqs.size(), 101); - for (auto i = start_seq; i < seq_end - 99; i++) { - // likely the first 100 entries reports 0 - ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), (i - start_seq) + 3000); - } - start_seq += 101; - for (auto i = start_seq; i < seq_end; i++) { - ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), - (i - start_seq) * 200 + 22200); + // aged out entries allowed to report time=0 + if ((seq_end - i) * 200 <= 10000) { + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 200 - 100); + } ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), - (i - start_seq) * 200 + 22600); + start_time + (i - start_seq) * 200); } checked_file_nums.insert(it->second->orig_file_number); start_seq = seq_end; + start_time = mock_clock_->NowSeconds(); // Write a key every 100 seconds for (int i = 0; i < 200; i++) { @@ -489,18 +503,15 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { seqs = tp_mapping.TEST_GetInternalMapping(); ASSERT_GE(seqs.size(), 99); ASSERT_LE(seqs.size(), 101); - for (auto i = start_seq; i < seq_end - 99; i++) { - // likely the first 100 entries reports 0 - ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), - (i - start_seq) * 100 + 50000); - } - start_seq += 101; - for (auto i = start_seq; i < seq_end; i++) { - ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), - (i - start_seq) * 100 + 52200); + // aged out entries allowed to report time=0 + // FIXME: should be <= + if ((seq_end - i) * 100 < 10000) { + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 100 - 100); + } ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), - (i - start_seq) * 100 + 52400); + start_time + (i - start_seq) * 100); } ASSERT_OK(db_->Close()); } @@ -625,14 +636,12 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { ASSERT_GE(seqs.size(), 99); ASSERT_LE(seqs.size(), 101); - for (int j = 0; j < 2; j++) { for (int i = 0; i < 200; i++) { ASSERT_OK(Put(0, Key(i), "value")); dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_OK(Flush(0)); - } ASSERT_OK(dbfull()->TEST_WaitForCompact()); tables_props.clear(); ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[0], &tables_props)); diff --git a/port/stack_trace.cc b/port/stack_trace.cc index 1ccf9d8040..a5a6d2e77c 100644 --- a/port/stack_trace.cc +++ b/port/stack_trace.cc @@ -191,6 +191,11 @@ void PrintStack(int first_frames_to_skip) { char* debug_env = getenv("ROCKSDB_DEBUG"); bool debug = debug_env != nullptr && strlen(debug_env) > 0; + if (!debug && getenv("ROCKSDB_NO_STACK") != nullptr) { + // Skip stack trace + return; + } + if (lldb_stack_trace || gdb_stack_trace || debug) { // Allow ouside debugger to attach, even with Yama security restrictions #ifdef PR_SET_PTRACER_ANY