dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add an unittest for Periodic compaction conflict with ongoing compaction (#10908) 

Summary:
Add a tiered storage migration test which would conflict with
an ongoing penultimate level compaction.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10908

Test Plan: Test only change

Reviewed By: anand1976

Differential Revision: D40864509

Pulled By: ajkr

fbshipit-source-id: e316e849a01a6c71a41be130101f909b6c0498cb
This commit is contained in:
Jay Zhuang 2022-12-12 10:37:55 -08:00 committed by Facebook GitHub Bot
parent 4d60cbc629
commit 1078d860a9
6 changed files with 211 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
db/compaction/tiered_compaction_test.cc
View File

@ -1251,7 +1251,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
int sst_num = 0; int sst_num = 0;
@ -1259,7 +1259,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
for (; sst_num < kNumTrigger; sst_num++) { for (; sst_num < kNumTrigger; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1313,7 +1313,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
int sst_num = 0; int sst_num = 0;
@ -1321,7 +1321,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
for (; sst_num < kNumTrigger; sst_num++) { for (; sst_num < kNumTrigger; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1355,7 +1355,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
// the value needs to be big enough to trigger full compaction // the value needs to be big enough to trigger full compaction
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100))); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100)));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1389,7 +1389,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
int sst_num = 0; int sst_num = 0;
@ -1397,7 +1397,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
for (; sst_num < kNumTrigger; sst_num++) { for (; sst_num < kNumTrigger; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1463,13 +1463,13 @@ TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) {
Random rnd(301); Random rnd(301);
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(10) + 1)); mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(10) + 1));
}); });
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(100))); ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(2))); mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(2)));
}); });
} }
@ -1516,7 +1516,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
int sst_num = 0; int sst_num = 0;
@ -1524,7 +1524,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
for (; sst_num < kNumTrigger; sst_num++) { for (; sst_num < kNumTrigger; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1594,7 +1594,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
Random rnd(301); Random rnd(301);
@ -1603,7 +1603,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
for (; sst_num < kNumTrigger; sst_num++) { for (; sst_num < kNumTrigger; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100))); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100)));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1696,7 +1696,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
// the value needs to be big enough to trigger full compaction // the value needs to be big enough to trigger full compaction
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -1721,6 +1721,133 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
Close(); Close();
} }
TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
// Test the last level only periodic compaction should also be blocked by an
// ongoing compaction in penultimate level if tiered compaction is enabled
// otherwise, the periodic compaction should just run for the last level.
const int kNumTrigger = 4;
const int kNumLevels = 7;
const int kPenultimateLevel = kNumLevels - 2;
const int kKeyPerSec = 1;
const int kNumKeys = 100;
bool enable_preclude_last_level = GetParam();
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.preserve_internal_time_seconds = 20000;
options.env = mock_env_.get();
options.level0_file_num_compaction_trigger = kNumTrigger;
options.num_levels = kNumLevels;
options.ignore_max_compaction_bytes_for_input = false;
options.periodic_compaction_seconds = 10000;
DestroyAndReopen(options);
Random rnd(301);
for (int i = 0; i < 3 * kNumKeys; i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
}
ASSERT_OK(Flush());
CompactRangeOptions cro;
cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
// make sure all data is compacted to the last level
ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
// enable preclude feature
if (enable_preclude_last_level) {
options.preclude_last_level_data_seconds = 20000;
}
options.max_background_jobs = 8;
options.last_level_temperature = Temperature::kCold;
Reopen(options);
std::atomic_bool is_size_ratio_compaction_running = false;
std::atomic_bool verified_last_level_compaction = false;
SyncPoint::GetInstance()->SetCallBack(
"CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) {
auto compaction = static_cast<Compaction*>(arg);
if (compaction->output_level() == kPenultimateLevel) {
is_size_ratio_compaction_running = true;
TEST_SYNC_POINT(
"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
"SizeRatioCompaction1");
TEST_SYNC_POINT(
"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
"SizeRatioCompaction2");
is_size_ratio_compaction_running = false;
}
});
SyncPoint::GetInstance()->SetCallBack(
"UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
auto compaction = static_cast<Compaction*>(arg);
if (is_size_ratio_compaction_running) {
if (enable_preclude_last_level) {
ASSERT_TRUE(compaction == nullptr);
} else {
ASSERT_TRUE(compaction != nullptr);
ASSERT_EQ(compaction->compaction_reason(),
CompactionReason::kPeriodicCompaction);
ASSERT_EQ(compaction->start_level(), kNumLevels - 1);
}
verified_last_level_compaction = true;
}
TEST_SYNC_POINT(
"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
"AutoCompactionPicked");
});
SyncPoint::GetInstance()->LoadDependency({
{"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
"SizeRatioCompaction1",
"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"},
{"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
"AutoCompactionPicked",
"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
"SizeRatioCompaction2"},
});
auto stop_token =
dbfull()->TEST_write_controler().GetCompactionPressureToken();
for (int i = 0; i < kNumTrigger - 1; i++) {
for (int j = 0; j < kNumKeys; j++) {
ASSERT_OK(Put(Key(i * (kNumKeys - 1) + i), rnd.RandomString(10)));
dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
}
ASSERT_OK(Flush());
}
TEST_SYNC_POINT(
"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite");
// wait for periodic compaction time and flush to trigger the periodic
// compaction, which should be blocked by ongoing compaction in the
// penultimate level
mock_clock_->MockSleepForSeconds(10000);
for (int i = 0; i < 3 * kNumKeys; i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(10)));
dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
}
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->WaitForCompact(true));
stop_token.reset();
Close();
}
INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms, INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms,
PrecludeLastLevelTestWithParms, testing::Bool()); PrecludeLastLevelTestWithParms, testing::Bool());
@ -1781,14 +1908,14 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
Random rnd(301); Random rnd(301);
for (int i = 0; i < 300; i++) { for (int i = 0; i < 300; i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(100))); ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -1892,7 +2019,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kSecondsPerKey)); mock_clock_->MockSleepForSeconds(static_cast<int>(kSecondsPerKey));
}); });
@ -1917,13 +2044,13 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
Random rnd(301); Random rnd(301);
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(i + 3), rnd.RandomString(kValueBytes))); ASSERT_OK(Put(Key(i + 3), rnd.RandomString(kValueBytes)));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
} }
auto* snap1 = db_->GetSnapshot(); auto* snap1 = db_->GetSnapshot();
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(kValueBytes))); ASSERT_OK(Put(Key(i), rnd.RandomString(kValueBytes)));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
} }
auto* snap2 = db_->GetSnapshot(); auto* snap2 = db_->GetSnapshot();
@ -1937,7 +2064,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
Key(2 * kNumKeysPerFile - 1), Key(2 * kNumKeysPerFile - 1),
Key(2 * kNumKeysPerFile + 1))); Key(2 * kNumKeysPerFile + 1)));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
verify_db(); verify_db();

2
db/db_impl/db_impl.h
View File

@ -1161,7 +1161,7 @@ class DBImpl : public DB {
int TEST_BGCompactionsAllowed() const; int TEST_BGCompactionsAllowed() const;
int TEST_BGFlushesAllowed() const; int TEST_BGFlushesAllowed() const;
size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const; size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
void TEST_WaitForPeridicTaskRun(std::function<void()> callback) const; void TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const;
SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const; SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const;
size_t TEST_EstimateInMemoryStatsHistorySize() const; size_t TEST_EstimateInMemoryStatsHistorySize() const;

2
db/db_impl/db_impl_debug.cc
View File

@ -290,7 +290,7 @@ size_t DBImpl::TEST_GetWalPreallocateBlockSize(
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
void DBImpl::TEST_WaitForPeridicTaskRun(std::function<void()> callback) const { void DBImpl::TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const {
periodic_task_scheduler_.TEST_WaitForRun(callback); periodic_task_scheduler_.TEST_WaitForRun(callback);
} }

20
db/periodic_task_scheduler_test.cc
View File

@ -65,7 +65,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec); ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);
ASSERT_GT(kPeriodSec, 1u); ASSERT_GT(kPeriodSec, 1u);
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec) - 1); mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec) - 1);
}); });
@ -77,14 +77,14 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
ASSERT_EQ(1, pst_st_counter); ASSERT_EQ(1, pst_st_counter);
ASSERT_EQ(1, flush_info_log_counter); ASSERT_EQ(1, flush_info_log_counter);
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
ASSERT_EQ(2, dump_st_counter); ASSERT_EQ(2, dump_st_counter);
ASSERT_EQ(2, pst_st_counter); ASSERT_EQ(2, pst_st_counter);
ASSERT_EQ(2, flush_info_log_counter); ASSERT_EQ(2, flush_info_log_counter);
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
ASSERT_EQ(3, dump_st_counter); ASSERT_EQ(3, dump_st_counter);
@ -98,7 +98,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec); ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec);
// Info log flush should still run. // Info log flush should still run.
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
ASSERT_EQ(3, dump_st_counter); ASSERT_EQ(3, dump_st_counter);
ASSERT_EQ(3, pst_st_counter); ASSERT_EQ(3, pst_st_counter);
@ -113,7 +113,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum()); ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum());
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
ASSERT_EQ(4, dump_st_counter); ASSERT_EQ(4, dump_st_counter);
ASSERT_EQ(3, pst_st_counter); ASSERT_EQ(3, pst_st_counter);
@ -154,19 +154,19 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum()); ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum());
int expected_run = kInstanceNum; int expected_run = kInstanceNum;
dbi->TEST_WaitForPeridicTaskRun( dbi->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter); ASSERT_EQ(expected_run, pst_st_counter);
expected_run += kInstanceNum; expected_run += kInstanceNum;
dbi->TEST_WaitForPeridicTaskRun( dbi->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter); ASSERT_EQ(expected_run, pst_st_counter);
expected_run += kInstanceNum; expected_run += kInstanceNum;
dbi->TEST_WaitForPeridicTaskRun( dbi->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter); ASSERT_EQ(expected_run, pst_st_counter);
@ -178,9 +178,9 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
expected_run += (kInstanceNum - half) * 2; expected_run += (kInstanceNum - half) * 2;
dbi->TEST_WaitForPeridicTaskRun( dbi->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
dbi->TEST_WaitForPeridicTaskRun( dbi->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter); ASSERT_EQ(expected_run, pst_st_counter);

52
db/seqno_time_test.cc
View File

@ -80,7 +80,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
int sst_num = 0; int sst_num = 0;
@ -88,7 +88,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
for (; sst_num < kNumTrigger; sst_num++) { for (; sst_num < kNumTrigger; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -110,7 +110,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
for (; sst_num < kNumTrigger * 2; sst_num++) { for (; sst_num < kNumTrigger * 2; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -124,7 +124,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
for (; sst_num < kNumTrigger * 3; sst_num++) { for (; sst_num < kNumTrigger * 3; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
}); });
} }
@ -143,7 +143,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
AssertKeyTemperature(20, Temperature::kCold); AssertKeyTemperature(20, Temperature::kCold);
for (int i = 0; i < 30; i++) { for (int i = 0; i < 30; i++) {
dbfull()->TEST_WaitForPeridicTaskRun([&] { dbfull()->TEST_WaitForPeriodicTaskRun([&] {
mock_clock_->MockSleepForSeconds(static_cast<int>(20 * kKeyPerSec)); mock_clock_->MockSleepForSeconds(static_cast<int>(20 * kKeyPerSec));
}); });
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@ -161,7 +161,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
// the compaction will not get the new seqno->time sampling to decide the last // the compaction will not get the new seqno->time sampling to decide the last
// a few data's time. // a few data's time.
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
} }
@ -192,7 +192,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
// pass some time first, otherwise the first a few keys write time are going // pass some time first, otherwise the first a few keys write time are going
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
int sst_num = 0; int sst_num = 0;
@ -200,7 +200,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
for (; sst_num < 4; sst_num++) { for (; sst_num < 4; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -222,7 +222,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
for (; sst_num < 14; sst_num++) { for (; sst_num < 14; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -243,7 +243,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
// Wait some time, with each wait, the cold data is increasing and hot data is // Wait some time, with each wait, the cold data is increasing and hot data is
// decreasing // decreasing
for (int i = 0; i < 30; i++) { for (int i = 0; i < 30; i++) {
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
uint64_t pre_hot = hot_data_size; uint64_t pre_hot = hot_data_size;
@ -262,7 +262,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
// hot data might not be empty, because if we don't write new data, there's // hot data might not be empty, because if we don't write new data, there's
// no seqno->time sampling available to the compaction // no seqno->time sampling available to the compaction
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
} }
@ -324,7 +324,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
// Write a key every 10 seconds // Write a key every 10 seconds
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(Key(i), "value")); ASSERT_OK(Put(Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -357,7 +357,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
// Write a key every 1 seconds // Write a key every 1 seconds
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(Key(i + 190), "value")); ASSERT_OK(Put(Key(i + 190), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1)); });
} }
seq_end = dbfull()->GetLatestSequenceNumber(); seq_end = dbfull()->GetLatestSequenceNumber();
@ -393,7 +393,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
// Write a key every 200 seconds // Write a key every 200 seconds
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(Key(i + 380), "value")); ASSERT_OK(Put(Key(i + 380), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
} }
seq_end = dbfull()->GetLatestSequenceNumber(); seq_end = dbfull()->GetLatestSequenceNumber();
@ -435,7 +435,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
// Write a key every 100 seconds // Write a key every 100 seconds
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(Key(i + 570), "value")); ASSERT_OK(Put(Key(i + 570), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
seq_end = dbfull()->GetLatestSequenceNumber(); seq_end = dbfull()->GetLatestSequenceNumber();
@ -516,7 +516,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
// Write some data and increase the current time // Write some data and increase the current time
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(Key(i), "value")); ASSERT_OK(Put(Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -536,7 +536,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
// Write some data to the default CF (without preclude_last_level feature) // Write some data to the default CF (without preclude_last_level feature)
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(Key(i), "value")); ASSERT_OK(Put(Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -544,7 +544,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
// Write some data to the CF one // Write some data to the CF one
for (int i = 0; i < 20; i++) { for (int i = 0; i < 20; i++) {
ASSERT_OK(Put(1, Key(i), "value")); ASSERT_OK(Put(1, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
} }
ASSERT_OK(Flush(1)); ASSERT_OK(Flush(1));
@ -568,7 +568,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
// Add more data to CF "two" to fill the in memory mapping // Add more data to CF "two" to fill the in memory mapping
for (int i = 0; i < 2000; i++) { for (int i = 0; i < 2000; i++) {
ASSERT_OK(Put(2, Key(i), "value")); ASSERT_OK(Put(2, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping(); seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
@ -592,7 +592,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
// enabled have flushed, the in-memory seqno->time mapping should be cleared // enabled have flushed, the in-memory seqno->time mapping should be cleared
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
ASSERT_OK(Put(0, Key(i), "value")); ASSERT_OK(Put(0, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping(); seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
@ -603,7 +603,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
for (int j = 0; j < 3; j++) { for (int j = 0; j < 3; j++) {
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(2, Key(i), "value")); ASSERT_OK(Put(2, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
ASSERT_OK(Flush(2)); ASSERT_OK(Flush(2));
@ -623,7 +623,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
for (int j = 0; j < 2; j++) { for (int j = 0; j < 2; j++) {
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(Put(0, Key(i), "value")); ASSERT_OK(Put(0, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
ASSERT_OK(Flush(0)); ASSERT_OK(Flush(0));
@ -638,7 +638,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
// Write some data to CF "two", but don't flush to accumulate // Write some data to CF "two", but don't flush to accumulate
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
ASSERT_OK(Put(2, Key(i), "value")); ASSERT_OK(Put(2, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
ASSERT_GE( ASSERT_GE(
@ -682,7 +682,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiInstancesBasic) {
WriteOptions wo; WriteOptions wo;
for (int i = 0; i < 200; i++) { for (int i = 0; i < 200; i++) {
ASSERT_OK(dbi->Put(wo, Key(i), "value")); ASSERT_OK(dbi->Put(wo, Key(i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
} }
SeqnoToTimeMapping seqno_to_time_mapping = dbi->TEST_GetSeqnoToTimeMapping(); SeqnoToTimeMapping seqno_to_time_mapping = dbi->TEST_GetSeqnoToTimeMapping();
@ -720,7 +720,7 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
for (; sst_num < kNumTrigger - 1; sst_num++) { for (; sst_num < kNumTrigger - 1; sst_num++) {
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -742,7 +742,7 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
// Trigger a compaction // Trigger a compaction
for (int i = 0; i < kNumKeys; i++) { for (int i = 0; i < kNumKeys; i++) {
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); }); [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
} }
sst_num++; sst_num++;

52
monitoring/stats_history_test.cc
View File

@ -66,10 +66,10 @@ TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) {
// Wait for the first stats persist to finish, as the initial delay could be // Wait for the first stats persist to finish, as the initial delay could be
// different. // different.
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
ASSERT_GE(counter, 1); ASSERT_GE(counter, 1);
@ -98,17 +98,17 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) {
// Wait for the first stats persist to finish, as the initial delay could be // Wait for the first stats persist to finish, as the initial delay could be
// different. // different.
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
ASSERT_GE(counter, 1); ASSERT_GE(counter, 1);
// Test cancel job through SetOptions // Test cancel job through SetOptions
ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}})); ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
int old_val = counter; int old_val = counter;
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); });
ASSERT_EQ(counter, old_val); ASSERT_EQ(counter, old_val);
@ -130,7 +130,7 @@ TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) {
{{"stats_persist_period_sec", std::to_string(kPeriodSec)}})); {{"stats_persist_period_sec", std::to_string(kPeriodSec)}}));
ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec); ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
ASSERT_GE(counter, 1); ASSERT_GE(counter, 1);
Close(); Close();
@ -149,11 +149,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
ReopenWithColumnFamilies({"default", "pikachu"}, options); ReopenWithColumnFamilies({"default", "pikachu"}, options);
// make sure the first stats persist to finish // make sure the first stats persist to finish
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
// Wait for stats persist to finish // Wait for stats persist to finish
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
std::unique_ptr<StatsHistoryIterator> stats_iter; std::unique_ptr<StatsHistoryIterator> stats_iter;
@ -171,7 +171,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
ASSERT_GT(stats_count, 0); ASSERT_GT(stats_count, 0);
// Wait a bit and verify no more stats are found // Wait a bit and verify no more stats are found
for (int i = 0; i < 10; ++i) { for (int i = 0; i < 10; ++i) {
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(1); }); [&] { mock_clock_->MockSleepForSeconds(1); });
} }
ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter)); ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
@ -226,7 +226,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
const int kIterations = 10; const int kIterations = 10;
for (int i = 0; i < kIterations; ++i) { for (int i = 0; i < kIterations; ++i) {
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
} }
@ -250,7 +250,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
// Wait for stats persist to finish // Wait for stats persist to finish
for (int i = 0; i < kIterations; ++i) { for (int i = 0; i < kIterations; ++i) {
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
} }
@ -299,11 +299,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
// Wait for the first stats persist to finish, as the initial delay could be // Wait for the first stats persist to finish, as the initial delay could be
// different. // different.
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
// Wait for stats persist to finish // Wait for stats persist to finish
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
auto iter = auto iter =
@ -311,14 +311,14 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
int key_count1 = countkeys(iter); int key_count1 = countkeys(iter);
delete iter; delete iter;
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
iter = iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
int key_count2 = countkeys(iter); int key_count2 = countkeys(iter);
delete iter; delete iter;
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
iter = iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
@ -392,32 +392,32 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {
// Wait for the first stats persist to finish, as the initial delay could be // Wait for the first stats persist to finish, as the initial delay could be
// different. // different.
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
// Wait for stats persist to finish // Wait for stats persist to finish
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
auto iter = auto iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
countkeys(iter); countkeys(iter);
delete iter; delete iter;
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
iter = iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
countkeys(iter); countkeys(iter);
delete iter; delete iter;
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
iter = iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
countkeys(iter); countkeys(iter);
delete iter; delete iter;
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
std::map<std::string, uint64_t> stats_map_after; std::map<std::string, uint64_t> stats_map_after;
@ -481,10 +481,10 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
ASSERT_EQ(Get(2, "foo"), "bar"); ASSERT_EQ(Get(2, "foo"), "bar");
// make sure the first stats persist to finish // make sure the first stats persist to finish
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
auto iter = auto iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
@ -581,7 +581,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
// Wait for the first stats persist to finish, as the initial delay could be // Wait for the first stats persist to finish, as the initial delay could be
// different. // different.
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
ColumnFamilyData* cfd_default = ColumnFamilyData* cfd_default =
@ -600,7 +600,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
ASSERT_OK(Put(1, "Eevee", "v0")); ASSERT_OK(Put(1, "Eevee", "v0"));
ASSERT_EQ("v0", Get(1, "Eevee")); ASSERT_EQ("v0", Get(1, "Eevee"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
// writing to all three cf, flush default cf // writing to all three cf, flush default cf
// LogNumbers: default: 16, stats: 10, pikachu: 5 // LogNumbers: default: 16, stats: 10, pikachu: 5
@ -629,7 +629,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
ASSERT_EQ("v2", Get("bar2")); ASSERT_EQ("v2", Get("bar2"));
ASSERT_EQ("v2", Get("foo2")); ASSERT_EQ("v2", Get("foo2"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
// writing to default and stats cf, flushing default cf // writing to default and stats cf, flushing default cf
// LogNumbers: default: 19, stats: 19, pikachu: 19 // LogNumbers: default: 19, stats: 19, pikachu: 19
@ -644,7 +644,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
ASSERT_OK(Put(1, "Jolteon", "v3")); ASSERT_OK(Put(1, "Jolteon", "v3"));
ASSERT_EQ("v3", Get(1, "Jolteon")); ASSERT_EQ("v3", Get(1, "Jolteon"));
dbfull()->TEST_WaitForPeridicTaskRun( dbfull()->TEST_WaitForPeriodicTaskRun(
[&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
// writing to all three cf, flushing test cf // writing to all three cf, flushing test cf
// LogNumbers: default: 19, stats: 19, pikachu: 22 // LogNumbers: default: 19, stats: 19, pikachu: 22