dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add test function MockTimeEnv.SleepForMicroseconds() (#7293) 

Summary:
And change the internal time value from seconds to microseconds.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7293

Reviewed By: pdillinger

Differential Revision: D23253751

Pulled By: jay-zhuang

fbshipit-source-id: 36aa9376b8801b85bd10163173590a17cf4f3a3a
This commit is contained in:
Jay Zhuang 2020-08-21 11:31:27 -07:00 committed by Facebook GitHub Bot
parent a1b5484811
commit 187964a039
4 changed files with 177 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
monitoring/stats_dump_scheduler_test.cc
View File

@ -38,8 +38,6 @@ TEST_F(StatsDumpSchedulerTest, Basic) {
options.stats_dump_period_sec = kPeriodSec;
options.stats_persist_period_sec = kPeriodSec;
options.create_if_missing = true;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
int dump_st_counter = 0;
@ -56,9 +54,8 @@ TEST_F(StatsDumpSchedulerTest, Basic) {
ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec);
ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec);
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
auto scheduler = dbfull()->TEST_GetStatsDumpScheduler();
ASSERT_NE(nullptr, scheduler);
@ -67,16 +64,14 @@ TEST_F(StatsDumpSchedulerTest, Basic) {
ASSERT_EQ(1, dump_st_counter);
ASSERT_EQ(1, pst_st_counter);
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(2, dump_st_counter);
ASSERT_EQ(2, pst_st_counter);
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(3, dump_st_counter);
ASSERT_EQ(3, pst_st_counter);
@ -100,9 +95,8 @@ TEST_F(StatsDumpSchedulerTest, Basic) {
ASSERT_EQ(1, scheduler->TEST_GetValidTaskNum());
dump_st_counter = 0;
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(1, dump_st_counter);
Close();
@ -117,8 +111,6 @@ TEST_F(StatsDumpSchedulerTest, MultiInstances) {
options.stats_dump_period_sec = kPeriodSec;
options.stats_persist_period_sec = kPeriodSec;
options.create_if_missing = true;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
int dump_st_counter = 0;
@ -141,23 +133,20 @@ TEST_F(StatsDumpSchedulerTest, MultiInstances) {
ASSERT_EQ(kInstanceNum * 2, scheduler->TEST_GetValidTaskNum());
int expected_run = kInstanceNum;
mock_time_sec += kPeriodSec - 1;
dbi->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter);
expected_run += kInstanceNum;
mock_time_sec += kPeriodSec;
dbi->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter);
expected_run += kInstanceNum;
mock_time_sec += kPeriodSec;
dbi->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter);
@ -168,12 +157,10 @@ TEST_F(StatsDumpSchedulerTest, MultiInstances) {
expected_run += (kInstanceNum - half) * 2;
mock_time_sec += kPeriodSec;
dbi->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
mock_time_sec += kPeriodSec;
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
dbi->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_EQ(expected_run, dump_st_counter);
ASSERT_EQ(expected_run, pst_st_counter);
@ -191,7 +178,6 @@ TEST_F(StatsDumpSchedulerTest, MultiEnv) {
options1.stats_dump_period_sec = kDumpPeriodSec;
options1.stats_persist_period_sec = kPersistPeriodSec;
options1.create_if_missing = true;
mock_env_->set_current_time(0);
options1.env = mock_env_.get();
Reopen(options1);
@ -201,7 +187,6 @@ TEST_F(StatsDumpSchedulerTest, MultiEnv) {
options2.stats_dump_period_sec = kDumpPeriodSec;
options2.stats_persist_period_sec = kPersistPeriodSec;
options2.create_if_missing = true;
mock_env2->set_current_time(0);
options1.env = mock_env2.get();
std::string dbname = test::PerThreadDBPath("multi_env_test");

139
monitoring/stats_history_test.cc
View File

@ -55,8 +55,6 @@ TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) {
Options options;
options.create_if_missing = true;
options.stats_dump_period_sec = kPeriodSec;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
int counter = 0;
SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:1",
@ -66,20 +64,18 @@ TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) {
// Wait for the first stats persist to finish, as the initial delay could be
// different.
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_GE(counter, 1);
// Test cancel job through SetOptions
ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "0"}}));
int old_val = counter;
for (int i = 1; i < 20; ++i) {
mock_env_->set_current_time(i + mock_time_sec);
mock_env_->MockSleepForSeconds(kPeriodSec);
}
ASSERT_EQ(counter, old_val);
Close();
@ -91,8 +87,6 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) {
Options options;
options.create_if_missing = true;
options.stats_persist_period_sec = kPeriodSec;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
int counter = 0;
SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry",
@ -102,21 +96,18 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) {
// Wait for the first stats persist to finish, as the initial delay could be
// different.
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_GE(counter, 1);
// Test cacel job through SetOptions
// Test cancel job through SetOptions
ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
int old_val = counter;
mock_time_sec += kPeriodSec * 2;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec * 2); });
ASSERT_EQ(counter, old_val);
Close();
@ -124,19 +115,21 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) {
// Test enabling persistent stats for the first time
TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) {
constexpr unsigned int kPeriodSec = 5;
Options options;
options.create_if_missing = true;
options.stats_persist_period_sec = 0;
mock_env_->set_current_time(0); // in seconds
options.env = mock_env_.get();
int counter = 0;
SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry",
[&](void* /*arg*/) { counter++; });
Reopen(options);
ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "5"}}));
ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec);
ASSERT_OK(dbfull()->SetDBOptions(
{{"stats_persist_period_sec", std::to_string(kPeriodSec)}}));
ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);
dbfull()->TEST_WaitForStatsDumpRun([&] { mock_env_->set_current_time(5); });
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
ASSERT_GE(counter, 1);
Close();
}
@ -148,41 +141,37 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
options.create_if_missing = true;
options.stats_persist_period_sec = kPeriodSec;
options.statistics = CreateDBStatistics();
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
CreateColumnFamilies({"pikachu"}, options);
ASSERT_OK(Put("foo", "bar"));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
// make sure the first stats persist to finish
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
// Wait for stats persist to finish
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
std::unique_ptr<StatsHistoryIterator> stats_iter;
db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
// disabled stats snapshots
ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
size_t stats_count = 0;
for (; stats_iter->Valid(); stats_iter->Next()) {
auto stats_map = stats_iter->GetStatsMap();
ASSERT_EQ(stats_iter->GetStatsTime(), mock_time_sec);
ASSERT_EQ(stats_iter->GetStatsTime(), mock_env_->NowSeconds());
stats_count += stats_map.size();
}
ASSERT_GT(stats_count, 0);
// Wait a bit and verify no more stats are found
for (; mock_time_sec < 30; ++mock_time_sec) {
for (int i = 0; i < 10; ++i) {
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(1); });
}
db_->GetStatsHistory(0, mock_time_sec, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds(), &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
size_t stats_count_new = 0;
for (; stats_iter->Valid(); stats_iter->Next()) {
@ -198,8 +187,6 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
options.create_if_missing = true;
options.statistics = CreateDBStatistics();
options.stats_persist_period_sec = kPeriodSec;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
CreateColumnFamilies({"pikachu"}, options);
@ -221,11 +208,6 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
ASSERT_OK(Flush());
ASSERT_OK(Delete("sol"));
db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
// Wait for stats persist to finish
for (mock_time_sec = 1; mock_time_sec < kPeriodSec; mock_time_sec++) {
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
}
// second round of ops
ASSERT_OK(Put("saigon", "saigon"));
@ -239,13 +221,14 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
ASSERT_OK(Flush());
db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
for (; mock_time_sec < 10; mock_time_sec++) {
const int kIterations = 10;
for (int i = 0; i < kIterations; ++i) {
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
}
std::unique_ptr<StatsHistoryIterator> stats_iter;
db_->GetStatsHistory(0, 10, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
size_t stats_count = 0;
int slice_count = 0;
@ -255,19 +238,19 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
stats_count += stats_map.size();
}
size_t stats_history_size = dbfull()->TEST_EstimateInMemoryStatsHistorySize();
ASSERT_GE(slice_count, 9);
ASSERT_GE(slice_count, kIterations - 1);
ASSERT_GE(stats_history_size, 13000);
// capping memory cost at 13000 bytes since one slice is around 10000~13000
ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "13000"}}));
ASSERT_EQ(13000, dbfull()->GetDBOptions().stats_history_buffer_size);
// Wait for stats persist to finish
for (; mock_time_sec < 20; mock_time_sec++) {
for (int i = 0; i < kIterations; ++i) {
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
}
db_->GetStatsHistory(0, 20, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
size_t stats_count_reopen = 0;
slice_count = 0;
@ -303,8 +286,6 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
options.stats_persist_period_sec = kPeriodSec;
options.statistics = CreateDBStatistics();
options.persist_stats_to_disk = true;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
CreateColumnFamilies({"pikachu"}, options);
ASSERT_OK(Put("foo", "bar"));
@ -313,31 +294,27 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
// Wait for the first stats persist to finish, as the initial delay could be
// different.
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
// Wait for stats persist to finish
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
auto iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
int key_count1 = countkeys(iter);
delete iter;
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
int key_count2 = countkeys(iter);
delete iter;
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
int key_count3 = countkeys(iter);
@ -346,7 +323,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
ASSERT_GE(key_count3, key_count2);
ASSERT_EQ(key_count3 - key_count2, key_count2 - key_count1);
std::unique_ptr<StatsHistoryIterator> stats_iter;
db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
size_t stats_count = 0;
int slice_count = 0;
@ -367,7 +344,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
ASSERT_EQ(stats_count, key_count3 - 2);
// verify reopen will not cause data loss
ReopenWithColumnFamilies({"default", "pikachu"}, options);
db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
size_t stats_count_reopen = 0;
int slice_count_reopen = 0;
@ -398,11 +375,8 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {
options.stats_persist_period_sec = kPeriodSec;
options.statistics = CreateDBStatistics();
options.persist_stats_to_disk = true;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
std::map<std::string, uint64_t> stats_map_before;
ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_before));
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
CreateColumnFamilies({"pikachu"}, options);
ASSERT_OK(Put("foo", "bar"));
@ -411,43 +385,38 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {
// Wait for the first stats persist to finish, as the initial delay could be
// different.
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
// Wait for stats persist to finish
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
auto iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
countkeys(iter);
delete iter;
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
countkeys(iter);
delete iter;
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
countkeys(iter);
delete iter;
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
std::map<std::string, uint64_t> stats_map_after;
ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_after));
std::unique_ptr<StatsHistoryIterator> stats_iter;
db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
std::string sample = "rocksdb.num.iterator.deleted";
uint64_t recovered_value = 0;
@ -464,7 +433,7 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {
// test stats value retains after recovery
ReopenWithColumnFamilies({"default", "pikachu"}, options);
db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds() + 1, &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
uint64_t new_recovered_value = 0;
for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) {
@ -492,8 +461,6 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
options.stats_persist_period_sec = kPeriodSec;
options.statistics = CreateDBStatistics();
options.persist_stats_to_disk = true;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
ASSERT_OK(TryReopen(options));
CreateColumnFamilies({"one", "two", "three"}, options);
@ -505,13 +472,11 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
ASSERT_EQ(Get(2, "foo"), "bar");
// make sure the first stats persist to finish
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
auto iter =
db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
int key_count = countkeys(iter);
@ -520,7 +485,7 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
uint64_t num_write_wal = 0;
std::string sample = "rocksdb.write.wal";
std::unique_ptr<StatsHistoryIterator> stats_iter;
db_->GetStatsHistory(0, mock_time_sec, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds(), &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
for (; stats_iter->Valid(); stats_iter->Next()) {
auto stats_map = stats_iter->GetStatsMap();
@ -556,7 +521,7 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName,
&handle));
// verify stats is not affected by prior failed CF creation
db_->GetStatsHistory(0, mock_time_sec, &stats_iter);
db_->GetStatsHistory(0, mock_env_->NowSeconds(), &stats_iter);
ASSERT_TRUE(stats_iter != nullptr);
num_write_wal = 0;
for (; stats_iter->Valid(); stats_iter->Next()) {
@ -601,17 +566,14 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
options.stats_persist_period_sec = kPeriodSec;
options.statistics = CreateDBStatistics();
options.persist_stats_to_disk = true;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
options.env = mock_env_.get();
CreateColumnFamilies({"pikachu"}, options);
ReopenWithColumnFamilies({"default", "pikachu"}, options);
// Wait for the first stats persist to finish, as the initial delay could be
// different.
mock_time_sec += kPeriodSec - 1;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec - 1); });
ColumnFamilyData* cfd_default =
static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily())
@ -629,9 +591,8 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
ASSERT_OK(Put(1, "Eevee", "v0"));
ASSERT_EQ("v0", Get(1, "Eevee"));
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
// writing to all three cf, flush default cf
// LogNumbers: default: 14, stats: 4, pikachu: 4
ASSERT_OK(Flush());
@ -655,9 +616,8 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
ASSERT_EQ("v2", Get("bar2"));
ASSERT_EQ("v2", Get("foo2"));
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
// writing to default and stats cf, flushing default cf
// LogNumbers: default: 19, stats: 19, pikachu: 19
ASSERT_OK(Flush());
@ -671,9 +631,8 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
ASSERT_OK(Put(1, "Jolteon", "v3"));
ASSERT_EQ("v3", Get(1, "Jolteon"));
mock_time_sec += kPeriodSec;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(mock_time_sec); });
[&] { mock_env_->MockSleepForSeconds(kPeriodSec); });
// writing to all three cf, flushing test cf
// LogNumbers: default: 19, stats: 19, pikachu: 22
ASSERT_OK(Flush(1));

48
test_util/mock_time_env.h
View File

@ -16,29 +16,47 @@ class MockTimeEnv : public EnvWrapper {
public:
explicit MockTimeEnv(Env* base) : EnvWrapper(base) {}
virtual Status GetCurrentTime(int64_t* time) override {
assert(time != nullptr);
assert(current_time_ <=
static_cast<uint64_t>(std::numeric_limits<int64_t>::max()));
*time = static_cast<int64_t>(current_time_);
virtual Status GetCurrentTime(int64_t* time_sec) override {
assert(time_sec != nullptr);
*time_sec = static_cast<int64_t>(current_time_us_ / kMicrosInSecond);
return Status::OK();
}
virtual uint64_t NowMicros() override {
assert(current_time_ <= std::numeric_limits<uint64_t>::max() / 1000000);
return current_time_ * 1000000;
}
virtual uint64_t NowSeconds() { return current_time_us_ / kMicrosInSecond; }
virtual uint64_t NowMicros() override { return current_time_us_; }
virtual uint64_t NowNanos() override {
assert(current_time_ <= std::numeric_limits<uint64_t>::max() / 1000000000);
return current_time_ * 1000000000;
assert(current_time_us_ <= std::numeric_limits<uint64_t>::max() / 1000);
return current_time_us_ * 1000;
}
uint64_t RealNowMicros() { return target()->NowMicros(); }
void set_current_time(uint64_t time) {
assert(time >= current_time_);
current_time_ = time;
void set_current_time(uint64_t time_sec) {
assert(time_sec < std::numeric_limits<uint64_t>::max() / kMicrosInSecond);
assert(time_sec * kMicrosInSecond >= current_time_us_);
current_time_us_ = time_sec * kMicrosInSecond;
}
// It's a fake sleep that just updates the Env current time, which is similar
// to `NoSleepEnv.SleepForMicroseconds()` and
// `SpecialEnv.MockSleepForMicroseconds()`.
// It's also similar to `set_current_time()`, which takes an absolute time in
// seconds, vs. this one takes the sleep in microseconds.
// Note: Not thread safe.
void MockSleepForMicroseconds(int micros) {
assert(micros >= 0);
assert(current_time_us_ + static_cast<uint64_t>(micros) >=
current_time_us_);
current_time_us_.fetch_add(micros);
}
void MockSleepForSeconds(int seconds) {
assert(seconds >= 0);
uint64_t micros = static_cast<uint64_t>(seconds) * kMicrosInSecond;
assert(current_time_us_ + micros >= current_time_us_);
current_time_us_.fetch_add(micros);
}
// TODO: this is a workaround for the different behavior on different platform
@ -66,7 +84,7 @@ class MockTimeEnv : public EnvWrapper {
}
private:
std::atomic<uint64_t> current_time_{0};
std::atomic<uint64_t> current_time_us_{0};
};
} // namespace ROCKSDB_NAMESPACE

186
util/timer_test.cc
View File

@ -15,56 +15,54 @@ class TimerTest : public testing::Test {
protected:
std::unique_ptr<MockTimeEnv> mock_env_;
const uint64_t kSecond = 1000000; // 1sec = 1000000us
void SetUp() override { mock_env_->InstallTimedWaitFixCallback(); }
const int kUsPerSec = 1000000;
};
TEST_F(TimerTest, SingleScheduleOnceTest) {
const int kInitDelaySec = 1;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
TEST_F(TimerTest, SingleScheduleOnce) {
const int kInitDelayUs = 1 * kUsPerSec;
Timer timer(mock_env_.get());
int count = 0;
timer.Add([&] { count++; }, "fn_sch_test", kInitDelaySec * kSecond, 0);
timer.Add([&] { count++; }, "fn_sch_test", kInitDelayUs, 0);
ASSERT_TRUE(timer.Start());
ASSERT_EQ(0, count);
// Wait for execution to finish
mock_time_sec += kInitDelaySec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kInitDelayUs); });
ASSERT_EQ(1, count);
ASSERT_TRUE(timer.Shutdown());
}
TEST_F(TimerTest, MultipleScheduleOnceTest) {
const int kInitDelay1Sec = 1;
const int kInitDelay2Sec = 3;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
TEST_F(TimerTest, MultipleScheduleOnce) {
const int kInitDelay1Us = 1 * kUsPerSec;
const int kInitDelay2Us = 3 * kUsPerSec;
Timer timer(mock_env_.get());
int count1 = 0;
timer.Add([&] { count1++; }, "fn_sch_test1", kInitDelay1Sec * kSecond, 0);
timer.Add([&] { count1++; }, "fn_sch_test1", kInitDelay1Us, 0);
int count2 = 0;
timer.Add([&] { count2++; }, "fn_sch_test2", kInitDelay2Sec * kSecond, 0);
timer.Add([&] { count2++; }, "fn_sch_test2", kInitDelay2Us, 0);
ASSERT_TRUE(timer.Start());
ASSERT_EQ(0, count1);
ASSERT_EQ(0, count2);
mock_time_sec = kInitDelay1Sec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kInitDelay1Us); });
ASSERT_EQ(1, count1);
ASSERT_EQ(0, count2);
mock_time_sec = kInitDelay2Sec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun([&] {
mock_env_->MockSleepForMicroseconds(kInitDelay2Us - kInitDelay1Us);
});
ASSERT_EQ(1, count1);
ASSERT_EQ(1, count2);
@ -72,70 +70,62 @@ TEST_F(TimerTest, MultipleScheduleOnceTest) {
ASSERT_TRUE(timer.Shutdown());
}
TEST_F(TimerTest, SingleScheduleRepeatedlyTest) {
TEST_F(TimerTest, SingleScheduleRepeatedly) {
const int kIterations = 5;
const int kInitDelaySec = 1;
const int kRepeatSec = 1;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
const int kInitDelayUs = 1 * kUsPerSec;
const int kRepeatUs = 1 * kUsPerSec;
Timer timer(mock_env_.get());
int count = 0;
timer.Add([&] { count++; }, "fn_sch_test", kInitDelaySec * kSecond,
kRepeatSec * kSecond);
timer.Add([&] { count++; }, "fn_sch_test", kInitDelayUs, kRepeatUs);
ASSERT_TRUE(timer.Start());
ASSERT_EQ(0, count);
mock_time_sec += kInitDelaySec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kInitDelayUs); });
ASSERT_EQ(1, count);
// Wait for execution to finish
for (int i = 1; i < kIterations; i++) {
mock_time_sec += kRepeatSec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kRepeatUs); });
}
ASSERT_EQ(kIterations, count);
ASSERT_TRUE(timer.Shutdown());
}
TEST_F(TimerTest, MultipleScheduleRepeatedlyTest) {
const int kInitDelay1Sec = 0;
const int kInitDelay2Sec = 1;
const int kInitDelay3Sec = 0;
const int kRepeatSec = 2;
const int kLargeRepeatSec = 100;
TEST_F(TimerTest, MultipleScheduleRepeatedly) {
const int kIterations = 5;
const int kInitDelay1Us = 0 * kUsPerSec;
const int kInitDelay2Us = 1 * kUsPerSec;
const int kInitDelay3Us = 0 * kUsPerSec;
const int kRepeatUs = 2 * kUsPerSec;
const int kLargeRepeatUs = 100 * kUsPerSec;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
Timer timer(mock_env_.get());
int count1 = 0;
timer.Add([&] { count1++; }, "fn_sch_test1", kInitDelay1Sec * kSecond,
kRepeatSec * kSecond);
timer.Add([&] { count1++; }, "fn_sch_test1", kInitDelay1Us, kRepeatUs);
int count2 = 0;
timer.Add([&] { count2++; }, "fn_sch_test2", kInitDelay2Sec * kSecond,
kRepeatSec * kSecond);
timer.Add([&] { count2++; }, "fn_sch_test2", kInitDelay2Us, kRepeatUs);
// Add a function with relatively large repeat interval
int count3 = 0;
timer.Add([&] { count3++; }, "fn_sch_test3", kInitDelay3Sec * kSecond,
kLargeRepeatSec * kSecond);
timer.Add([&] { count3++; }, "fn_sch_test3", kInitDelay3Us, kLargeRepeatUs);
ASSERT_TRUE(timer.Start());
ASSERT_EQ(0, count2);
ASSERT_EQ(0, count3);
// Wait for execution to finish
for (; count1 < kIterations; mock_time_sec++) {
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
ASSERT_EQ((mock_time_sec + 2) / kRepeatSec, count1);
ASSERT_EQ((mock_time_sec + 1) / kRepeatSec, count2);
for (int i = 1; i < kIterations * (kRepeatUs / kUsPerSec); i++) {
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(1 * kUsPerSec); });
ASSERT_EQ((i + 2) / (kRepeatUs / kUsPerSec), count1);
ASSERT_EQ((i + 1) / (kRepeatUs / kUsPerSec), count2);
// large interval function should only run once (the first one).
ASSERT_EQ(1, count3);
@ -144,8 +134,8 @@ TEST_F(TimerTest, MultipleScheduleRepeatedlyTest) {
timer.Cancel("fn_sch_test1");
// Wait for execution to finish
mock_time_sec++;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(1 * kUsPerSec); });
ASSERT_EQ(kIterations, count1);
ASSERT_EQ(kIterations, count2);
ASSERT_EQ(1, count3);
@ -156,8 +146,10 @@ TEST_F(TimerTest, MultipleScheduleRepeatedlyTest) {
ASSERT_EQ(kIterations, count2);
// execute the long interval one
mock_time_sec = kLargeRepeatSec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun([&] {
mock_env_->MockSleepForMicroseconds(
kLargeRepeatUs - static_cast<int>(mock_env_->NowMicros()));
});
ASSERT_EQ(2, count3);
ASSERT_TRUE(timer.Shutdown());
@ -165,33 +157,30 @@ TEST_F(TimerTest, MultipleScheduleRepeatedlyTest) {
TEST_F(TimerTest, AddAfterStartTest) {
const int kIterations = 5;
const int kInitDelaySec = 1;
const int kRepeatSec = 1;
const int kInitDelayUs = 1 * kUsPerSec;
const int kRepeatUs = 1 * kUsPerSec;
// wait timer to run and then add a new job
SyncPoint::GetInstance()->LoadDependency(
{{"Timer::Run::Waiting", "TimerTest:AddAfterStartTest:1"}});
SyncPoint::GetInstance()->EnableProcessing();
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
Timer timer(mock_env_.get());
ASSERT_TRUE(timer.Start());
TEST_SYNC_POINT("TimerTest:AddAfterStartTest:1");
int count = 0;
timer.Add([&] { count++; }, "fn_sch_test", kInitDelaySec * kSecond,
kRepeatSec * kSecond);
timer.Add([&] { count++; }, "fn_sch_test", kInitDelayUs, kRepeatUs);
ASSERT_EQ(0, count);
// Wait for execution to finish
mock_time_sec += kInitDelaySec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kInitDelayUs); });
ASSERT_EQ(1, count);
for (int i = 1; i < kIterations; i++) {
mock_time_sec += kRepeatSec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kRepeatUs); });
}
ASSERT_EQ(kIterations, count);
@ -199,8 +188,8 @@ TEST_F(TimerTest, AddAfterStartTest) {
}
TEST_F(TimerTest, CancelRunningTask) {
const int kRepeatUs = 1 * kUsPerSec;
constexpr char kTestFuncName[] = "test_func";
mock_env_->set_current_time(0);
Timer timer(mock_env_.get());
ASSERT_TRUE(timer.Start());
int* value = new int;
@ -219,7 +208,7 @@ TEST_F(TimerTest, CancelRunningTask) {
TEST_SYNC_POINT("TimerTest::CancelRunningTask:test_func:0");
TEST_SYNC_POINT("TimerTest::CancelRunningTask:test_func:1");
},
kTestFuncName, 0, 1 * kSecond);
kTestFuncName, 0, kRepeatUs);
port::Thread control_thr([&]() {
TEST_SYNC_POINT("TimerTest::CancelRunningTask:BeforeCancel");
timer.Cancel(kTestFuncName);
@ -228,15 +217,15 @@ TEST_F(TimerTest, CancelRunningTask) {
delete value;
value = nullptr;
});
mock_env_->set_current_time(1);
mock_env_->MockSleepForMicroseconds(kRepeatUs);
control_thr.join();
ASSERT_TRUE(timer.Shutdown());
}
TEST_F(TimerTest, ShutdownRunningTask) {
const int kRepeatUs = 1 * kUsPerSec;
constexpr char kTestFunc1Name[] = "test_func1";
constexpr char kTestFunc2Name[] = "test_func2";
mock_env_->set_current_time(0);
Timer timer(mock_env_.get());
SyncPoint::GetInstance()->DisableProcessing();
@ -258,56 +247,52 @@ TEST_F(TimerTest, ShutdownRunningTask) {
*value = 1;
TEST_SYNC_POINT("TimerTest::ShutdownRunningTest:test_func:1");
},
kTestFunc1Name, 0, 1 * kSecond);
kTestFunc1Name, 0, kRepeatUs);
timer.Add([&]() { ++(*value); }, kTestFunc2Name, 0, 1 * kSecond);
timer.Add([&]() { ++(*value); }, kTestFunc2Name, 0, kRepeatUs);
port::Thread control_thr([&]() {
TEST_SYNC_POINT("TimerTest::ShutdownRunningTest:BeforeShutdown");
timer.Shutdown();
});
mock_env_->set_current_time(1);
mock_env_->MockSleepForMicroseconds(kRepeatUs);
control_thr.join();
delete value;
}
TEST_F(TimerTest, AddSameFuncName) {
const int kInitDelaySec = 1;
const int kRepeat1Sec = 5;
const int kRepeat2Sec = 4;
const int kInitDelayUs = 1 * kUsPerSec;
const int kRepeat1Us = 5 * kUsPerSec;
const int kRepeat2Us = 4 * kUsPerSec;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
Timer timer(mock_env_.get());
ASSERT_TRUE(timer.Start());
int func_counter1 = 0;
timer.Add([&] { func_counter1++; }, "duplicated_func",
kInitDelaySec * kSecond, kRepeat1Sec * kSecond);
timer.Add([&] { func_counter1++; }, "duplicated_func", kInitDelayUs,
kRepeat1Us);
int func2_counter = 0;
timer.Add([&] { func2_counter++; }, "func2", kInitDelaySec * kSecond,
kRepeat2Sec * kSecond);
timer.Add([&] { func2_counter++; }, "func2", kInitDelayUs, kRepeat2Us);
// New function with the same name should override the existing one
int func_counter2 = 0;
timer.Add([&] { func_counter2++; }, "duplicated_func",
kInitDelaySec * kSecond, kRepeat1Sec * kSecond);
timer.Add([&] { func_counter2++; }, "duplicated_func", kInitDelayUs,
kRepeat1Us);
ASSERT_EQ(0, func_counter1);
ASSERT_EQ(0, func2_counter);
ASSERT_EQ(0, func_counter2);
mock_time_sec += kInitDelaySec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kInitDelayUs); });
ASSERT_EQ(0, func_counter1);
ASSERT_EQ(1, func2_counter);
ASSERT_EQ(1, func_counter2);
mock_time_sec += kRepeat1Sec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kRepeat1Us); });
ASSERT_EQ(0, func_counter1);
ASSERT_EQ(2, func2_counter);
@ -317,39 +302,40 @@ TEST_F(TimerTest, AddSameFuncName) {
}
TEST_F(TimerTest, RepeatIntervalWithFuncRunningTime) {
const int kInitDelaySec = 1;
const int kRepeatSec = 5;
const int kFuncRunningTimeSec = 1;
const int kInitDelayUs = 1 * kUsPerSec;
const int kRepeatUs = 5 * kUsPerSec;
const int kFuncRunningTimeUs = 1 * kUsPerSec;
int mock_time_sec = 0;
mock_env_->set_current_time(mock_time_sec);
Timer timer(mock_env_.get());
ASSERT_TRUE(timer.Start());
int func_counter = 0;
timer.Add(
[&] {
mock_env_->set_current_time(mock_time_sec + kFuncRunningTimeSec);
mock_env_->MockSleepForMicroseconds(kFuncRunningTimeUs);
func_counter++;
},
"func", kInitDelaySec * kSecond, kRepeatSec * kSecond);
"func", kInitDelayUs, kRepeatUs);
ASSERT_EQ(0, func_counter);
mock_time_sec += kInitDelaySec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kInitDelayUs); });
ASSERT_EQ(1, func_counter);
ASSERT_EQ(kInitDelayUs + kFuncRunningTimeUs, mock_env_->NowMicros());
// After repeat interval time, the function is not executed, as running
// the function takes some time (`kFuncRunningTimeSec`). The repeat interval
// is the time between ending time of the last call and starting time of the
// next call.
mock_time_sec += kRepeatSec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
uint64_t next_abs_interval_time_us = kInitDelayUs + kRepeatUs;
timer.TEST_WaitForRun([&] {
mock_env_->set_current_time(next_abs_interval_time_us / kUsPerSec);
});
ASSERT_EQ(1, func_counter);
mock_time_sec += kFuncRunningTimeSec;
timer.TEST_WaitForRun([&] { mock_env_->set_current_time(mock_time_sec); });
// After the function running time, it's executed again
timer.TEST_WaitForRun(
[&] { mock_env_->MockSleepForMicroseconds(kFuncRunningTimeUs); });
ASSERT_EQ(2, func_counter);
ASSERT_TRUE(timer.Shutdown());