dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mark more files for periodic compaction during offpeak (#12031) 

Summary:
- The struct previously named `OffpeakTimeInfo` has been renamed to `OffpeakTimeOption` to indicate that it's a user-configurable option. Additionally, a new struct, `OffpeakTimeInfo`, has been introduced, which includes two fields: `is_now_offpeak` and `seconds_till_next_offpeak_start`. This change prevents the need to parse the `daily_offpeak_time_utc` string twice.
- It's worth noting that we may consider adding more fields to the `OffpeakTimeInfo` struct, such as `elapsed_seconds` and `total_seconds`, as needed for further optimization.
- Within `VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction()`, we've adjusted the `allowed_time_limit` to include files that are expected to expire by the next offpeak start.
- We might explore further optimizations, such as evenly distributing files to mark during offpeak hours, if the initial approach results in marking too many files simultaneously during the first scoring in offpeak hours. The primary objective of this PR is to prevent periodic compactions during non-offpeak hours when offpeak hours are configured. We'll start with this straightforward solution and assess whether it suffices for now.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/12031

Test Plan:
Unit Tests added
- `DBCompactionTest::LevelPeriodicCompactionOffpeak` for Leveled
- `DBTestUniversalCompaction2::PeriodicCompaction` for Universal

Reviewed By: cbi42

Differential Revision: D50900292

Pulled By: jaykorean

fbshipit-source-id: 267e7d3332d45a5d9881796786c8650fa0a3b43d
This commit is contained in:
Jay Huh 2023-11-06 11:43:59 -08:00 committed by Facebook GitHub Bot
parent a399bbc037
commit 2dab137182
14 changed files with 470 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
db/compaction/compaction_picker_test.cc
View File

@ -86,7 +86,7 @@ class CompactionPickerTestBase : public testing::Test {
&icmp_, ucmp_, options_.num_levels, style, nullptr, false,
EpochNumberRequirement::kMustPresent, ioptions_.clock,
options_.bottommost_file_compaction_delay,
OffpeakTimeInfo(mutable_db_options_.daily_offpeak_time_utc)));
OffpeakTimeOption(mutable_db_options_.daily_offpeak_time_utc)));
vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
}
@ -97,7 +97,7 @@ class CompactionPickerTestBase : public testing::Test {
&icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style,
vstorage_.get(), false, EpochNumberRequirement::kMustPresent,
ioptions_.clock, options_.bottommost_file_compaction_delay,
OffpeakTimeInfo(mutable_db_options_.daily_offpeak_time_utc)));
OffpeakTimeOption(mutable_db_options_.daily_offpeak_time_utc)));
}
void DeleteVersionStorage() {

114
db/db_compaction_test.cc
View File

@ -19,7 +19,7 @@
#include "rocksdb/concurrent_task_limiter.h"
#include "rocksdb/experimental.h"
#include "rocksdb/sst_file_writer.h"
#include "rocksdb/utilities/convenience.h"
#include "test_util/mock_time_env.h"
#include "test_util/sync_point.h"
#include "test_util/testutil.h"
#include "util/concurrent_task_limiter_impl.h"
@ -4794,9 +4794,9 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"VersionEdit::EncodeTo:VarintOldestAncesterTime", [&](void* arg) {
if (if_restart && if_open_all_files) {
std::string* encoded_fieled = static_cast<std::string*>(arg);
*encoded_fieled = "";
PutVarint64(encoded_fieled, 0);
std::string* encoded_field = static_cast<std::string*>(arg);
*encoded_field = "";
PutVarint64(encoded_field, 0);
}
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@ -4942,9 +4942,9 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) {
if (if_restart && if_open_all_files) {
std::string* encoded_fieled = static_cast<std::string*>(arg);
*encoded_fieled = "";
PutVarint64(encoded_fieled, 0);
std::string* encoded_field = static_cast<std::string*>(arg);
*encoded_field = "";
PutVarint64(encoded_field, 0);
}
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@ -5020,6 +5020,106 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) {
}
}
TEST_F(DBCompactionTest, LevelPeriodicCompactionOffpeak) {
// This test simply checks if offpeak adjustment works in Leveled
// Compactions. For testing offpeak periodic compactions in various
// scenarios, please refer to
// DBTestUniversalCompaction2::PeriodicCompactionOffpeak
constexpr int kNumKeysPerFile = 32;
constexpr int kNumLevelFiles = 2;
constexpr int kValueSize = 100;
constexpr int kSecondsPerDay = 86400;
constexpr int kSecondsPerHour = 3600;
constexpr int kSecondsPerMinute = 60;
for (bool if_restart : {false, true}) {
SCOPED_TRACE("if_restart=" + std::to_string(if_restart));
Options options = CurrentOptions();
options.ttl = 0;
options.periodic_compaction_seconds = 5 * kSecondsPerDay; // 5 days
// In the case where all files are opened and doing DB restart
// forcing the file creation time in manifest file to be 0 to
// simulate the case of reading from an old version.
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) {
if (if_restart) {
std::string* encoded_field = static_cast<std::string*>(arg);
*encoded_field = "";
PutVarint64(encoded_field, 0);
}
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
// Just to add some extra random days to current time
Random rnd(test::RandomSeed());
int days = rnd.Uniform(100);
int periodic_compactions = 0;
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
Compaction* compaction = static_cast<Compaction*>(arg);
auto compaction_reason = compaction->compaction_reason();
if (compaction_reason == CompactionReason::kPeriodicCompaction) {
periodic_compactions++;
}
});
// Starting at 12:15AM
int now_hour = 0;
int now_minute = 15;
auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
auto mock_env = std::make_unique<CompositeEnvWrapper>(env_, mock_clock);
options.env = mock_env.get();
mock_clock->SetCurrentTime(days * kSecondsPerDay +
now_hour * kSecondsPerHour +
now_minute * kSecondsPerMinute);
// Offpeak is set from 12:30AM to 4:30AM
options.daily_offpeak_time_utc = "00:30-04:30";
Reopen(options);
for (int i = 0; i < kNumLevelFiles; ++i) {
for (int j = 0; j < kNumKeysPerFile; ++j) {
ASSERT_OK(
Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize)));
}
ASSERT_OK(Flush());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ("2", FilesPerLevel());
ASSERT_EQ(0, periodic_compactions);
// Move clock forward by 1 hour. Now at 1:15AM Day 0. No compaction.
mock_clock->MockSleepForSeconds(1 * kSecondsPerHour);
ASSERT_OK(Put("a", "1"));
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Assert that the files stay in the same level
ASSERT_EQ("3", FilesPerLevel());
ASSERT_EQ(0, periodic_compactions);
MoveFilesToLevel(1);
ASSERT_EQ("0,3", FilesPerLevel());
// Move clock forward by 4 days and check if it triggers periodic
// comapaction at 1:15AM Day 4. Files created on Day 0 at 12:15AM is
// expected to expire before the offpeak starts next day at 12:30AM
mock_clock->MockSleepForSeconds(4 * kSecondsPerDay);
ASSERT_OK(Put("b", "2"));
if (if_restart) {
Reopen(options);
} else {
ASSERT_OK(Flush());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ("1,3", FilesPerLevel());
// The two old files go through the periodic compaction process
ASSERT_EQ(2, periodic_compactions);
Destroy(options);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
}
}
TEST_F(DBCompactionTest, LevelPeriodicCompactionWithOldDB) {
// This test makes sure that periodic compactions are working with a DB
// where file_creation_time of some files is 0.

4
db/db_impl/db_impl.cc
View File

@ -1329,7 +1329,7 @@ Status DBImpl::SetDBOptions(
new_bg_job_limits.max_compactions >
current_bg_job_limits.max_compactions;
const bool offpeak_time_changed =
versions_->offpeak_time_info().daily_offpeak_time_utc !=
versions_->offpeak_time_option().daily_offpeak_time_utc !=
new_db_options.daily_offpeak_time_utc;
if (max_flushes_increased || max_compactions_increased ||
@ -1343,7 +1343,7 @@ Status DBImpl::SetDBOptions(
Env::Priority::LOW);
}
if (offpeak_time_changed) {
versions_->ChangeOffpeakTimeInfo(
versions_->ChangeOffpeakTimeOption(
new_db_options.daily_offpeak_time_utc);
}

105
db/db_options_test.cc
View File

@ -1155,48 +1155,67 @@ TEST_F(DBOptionsTest, OffpeakTimes) {
verify_valid();
}
auto verify_is_now_offpeak = [&](bool expected, int now_utc_hour,
int now_utc_minute, int now_utc_second = 0) {
auto verify_offpeak_info = [&](bool expected_is_now_off_peak,
int expected_seconds_till_next_offpeak_start,
int now_utc_hour, int now_utc_minute,
int now_utc_second = 0) {
auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
// Add some extra random days to current time
int days = rnd.Uniform(100);
mock_clock->SetCurrentTime(days * 86400 + now_utc_hour * 3600 +
now_utc_minute * 60 + now_utc_second);
mock_clock->SetCurrentTime(
days * OffpeakTimeOption::kSecondsPerDay +
now_utc_hour * OffpeakTimeOption::kSecondsPerHour +
now_utc_minute * OffpeakTimeOption::kSecondsPerMinute + now_utc_second);
Status s = DBImpl::TEST_ValidateOptions(options);
ASSERT_OK(s);
auto offpeak_info = OffpeakTimeInfo(options.daily_offpeak_time_utc);
ASSERT_EQ(expected, offpeak_info.IsNowOffpeak(mock_clock.get()));
auto offpeak_option = OffpeakTimeOption(options.daily_offpeak_time_utc);
int64_t now;
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
auto offpeak_info = offpeak_option.GetOffpeakTimeInfo(now);
ASSERT_EQ(expected_is_now_off_peak, offpeak_info.is_now_offpeak);
ASSERT_EQ(expected_seconds_till_next_offpeak_start,
offpeak_info.seconds_till_next_offpeak_start);
};
options.daily_offpeak_time_utc = "";
verify_is_now_offpeak(false, 12, 30);
verify_offpeak_info(false, 0, 12, 30);
options.daily_offpeak_time_utc = "06:30-11:30";
verify_is_now_offpeak(false, 5, 30);
verify_is_now_offpeak(true, 6, 30);
verify_is_now_offpeak(true, 10, 30);
verify_is_now_offpeak(true, 11, 30);
verify_is_now_offpeak(false, 13, 30);
verify_offpeak_info(false, 1 * OffpeakTimeOption::kSecondsPerHour, 5, 30);
verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 6, 30);
verify_offpeak_info(true, 20 * OffpeakTimeOption::kSecondsPerHour, 10, 30);
verify_offpeak_info(true, 19 * OffpeakTimeOption::kSecondsPerHour, 11, 30);
verify_offpeak_info(false, 17 * OffpeakTimeOption::kSecondsPerHour, 13, 30);
options.daily_offpeak_time_utc = "23:30-04:30";
verify_is_now_offpeak(false, 6, 30);
verify_is_now_offpeak(true, 23, 30);
verify_is_now_offpeak(true, 0, 0);
verify_is_now_offpeak(true, 1, 0);
verify_is_now_offpeak(true, 4, 30);
verify_is_now_offpeak(false, 4, 31);
verify_offpeak_info(false, 17 * OffpeakTimeOption::kSecondsPerHour, 6, 30);
verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 23, 30);
verify_offpeak_info(true,
23 * OffpeakTimeOption::kSecondsPerHour +
30 * OffpeakTimeOption::kSecondsPerMinute,
0, 0);
verify_offpeak_info(true,
22 * OffpeakTimeOption::kSecondsPerHour +
30 * OffpeakTimeOption::kSecondsPerMinute,
1, 0);
verify_offpeak_info(true, 19 * OffpeakTimeOption::kSecondsPerHour, 4, 30);
verify_offpeak_info(false,
18 * OffpeakTimeOption::kSecondsPerHour +
59 * OffpeakTimeOption::kSecondsPerMinute,
4, 31);
// Entire day offpeak
options.daily_offpeak_time_utc = "00:00-23:59";
verify_is_now_offpeak(true, 0, 0);
verify_is_now_offpeak(true, 12, 00);
verify_is_now_offpeak(true, 23, 59);
verify_is_now_offpeak(true, 23, 59, 1);
verify_is_now_offpeak(true, 23, 59, 59);
verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 0, 0);
verify_offpeak_info(true, 12 * OffpeakTimeOption::kSecondsPerHour, 12, 00);
verify_offpeak_info(true, 1 * OffpeakTimeOption::kSecondsPerMinute, 23, 59);
verify_offpeak_info(true, 59, 23, 59, 1);
verify_offpeak_info(true, 1, 23, 59, 59);
options.daily_offpeak_time_utc = "";
// Start with a valid option
options.daily_offpeak_time_utc = "01:30-04:15";
DestroyAndReopen(options);
ASSERT_EQ("", dbfull()->GetDBOptions().daily_offpeak_time_utc);
ASSERT_EQ("01:30-04:15", dbfull()->GetDBOptions().daily_offpeak_time_utc);
int may_schedule_compaction_called = 0;
SyncPoint::GetInstance()->SetCallBack(
@ -1204,30 +1223,44 @@ TEST_F(DBOptionsTest, OffpeakTimes) {
[&](void*) { may_schedule_compaction_called++; });
SyncPoint::GetInstance()->EnableProcessing();
// Make sure calling SetDBOptions with invalid option does not set the value
// nor call MaybeScheduleFlushOrCompaction()
// Make sure calling SetDBOptions with invalid option does not change the
// value nor call MaybeScheduleFlushOrCompaction()
for (std::string invalid_case : invalid_cases) {
ASSERT_NOK(
dbfull()->SetDBOptions({{"daily_offpeak_time_utc", invalid_case}}));
ASSERT_EQ(
"",
dbfull()->GetVersionSet()->offpeak_time_info().daily_offpeak_time_utc);
ASSERT_EQ("01:30-04:15", dbfull()
->GetVersionSet()
->offpeak_time_option()
.daily_offpeak_time_utc);
ASSERT_EQ(1 * kSecondInHour + 30 * kSecondInMinute,
dbfull()
->GetVersionSet()
->offpeak_time_option()
.daily_offpeak_start_time_utc);
ASSERT_EQ(4 * kSecondInHour + 15 * kSecondInMinute,
dbfull()
->GetVersionSet()
->offpeak_time_option()
.daily_offpeak_end_time_utc);
}
ASSERT_EQ(0, may_schedule_compaction_called);
// Changing to new valid values should call MaybeScheduleFlushOrCompaction()
// and sets the offpeak_time_info in VersionSet
// and sets the offpeak_time_option in VersionSet
int expected_count = 0;
for (std::string valid_case : valid_cases) {
if (dbfull()->GetVersionSet()->offpeak_time_info().daily_offpeak_time_utc !=
valid_case) {
if (dbfull()
->GetVersionSet()
->offpeak_time_option()
.daily_offpeak_time_utc != valid_case) {
expected_count++;
}
ASSERT_OK(dbfull()->SetDBOptions({{"daily_offpeak_time_utc", valid_case}}));
ASSERT_EQ(valid_case, dbfull()->GetDBOptions().daily_offpeak_time_utc);
ASSERT_EQ(
valid_case,
dbfull()->GetVersionSet()->offpeak_time_info().daily_offpeak_time_utc);
ASSERT_EQ(valid_case, dbfull()
->GetVersionSet()
->offpeak_time_option()
.daily_offpeak_time_utc);
}
ASSERT_EQ(expected_count, may_schedule_compaction_called);

138
db/db_universal_compaction_test.cc
View File

@ -7,9 +7,12 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <string>
#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "rocksdb/utilities/table_properties_collectors.h"
#include "test_util/mock_time_env.h"
#include "test_util/sync_point.h"
#include "test_util/testutil.h"
#include "util/random.h"
@ -2230,6 +2233,141 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) {
ASSERT_EQ(4, output_level);
}
TEST_F(DBTestUniversalCompaction2, PeriodicCompactionOffpeak) {
constexpr int kSecondsPerDay = 86400;
constexpr int kSecondsPerHour = 3600;
constexpr int kSecondsPerMinute = 60;
Options opts = CurrentOptions();
opts.compaction_style = kCompactionStyleUniversal;
opts.level0_file_num_compaction_trigger = 10;
opts.max_open_files = -1;
opts.compaction_options_universal.size_ratio = 10;
opts.compaction_options_universal.min_merge_width = 2;
opts.compaction_options_universal.max_size_amplification_percent = 200;
opts.periodic_compaction_seconds = 5 * kSecondsPerDay; // 5 days
opts.num_levels = 5;
// Just to add some extra random days to current time
Random rnd(test::RandomSeed());
int days = rnd.Uniform(100);
int periodic_compactions = 0;
int start_level = -1;
int output_level = -1;
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"UniversalCompactionPicker::PickPeriodicCompaction:Return",
[&](void* arg) {
Compaction* compaction = reinterpret_cast<Compaction*>(arg);
ASSERT_TRUE(arg != nullptr);
ASSERT_TRUE(compaction->compaction_reason() ==
CompactionReason::kPeriodicCompaction);
start_level = compaction->start_level();
output_level = compaction->output_level();
periodic_compactions++;
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
for (std::string preset_offpeak_time : {"", "00:30-04:30", "10:30-02:30"}) {
SCOPED_TRACE("preset_offpeak_time=" + preset_offpeak_time);
for (std::string new_offpeak_time : {"", "23:30-02:30"}) {
SCOPED_TRACE("new_offpeak_time=" + new_offpeak_time);
std::vector<std::pair<int, int>> times_to_test = {
{0, 0}, {2, 30}, {3, 15}, {5, 10}, {13, 30}, {23, 30}};
for (std::pair<int, int> now : times_to_test) {
int now_hour = now.first;
int now_minute = now.second;
SCOPED_TRACE("now=" + std::to_string(now_hour) + ":" +
std::to_string(now_minute));
auto mock_clock =
std::make_shared<MockSystemClock>(env_->GetSystemClock());
auto mock_env = std::make_unique<CompositeEnvWrapper>(env_, mock_clock);
opts.env = mock_env.get();
mock_clock->SetCurrentTime(days * kSecondsPerDay +
now_hour * kSecondsPerHour +
now_minute * kSecondsPerMinute);
opts.daily_offpeak_time_utc = preset_offpeak_time;
Reopen(opts);
ASSERT_OK(Put("foo", "bar1"));
ASSERT_OK(Flush());
ASSERT_EQ(0, periodic_compactions);
// Move clock forward by 8 hours. There should be no periodic
// compaction, yet.
mock_clock->MockSleepForSeconds(8 * kSecondsPerHour);
ASSERT_OK(Put("foo", "bar2"));
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, periodic_compactions);
// Move clock forward by 4 days
mock_clock->MockSleepForSeconds(4 * kSecondsPerDay);
ASSERT_OK(Put("foo", "bar3"));
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
int64_t mock_now;
ASSERT_OK(mock_clock->GetCurrentTime(&mock_now));
auto offpeak_time_info =
dbfull()->GetVersionSet()->offpeak_time_option().GetOffpeakTimeInfo(
mock_now);
// At this point, the first file is 4 days and 8 hours old.
// If it's offpeak now and the file is expected to expire before the
// next offpeak starts
if (offpeak_time_info.is_now_offpeak &&
offpeak_time_info.seconds_till_next_offpeak_start /
kSecondsPerHour >
16) {
ASSERT_EQ(1, periodic_compactions);
} else {
ASSERT_EQ(0, periodic_compactions);
// Change offpeak option by SetDBOption()
if (preset_offpeak_time != new_offpeak_time) {
ASSERT_OK(dbfull()->SetDBOptions(
{{"daily_offpeak_time_utc", new_offpeak_time}}));
ASSERT_OK(Put("foo", "bar4"));
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
offpeak_time_info = dbfull()
->GetVersionSet()
->offpeak_time_option()
.GetOffpeakTimeInfo(mock_now);
// if the first file is now eligible to be picked up
if (offpeak_time_info.is_now_offpeak &&
offpeak_time_info.seconds_till_next_offpeak_start /
kSecondsPerHour >
16) {
ASSERT_OK(Put("foo", "bar5"));
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, periodic_compactions);
}
}
// If the file has not been picked up yet (no offpeak set, or offpeak
// set but then unset before the file becomes eligible)
if (periodic_compactions == 0) {
// move clock forward by one more day
mock_clock->MockSleepForSeconds(1 * kSecondsPerDay);
ASSERT_OK(Put("foo", "bar6"));
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
}
ASSERT_EQ(1, periodic_compactions);
ASSERT_EQ(0, start_level);
ASSERT_EQ(4, output_level);
Destroy(opts);
periodic_compactions = 0;
}
}
}
}
} // namespace ROCKSDB_NAMESPACE

2
db/import_column_family_job.cc
View File

@ -187,7 +187,7 @@ Status ImportColumnFamilyJob::Run() {
nullptr /* src_vstorage */, cfd_->ioptions()->force_consistency_checks,
EpochNumberRequirement::kMightMissing, cfd_->ioptions()->clock,
cfd_->GetLatestMutableCFOptions()->bottommost_file_compaction_delay,
cfd_->current()->version_set()->offpeak_time_info());
cfd_->current()->version_set()->offpeak_time_option());
Status s;
for (size_t i = 0; s.ok() && i < files_to_import_.size(); ++i) {

2
db/repair.cc
View File

@ -695,7 +695,7 @@ class Repairer {
nullptr /* src_vstorage */, cfd->ioptions()->force_consistency_checks,
EpochNumberRequirement::kMightMissing, cfd->ioptions()->clock,
/*bottommost_file_compaction_delay=*/0,
cfd->current()->version_set()->offpeak_time_info());
cfd->current()->version_set()->offpeak_time_option());
Status s;
VersionEdit dummy_edit;
for (const auto* table : cf_id_and_tables.second) {

46
db/version_builder_test.cc
View File

@ -39,7 +39,7 @@ class VersionBuilderTest : public testing::Test {
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
nullptr, false, EpochNumberRequirement::kMustPresent,
ioptions_.clock, options_.bottommost_file_compaction_delay,
OffpeakTimeInfo(options_.daily_offpeak_time_utc)),
OffpeakTimeOption(options_.daily_offpeak_time_utc)),
file_num_(1) {
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
size_being_compacted_.resize(options_.num_levels);
@ -204,7 +204,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -256,7 +256,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -312,7 +312,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -370,7 +370,7 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -394,7 +394,7 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
VersionEdit version_edit;
version_edit.AddFile(
@ -563,7 +563,7 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -708,7 +708,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -753,7 +753,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -893,7 +893,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -967,7 +967,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -1148,7 +1148,7 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -1197,7 +1197,7 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
VersionStorageInfo newer_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &new_vstorage,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(second_builder.SaveTo(&newer_vstorage));
@ -1284,7 +1284,7 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -1388,7 +1388,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -1428,7 +1428,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
const Status s = builder.SaveTo(&new_vstorage);
ASSERT_TRUE(s.IsCorruption());
@ -1470,7 +1470,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
const Status s = builder.SaveTo(&new_vstorage);
ASSERT_TRUE(s.IsCorruption());
@ -1520,7 +1520,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbageLinkedSsts) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
const Status s = builder.SaveTo(&new_vstorage);
ASSERT_TRUE(s.IsCorruption());
@ -1684,7 +1684,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
VersionStorageInfo new_vstorage(
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_,
force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr,
0, OffpeakTimeInfo(options_.daily_offpeak_time_utc));
0, OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(builder.SaveTo(&new_vstorage));
@ -1737,7 +1737,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr,
true /* force_consistency_checks */,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -1749,7 +1749,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr,
true /* force_consistency_checks */,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_NOK(version_builder2.Apply(&version_edit));
UnrefFilesInVersion(&new_vstorage);
@ -1789,7 +1789,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForL0FilesSortedByEpochNumber) {
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
nullptr /* src_vstorage */, true /* force_consistency_checks */,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder_1.Apply(&version_edit_1));
s = version_builder_1.SaveTo(&new_vstorage_1);
@ -1828,7 +1828,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForL0FilesSortedByEpochNumber) {
&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
nullptr /* src_vstorage */, true /* force_consistency_checks */,
EpochNumberRequirement::kMightMissing, nullptr, 0,
OffpeakTimeInfo(options_.daily_offpeak_time_utc));
OffpeakTimeOption(options_.daily_offpeak_time_utc));
ASSERT_OK(version_builder_2.Apply(&version_edit_2));
s = version_builder_2.SaveTo(&new_vstorage_2);

20
db/version_set.cc
View File

@ -2125,7 +2125,7 @@ VersionStorageInfo::VersionStorageInfo(
bool _force_consistency_checks,
EpochNumberRequirement epoch_number_requirement, SystemClock* clock,
uint32_t bottommost_file_compaction_delay,
OffpeakTimeInfo offpeak_time_info)
OffpeakTimeOption offpeak_time_option)
: internal_comparator_(internal_comparator),
user_comparator_(user_comparator),
// cfd is nullptr if Version is dummy
@ -2158,7 +2158,7 @@ VersionStorageInfo::VersionStorageInfo(
finalized_(false),
force_consistency_checks_(_force_consistency_checks),
epoch_number_requirement_(epoch_number_requirement),
offpeak_time_info_(offpeak_time_info) {
offpeak_time_option_(std::move(offpeak_time_option)) {
if (ref_vstorage != nullptr) {
accumulated_file_size_ = ref_vstorage->accumulated_file_size_;
accumulated_raw_key_size_ = ref_vstorage->accumulated_raw_key_size_;
@ -2204,7 +2204,7 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
cfd_ == nullptr ? nullptr : cfd_->ioptions()->clock,
cfd_ == nullptr ? 0
: mutable_cf_options.bottommost_file_compaction_delay,
vset->offpeak_time_info()),
vset->offpeak_time_option()),
vset_(vset),
next_(this),
prev_(this),
@ -3672,6 +3672,16 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction(
const uint64_t allowed_time_limit =
current_time - periodic_compaction_seconds;
// Find the adjust_allowed_time_limit such that it includes files that are
// going to expire by the time next daily offpeak starts.
const OffpeakTimeInfo offpeak_time_info =
offpeak_time_option_.GetOffpeakTimeInfo(current_time);
const uint64_t adjusted_allowed_time_limit =
allowed_time_limit +
(offpeak_time_info.is_now_offpeak
? offpeak_time_info.seconds_till_next_offpeak_start
: 0);
for (int level = 0; level <= last_level; level++) {
for (auto f : files_[level]) {
if (!f->being_compacted) {
@ -3698,7 +3708,7 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction(
}
}
if (file_modification_time > 0 &&
file_modification_time < allowed_time_limit) {
file_modification_time < adjusted_allowed_time_limit) {
files_marked_for_periodic_compaction_.emplace_back(level, f);
}
}
@ -5077,7 +5087,7 @@ VersionSet::VersionSet(
block_cache_tracer_(block_cache_tracer),
io_tracer_(io_tracer),
db_session_id_(db_session_id),
offpeak_time_info_(OffpeakTimeInfo(daily_offpeak_time_utc)) {}
offpeak_time_option_(OffpeakTimeOption(daily_offpeak_time_utc)) {}
VersionSet::~VersionSet() {
// we need to delete column_family_set_ because its destructor depends on

16
db/version_set.h
View File

@ -136,7 +136,7 @@ class VersionStorageInfo {
EpochNumberRequirement epoch_number_requirement,
SystemClock* clock,
uint32_t bottommost_file_compaction_delay,
OffpeakTimeInfo offpeak_time_info);
OffpeakTimeOption offpeak_time_option);
// No copying allowed
VersionStorageInfo(const VersionStorageInfo&) = delete;
void operator=(const VersionStorageInfo&) = delete;
@ -766,7 +766,7 @@ class VersionStorageInfo {
EpochNumberRequirement epoch_number_requirement_;
OffpeakTimeInfo offpeak_time_info_;
OffpeakTimeOption offpeak_time_option_;
friend class Version;
friend class VersionSet;
@ -1508,9 +1508,11 @@ class VersionSet {
}
// TODO - Consider updating together when file options change in SetDBOptions
const OffpeakTimeInfo& offpeak_time_info() { return offpeak_time_info_; }
void ChangeOffpeakTimeInfo(const std::string& daily_offpeak_time_utc) {
offpeak_time_info_.daily_offpeak_time_utc = daily_offpeak_time_utc;
const OffpeakTimeOption& offpeak_time_option() {
return offpeak_time_option_;
}
void ChangeOffpeakTimeOption(const std::string& daily_offpeak_time_utc) {
offpeak_time_option_.SetFromOffpeakTimeString(daily_offpeak_time_utc);
}
const ImmutableDBOptions* db_options() const { return db_options_; }
@ -1663,8 +1665,8 @@ class VersionSet {
std::string db_session_id_;
// Off-peak time information used for compaction scoring
OffpeakTimeInfo offpeak_time_info_;
// Off-peak time option used for compaction scoring
OffpeakTimeOption offpeak_time_option_;
private:
// REQUIRES db mutex at beginning. may release and re-acquire db mutex

58
db/version_set_test.cc
View File

@ -134,7 +134,7 @@ class VersionStorageInfoTestBase : public testing::Test {
/*_force_consistency_checks=*/false,
EpochNumberRequirement::kMustPresent, ioptions_.clock,
mutable_cf_options_.bottommost_file_compaction_delay,
OffpeakTimeInfo()) {}
OffpeakTimeOption()) {}
~VersionStorageInfoTestBase() override {
for (int i = 0; i < vstorage_.num_levels(); ++i) {
@ -2200,63 +2200,89 @@ TEST_F(VersionSetTest, OffpeakTimeInfoTest) {
// and see if IsNowOffpeak() returns correctly per time changes
int now_hour = 13;
int now_minute = 30;
versions_->ChangeOffpeakTimeInfo("23:30-04:30");
versions_->ChangeOffpeakTimeOption("23:30-04:30");
auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
// Add some extra random days to current time
int days = rnd.Uniform(100);
mock_clock->SetCurrentTime(days * 86400 + now_hour * 3600 + now_minute * 60);
int64_t now;
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
// Starting at 1:30PM. It's not off-peak
ASSERT_FALSE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_FALSE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Now it's at 4:30PM. Still not off-peak
mock_clock->MockSleepForSeconds(3 * 3600);
ASSERT_FALSE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_FALSE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Now it's at 11:30PM. It's off-peak
mock_clock->MockSleepForSeconds(7 * 3600);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Now it's at 2:30AM next day. It's still off-peak
mock_clock->MockSleepForSeconds(3 * 3600);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Now it's at 4:30AM. It's still off-peak
mock_clock->MockSleepForSeconds(2 * 3600);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Sleep for one more minute. It's at 4:31AM It's no longer off-peak
mock_clock->MockSleepForSeconds(60);
ASSERT_FALSE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_FALSE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Entire day offpeak
versions_->ChangeOffpeakTimeInfo("00:00-23:59");
versions_->ChangeOffpeakTimeOption("00:00-23:59");
// It doesn't matter what time it is. It should be just offpeak.
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Mock Sleep for 3 hours. It's still off-peak
mock_clock->MockSleepForSeconds(3 * 3600);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Mock Sleep for 20 hours. It's still off-peak
mock_clock->MockSleepForSeconds(20 * 3600);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Mock Sleep for 59 minutes. It's still off-peak
mock_clock->MockSleepForSeconds(59 * 60);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Mock Sleep for 59 seconds. It's still off-peak
mock_clock->MockSleepForSeconds(59);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Mock Sleep for 1 second (exactly 24h passed). It's still off-peak
mock_clock->MockSleepForSeconds(1);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
// Another second for sanity check
mock_clock->MockSleepForSeconds(1);
ASSERT_TRUE(versions_->offpeak_time_info().IsNowOffpeak(mock_clock.get()));
ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
ASSERT_TRUE(
versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak);
}
TEST_F(VersionStorageInfoTest, AddRangeDeletionCompensatedFileSize) {

73
options/offpeak_time_info.cc
View File

@ -9,40 +9,51 @@
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
OffpeakTimeInfo::OffpeakTimeInfo() : daily_offpeak_time_utc("") {}
OffpeakTimeInfo::OffpeakTimeInfo(const std::string& offpeak_time)
: daily_offpeak_time_utc(offpeak_time) {}
OffpeakTimeOption::OffpeakTimeOption() : OffpeakTimeOption("") {}
OffpeakTimeOption::OffpeakTimeOption(const std::string& offpeak_time_string) {
SetFromOffpeakTimeString(offpeak_time_string);
}
bool OffpeakTimeInfo::IsNowOffpeak(SystemClock* clock) const {
if (daily_offpeak_time_utc.empty()) {
return false;
void OffpeakTimeOption::SetFromOffpeakTimeString(
const std::string& offpeak_time_string) {
const int old_start_time = daily_offpeak_start_time_utc;
const int old_end_time = daily_offpeak_end_time_utc;
if (TryParseTimeRangeString(offpeak_time_string, daily_offpeak_start_time_utc,
daily_offpeak_end_time_utc)) {
daily_offpeak_time_utc = offpeak_time_string;
} else {
daily_offpeak_start_time_utc = old_start_time;
daily_offpeak_end_time_utc = old_end_time;
}
int64_t now;
if (clock->GetCurrentTime(&now).ok()) {
constexpr int kSecondsPerDay = 86400;
constexpr int kSecondsPerMinute = 60;
int seconds_since_midnight_to_nearest_minute =
(static_cast<int>(now % kSecondsPerDay) / kSecondsPerMinute) *
kSecondsPerMinute;
int start_time = 0, end_time = 0;
bool success =
TryParseTimeRangeString(daily_offpeak_time_utc, start_time, end_time);
assert(success);
assert(start_time != end_time);
if (!success) {
// If the validation was done properly, we should never reach here
return false;
}
// if the offpeak duration spans overnight (i.e. 23:30 - 4:30 next day)
if (start_time > end_time) {
return start_time <= seconds_since_midnight_to_nearest_minute ||
seconds_since_midnight_to_nearest_minute <= end_time;
} else {
return start_time <= seconds_since_midnight_to_nearest_minute &&
seconds_since_midnight_to_nearest_minute <= end_time;
}
}
OffpeakTimeInfo OffpeakTimeOption::GetOffpeakTimeInfo(
const int64_t& current_time) const {
OffpeakTimeInfo offpeak_time_info;
if (daily_offpeak_start_time_utc == daily_offpeak_end_time_utc) {
return offpeak_time_info;
}
return false;
int seconds_since_midnight = static_cast<int>(current_time % kSecondsPerDay);
int seconds_since_midnight_to_nearest_minute =
(seconds_since_midnight / kSecondsPerMinute) * kSecondsPerMinute;
// if the offpeak duration spans overnight (i.e. 23:30 - 4:30 next day)
if (daily_offpeak_start_time_utc > daily_offpeak_end_time_utc) {
offpeak_time_info.is_now_offpeak =
daily_offpeak_start_time_utc <=
seconds_since_midnight_to_nearest_minute ||
seconds_since_midnight_to_nearest_minute <= daily_offpeak_end_time_utc;
} else {
offpeak_time_info.is_now_offpeak =
daily_offpeak_start_time_utc <=
seconds_since_midnight_to_nearest_minute &&
seconds_since_midnight_to_nearest_minute <= daily_offpeak_end_time_utc;
}
offpeak_time_info.seconds_till_next_offpeak_start =
seconds_since_midnight < daily_offpeak_start_time_utc
? daily_offpeak_start_time_utc - seconds_since_midnight
: ((daily_offpeak_start_time_utc + kSecondsPerDay) -
seconds_since_midnight);
return offpeak_time_info;
}
} // namespace ROCKSDB_NAMESPACE

22
options/offpeak_time_info.h
View File

@ -13,10 +13,24 @@ namespace ROCKSDB_NAMESPACE {
class SystemClock;
struct OffpeakTimeInfo {
OffpeakTimeInfo();
explicit OffpeakTimeInfo(const std::string& offpeak_time);
std::string daily_offpeak_time_utc;
bool IsNowOffpeak(SystemClock* clock) const;
bool is_now_offpeak = false;
int seconds_till_next_offpeak_start = 0;
};
struct OffpeakTimeOption {
static constexpr int kSecondsPerDay = 86400;
static constexpr int kSecondsPerHour = 3600;
static constexpr int kSecondsPerMinute = 60;
OffpeakTimeOption();
explicit OffpeakTimeOption(const std::string& offpeak_time_string);
std::string daily_offpeak_time_utc = "";
int daily_offpeak_start_time_utc = 0;
int daily_offpeak_end_time_utc = 0;
void SetFromOffpeakTimeString(const std::string& offpeak_time_string);
OffpeakTimeInfo GetOffpeakTimeInfo(const int64_t& current_time) const;
};
} // namespace ROCKSDB_NAMESPACE

1
unreleased_history/behavior_changes/periodic_compaction_during_offpeak.md Normal file
View File

@ -0,0 +1 @@
During off-peak hours defined by `daily_offpeak_time_utc`, the compaction picker will select a larger number of files for periodic compaction. This selection will include files that are projected to expire by the next off-peak start time, ensuring that these files are not chosen for periodic compaction outside of off-peak hours.