dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Limit compaction input files expansion (#12484) 

Summary:
We removed the limit in https://github.com/facebook/rocksdb/issues/10835 and the option in https://github.com/facebook/rocksdb/issues/12323. Usually input level is much smaller than output level, which is likely why we have not seen issues with not applying a limit. It should be safer to add a safe guard as suggested in https://github.com/facebook/rocksdb/pull/12323#issuecomment-2016687321.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/12484

Test Plan: * new and existing UT

Reviewed By: ajkr

Differential Revision: D55438870

Pulled By: cbi42

fbshipit-source-id: 0511d0465a70398c36230ed7cced5291ff1a6c19
This commit is contained in:
Changyu Bi 2024-03-29 11:34:29 -07:00 committed by Facebook GitHub Bot
parent d985902ef4
commit 796011e5ad
5 changed files with 61 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
db/compaction/compaction_picker.cc
View File

@ -457,9 +457,10 @@ bool CompactionPicker::IsRangeInCompaction(VersionStorageInfo* vstorage,
// Returns false if files on parent level are currently in compaction, which
// means that we can't compact them
bool CompactionPicker::SetupOtherInputs(
const std::string& cf_name, VersionStorageInfo* vstorage,
CompactionInputFiles* inputs, CompactionInputFiles* output_level_inputs,
int* parent_index, int base_index, bool only_expand_towards_right) {
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, CompactionInputFiles* inputs,
CompactionInputFiles* output_level_inputs, int* parent_index,
int base_index, bool only_expand_towards_right) {
assert(!inputs->empty());
assert(output_level_inputs->empty());
const int input_level = inputs->level;
@ -525,8 +526,15 @@ bool CompactionPicker::SetupOtherInputs(
if (!ExpandInputsToCleanCut(cf_name, vstorage, &expanded_inputs)) {
try_overlapping_inputs = false;
}
// It helps to reduce write amp and avoid a further separate compaction
// to include more input level files without expanding output level files.
// So we apply a softer limit. We still need a limit to avoid overly large
// compactions and potential high space amp spikes.
const uint64_t limit =
MultiplyCheckOverflow(mutable_cf_options.max_compaction_bytes, 2.0);
if (try_overlapping_inputs && expanded_inputs.size() > inputs->size() &&
!AreFilesInCompaction(expanded_inputs.files)) {
!AreFilesInCompaction(expanded_inputs.files) &&
output_level_inputs_size + expanded_inputs_size < limit) {
InternalKey new_start, new_limit;
GetRange(expanded_inputs, &new_start, &new_limit);
CompactionInputFiles expanded_output_level_inputs;
@ -548,7 +556,8 @@ bool CompactionPicker::SetupOtherInputs(
base_index, nullptr);
expanded_inputs_size = TotalFileSize(expanded_inputs.files);
if (expanded_inputs.size() > inputs->size() &&
!AreFilesInCompaction(expanded_inputs.files)) {
!AreFilesInCompaction(expanded_inputs.files) &&
(output_level_inputs_size + expanded_inputs_size) < limit) {
expand_inputs = true;
}
}
@ -808,8 +817,8 @@ Compaction* CompactionPicker::CompactRange(
output_level_inputs.level = output_level;
if (input_level != output_level) {
int parent_index = -1;
if (!SetupOtherInputs(cf_name, vstorage, &inputs, &output_level_inputs,
&parent_index, -1)) {
if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage, &inputs,
&output_level_inputs, &parent_index, -1)) {
// manual compaction is now multi-threaded, so it can
// happen that SetupOtherInputs fails
// we handle it higher in RunManualCompaction

1
db/compaction/compaction_picker.h
View File

@ -186,6 +186,7 @@ class CompactionPicker {
int penultimate_level) const;
bool SetupOtherInputs(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
CompactionInputFiles* inputs,
CompactionInputFiles* output_level_inputs,

5
db/compaction/compaction_picker_level.cc
View File

@ -467,8 +467,9 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
}
if (!is_l0_trivial_move_ &&
!compaction_picker_->SetupOtherInputs(
cf_name_, vstorage_, &start_level_inputs_, &output_level_inputs_,
&parent_index_, base_index_, round_robin_expanding)) {
cf_name_, mutable_cf_options_, vstorage_, &start_level_inputs_,
&output_level_inputs_, &parent_index_, base_index_,
round_robin_expanding)) {
return false;
}

38
db/compaction/compaction_picker_test.cc
View File

@ -2465,7 +2465,7 @@ TEST_F(CompactionPickerTest, IsBottommostLevelTest) {
DeleteVersionStorage();
}
TEST_F(CompactionPickerTest, IgnoreCompactionLimitWhenAddFileFromInputLevel) {
TEST_F(CompactionPickerTest, CompactionLimitWhenAddFileFromInputLevel) {
mutable_cf_options_.max_bytes_for_level_base = 1000000u;
mutable_cf_options_.max_compaction_bytes = 800000u;
ioptions_.level_compaction_dynamic_level_bytes = false;
@ -2473,8 +2473,10 @@ TEST_F(CompactionPickerTest, IgnoreCompactionLimitWhenAddFileFromInputLevel) {
// A compaction should be triggered and pick file 2 and 5.
// It pulls in other compaction input file from the input level L1
// without pulling in more output level files.
// Files 1, 3, 4 will be included in the compaction.
// Files 1, 3, 4 are eligible.
// File 6 is excluded since it overlaps with file 7.
// It can expand input level since in this case, the limit on compaction size
// is 2 * max_compaction_bytes.
Add(1, 1U, "100", "150", 300000U);
Add(1, 2U, "151", "200", 300001U, 0, 0);
Add(1, 3U, "201", "250", 300000U, 0, 0);
@ -2498,6 +2500,38 @@ TEST_F(CompactionPickerTest, IgnoreCompactionLimitWhenAddFileFromInputLevel) {
ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
}
TEST_F(CompactionPickerTest, HitCompactionLimitWhenAddFileFromInputLevel) {
mutable_cf_options_.max_bytes_for_level_base = 1000000u;
mutable_cf_options_.max_compaction_bytes = 800000u;
ioptions_.level_compaction_dynamic_level_bytes = false;
NewVersionStorage(6, kCompactionStyleLevel);
// A compaction should be triggered and pick file 2 and 5.
// It pulls in other compaction input file from the input level L1
// without pulling in more output level files.
// Files 1, 3, 4 are eligible.
// File 6 is excluded since it overlaps with file 7.
// It can not expand input level since total compaction size hit the limit
// 2 * max_compaction_bytes.
Add(1, 1U, "100", "150", 400000U);
Add(1, 2U, "151", "200", 400001U, 0, 0);
Add(1, 3U, "201", "250", 400000U, 0, 0);
Add(1, 4U, "251", "300", 400000U, 0, 0);
Add(1, 6U, "325", "400", 400000U, 0, 0);
Add(2, 5U, "100", "350", 1U);
Add(2, 7U, "375", "425", 1U);
UpdateVersionStorageInfo();
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
&log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(2U, compaction->num_input_levels());
ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(1U, compaction->num_input_files(1));
ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
}
TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
mutable_cf_options_.max_bytes_for_level_base = 10000u;
mutable_cf_options_.max_compaction_bytes = 10001u;

8
db/compaction/compaction_picker_universal.cc
View File

@ -1139,7 +1139,8 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
// from bottom_start_idx and bottom_end_idx, but for now, we use
// SetupOtherInputs() for simplicity.
int parent_index = -1; // Create and use bottom_start_idx?
if (!picker_->SetupOtherInputs(cf_name_, vstorage_, &second_last_level_inputs,
if (!picker_->SetupOtherInputs(cf_name_, mutable_cf_options_, vstorage_,
&second_last_level_inputs,
&bottom_level_inputs, &parent_index,
/*base_index=*/-1)) {
return nullptr;
@ -1310,8 +1311,9 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
int parent_index = -1;
output_level_inputs.level = output_level;
if (!picker_->SetupOtherInputs(cf_name_, vstorage_, &start_level_inputs,
&output_level_inputs, &parent_index, -1)) {
if (!picker_->SetupOtherInputs(cf_name_, mutable_cf_options_, vstorage_,
&start_level_inputs, &output_level_inputs,
&parent_index, -1)) {
return nullptr;
}
inputs.push_back(start_level_inputs);