dolysis/rocksdb
3
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2024-11-26 16:30:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow sufficient subcompactions under round-robin compaction priority (#10422)

Browse source 
Summary:
Allow sufficient subcompactions can be used when the number of input files is less than `max_subcompactions` under round-robin compaction priority.

Test Case:
Add `RoundRobinWithoutAdditionalResources` into `db_compaction_test`

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10422

Reviewed By: ajkr

Differential Revision: D38186545

Pulled By: littlepig2013

fbshipit-source-id: b8e5098306f1e5b9561dfafafc8300a38f7fe88e
This commit is contained in:
Zichen Zhu 2022-07-26 20:37:34 -07:00 committed by Facebook GitHub Bot
parent 9d7de6517c
commit c945a9a664
2 changed files with 70 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
db/compaction/compaction_job.cc
View file

@ -508,6 +508,8 @@ void CompactionJob::GenSubcompactionBoundaries() {
// of planned subcompactions // of planned subcompactions
num_planned_subcompactions = num_planned_subcompactions =
std::min(num_planned_subcompactions, GetSubcompactionsLimit()); std::min(num_planned_subcompactions, GetSubcompactionsLimit());
} else {
num_planned_subcompactions = max_subcompactions_limit;
} }
} else { } else {
num_planned_subcompactions = GetSubcompactionsLimit(); num_planned_subcompactions = GetSubcompactionsLimit();

68
db/db_compaction_test.cc
View file

@ -5511,6 +5511,74 @@ INSTANTIATE_TEST_CASE_P(RoundRobinSubcompactionsAgainstResources,
std::make_tuple(5, 10), std::make_tuple(5, 10),
std::make_tuple(10, 10))); std::make_tuple(10, 10)));
TEST_P(DBCompactionTestWithParam, RoundRobinWithoutAdditionalResources) {
const int kKeysPerBuffer = 200;
Options options = CurrentOptions();
options.num_levels = 4;
options.level0_file_num_compaction_trigger = 3;
options.target_file_size_base = kKeysPerBuffer * 1024;
options.compaction_pri = CompactionPri::kRoundRobin;
options.max_bytes_for_level_base = 30 * kKeysPerBuffer * 1024;
options.disable_auto_compactions = true;
options.max_subcompactions = max_subcompactions_;
options.max_background_compactions = 1;
options.max_compaction_bytes = 100000000;
// Similar experiment setting as above except the max_subcompactions
// is given by max_subcompactions_ (1 or 4), and we fix the
// additional resources as (1, 1) and thus no more extra resources
// can be used
DestroyAndReopen(options);
env_->SetBackgroundThreads(1, Env::LOW);
Random rnd(301);
const std::vector<int> files_per_level = {0, 33, 100};
for (int lvl = 2; lvl > 0; lvl--) {
for (int i = 0; i < files_per_level[lvl]; i++) {
for (int j = 0; j < kKeysPerBuffer; j++) {
// Add (lvl-1) to ensure nearly equivallent number of files
// in L2 are overlapped with fils selected to compact from
// L1
ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)),
rnd.RandomString(1010)));
}
ASSERT_OK(Flush());
}
MoveFilesToLevel(lvl);
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0));
}
// 33 files in L1; 100 files in L2
// This is a variable for making sure the following callback is called
// and the assertions in it are indeed excuted.
bool num_planned_subcompactions_verified = false;
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) {
uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
// At most 4 files are selected for round-robin under auto
// compaction. The number of planned subcompaction is restricted by
// the max_subcompactions since no extra resources can be used
ASSERT_EQ(num_planned_subcompactions, options.max_subcompactions);
num_planned_subcompactions_verified = true;
});
// No need to setup dependency for pressure token since
// AcquireSubcompactionResources may not be called and it anyway cannot
// reserve any additional resources
SyncPoint::GetInstance()->LoadDependency(
{{"DBCompactionTest::RoundRobinWithoutAdditionalResources:0",
"BackgroundCallCompaction:0"}});
SyncPoint::GetInstance()->EnableProcessing();
ASSERT_OK(dbfull()->WaitForCompact());
ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()}));
TEST_SYNC_POINT("DBCompactionTest::RoundRobinWithoutAdditionalResources:0");
ASSERT_OK(dbfull()->WaitForCompact());
ASSERT_TRUE(num_planned_subcompactions_verified);
SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->ClearAllCallBacks();
}
TEST_F(DBCompactionTest, RoundRobinCutOutputAtCompactCursor) { TEST_F(DBCompactionTest, RoundRobinCutOutputAtCompactCursor) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.num_levels = 3; options.num_levels = 3;