2014-10-27 22:49:46 +00:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
|
|
|
|
|
#include "db/compaction_picker.h"
|
2014-12-16 05:48:16 +00:00
|
|
|
#include <limits>
|
2014-10-27 22:49:46 +00:00
|
|
|
#include <string>
|
|
|
|
|
#include "util/logging.h"
|
2015-03-20 00:29:37 +00:00
|
|
|
#include "util/string_util.h"
|
2014-10-27 22:49:46 +00:00
|
|
|
#include "util/testharness.h"
|
|
|
|
|
#include "util/testutil.h"
|
|
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
|
|
class CountingLogger : public Logger {
|
|
|
|
|
public:
|
2015-02-01 19:08:19 +00:00
|
|
|
using Logger::Logv;
|
2014-10-27 22:49:46 +00:00
|
|
|
virtual void Logv(const char* format, va_list ap) override { log_count++; }
|
|
|
|
|
size_t log_count;
|
|
|
|
|
};
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
class CompactionPickerTest : public testing::Test {
|
2014-10-27 22:49:46 +00:00
|
|
|
public:
|
2014-11-13 21:41:43 +00:00
|
|
|
const Comparator* ucmp_;
|
|
|
|
|
InternalKeyComparator icmp_;
|
|
|
|
|
Options options_;
|
|
|
|
|
ImmutableCFOptions ioptions_;
|
|
|
|
|
MutableCFOptions mutable_cf_options_;
|
2014-10-27 22:49:46 +00:00
|
|
|
LevelCompactionPicker level_compaction_picker;
|
2014-11-13 21:41:43 +00:00
|
|
|
std::string cf_name_;
|
|
|
|
|
CountingLogger logger_;
|
|
|
|
|
LogBuffer log_buffer_;
|
|
|
|
|
uint32_t file_num_;
|
|
|
|
|
CompactionOptionsFIFO fifo_options_;
|
|
|
|
|
std::unique_ptr<VersionStorageInfo> vstorage_;
|
|
|
|
|
std::vector<std::unique_ptr<FileMetaData>> files_;
|
2014-10-27 22:49:46 +00:00
|
|
|
|
|
|
|
|
CompactionPickerTest()
|
2014-11-13 21:41:43 +00:00
|
|
|
: ucmp_(BytewiseComparator()),
|
|
|
|
|
icmp_(ucmp_),
|
|
|
|
|
ioptions_(options_),
|
|
|
|
|
mutable_cf_options_(options_, ioptions_),
|
|
|
|
|
level_compaction_picker(ioptions_, &icmp_),
|
|
|
|
|
cf_name_("dummy"),
|
|
|
|
|
log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
|
|
|
|
|
file_num_(1),
|
|
|
|
|
vstorage_(nullptr) {
|
|
|
|
|
fifo_options_.max_table_files_size = 1;
|
|
|
|
|
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
2014-12-16 05:48:16 +00:00
|
|
|
ioptions_.db_paths.emplace_back("dummy",
|
|
|
|
|
std::numeric_limits<uint64_t>::max());
|
2014-10-27 22:49:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
~CompactionPickerTest() {
|
2014-11-13 21:41:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void NewVersionStorage(int num_levels, CompactionStyle style) {
|
|
|
|
|
DeleteVersionStorage();
|
|
|
|
|
options_.num_levels = num_levels;
|
|
|
|
|
vstorage_.reset(new VersionStorageInfo(
|
|
|
|
|
&icmp_, ucmp_, options_.num_levels, style, nullptr));
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
|
2014-11-13 21:41:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void DeleteVersionStorage() {
|
|
|
|
|
vstorage_.reset();
|
|
|
|
|
files_.clear();
|
2014-10-27 22:49:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void Add(int level, uint32_t file_number, const char* smallest,
|
|
|
|
|
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
|
|
|
|
|
SequenceNumber smallest_seq = 100,
|
|
|
|
|
SequenceNumber largest_seq = 100) {
|
2014-11-13 21:41:43 +00:00
|
|
|
assert(level < vstorage_->num_levels());
|
2014-10-27 22:49:46 +00:00
|
|
|
FileMetaData* f = new FileMetaData;
|
|
|
|
|
f->fd = FileDescriptor(file_number, path_id, file_size);
|
|
|
|
|
f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
|
|
|
|
|
f->largest = InternalKey(largest, largest_seq, kTypeValue);
|
|
|
|
|
f->compensated_file_size = file_size;
|
2014-10-31 15:48:19 +00:00
|
|
|
f->refs = 0;
|
2014-11-13 21:41:43 +00:00
|
|
|
vstorage_->AddFile(level, f);
|
|
|
|
|
files_.emplace_back(f);
|
2014-10-27 22:49:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void UpdateVersionStorageInfo() {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
|
2014-11-13 21:41:43 +00:00
|
|
|
vstorage_->UpdateFilesBySize();
|
|
|
|
|
vstorage_->UpdateNumNonEmptyLevels();
|
|
|
|
|
vstorage_->GenerateFileIndexer();
|
|
|
|
|
vstorage_->GenerateLevelFilesBrief();
|
2015-02-05 00:20:25 +00:00
|
|
|
vstorage_->ComputeCompactionScore(mutable_cf_options_, fifo_options_);
|
2014-11-13 21:41:43 +00:00
|
|
|
vstorage_->SetFinalized();
|
2014-10-27 22:49:46 +00:00
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Empty) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(6, kCompactionStyleLevel);
|
2014-10-27 22:49:46 +00:00
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
2014-11-13 21:41:43 +00:00
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Single) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(6, kCompactionStyleLevel);
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
2014-10-27 22:49:46 +00:00
|
|
|
Add(0, 1U, "p", "q");
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
2014-11-13 21:41:43 +00:00
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level0Trigger) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(6, kCompactionStyleLevel);
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
2014-10-27 22:49:46 +00:00
|
|
|
Add(0, 1U, "150", "200");
|
|
|
|
|
Add(0, 2U, "200", "250");
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
2014-11-13 21:41:43 +00:00
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
2014-11-11 21:47:22 +00:00
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level1Trigger) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(6, kCompactionStyleLevel);
|
2014-10-27 22:49:46 +00:00
|
|
|
Add(1, 66U, "150", "200", 1000000000U);
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
2014-11-13 21:41:43 +00:00
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
2014-11-11 21:47:22 +00:00
|
|
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level1Trigger2) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(6, kCompactionStyleLevel);
|
2014-10-30 04:03:45 +00:00
|
|
|
Add(1, 66U, "150", "200", 1000000001U);
|
2014-10-27 22:49:46 +00:00
|
|
|
Add(1, 88U, "201", "300", 1000000000U);
|
2014-10-29 23:45:07 +00:00
|
|
|
Add(2, 6U, "150", "179", 1000000000U);
|
2014-10-27 22:49:46 +00:00
|
|
|
Add(2, 7U, "180", "220", 1000000000U);
|
2014-10-29 23:45:07 +00:00
|
|
|
Add(2, 8U, "221", "300", 1000000000U);
|
2014-10-27 22:49:46 +00:00
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
2014-11-13 21:41:43 +00:00
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
2014-11-11 21:47:22 +00:00
|
|
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(1));
|
2014-10-27 22:49:46 +00:00
|
|
|
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, LevelMaxScore) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(6, kCompactionStyleLevel);
|
|
|
|
|
mutable_cf_options_.target_file_size_base = 10000000;
|
|
|
|
|
mutable_cf_options_.target_file_size_multiplier = 10;
|
2014-10-29 23:45:07 +00:00
|
|
|
Add(0, 1U, "150", "200", 1000000000U);
|
|
|
|
|
// Level 1 score 1.2
|
|
|
|
|
Add(1, 66U, "150", "200", 6000000U);
|
|
|
|
|
Add(1, 88U, "201", "300", 6000000U);
|
|
|
|
|
// Level 2 score 1.8. File 7 is the largest. Should be picked
|
|
|
|
|
Add(2, 6U, "150", "179", 60000000U);
|
|
|
|
|
Add(2, 7U, "180", "220", 60000001U);
|
|
|
|
|
Add(2, 8U, "221", "300", 60000000U);
|
|
|
|
|
// Level 3 score slightly larger than 1
|
|
|
|
|
Add(3, 26U, "150", "170", 260000000U);
|
|
|
|
|
Add(3, 27U, "171", "179", 260000000U);
|
|
|
|
|
Add(3, 28U, "191", "220", 260000000U);
|
|
|
|
|
Add(3, 29U, "221", "300", 260000000U);
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
2014-11-13 21:41:43 +00:00
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
2014-10-29 23:45:07 +00:00
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
2014-11-11 21:47:22 +00:00
|
|
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
2014-10-29 23:45:07 +00:00
|
|
|
ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
|
2014-11-13 21:41:43 +00:00
|
|
|
const int kLevels = 6;
|
|
|
|
|
const int kFileCount = 20;
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
|
2014-11-13 21:41:43 +00:00
|
|
|
for (int level = 0; level < kLevels - 1; ++level) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
NewVersionStorage(kLevels, kCompactionStyleLevel);
|
|
|
|
|
uint64_t file_size = vstorage_->MaxBytesForLevel(level) * 2 / kFileCount;
|
2014-11-13 21:41:43 +00:00
|
|
|
for (int file_count = 1; file_count <= kFileCount; ++file_count) {
|
|
|
|
|
// start a brand new version in each test.
|
|
|
|
|
NewVersionStorage(kLevels, kCompactionStyleLevel);
|
|
|
|
|
for (int i = 0; i < file_count; ++i) {
|
2014-11-25 04:44:49 +00:00
|
|
|
Add(level, i, ToString((i + 100) * 1000).c_str(),
|
|
|
|
|
ToString((i + 100) * 1000 + 999).c_str(),
|
2014-11-13 21:41:43 +00:00
|
|
|
file_size, 0, i * 100, i * 100 + 99);
|
|
|
|
|
}
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
|
2014-11-13 23:21:04 +00:00
|
|
|
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
|
|
|
|
|
vstorage_->CompactionScore(0) >= 1);
|
2014-11-13 21:41:43 +00:00
|
|
|
// release the version storage
|
|
|
|
|
DeleteVersionStorage();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level0TriggerDynamic) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
int num_levels = ioptions_.num_levels;
|
|
|
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_base = 200;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
|
|
|
|
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
|
|
|
|
Add(0, 1U, "150", "200");
|
|
|
|
|
Add(0, 2U, "200", "250");
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
|
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
|
|
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
2015-04-10 22:01:54 +00:00
|
|
|
ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels()));
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
ASSERT_EQ(num_levels - 1, compaction->output_level());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level0TriggerDynamic2) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
int num_levels = ioptions_.num_levels;
|
|
|
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_base = 200;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
|
|
|
|
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
|
|
|
|
Add(0, 1U, "150", "200");
|
|
|
|
|
Add(0, 2U, "200", "250");
|
|
|
|
|
Add(num_levels - 1, 3U, "200", "250", 300U);
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
ASSERT_EQ(vstorage_->base_level(), num_levels - 2);
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
|
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
|
|
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
2015-04-10 22:01:54 +00:00
|
|
|
ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels()));
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
ASSERT_EQ(num_levels - 2, compaction->output_level());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level0TriggerDynamic3) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
int num_levels = ioptions_.num_levels;
|
|
|
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_base = 200;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
|
|
|
|
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
|
|
|
|
Add(0, 1U, "150", "200");
|
|
|
|
|
Add(0, 2U, "200", "250");
|
|
|
|
|
Add(num_levels - 1, 3U, "200", "250", 300U);
|
|
|
|
|
Add(num_levels - 1, 4U, "300", "350", 3000U);
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
ASSERT_EQ(vstorage_->base_level(), num_levels - 3);
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
|
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
|
|
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
2015-04-10 22:01:54 +00:00
|
|
|
ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels()));
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
ASSERT_EQ(num_levels - 3, compaction->output_level());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, Level0TriggerDynamic4) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
int num_levels = ioptions_.num_levels;
|
|
|
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_base = 200;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
|
|
|
|
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
|
|
|
|
Add(0, 1U, "150", "200");
|
|
|
|
|
Add(0, 2U, "200", "250");
|
|
|
|
|
Add(num_levels - 1, 3U, "200", "250", 300U);
|
|
|
|
|
Add(num_levels - 1, 4U, "300", "350", 3000U);
|
|
|
|
|
Add(num_levels - 3, 5U, "150", "180", 3U);
|
|
|
|
|
Add(num_levels - 3, 6U, "181", "300", 3U);
|
|
|
|
|
Add(num_levels - 3, 7U, "400", "450", 3U);
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
ASSERT_EQ(vstorage_->base_level(), num_levels - 3);
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
|
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
|
|
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
2015-04-10 22:01:54 +00:00
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(1));
|
|
|
|
|
ASSERT_EQ(num_levels - 3, compaction->level(1));
|
|
|
|
|
ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(6U, compaction->input(1, 1)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2, static_cast<int>(compaction->num_input_levels()));
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
ASSERT_EQ(num_levels - 3, compaction->output_level());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 19:44:17 +00:00
|
|
|
int num_levels = ioptions_.num_levels;
|
|
|
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_base = 200;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
|
|
|
|
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
|
|
|
|
Add(0, 1U, "150", "200");
|
|
|
|
|
Add(num_levels - 1, 3U, "200", "250", 300U);
|
|
|
|
|
Add(num_levels - 1, 4U, "300", "350", 3000U);
|
|
|
|
|
Add(num_levels - 1, 4U, "400", "450", 3U);
|
|
|
|
|
Add(num_levels - 2, 5U, "150", "180", 300U);
|
|
|
|
|
Add(num_levels - 2, 6U, "181", "350", 500U);
|
|
|
|
|
Add(num_levels - 2, 7U, "400", "450", 200U);
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
|
|
|
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
|
|
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
|
|
|
ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_files(1));
|
|
|
|
|
ASSERT_EQ(3U, compaction->input(1, 0)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(4U, compaction->input(1, 1)->fd.GetNumber());
|
|
|
|
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
|
|
|
ASSERT_EQ(num_levels - 1, compaction->output_level());
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(1, kCompactionStyleUniversal);
|
|
|
|
|
UniversalCompactionPicker universal_compaction_picker(
|
|
|
|
|
ioptions_, &icmp_);
|
|
|
|
|
// must return false when there's no files.
|
2014-11-13 23:21:04 +00:00
|
|
|
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
|
|
|
|
|
false);
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
2015-04-17 23:44:45 +00:00
|
|
|
UpdateVersionStorageInfo();
|
2014-11-13 21:41:43 +00:00
|
|
|
|
|
|
|
|
// verify the trigger given different number of L0 files.
|
|
|
|
|
for (int i = 1;
|
2014-11-13 23:21:04 +00:00
|
|
|
i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
2015-04-17 23:44:45 +00:00
|
|
|
NewVersionStorage(1, kCompactionStyleUniversal);
|
2014-11-25 04:44:49 +00:00
|
|
|
Add(0, i, ToString((i + 100) * 1000).c_str(),
|
|
|
|
|
ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
|
2014-11-13 23:21:04 +00:00
|
|
|
i * 100 + 99);
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
2015-04-17 23:44:45 +00:00
|
|
|
UpdateVersionStorageInfo();
|
2014-11-13 23:21:04 +00:00
|
|
|
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
|
|
|
|
|
vstorage_->CompactionScore(0) >= 1);
|
2014-11-13 21:41:43 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
|
2014-11-13 21:41:43 +00:00
|
|
|
NewVersionStorage(1, kCompactionStyleFIFO);
|
|
|
|
|
const int kFileCount =
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger * 3;
|
|
|
|
|
const uint64_t kFileSize = 100000;
|
|
|
|
|
const uint64_t kMaxSize = kFileSize * kFileCount / 2;
|
|
|
|
|
|
|
|
|
|
fifo_options_.max_table_files_size = kMaxSize;
|
|
|
|
|
ioptions_.compaction_options_fifo = fifo_options_;
|
|
|
|
|
FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
|
|
|
|
|
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
2015-04-17 23:44:45 +00:00
|
|
|
UpdateVersionStorageInfo();
|
2014-11-13 21:41:43 +00:00
|
|
|
// must return false when there's no files.
|
2014-11-13 23:21:04 +00:00
|
|
|
ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false);
|
2014-11-13 21:41:43 +00:00
|
|
|
|
|
|
|
|
// verify whether compaction is needed based on the current
|
|
|
|
|
// size of L0 files.
|
|
|
|
|
uint64_t current_size = 0;
|
|
|
|
|
for (int i = 1; i <= kFileCount; ++i) {
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
2015-04-17 23:44:45 +00:00
|
|
|
NewVersionStorage(1, kCompactionStyleFIFO);
|
2014-11-25 04:44:49 +00:00
|
|
|
Add(0, i, ToString((i + 100) * 1000).c_str(),
|
|
|
|
|
ToString((i + 100) * 1000 + 999).c_str(),
|
2014-11-13 21:41:43 +00:00
|
|
|
kFileSize, 0, i * 100, i * 100 + 99);
|
|
|
|
|
current_size += kFileSize;
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
2015-04-17 23:44:45 +00:00
|
|
|
UpdateVersionStorageInfo();
|
2014-11-13 23:21:04 +00:00
|
|
|
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
|
|
|
|
|
vstorage_->CompactionScore(0) >= 1);
|
2014-11-13 21:41:43 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-05-12 18:16:25 +00:00
|
|
|
// This test exhibits the bug where we don't properly reset parent_index in
|
|
|
|
|
// PickCompaction()
|
|
|
|
|
TEST_F(CompactionPickerTest, ParentIndexResetBug) {
|
|
|
|
|
int num_levels = ioptions_.num_levels;
|
|
|
|
|
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
|
|
|
mutable_cf_options_.max_bytes_for_level_base = 200;
|
|
|
|
|
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
|
|
|
|
Add(0, 1U, "150", "200"); // <- marked for compaction
|
|
|
|
|
Add(1, 3U, "400", "500", 600); // <- this one needs compacting
|
|
|
|
|
Add(2, 4U, "150", "200");
|
|
|
|
|
Add(2, 5U, "201", "210");
|
|
|
|
|
Add(2, 6U, "300", "310");
|
|
|
|
|
Add(2, 7U, "400", "500"); // <- being compacted
|
|
|
|
|
|
|
|
|
|
vstorage_->LevelFiles(2)[3]->being_compacted = true;
|
|
|
|
|
vstorage_->LevelFiles(0)[0]->marked_for_compaction = true;
|
|
|
|
|
|
|
|
|
|
UpdateVersionStorageInfo();
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
|
|
|
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
|
|
|
|
}
|
2014-11-13 21:41:43 +00:00
|
|
|
|
2014-10-27 22:49:46 +00:00
|
|
|
} // namespace rocksdb
|
|
|
|
|
|
2015-03-17 21:08:00 +00:00
|
|
|
int main(int argc, char** argv) {
|
|
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
|
|
|
return RUN_ALL_TESTS();
|
|
|
|
|
}
|