mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-27 11:43:49 +00:00
d985902ef4
Summary: **Context/Summary:** We recently discovered that `CompactRange(change_level=true, target_level=0)` can possibly refit more than 1 files to L0. This refitting can cause read performance regression as we need to go through every file in L0, corruption in some edge case and false positive corruption caught by force consistency check. We decided to explicitly disallow such behavior. A related change to OptionChangeMigration(): - When migrating to FIFO with `compaction_options_fifo.max_table_files_size > 0`, RocksDB will [CompactRange() all the to-be-migrate data into a couple L0 files](https://github.com/facebook/rocksdb/blob/main/utilities/option_change_migration/option_change_migration.cc#L164-L169) to avoid dropping all the data upon migration finishes when the migrated data is larger than max_table_files_size. This is achieved by first compacting all the data into a couple non-L0 files and refitting those files from non-L0 to L0 if needed. In that way, only some data instead of all data will be dropped immediately after migration to FIFO with a max_table_files_size. - Since this type of refitting behavior is disallowed from now on, we won't do this trick anymore and explicitly state such risk in API comment. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12481 Test Plan: - New UT - Modified UT Reviewed By: cbi42 Differential Revision: D55351178 Pulled By: hx235 fbshipit-source-id: 9d8854f2f81d7e8aff859c3a4e53b7d688048e80
566 lines
22 KiB
C++
566 lines
22 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#include "rocksdb/utilities/option_change_migration.h"
|
|
|
|
#include <cstdint>
|
|
#include <limits>
|
|
#include <set>
|
|
|
|
#include "db/db_test_util.h"
|
|
#include "port/stack_trace.h"
|
|
#include "util/random.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
class DBOptionChangeMigrationTests
|
|
: public DBTestBase,
|
|
public testing::WithParamInterface<
|
|
std::tuple<int, int, bool, int, int, bool, uint64_t>> {
|
|
public:
|
|
DBOptionChangeMigrationTests()
|
|
: DBTestBase("db_option_change_migration_test", /*env_do_fsync=*/true) {
|
|
level1_ = std::get<0>(GetParam());
|
|
compaction_style1_ = std::get<1>(GetParam());
|
|
is_dynamic1_ = std::get<2>(GetParam());
|
|
|
|
level2_ = std::get<3>(GetParam());
|
|
compaction_style2_ = std::get<4>(GetParam());
|
|
is_dynamic2_ = std::get<5>(GetParam());
|
|
// This is set to be extremely large if not zero to avoid dropping any data
|
|
// right after migration, which makes test verification difficult
|
|
fifo_max_table_files_size_ = std::get<6>(GetParam());
|
|
}
|
|
|
|
// Required if inheriting from testing::WithParamInterface<>
|
|
static void SetUpTestCase() {}
|
|
static void TearDownTestCase() {}
|
|
|
|
int level1_;
|
|
int compaction_style1_;
|
|
bool is_dynamic1_;
|
|
|
|
int level2_;
|
|
int compaction_style2_;
|
|
bool is_dynamic2_;
|
|
|
|
uint64_t fifo_max_table_files_size_;
|
|
};
|
|
|
|
TEST_P(DBOptionChangeMigrationTests, Migrate1) {
|
|
Options old_options = CurrentOptions();
|
|
old_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style1_);
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
|
|
}
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
old_options.max_open_files = -1;
|
|
}
|
|
old_options.level0_file_num_compaction_trigger = 3;
|
|
old_options.write_buffer_size = 64 * 1024;
|
|
old_options.target_file_size_base = 128 * 1024;
|
|
// Make level target of L1, L2 to be 200KB and 600KB
|
|
old_options.num_levels = level1_;
|
|
old_options.max_bytes_for_level_multiplier = 3;
|
|
old_options.max_bytes_for_level_base = 200 * 1024;
|
|
|
|
Reopen(old_options);
|
|
|
|
Random rnd(301);
|
|
int key_idx = 0;
|
|
|
|
// Generate at least 2MB of data
|
|
for (int num = 0; num < 20; num++) {
|
|
GenerateNewFile(&rnd, &key_idx);
|
|
}
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
|
|
// Will make sure exactly those keys are in the DB after migration.
|
|
std::set<std::string> keys;
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (; it->Valid(); it->Next()) {
|
|
keys.insert(it->key().ToString());
|
|
}
|
|
ASSERT_OK(it->status());
|
|
}
|
|
Close();
|
|
|
|
Options new_options = old_options;
|
|
new_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style2_);
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
|
|
}
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
new_options.max_open_files = -1;
|
|
}
|
|
if (fifo_max_table_files_size_ != 0) {
|
|
new_options.compaction_options_fifo.max_table_files_size =
|
|
fifo_max_table_files_size_;
|
|
}
|
|
new_options.target_file_size_base = 256 * 1024;
|
|
new_options.num_levels = level2_;
|
|
new_options.max_bytes_for_level_base = 150 * 1024;
|
|
new_options.max_bytes_for_level_multiplier = 4;
|
|
ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
|
|
Reopen(new_options);
|
|
|
|
// Wait for compaction to finish and make sure it can reopen
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
Reopen(new_options);
|
|
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (const std::string& key : keys) {
|
|
ASSERT_TRUE(it->Valid());
|
|
ASSERT_EQ(key, it->key().ToString());
|
|
it->Next();
|
|
}
|
|
ASSERT_TRUE(!it->Valid());
|
|
ASSERT_OK(it->status());
|
|
}
|
|
}
|
|
|
|
TEST_P(DBOptionChangeMigrationTests, Migrate2) {
|
|
Options old_options = CurrentOptions();
|
|
old_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style2_);
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
|
|
}
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
old_options.max_open_files = -1;
|
|
}
|
|
old_options.level0_file_num_compaction_trigger = 3;
|
|
old_options.write_buffer_size = 64 * 1024;
|
|
old_options.target_file_size_base = 128 * 1024;
|
|
// Make level target of L1, L2 to be 200KB and 600KB
|
|
old_options.num_levels = level2_;
|
|
old_options.max_bytes_for_level_multiplier = 3;
|
|
old_options.max_bytes_for_level_base = 200 * 1024;
|
|
|
|
Reopen(old_options);
|
|
|
|
Random rnd(301);
|
|
int key_idx = 0;
|
|
|
|
// Generate at least 2MB of data
|
|
for (int num = 0; num < 20; num++) {
|
|
GenerateNewFile(&rnd, &key_idx);
|
|
}
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
|
|
// Will make sure exactly those keys are in the DB after migration.
|
|
std::set<std::string> keys;
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (; it->Valid(); it->Next()) {
|
|
keys.insert(it->key().ToString());
|
|
}
|
|
ASSERT_OK(it->status());
|
|
}
|
|
|
|
Close();
|
|
|
|
Options new_options = old_options;
|
|
new_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style1_);
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
|
|
}
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
new_options.max_open_files = -1;
|
|
}
|
|
if (fifo_max_table_files_size_ != 0) {
|
|
new_options.compaction_options_fifo.max_table_files_size =
|
|
fifo_max_table_files_size_;
|
|
}
|
|
new_options.target_file_size_base = 256 * 1024;
|
|
new_options.num_levels = level1_;
|
|
new_options.max_bytes_for_level_base = 150 * 1024;
|
|
new_options.max_bytes_for_level_multiplier = 4;
|
|
ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
|
|
Reopen(new_options);
|
|
// Wait for compaction to finish and make sure it can reopen
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
Reopen(new_options);
|
|
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (const std::string& key : keys) {
|
|
ASSERT_TRUE(it->Valid());
|
|
ASSERT_EQ(key, it->key().ToString());
|
|
it->Next();
|
|
}
|
|
ASSERT_TRUE(!it->Valid());
|
|
ASSERT_OK(it->status());
|
|
}
|
|
}
|
|
|
|
TEST_P(DBOptionChangeMigrationTests, Migrate3) {
|
|
Options old_options = CurrentOptions();
|
|
old_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style1_);
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
|
|
}
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
old_options.max_open_files = -1;
|
|
}
|
|
old_options.level0_file_num_compaction_trigger = 3;
|
|
old_options.write_buffer_size = 64 * 1024;
|
|
old_options.target_file_size_base = 128 * 1024;
|
|
// Make level target of L1, L2 to be 200KB and 600KB
|
|
old_options.num_levels = level1_;
|
|
old_options.max_bytes_for_level_multiplier = 3;
|
|
old_options.max_bytes_for_level_base = 200 * 1024;
|
|
|
|
Reopen(old_options);
|
|
Random rnd(301);
|
|
for (int num = 0; num < 20; num++) {
|
|
for (int i = 0; i < 50; i++) {
|
|
ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
|
|
}
|
|
ASSERT_OK(Flush());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
if (num == 9) {
|
|
// Issue a full compaction to generate some zero-out files
|
|
CompactRangeOptions cro;
|
|
cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
|
|
ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
|
|
}
|
|
}
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
|
|
// Will make sure exactly those keys are in the DB after migration.
|
|
std::set<std::string> keys;
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (; it->Valid(); it->Next()) {
|
|
keys.insert(it->key().ToString());
|
|
}
|
|
ASSERT_OK(it->status());
|
|
}
|
|
Close();
|
|
|
|
Options new_options = old_options;
|
|
new_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style2_);
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
|
|
}
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
new_options.max_open_files = -1;
|
|
}
|
|
if (fifo_max_table_files_size_ != 0) {
|
|
new_options.compaction_options_fifo.max_table_files_size =
|
|
fifo_max_table_files_size_;
|
|
}
|
|
new_options.target_file_size_base = 256 * 1024;
|
|
new_options.num_levels = level2_;
|
|
new_options.max_bytes_for_level_base = 150 * 1024;
|
|
new_options.max_bytes_for_level_multiplier = 4;
|
|
ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
|
|
Reopen(new_options);
|
|
|
|
// Wait for compaction to finish and make sure it can reopen
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
Reopen(new_options);
|
|
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (const std::string& key : keys) {
|
|
ASSERT_TRUE(it->Valid());
|
|
ASSERT_EQ(key, it->key().ToString());
|
|
it->Next();
|
|
}
|
|
ASSERT_TRUE(!it->Valid());
|
|
ASSERT_OK(it->status());
|
|
}
|
|
}
|
|
|
|
TEST_P(DBOptionChangeMigrationTests, Migrate4) {
|
|
Options old_options = CurrentOptions();
|
|
old_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style2_);
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
|
|
}
|
|
if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
old_options.max_open_files = -1;
|
|
}
|
|
old_options.level0_file_num_compaction_trigger = 3;
|
|
old_options.write_buffer_size = 64 * 1024;
|
|
old_options.target_file_size_base = 128 * 1024;
|
|
// Make level target of L1, L2 to be 200KB and 600KB
|
|
old_options.num_levels = level2_;
|
|
old_options.max_bytes_for_level_multiplier = 3;
|
|
old_options.max_bytes_for_level_base = 200 * 1024;
|
|
|
|
Reopen(old_options);
|
|
Random rnd(301);
|
|
for (int num = 0; num < 20; num++) {
|
|
for (int i = 0; i < 50; i++) {
|
|
ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
|
|
}
|
|
ASSERT_OK(Flush());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
if (num == 9) {
|
|
// Issue a full compaction to generate some zero-out files
|
|
CompactRangeOptions cro;
|
|
cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
|
|
ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
|
|
}
|
|
}
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
|
|
// Will make sure exactly those keys are in the DB after migration.
|
|
std::set<std::string> keys;
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (; it->Valid(); it->Next()) {
|
|
keys.insert(it->key().ToString());
|
|
}
|
|
ASSERT_OK(it->status());
|
|
}
|
|
|
|
Close();
|
|
|
|
Options new_options = old_options;
|
|
new_options.compaction_style =
|
|
static_cast<CompactionStyle>(compaction_style1_);
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
|
|
}
|
|
if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
new_options.max_open_files = -1;
|
|
}
|
|
if (fifo_max_table_files_size_ != 0) {
|
|
new_options.compaction_options_fifo.max_table_files_size =
|
|
fifo_max_table_files_size_;
|
|
}
|
|
new_options.target_file_size_base = 256 * 1024;
|
|
new_options.num_levels = level1_;
|
|
new_options.max_bytes_for_level_base = 150 * 1024;
|
|
new_options.max_bytes_for_level_multiplier = 4;
|
|
ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
|
|
Reopen(new_options);
|
|
// Wait for compaction to finish and make sure it can reopen
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
Reopen(new_options);
|
|
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (const std::string& key : keys) {
|
|
ASSERT_TRUE(it->Valid());
|
|
ASSERT_EQ(key, it->key().ToString());
|
|
it->Next();
|
|
}
|
|
ASSERT_TRUE(!it->Valid());
|
|
ASSERT_OK(it->status());
|
|
}
|
|
}
|
|
|
|
INSTANTIATE_TEST_CASE_P(
|
|
DBOptionChangeMigrationTests, DBOptionChangeMigrationTests,
|
|
::testing::Values(
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 0 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
true /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 0 /* new compaction style */,
|
|
true /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
true /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 0 /* new compaction style */,
|
|
false, 0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 0 /* new compaction style */,
|
|
true /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 1 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 1 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 1 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
1 /* old num_levels */, 1 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
true /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 1 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
true /* is dynamic leveling in old option */,
|
|
1 /* old num_levels */, 1 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 0 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(4 /* old num_levels */, 0 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
1 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
|
|
true /* is dynamic leveling in old option */,
|
|
2 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(3 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
3 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
0 /*fifo max_table_files_size*/),
|
|
std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */, 0),
|
|
std::make_tuple(
|
|
4 /* old num_levels */, 0 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
1 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
std::numeric_limits<uint64_t>::max() /*fifo max_table_files_size*/),
|
|
std::make_tuple(
|
|
3 /* old num_levels */, 0 /* old compaction style */,
|
|
true /* is dynamic leveling in old option */,
|
|
2 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
std::numeric_limits<uint64_t>::max() /*fifo max_table_files_size*/),
|
|
std::make_tuple(
|
|
3 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
3 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
std::numeric_limits<uint64_t>::max() /*fifo max_table_files_size*/),
|
|
std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
|
|
false /* is dynamic leveling in old option */,
|
|
4 /* old num_levels */, 2 /* new compaction style */,
|
|
false /* is dynamic leveling in new option */,
|
|
std::numeric_limits<
|
|
uint64_t>::max() /*fifo max_table_files_size*/)));
|
|
|
|
class DBOptionChangeMigrationTest : public DBTestBase {
|
|
public:
|
|
DBOptionChangeMigrationTest()
|
|
: DBTestBase("db_option_change_migration_test2", /*env_do_fsync=*/true) {}
|
|
};
|
|
|
|
TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
|
|
Options old_options = CurrentOptions();
|
|
old_options.compaction_style = CompactionStyle::kCompactionStyleLevel;
|
|
old_options.max_compaction_bytes = 200 * 1024;
|
|
old_options.level_compaction_dynamic_level_bytes = false;
|
|
old_options.level0_file_num_compaction_trigger = 3;
|
|
old_options.write_buffer_size = 64 * 1024;
|
|
old_options.target_file_size_base = 128 * 1024;
|
|
// Make level target of L1, L2 to be 200KB and 600KB
|
|
old_options.num_levels = 4;
|
|
old_options.max_bytes_for_level_multiplier = 3;
|
|
old_options.max_bytes_for_level_base = 200 * 1024;
|
|
|
|
Reopen(old_options);
|
|
Random rnd(301);
|
|
for (int num = 0; num < 20; num++) {
|
|
for (int i = 0; i < 50; i++) {
|
|
ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
|
|
}
|
|
}
|
|
ASSERT_OK(Flush());
|
|
CompactRangeOptions cro;
|
|
cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
|
|
ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
|
|
|
|
// Will make sure exactly those keys are in the DB after migration.
|
|
std::set<std::string> keys;
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (; it->Valid(); it->Next()) {
|
|
keys.insert(it->key().ToString());
|
|
}
|
|
ASSERT_OK(it->status());
|
|
}
|
|
|
|
Close();
|
|
|
|
Options new_options = old_options;
|
|
new_options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
|
|
new_options.target_file_size_base = 256 * 1024;
|
|
new_options.num_levels = 1;
|
|
new_options.max_bytes_for_level_base = 150 * 1024;
|
|
new_options.max_bytes_for_level_multiplier = 4;
|
|
ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
|
|
Reopen(new_options);
|
|
// Wait for compaction to finish and make sure it can reopen
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
Reopen(new_options);
|
|
|
|
{
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
|
|
it->SeekToFirst();
|
|
for (const std::string& key : keys) {
|
|
ASSERT_TRUE(it->Valid());
|
|
ASSERT_EQ(key, it->key().ToString());
|
|
it->Next();
|
|
}
|
|
ASSERT_TRUE(!it->Valid());
|
|
ASSERT_OK(it->status());
|
|
}
|
|
}
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
|
|
int main(int argc, char** argv) {
|
|
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|