mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-27 11:43:49 +00:00
4d16a9a633
Summary: During recovery, VersionBuilder::Apply() was called multiple times. If the DB is open for long enough, most of files added earlier will be deleted by later deletes. In current solution, sorting added file happens first and then deletes are applied. In this patch, deletes are applied when possible inside Apply(), which can significantly reduce the sorting time in some cases. Test Plan: Add unit tests in version_builder valgrind_check Open a manifest of 50MB, with 9K live files. The manifest read time reduced from 1.6 seconds to 0.7 seconds. Reviewers: rven, yhchiang, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D30765
232 lines
7.8 KiB
C++
232 lines
7.8 KiB
C++
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
#include <string>
|
|
#include "db/version_edit.h"
|
|
#include "db/version_set.h"
|
|
#include "util/logging.h"
|
|
#include "util/testharness.h"
|
|
#include "util/testutil.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class VersionBuilderTest {
|
|
public:
|
|
const Comparator* ucmp_;
|
|
InternalKeyComparator icmp_;
|
|
Options options_;
|
|
ImmutableCFOptions ioptions_;
|
|
MutableCFOptions mutable_cf_options_;
|
|
VersionStorageInfo vstorage_;
|
|
uint32_t file_num_;
|
|
CompactionOptionsFIFO fifo_options_;
|
|
std::vector<uint64_t> size_being_compacted_;
|
|
|
|
VersionBuilderTest()
|
|
: ucmp_(BytewiseComparator()),
|
|
icmp_(ucmp_),
|
|
ioptions_(options_),
|
|
mutable_cf_options_(options_, ioptions_),
|
|
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
|
|
nullptr),
|
|
file_num_(1) {
|
|
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
|
size_being_compacted_.resize(options_.num_levels);
|
|
}
|
|
|
|
~VersionBuilderTest() {
|
|
for (int i = 0; i < vstorage_.num_levels(); i++) {
|
|
for (auto* f : vstorage_.LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
InternalKey GetInternalKey(const char* ukey,
|
|
SequenceNumber smallest_seq = 100) {
|
|
return InternalKey(ukey, smallest_seq, kTypeValue);
|
|
}
|
|
|
|
void Add(int level, uint32_t file_number, const char* smallest,
|
|
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
|
|
SequenceNumber smallest_seq = 100,
|
|
SequenceNumber largest_seq = 100,
|
|
uint64_t num_entries = 0, uint64_t num_deletions = 0,
|
|
bool sampled = false) {
|
|
assert(level < vstorage_.num_levels());
|
|
FileMetaData* f = new FileMetaData;
|
|
f->fd = FileDescriptor(file_number, path_id, file_size);
|
|
f->smallest = GetInternalKey(smallest, smallest_seq);
|
|
f->largest = GetInternalKey(largest, largest_seq);
|
|
f->compensated_file_size = file_size;
|
|
f->refs = 0;
|
|
f->num_entries = num_entries;
|
|
f->num_deletions = num_deletions;
|
|
vstorage_.AddFile(level, f);
|
|
if (sampled) {
|
|
f->init_stats_from_file = true;
|
|
vstorage_.UpdateAccumulatedStats(f);
|
|
}
|
|
}
|
|
|
|
void UpdateVersionStorageInfo() {
|
|
vstorage_.ComputeCompactionScore(mutable_cf_options_, fifo_options_,
|
|
size_being_compacted_);
|
|
vstorage_.UpdateFilesBySize();
|
|
vstorage_.UpdateNumNonEmptyLevels();
|
|
vstorage_.GenerateFileIndexer();
|
|
vstorage_.GenerateLevelFilesBrief();
|
|
vstorage_.SetFinalized();
|
|
}
|
|
};
|
|
|
|
TEST(VersionBuilderTest, ApplyAndSaveTo) {
|
|
Add(0, 1U, "150", "200", 100U);
|
|
// Level 1 score 1.2
|
|
Add(1, 66U, "150", "200", 100U);
|
|
Add(1, 88U, "201", "300", 100U);
|
|
// Level 2 score 1.8. File 7 is the largest. Should be picked
|
|
Add(2, 6U, "150", "179", 100U);
|
|
Add(2, 7U, "180", "220", 100U);
|
|
Add(2, 8U, "221", "300", 100U);
|
|
// Level 3 score slightly larger than 1
|
|
Add(3, 26U, "150", "170", 100U);
|
|
Add(3, 27U, "171", "179", 100U);
|
|
Add(3, 28U, "191", "220", 100U);
|
|
Add(3, 29U, "221", "300", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.DeleteFile(3, 27U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
|
|
|
|
for (int i = 0; i < new_vstorage.num_levels(); i++) {
|
|
for (auto* f : new_vstorage.LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(VersionBuilderTest, ApplyMultipleAndSaveTo) {
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
|
|
GetInternalKey("450"), 200, 200);
|
|
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
|
|
GetInternalKey("650"), 200, 200);
|
|
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
|
|
GetInternalKey("550"), 200, 200);
|
|
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
|
|
GetInternalKey("750"), 200, 200);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2));
|
|
|
|
for (int i = 0; i < new_vstorage.num_levels(); i++) {
|
|
for (auto* f : new_vstorage.LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(VersionBuilderTest, ApplyDeleteAndSaveTo) {
|
|
UpdateVersionStorageInfo();
|
|
|
|
EnvOptions env_options;
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
|
|
GetInternalKey("450"), 200, 200);
|
|
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
|
|
GetInternalKey("650"), 200, 200);
|
|
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
|
|
GetInternalKey("550"), 200, 200);
|
|
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
|
|
GetInternalKey("750"), 200, 200);
|
|
version_builder.Apply(&version_edit);
|
|
|
|
VersionEdit version_edit2;
|
|
version_edit.AddFile(2, 808, 0, 100U, GetInternalKey("901"),
|
|
GetInternalKey("950"), 200, 200);
|
|
version_edit2.DeleteFile(2, 616);
|
|
version_edit2.DeleteFile(2, 636);
|
|
version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
|
|
GetInternalKey("850"), 200, 200);
|
|
version_builder.Apply(&version_edit2);
|
|
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2));
|
|
|
|
for (int i = 0; i < new_vstorage.num_levels(); i++) {
|
|
for (auto* f : new_vstorage.LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(VersionBuilderTest, EstimatedActiveKeys) {
|
|
const uint32_t kTotalSamples = 20;
|
|
const uint32_t kNumLevels = 5;
|
|
const uint32_t kFilesPerLevel = 8;
|
|
const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
|
|
const uint32_t kEntriesPerFile = 1000;
|
|
const uint32_t kDeletionsPerFile = 100;
|
|
for (uint32_t i = 0; i < kNumFiles; ++i) {
|
|
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
|
|
ToString((i + 100) * 1000).c_str(),
|
|
ToString((i + 100) * 1000 + 999).c_str(),
|
|
100U, 0, 100, 100,
|
|
kEntriesPerFile, kDeletionsPerFile,
|
|
(i < kTotalSamples));
|
|
}
|
|
// minus 2X for the number of deletion entries because:
|
|
// 1x for deletion entry does not count as a data entry.
|
|
// 1x for each deletion entry will actually remove one data entry.
|
|
ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
|
|
(kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
|
|
}
|
|
|
|
} // namespace rocksdb
|
|
|
|
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }
|