dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add compaction stats for filtered files (#13136) 

Summary:
As titled. This PR adds some compaction job stats, internal stats and some logging for filtered files.

Example logging:
[default] compacted to: files[0 0 0 0 2 0 0] max score 0.25, estimated pending compaction bytes 0, MB/sec: 0.3 rd, 0.2 wr, level 6, files in(1, 0) filtered(0, 2) out(1 +0 blob) MB in(0.0, 0.0 +0.0 blob) filtered(0.0, 0.0) out(0.0 +0.0 blob), read-write-amplify(2.0) write-amplify(1.0) OK, records in: 1, records dropped: 1 output_compression: Snappy

Pull Request resolved: https://github.com/facebook/rocksdb/pull/13136

Test Plan: Added unit tests

Reviewed By: cbi42

Differential Revision: D65855380

Pulled By: jowlyzhang

fbshipit-source-id: a4d8eef66f8d999ca5c3d9472aeeae98d7bb03ab
This commit is contained in:
Yu Zhang 2024-11-14 10:10:38 -08:00 committed by Facebook GitHub Bot
parent 9a136e18b3
commit ef119c9811
8 changed files with 294 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
db/compaction/compaction.cc
View File

@ -368,9 +368,10 @@ Compaction::Compaction(
}
#endif
// setup input_levels_
// setup input_levels_ and filtered_input_levels_
{
input_levels_.resize(num_input_levels());
filtered_input_levels_.resize(num_input_levels());
if (earliest_snapshot_.has_value()) {
FilterInputsForCompactionIterator();
} else {
@ -1085,6 +1086,7 @@ void Compaction::FilterInputsForCompactionIterator() {
ucmp->CompareWithoutTimestamp(rangedel_end_ukey,
file->largest.user_key()) > 0) {
non_start_level_input_files_filtered_.back().back() = true;
filtered_input_levels_[level].push_back(file);
} else {
non_start_level_input_files.back().push_back(file);
}

17
db/compaction/compaction.h
View File

@ -184,6 +184,16 @@ class Compaction {
return &input_levels_[compaction_input_level];
}
// Returns the filtered input files of the specified compaction input level.
// For now, only non start level is filtered.
const std::vector<FileMetaData*>& filtered_input_levels(
size_t compaction_input_level) const {
const std::vector<FileMetaData*>& filtered_input_level =
filtered_input_levels_[compaction_input_level];
assert(compaction_input_level != 0 || filtered_input_level.size() == 0);
return filtered_input_level;
}
// Maximum size of files to build during this compaction.
uint64_t max_output_file_size() const { return max_output_file_size_; }
@ -545,10 +555,13 @@ class Compaction {
// Markers for which non start level input files are filtered out if
// applicable. Only applicable if earliest_snapshot_ is provided and input
// start level has a standalone range deletion file.
// start level has a standalone range deletion file. Filtered files are
// tracked in `filtered_input_levels_`.
std::vector<std::vector<bool>> non_start_level_input_files_filtered_;
// bool standalone_range_tombstones_used_for_filtering_inputs_;
// All files from inputs_ that are filtered.
std::vector<std::vector<FileMetaData*>> filtered_input_levels_;
const double score_; // score that was used to pick this compaction.
// Is this compaction creating a file in the bottom most level?

39
db/compaction/compaction_job.cc
View File

@ -911,19 +911,23 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
ROCKS_LOG_BUFFER(
log_buffer_,
"[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
"files in(%d, %d) out(%d +%d blob) "
"MB in(%.1f, %.1f +%.1f blob) out(%.1f +%.1f blob), "
"files in(%d, %d) filtered(%d, %d) out(%d +%d blob) "
"MB in(%.1f, %.1f +%.1f blob) filtered(%.1f, %.1f) out(%.1f +%.1f blob), "
"read-write-amplify(%.1f) write-amplify(%.1f) %s, records in: %" PRIu64
", records dropped: %" PRIu64 " output_compression: %s\n",
column_family_name.c_str(), vstorage->LevelSummary(&tmp),
bytes_read_per_sec, bytes_written_per_sec,
compact_->compaction->output_level(),
stats.num_input_files_in_non_output_levels,
stats.num_input_files_in_output_level, stats.num_output_files,
stats.num_input_files_in_output_level,
stats.num_filtered_input_files_in_non_output_levels,
stats.num_filtered_input_files_in_output_level, stats.num_output_files,
stats.num_output_files_blob, stats.bytes_read_non_output_levels / kMB,
stats.bytes_read_output_level / kMB, stats.bytes_read_blob / kMB,
stats.bytes_written / kMB, stats.bytes_written_blob / kMB, read_write_amp,
write_amp, status.ToString().c_str(), stats.num_input_records,
stats.bytes_skipped_non_output_levels / kMB,
stats.bytes_skipped_output_level / kMB, stats.bytes_written / kMB,
stats.bytes_written_blob / kMB, read_write_amp, write_amp,
status.ToString().c_str(), stats.num_input_records,
stats.num_dropped_records,
CompressionTypeToString(compact_->compaction->output_compression())
.c_str());
@ -2007,7 +2011,6 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
bool has_error = false;
const ReadOptions read_options(Env::IOActivity::kCompaction);
const auto& input_table_properties = compaction->GetInputTableProperties();
// TODO(yuzhangyu): add dedicated stats for filtered files.
for (int input_level = 0;
input_level < static_cast<int>(compaction->num_input_levels());
++input_level) {
@ -2047,6 +2050,23 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
*num_input_range_del += file_num_range_del;
}
}
const std::vector<FileMetaData*>& filtered_flevel =
compaction->filtered_input_levels(input_level);
size_t num_filtered_input_files = filtered_flevel.size();
uint64_t* bytes_skipped;
if (compaction->level(input_level) != compaction->output_level()) {
compaction_stats_.stats.num_filtered_input_files_in_non_output_levels +=
static_cast<int>(num_filtered_input_files);
bytes_skipped = &compaction_stats_.stats.bytes_skipped_non_output_levels;
} else {
compaction_stats_.stats.num_filtered_input_files_in_output_level +=
static_cast<int>(num_filtered_input_files);
bytes_skipped = &compaction_stats_.stats.bytes_skipped_output_level;
}
for (const FileMetaData* filtered_file_meta : filtered_flevel) {
*bytes_skipped += filtered_file_meta->fd.GetFileSize();
}
}
assert(compaction_job_stats_);
@ -2071,6 +2091,13 @@ void CompactionJob::UpdateCompactionJobStats(
stats.num_input_files_in_output_level;
compaction_job_stats_->num_input_files_at_output_level =
stats.num_input_files_in_output_level;
compaction_job_stats_->num_filtered_input_files =
stats.num_filtered_input_files_in_non_output_levels +
stats.num_filtered_input_files_in_output_level;
compaction_job_stats_->num_filtered_input_files_at_output_level =
stats.num_filtered_input_files_in_output_level;
compaction_job_stats_->total_skipped_input_bytes =
stats.bytes_skipped_non_output_levels + stats.bytes_skipped_output_level;
// output information
compaction_job_stats_->total_output_bytes = stats.bytes_written;

3
db/compaction/compaction_job.h
View File

@ -215,8 +215,7 @@ class CompactionJob {
virtual void RecordCompactionIOStats();
void CleanupCompaction();
// Call compaction filter. Then iterate through input and compact the
// kv-pairs
// Iterate through input and compact the kv-pairs.
void ProcessKeyValueCompaction(SubcompactionState* sub_compact);
CompactionState* compact_;

174
db/external_sst_file_basic_test.cc
View File

@ -182,6 +182,36 @@ class ExternalSSTFileBasicTest
write_global_seqno, verify_checksums_before_ingest, true_data);
}
void VerifyInputFilesInternalStatsForOutputLevel(
int output_level, int num_input_files_in_non_output_levels,
int num_input_files_in_output_level,
int num_filtered_input_files_in_non_output_levels,
int num_filtered_input_files_in_output_level,
uint64_t bytes_skipped_non_output_levels,
uint64_t bytes_skipped_output_level) {
ColumnFamilyHandleImpl* cfh =
static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily());
ColumnFamilyData* cfd = cfh->cfd();
const InternalStats* internal_stats_ptr = cfd->internal_stats();
const std::vector<InternalStats::CompactionStats>& comp_stats =
internal_stats_ptr->TEST_GetCompactionStats();
EXPECT_EQ(num_input_files_in_non_output_levels,
comp_stats[output_level].num_input_files_in_non_output_levels);
EXPECT_EQ(num_input_files_in_output_level,
comp_stats[output_level].num_input_files_in_output_level);
EXPECT_EQ(
num_filtered_input_files_in_non_output_levels,
comp_stats[output_level].num_filtered_input_files_in_non_output_levels);
EXPECT_EQ(
num_filtered_input_files_in_output_level,
comp_stats[output_level].num_filtered_input_files_in_output_level);
EXPECT_EQ(bytes_skipped_non_output_levels,
comp_stats[output_level].bytes_skipped_non_output_levels);
EXPECT_EQ(bytes_skipped_output_level,
comp_stats[output_level].bytes_skipped_output_level);
}
~ExternalSSTFileBasicTest() override {
DestroyDir(env_, sst_files_dir_).PermitUncheckedError();
}
@ -1841,11 +1871,60 @@ TEST_F(ExternalSSTFileBasicTest, OverlappingFiles) {
ASSERT_EQ(2, NumTableFilesAtLevel(5));
}
class CompactionJobStatsCheckerForFilteredFiles : public EventListener {
public:
CompactionJobStatsCheckerForFilteredFiles(
int num_input_files, int num_input_files_at_output_level,
int num_filtered_input_files,
int num_filtered_input_files_at_output_level)
: num_input_files_(num_input_files),
num_input_files_at_output_level_(num_input_files_at_output_level),
num_filtered_input_files_(num_filtered_input_files),
num_filtered_input_files_at_output_level_(
num_filtered_input_files_at_output_level) {}
void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
std::lock_guard<std::mutex> lock(mutex_);
ASSERT_EQ(num_input_files_, ci.stats.num_input_files);
ASSERT_EQ(num_input_files_at_output_level_,
ci.stats.num_input_files_at_output_level);
ASSERT_EQ(num_filtered_input_files_, ci.stats.num_filtered_input_files);
ASSERT_EQ(num_filtered_input_files_at_output_level_,
ci.stats.num_filtered_input_files_at_output_level);
ASSERT_EQ(ci.stats.total_skipped_input_bytes,
expected_compaction_skipped_file_size_);
}
void SetExpectedCompactionSkippedFileSize(uint64_t expected_size) {
std::lock_guard<std::mutex> lock(mutex_);
expected_compaction_skipped_file_size_ = expected_size;
}
private:
int num_input_files_ = 0;
int num_input_files_at_output_level_ = 0;
int num_filtered_input_files_ = 0;
int num_filtered_input_files_at_output_level_ = 0;
std::mutex mutex_;
uint64_t expected_compaction_skipped_file_size_ = 0;
};
TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
Options options = CurrentOptions();
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
int kCompactionNumInputFiles = 1;
int kCompactionNumInputFilesAtOutputLevel = 0;
int kCompactionNumFilteredInputFiles = 2;
int kCompactionNumFilteredInputFilesAtOutputLevel = 2;
auto compaction_listener =
std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles,
kCompactionNumFilteredInputFilesAtOutputLevel);
options.listeners.push_back(compaction_listener);
DestroyAndReopen(options);
size_t compaction_skipped_file_size = 0;
std::vector<std::string> files;
{
// Writes first version of data in range partitioned files.
@ -1856,6 +1935,7 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
ASSERT_OK(sst_file_writer.Put("b", "b1"));
ExternalSstFileInfo file1_info;
ASSERT_OK(sst_file_writer.Finish(&file1_info));
compaction_skipped_file_size += file1_info.file_size;
files.push_back(std::move(file1));
std::string file2 = sst_files_dir_ + "file2.sst";
@ -1864,7 +1944,10 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
ASSERT_OK(sst_file_writer.Put("y", "y1"));
ExternalSstFileInfo file2_info;
ASSERT_OK(sst_file_writer.Finish(&file2_info));
compaction_skipped_file_size += file2_info.file_size;
files.push_back(std::move(file2));
compaction_listener->SetExpectedCompactionSkippedFileSize(
compaction_skipped_file_size);
}
IngestExternalFileOptions ifo;
@ -1944,6 +2027,16 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
ASSERT_EQ(Get("x"), "x2");
ASSERT_EQ(Get("y"), "y2");
VerifyInputFilesInternalStatsForOutputLevel(
/*output_level*/ 6,
kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles -
kCompactionNumFilteredInputFilesAtOutputLevel,
kCompactionNumFilteredInputFilesAtOutputLevel,
/*bytes_skipped_non_output_levels*/ 0,
/*bytes_skipped_output_level*/ compaction_skipped_file_size);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
}
@ -1951,9 +2044,20 @@ TEST_F(ExternalSSTFileBasicTest,
PartiallyReplaceDataWithOneStandaloneRangeDeletion) {
Options options = CurrentOptions();
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
int kCompactionNumInputFiles = 2;
int kCompactionNumInputFilesAtOutputLevel = 1;
int kCompactionNumFilteredInputFiles = 1;
int kCompactionNumFilteredInputFilesAtOutputLevel = 1;
auto compaction_listener =
std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles,
kCompactionNumFilteredInputFilesAtOutputLevel);
options.listeners.push_back(compaction_listener);
DestroyAndReopen(options);
std::vector<std::string> files;
size_t compaction_skipped_file_size = 0;
{
// Writes first version of data in range partitioned files.
SstFileWriter sst_file_writer(EnvOptions(), options);
@ -1963,7 +2067,10 @@ TEST_F(ExternalSSTFileBasicTest,
ASSERT_OK(sst_file_writer.Put("b", "b1"));
ExternalSstFileInfo file1_info;
ASSERT_OK(sst_file_writer.Finish(&file1_info));
compaction_skipped_file_size += file1_info.file_size;
files.push_back(std::move(file1));
compaction_listener->SetExpectedCompactionSkippedFileSize(
compaction_skipped_file_size);
std::string file2 = sst_files_dir_ + "file2.sst";
ASSERT_OK(sst_file_writer.Open(file2));
@ -2032,6 +2139,17 @@ TEST_F(ExternalSSTFileBasicTest,
ASSERT_EQ(Get("h"), "h1");
ASSERT_EQ(Get("x"), "x2");
ASSERT_EQ(Get("y"), "y");
VerifyInputFilesInternalStatsForOutputLevel(
/*output_level*/ 6,
kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles -
kCompactionNumFilteredInputFilesAtOutputLevel,
kCompactionNumFilteredInputFilesAtOutputLevel,
/*bytes_skipped_non_output_levels*/ 0,
/*bytes_skipped_output_level*/ compaction_skipped_file_size);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
}
@ -2039,15 +2157,29 @@ TEST_F(ExternalSSTFileBasicTest,
PartiallyReplaceDataWithMultipleStandaloneRangeDeletions) {
Options options = CurrentOptions();
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
int kCompactionNumInputFiles = 2;
int kCompactionNumInputFilesAtOutputLevel = 0;
int kCompactionNumFilteredInputFiles = 2;
int kCompactionNumFilteredInputFilesAtOutputLevel = 2;
// Two compactions each included on standalone range deletion file that
// filters input file on the non start level.
auto compaction_listener =
std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
kCompactionNumInputFiles / 2,
kCompactionNumInputFilesAtOutputLevel / 2,
kCompactionNumFilteredInputFiles / 2,
kCompactionNumFilteredInputFilesAtOutputLevel / 2);
options.listeners.push_back(compaction_listener);
DestroyAndReopen(options);
std::vector<std::string> files;
ExternalSstFileInfo file1_info;
ExternalSstFileInfo file3_info;
{
SstFileWriter sst_file_writer(EnvOptions(), options);
std::string file1 = sst_files_dir_ + "file1.sst";
ASSERT_OK(sst_file_writer.Open(file1));
ASSERT_OK(sst_file_writer.Put("a", "a1"));
ExternalSstFileInfo file1_info;
ASSERT_OK(sst_file_writer.Finish(&file1_info));
files.push_back(std::move(file1));
std::string file2 = sst_files_dir_ + "file2.sst";
@ -2059,7 +2191,6 @@ TEST_F(ExternalSSTFileBasicTest,
std::string file3 = sst_files_dir_ + "file3.sst";
ASSERT_OK(sst_file_writer.Open(file3));
ASSERT_OK(sst_file_writer.Put("x", "x1"));
ExternalSstFileInfo file3_info;
ASSERT_OK(sst_file_writer.Finish(&file3_info));
files.push_back(std::move(file3));
}
@ -2107,9 +2238,15 @@ TEST_F(ExternalSSTFileBasicTest,
size_t* num_input_files = static_cast<size_t*>(arg);
EXPECT_EQ(1, *num_input_files);
num_compactions += 1;
if (num_compactions == 2) {
compaction_listener->SetExpectedCompactionSkippedFileSize(
file3_info.file_size);
}
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
compaction_listener->SetExpectedCompactionSkippedFileSize(
file1_info.file_size);
ASSERT_OK(db_->IngestExternalFile(files, ifo));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
@ -2121,12 +2258,35 @@ TEST_F(ExternalSSTFileBasicTest,
ASSERT_EQ(Get("a"), "a2");
ASSERT_EQ(Get("h"), "h");
ASSERT_EQ(Get("x"), "x2");
VerifyInputFilesInternalStatsForOutputLevel(
/*output_level*/ 6,
kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles -
kCompactionNumFilteredInputFilesAtOutputLevel,
kCompactionNumFilteredInputFilesAtOutputLevel,
/*bytes_skipped_non_output_levels*/ 0,
/*bytes_skipped_output_level*/ file1_info.file_size +
file3_info.file_size);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
}
TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) {
Options options = CurrentOptions();
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
int kCompactionNumInputFiles = 2;
int kCompactionNumInputFilesAtOutputLevel = 1;
int kCompactionNumFilteredInputFiles = 0;
int kCompactionNumFilteredInputFilesAtOutputLevel = 0;
auto compaction_listener =
std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles,
kCompactionNumFilteredInputFilesAtOutputLevel);
options.listeners.push_back(compaction_listener);
// No compaction input files are filtered because the range deletion file's
// end is exclusive, so it cannot cover the whole file.
compaction_listener->SetExpectedCompactionSkippedFileSize(0);
DestroyAndReopen(options);
std::vector<std::string> files;
@ -2181,6 +2341,16 @@ TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) {
ASSERT_EQ(Get("a"), "NOT_FOUND");
ASSERT_EQ(Get("b"), "b");
VerifyInputFilesInternalStatsForOutputLevel(
/*output_level*/ 6,
kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
kCompactionNumInputFilesAtOutputLevel,
kCompactionNumFilteredInputFiles -
kCompactionNumFilteredInputFilesAtOutputLevel,
kCompactionNumFilteredInputFilesAtOutputLevel,
/*bytes_skipped_non_output_levels*/ 0,
/*bytes_skipped_output_level*/ 0);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
}

54
db/internal_stats.h
View File

@ -182,6 +182,14 @@ class InternalStats {
// The number of bytes read from the compaction output level (table files)
uint64_t bytes_read_output_level;
// The number of bytes skipped from all non-output levels because the input
// files are filtered by compaction optimizations.
uint64_t bytes_skipped_non_output_levels;
// The number of bytes skipped from the compaction output level because the
// input files are filtered by compaction optimizations.
uint64_t bytes_skipped_output_level;
// The number of bytes read from blob files
uint64_t bytes_read_blob;
@ -201,6 +209,14 @@ class InternalStats {
// The number of compaction input files in the output level (table files)
int num_input_files_in_output_level;
// The number of non output level compaction input files that are filtered
// by compaction optimizations.
int num_filtered_input_files_in_non_output_levels;
// The number of output level compaction input files that are filtered by
// compaction optimizations.
int num_filtered_input_files_in_output_level;
// The number of compaction output files (table files)
int num_output_files;
@ -228,12 +244,16 @@ class InternalStats {
cpu_micros(0),
bytes_read_non_output_levels(0),
bytes_read_output_level(0),
bytes_skipped_non_output_levels(0),
bytes_skipped_output_level(0),
bytes_read_blob(0),
bytes_written(0),
bytes_written_blob(0),
bytes_moved(0),
num_input_files_in_non_output_levels(0),
num_input_files_in_output_level(0),
num_filtered_input_files_in_non_output_levels(0),
num_filtered_input_files_in_output_level(0),
num_output_files(0),
num_output_files_blob(0),
num_input_records(0),
@ -251,12 +271,16 @@ class InternalStats {
cpu_micros(0),
bytes_read_non_output_levels(0),
bytes_read_output_level(0),
bytes_skipped_non_output_levels(0),
bytes_skipped_output_level(0),
bytes_read_blob(0),
bytes_written(0),
bytes_written_blob(0),
bytes_moved(0),
num_input_files_in_non_output_levels(0),
num_input_files_in_output_level(0),
num_filtered_input_files_in_non_output_levels(0),
num_filtered_input_files_in_output_level(0),
num_output_files(0),
num_output_files_blob(0),
num_input_records(0),
@ -280,6 +304,8 @@ class InternalStats {
cpu_micros(c.cpu_micros),
bytes_read_non_output_levels(c.bytes_read_non_output_levels),
bytes_read_output_level(c.bytes_read_output_level),
bytes_skipped_non_output_levels(c.bytes_skipped_non_output_levels),
bytes_skipped_output_level(c.bytes_skipped_output_level),
bytes_read_blob(c.bytes_read_blob),
bytes_written(c.bytes_written),
bytes_written_blob(c.bytes_written_blob),
@ -287,6 +313,10 @@ class InternalStats {
num_input_files_in_non_output_levels(
c.num_input_files_in_non_output_levels),
num_input_files_in_output_level(c.num_input_files_in_output_level),
num_filtered_input_files_in_non_output_levels(
c.num_filtered_input_files_in_non_output_levels),
num_filtered_input_files_in_output_level(
c.num_filtered_input_files_in_output_level),
num_output_files(c.num_output_files),
num_output_files_blob(c.num_output_files_blob),
num_input_records(c.num_input_records),
@ -304,6 +334,8 @@ class InternalStats {
cpu_micros = c.cpu_micros;
bytes_read_non_output_levels = c.bytes_read_non_output_levels;
bytes_read_output_level = c.bytes_read_output_level;
bytes_skipped_non_output_levels = c.bytes_skipped_non_output_levels;
bytes_skipped_output_level = c.bytes_skipped_output_level;
bytes_read_blob = c.bytes_read_blob;
bytes_written = c.bytes_written;
bytes_written_blob = c.bytes_written_blob;
@ -311,6 +343,10 @@ class InternalStats {
num_input_files_in_non_output_levels =
c.num_input_files_in_non_output_levels;
num_input_files_in_output_level = c.num_input_files_in_output_level;
num_filtered_input_files_in_non_output_levels =
c.num_filtered_input_files_in_non_output_levels;
num_filtered_input_files_in_output_level =
c.num_filtered_input_files_in_output_level;
num_output_files = c.num_output_files;
num_output_files_blob = c.num_output_files_blob;
num_input_records = c.num_input_records;
@ -330,12 +366,16 @@ class InternalStats {
this->cpu_micros = 0;
this->bytes_read_non_output_levels = 0;
this->bytes_read_output_level = 0;
this->bytes_skipped_non_output_levels = 0;
this->bytes_skipped_output_level = 0;
this->bytes_read_blob = 0;
this->bytes_written = 0;
this->bytes_written_blob = 0;
this->bytes_moved = 0;
this->num_input_files_in_non_output_levels = 0;
this->num_input_files_in_output_level = 0;
this->num_filtered_input_files_in_non_output_levels = 0;
this->num_filtered_input_files_in_output_level = 0;
this->num_output_files = 0;
this->num_output_files_blob = 0;
this->num_input_records = 0;
@ -353,6 +393,9 @@ class InternalStats {
this->cpu_micros += c.cpu_micros;
this->bytes_read_non_output_levels += c.bytes_read_non_output_levels;
this->bytes_read_output_level += c.bytes_read_output_level;
this->bytes_skipped_non_output_levels +=
c.bytes_skipped_non_output_levels;
this->bytes_skipped_output_level += c.bytes_skipped_output_level;
this->bytes_read_blob += c.bytes_read_blob;
this->bytes_written += c.bytes_written;
this->bytes_written_blob += c.bytes_written_blob;
@ -361,6 +404,10 @@ class InternalStats {
c.num_input_files_in_non_output_levels;
this->num_input_files_in_output_level +=
c.num_input_files_in_output_level;
this->num_filtered_input_files_in_non_output_levels +=
c.num_filtered_input_files_in_non_output_levels;
this->num_filtered_input_files_in_output_level +=
c.num_filtered_input_files_in_output_level;
this->num_output_files += c.num_output_files;
this->num_output_files_blob += c.num_output_files_blob;
this->num_input_records += c.num_input_records;
@ -387,6 +434,9 @@ class InternalStats {
this->cpu_micros -= c.cpu_micros;
this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels;
this->bytes_read_output_level -= c.bytes_read_output_level;
this->bytes_skipped_non_output_levels -=
c.bytes_skipped_non_output_levels;
this->bytes_skipped_output_level -= c.bytes_skipped_output_level;
this->bytes_read_blob -= c.bytes_read_blob;
this->bytes_written -= c.bytes_written;
this->bytes_written_blob -= c.bytes_written_blob;
@ -395,6 +445,10 @@ class InternalStats {
c.num_input_files_in_non_output_levels;
this->num_input_files_in_output_level -=
c.num_input_files_in_output_level;
this->num_filtered_input_files_in_non_output_levels -=
c.num_filtered_input_files_in_non_output_levels;
this->num_filtered_input_files_in_output_level -=
c.num_filtered_input_files_in_output_level;
this->num_output_files -= c.num_output_files;
this->num_output_files_blob -= c.num_output_files_blob;
this->num_input_records -= c.num_input_records;

9
include/rocksdb/compaction_job_stats.h
View File

@ -35,6 +35,12 @@ struct CompactionJobStats {
size_t num_input_files = 0;
// the number of compaction input files at the output level (table files)
size_t num_input_files_at_output_level = 0;
// the number of compaction input files that are filtered out by compaction
// optimizations
size_t num_filtered_input_files = 0;
// the number of compaction input files at the output level that are filtered
// out by compaction optimizations
size_t num_filtered_input_files_at_output_level = 0;
// the number of compaction output records.
uint64_t num_output_records = 0;
@ -58,6 +64,9 @@ struct CompactionJobStats {
uint64_t total_output_bytes = 0;
// the total size of blob files in the compaction output
uint64_t total_output_bytes_blob = 0;
// the total size of table files for compaction input files that are skipped
// because input files are filtered out by compaction optimizations.
uint64_t total_skipped_input_bytes = 0;
// number of records being replaced by newer record associated with same key.
// this could be a new value or a deletion entry for that key so this field

7
util/compaction_job_stats_impl.cc
View File

@ -17,6 +17,8 @@ void CompactionJobStats::Reset() {
num_blobs_read = 0;
num_input_files = 0;
num_input_files_at_output_level = 0;
num_filtered_input_files = 0;
num_filtered_input_files_at_output_level = 0;
num_output_records = 0;
num_output_files = 0;
@ -30,6 +32,7 @@ void CompactionJobStats::Reset() {
total_blob_bytes_read = 0;
total_output_bytes = 0;
total_output_bytes_blob = 0;
total_skipped_input_bytes = 0;
num_records_replaced = 0;
@ -62,6 +65,9 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
num_blobs_read += stats.num_blobs_read;
num_input_files += stats.num_input_files;
num_input_files_at_output_level += stats.num_input_files_at_output_level;
num_filtered_input_files += stats.num_filtered_input_files;
num_filtered_input_files_at_output_level +=
stats.num_filtered_input_files_at_output_level;
num_output_records += stats.num_output_records;
num_output_files += stats.num_output_files;
@ -71,6 +77,7 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
total_blob_bytes_read += stats.total_blob_bytes_read;
total_output_bytes += stats.total_output_bytes;
total_output_bytes_blob += stats.total_output_bytes_blob;
total_skipped_input_bytes += stats.total_skipped_input_bytes;
num_records_replaced += stats.num_records_replaced;