diff --git a/db/compaction/compaction.cc b/db/compaction/compaction.cc index 8444e87c95..44ab72c9cd 100644 --- a/db/compaction/compaction.cc +++ b/db/compaction/compaction.cc @@ -368,9 +368,10 @@ Compaction::Compaction( } #endif - // setup input_levels_ + // setup input_levels_ and filtered_input_levels_ { input_levels_.resize(num_input_levels()); + filtered_input_levels_.resize(num_input_levels()); if (earliest_snapshot_.has_value()) { FilterInputsForCompactionIterator(); } else { @@ -1085,6 +1086,7 @@ void Compaction::FilterInputsForCompactionIterator() { ucmp->CompareWithoutTimestamp(rangedel_end_ukey, file->largest.user_key()) > 0) { non_start_level_input_files_filtered_.back().back() = true; + filtered_input_levels_[level].push_back(file); } else { non_start_level_input_files.back().push_back(file); } diff --git a/db/compaction/compaction.h b/db/compaction/compaction.h index 9e045fb58f..1604635d86 100644 --- a/db/compaction/compaction.h +++ b/db/compaction/compaction.h @@ -184,6 +184,16 @@ class Compaction { return &input_levels_[compaction_input_level]; } + // Returns the filtered input files of the specified compaction input level. + // For now, only non start level is filtered. + const std::vector& filtered_input_levels( + size_t compaction_input_level) const { + const std::vector& filtered_input_level = + filtered_input_levels_[compaction_input_level]; + assert(compaction_input_level != 0 || filtered_input_level.size() == 0); + return filtered_input_level; + } + // Maximum size of files to build during this compaction. uint64_t max_output_file_size() const { return max_output_file_size_; } @@ -545,10 +555,13 @@ class Compaction { // Markers for which non start level input files are filtered out if // applicable. Only applicable if earliest_snapshot_ is provided and input - // start level has a standalone range deletion file. + // start level has a standalone range deletion file. Filtered files are + // tracked in `filtered_input_levels_`. std::vector> non_start_level_input_files_filtered_; - // bool standalone_range_tombstones_used_for_filtering_inputs_; + // All files from inputs_ that are filtered. + std::vector> filtered_input_levels_; + const double score_; // score that was used to pick this compaction. // Is this compaction creating a file in the bottom most level? diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index cc31c5d595..31365d8fe4 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -911,19 +911,23 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options, ROCKS_LOG_BUFFER( log_buffer_, "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, " - "files in(%d, %d) out(%d +%d blob) " - "MB in(%.1f, %.1f +%.1f blob) out(%.1f +%.1f blob), " + "files in(%d, %d) filtered(%d, %d) out(%d +%d blob) " + "MB in(%.1f, %.1f +%.1f blob) filtered(%.1f, %.1f) out(%.1f +%.1f blob), " "read-write-amplify(%.1f) write-amplify(%.1f) %s, records in: %" PRIu64 ", records dropped: %" PRIu64 " output_compression: %s\n", column_family_name.c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec, bytes_written_per_sec, compact_->compaction->output_level(), stats.num_input_files_in_non_output_levels, - stats.num_input_files_in_output_level, stats.num_output_files, + stats.num_input_files_in_output_level, + stats.num_filtered_input_files_in_non_output_levels, + stats.num_filtered_input_files_in_output_level, stats.num_output_files, stats.num_output_files_blob, stats.bytes_read_non_output_levels / kMB, stats.bytes_read_output_level / kMB, stats.bytes_read_blob / kMB, - stats.bytes_written / kMB, stats.bytes_written_blob / kMB, read_write_amp, - write_amp, status.ToString().c_str(), stats.num_input_records, + stats.bytes_skipped_non_output_levels / kMB, + stats.bytes_skipped_output_level / kMB, stats.bytes_written / kMB, + stats.bytes_written_blob / kMB, read_write_amp, write_amp, + status.ToString().c_str(), stats.num_input_records, stats.num_dropped_records, CompressionTypeToString(compact_->compaction->output_compression()) .c_str()); @@ -2007,7 +2011,6 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) { bool has_error = false; const ReadOptions read_options(Env::IOActivity::kCompaction); const auto& input_table_properties = compaction->GetInputTableProperties(); - // TODO(yuzhangyu): add dedicated stats for filtered files. for (int input_level = 0; input_level < static_cast(compaction->num_input_levels()); ++input_level) { @@ -2047,6 +2050,23 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) { *num_input_range_del += file_num_range_del; } } + + const std::vector& filtered_flevel = + compaction->filtered_input_levels(input_level); + size_t num_filtered_input_files = filtered_flevel.size(); + uint64_t* bytes_skipped; + if (compaction->level(input_level) != compaction->output_level()) { + compaction_stats_.stats.num_filtered_input_files_in_non_output_levels += + static_cast(num_filtered_input_files); + bytes_skipped = &compaction_stats_.stats.bytes_skipped_non_output_levels; + } else { + compaction_stats_.stats.num_filtered_input_files_in_output_level += + static_cast(num_filtered_input_files); + bytes_skipped = &compaction_stats_.stats.bytes_skipped_output_level; + } + for (const FileMetaData* filtered_file_meta : filtered_flevel) { + *bytes_skipped += filtered_file_meta->fd.GetFileSize(); + } } assert(compaction_job_stats_); @@ -2071,6 +2091,13 @@ void CompactionJob::UpdateCompactionJobStats( stats.num_input_files_in_output_level; compaction_job_stats_->num_input_files_at_output_level = stats.num_input_files_in_output_level; + compaction_job_stats_->num_filtered_input_files = + stats.num_filtered_input_files_in_non_output_levels + + stats.num_filtered_input_files_in_output_level; + compaction_job_stats_->num_filtered_input_files_at_output_level = + stats.num_filtered_input_files_in_output_level; + compaction_job_stats_->total_skipped_input_bytes = + stats.bytes_skipped_non_output_levels + stats.bytes_skipped_output_level; // output information compaction_job_stats_->total_output_bytes = stats.bytes_written; diff --git a/db/compaction/compaction_job.h b/db/compaction/compaction_job.h index 82397a8210..b45c3494a7 100644 --- a/db/compaction/compaction_job.h +++ b/db/compaction/compaction_job.h @@ -215,8 +215,7 @@ class CompactionJob { virtual void RecordCompactionIOStats(); void CleanupCompaction(); - // Call compaction filter. Then iterate through input and compact the - // kv-pairs + // Iterate through input and compact the kv-pairs. void ProcessKeyValueCompaction(SubcompactionState* sub_compact); CompactionState* compact_; diff --git a/db/external_sst_file_basic_test.cc b/db/external_sst_file_basic_test.cc index cccf84efde..e65742bbf8 100644 --- a/db/external_sst_file_basic_test.cc +++ b/db/external_sst_file_basic_test.cc @@ -182,6 +182,36 @@ class ExternalSSTFileBasicTest write_global_seqno, verify_checksums_before_ingest, true_data); } + void VerifyInputFilesInternalStatsForOutputLevel( + int output_level, int num_input_files_in_non_output_levels, + int num_input_files_in_output_level, + int num_filtered_input_files_in_non_output_levels, + int num_filtered_input_files_in_output_level, + uint64_t bytes_skipped_non_output_levels, + uint64_t bytes_skipped_output_level) { + ColumnFamilyHandleImpl* cfh = + static_cast(dbfull()->DefaultColumnFamily()); + ColumnFamilyData* cfd = cfh->cfd(); + const InternalStats* internal_stats_ptr = cfd->internal_stats(); + const std::vector& comp_stats = + internal_stats_ptr->TEST_GetCompactionStats(); + + EXPECT_EQ(num_input_files_in_non_output_levels, + comp_stats[output_level].num_input_files_in_non_output_levels); + EXPECT_EQ(num_input_files_in_output_level, + comp_stats[output_level].num_input_files_in_output_level); + EXPECT_EQ( + num_filtered_input_files_in_non_output_levels, + comp_stats[output_level].num_filtered_input_files_in_non_output_levels); + EXPECT_EQ( + num_filtered_input_files_in_output_level, + comp_stats[output_level].num_filtered_input_files_in_output_level); + EXPECT_EQ(bytes_skipped_non_output_levels, + comp_stats[output_level].bytes_skipped_non_output_levels); + EXPECT_EQ(bytes_skipped_output_level, + comp_stats[output_level].bytes_skipped_output_level); + } + ~ExternalSSTFileBasicTest() override { DestroyDir(env_, sst_files_dir_).PermitUncheckedError(); } @@ -1841,11 +1871,60 @@ TEST_F(ExternalSSTFileBasicTest, OverlappingFiles) { ASSERT_EQ(2, NumTableFilesAtLevel(5)); } +class CompactionJobStatsCheckerForFilteredFiles : public EventListener { + public: + CompactionJobStatsCheckerForFilteredFiles( + int num_input_files, int num_input_files_at_output_level, + int num_filtered_input_files, + int num_filtered_input_files_at_output_level) + : num_input_files_(num_input_files), + num_input_files_at_output_level_(num_input_files_at_output_level), + num_filtered_input_files_(num_filtered_input_files), + num_filtered_input_files_at_output_level_( + num_filtered_input_files_at_output_level) {} + + void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { + std::lock_guard lock(mutex_); + ASSERT_EQ(num_input_files_, ci.stats.num_input_files); + ASSERT_EQ(num_input_files_at_output_level_, + ci.stats.num_input_files_at_output_level); + ASSERT_EQ(num_filtered_input_files_, ci.stats.num_filtered_input_files); + ASSERT_EQ(num_filtered_input_files_at_output_level_, + ci.stats.num_filtered_input_files_at_output_level); + ASSERT_EQ(ci.stats.total_skipped_input_bytes, + expected_compaction_skipped_file_size_); + } + + void SetExpectedCompactionSkippedFileSize(uint64_t expected_size) { + std::lock_guard lock(mutex_); + expected_compaction_skipped_file_size_ = expected_size; + } + + private: + int num_input_files_ = 0; + int num_input_files_at_output_level_ = 0; + int num_filtered_input_files_ = 0; + int num_filtered_input_files_at_output_level_ = 0; + std::mutex mutex_; + uint64_t expected_compaction_skipped_file_size_ = 0; +}; + TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) { Options options = CurrentOptions(); options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + int kCompactionNumInputFiles = 1; + int kCompactionNumInputFilesAtOutputLevel = 0; + int kCompactionNumFilteredInputFiles = 2; + int kCompactionNumFilteredInputFilesAtOutputLevel = 2; + auto compaction_listener = + std::make_shared( + kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles, + kCompactionNumFilteredInputFilesAtOutputLevel); + options.listeners.push_back(compaction_listener); DestroyAndReopen(options); + size_t compaction_skipped_file_size = 0; std::vector files; { // Writes first version of data in range partitioned files. @@ -1856,6 +1935,7 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) { ASSERT_OK(sst_file_writer.Put("b", "b1")); ExternalSstFileInfo file1_info; ASSERT_OK(sst_file_writer.Finish(&file1_info)); + compaction_skipped_file_size += file1_info.file_size; files.push_back(std::move(file1)); std::string file2 = sst_files_dir_ + "file2.sst"; @@ -1864,7 +1944,10 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) { ASSERT_OK(sst_file_writer.Put("y", "y1")); ExternalSstFileInfo file2_info; ASSERT_OK(sst_file_writer.Finish(&file2_info)); + compaction_skipped_file_size += file2_info.file_size; files.push_back(std::move(file2)); + compaction_listener->SetExpectedCompactionSkippedFileSize( + compaction_skipped_file_size); } IngestExternalFileOptions ifo; @@ -1944,6 +2027,16 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) { ASSERT_EQ(Get("x"), "x2"); ASSERT_EQ(Get("y"), "y2"); + VerifyInputFilesInternalStatsForOutputLevel( + /*output_level*/ 6, + kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel, + kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles - + kCompactionNumFilteredInputFilesAtOutputLevel, + kCompactionNumFilteredInputFilesAtOutputLevel, + /*bytes_skipped_non_output_levels*/ 0, + /*bytes_skipped_output_level*/ compaction_skipped_file_size); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } @@ -1951,9 +2044,20 @@ TEST_F(ExternalSSTFileBasicTest, PartiallyReplaceDataWithOneStandaloneRangeDeletion) { Options options = CurrentOptions(); options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + int kCompactionNumInputFiles = 2; + int kCompactionNumInputFilesAtOutputLevel = 1; + int kCompactionNumFilteredInputFiles = 1; + int kCompactionNumFilteredInputFilesAtOutputLevel = 1; + auto compaction_listener = + std::make_shared( + kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles, + kCompactionNumFilteredInputFilesAtOutputLevel); + options.listeners.push_back(compaction_listener); DestroyAndReopen(options); std::vector files; + size_t compaction_skipped_file_size = 0; { // Writes first version of data in range partitioned files. SstFileWriter sst_file_writer(EnvOptions(), options); @@ -1963,7 +2067,10 @@ TEST_F(ExternalSSTFileBasicTest, ASSERT_OK(sst_file_writer.Put("b", "b1")); ExternalSstFileInfo file1_info; ASSERT_OK(sst_file_writer.Finish(&file1_info)); + compaction_skipped_file_size += file1_info.file_size; files.push_back(std::move(file1)); + compaction_listener->SetExpectedCompactionSkippedFileSize( + compaction_skipped_file_size); std::string file2 = sst_files_dir_ + "file2.sst"; ASSERT_OK(sst_file_writer.Open(file2)); @@ -2032,6 +2139,17 @@ TEST_F(ExternalSSTFileBasicTest, ASSERT_EQ(Get("h"), "h1"); ASSERT_EQ(Get("x"), "x2"); ASSERT_EQ(Get("y"), "y"); + + VerifyInputFilesInternalStatsForOutputLevel( + /*output_level*/ 6, + kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel, + kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles - + kCompactionNumFilteredInputFilesAtOutputLevel, + kCompactionNumFilteredInputFilesAtOutputLevel, + /*bytes_skipped_non_output_levels*/ 0, + /*bytes_skipped_output_level*/ compaction_skipped_file_size); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } @@ -2039,15 +2157,29 @@ TEST_F(ExternalSSTFileBasicTest, PartiallyReplaceDataWithMultipleStandaloneRangeDeletions) { Options options = CurrentOptions(); options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + int kCompactionNumInputFiles = 2; + int kCompactionNumInputFilesAtOutputLevel = 0; + int kCompactionNumFilteredInputFiles = 2; + int kCompactionNumFilteredInputFilesAtOutputLevel = 2; + // Two compactions each included on standalone range deletion file that + // filters input file on the non start level. + auto compaction_listener = + std::make_shared( + kCompactionNumInputFiles / 2, + kCompactionNumInputFilesAtOutputLevel / 2, + kCompactionNumFilteredInputFiles / 2, + kCompactionNumFilteredInputFilesAtOutputLevel / 2); + options.listeners.push_back(compaction_listener); DestroyAndReopen(options); std::vector files; + ExternalSstFileInfo file1_info; + ExternalSstFileInfo file3_info; { SstFileWriter sst_file_writer(EnvOptions(), options); std::string file1 = sst_files_dir_ + "file1.sst"; ASSERT_OK(sst_file_writer.Open(file1)); ASSERT_OK(sst_file_writer.Put("a", "a1")); - ExternalSstFileInfo file1_info; ASSERT_OK(sst_file_writer.Finish(&file1_info)); files.push_back(std::move(file1)); std::string file2 = sst_files_dir_ + "file2.sst"; @@ -2059,7 +2191,6 @@ TEST_F(ExternalSSTFileBasicTest, std::string file3 = sst_files_dir_ + "file3.sst"; ASSERT_OK(sst_file_writer.Open(file3)); ASSERT_OK(sst_file_writer.Put("x", "x1")); - ExternalSstFileInfo file3_info; ASSERT_OK(sst_file_writer.Finish(&file3_info)); files.push_back(std::move(file3)); } @@ -2107,9 +2238,15 @@ TEST_F(ExternalSSTFileBasicTest, size_t* num_input_files = static_cast(arg); EXPECT_EQ(1, *num_input_files); num_compactions += 1; + if (num_compactions == 2) { + compaction_listener->SetExpectedCompactionSkippedFileSize( + file3_info.file_size); + } }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + compaction_listener->SetExpectedCompactionSkippedFileSize( + file1_info.file_size); ASSERT_OK(db_->IngestExternalFile(files, ifo)); ASSERT_OK(dbfull()->TEST_WaitForCompact()); @@ -2121,12 +2258,35 @@ TEST_F(ExternalSSTFileBasicTest, ASSERT_EQ(Get("a"), "a2"); ASSERT_EQ(Get("h"), "h"); ASSERT_EQ(Get("x"), "x2"); + VerifyInputFilesInternalStatsForOutputLevel( + /*output_level*/ 6, + kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel, + kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles - + kCompactionNumFilteredInputFilesAtOutputLevel, + kCompactionNumFilteredInputFilesAtOutputLevel, + /*bytes_skipped_non_output_levels*/ 0, + /*bytes_skipped_output_level*/ file1_info.file_size + + file3_info.file_size); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) { Options options = CurrentOptions(); options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + int kCompactionNumInputFiles = 2; + int kCompactionNumInputFilesAtOutputLevel = 1; + int kCompactionNumFilteredInputFiles = 0; + int kCompactionNumFilteredInputFilesAtOutputLevel = 0; + auto compaction_listener = + std::make_shared( + kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles, + kCompactionNumFilteredInputFilesAtOutputLevel); + options.listeners.push_back(compaction_listener); + // No compaction input files are filtered because the range deletion file's + // end is exclusive, so it cannot cover the whole file. + compaction_listener->SetExpectedCompactionSkippedFileSize(0); DestroyAndReopen(options); std::vector files; @@ -2181,6 +2341,16 @@ TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) { ASSERT_EQ(Get("a"), "NOT_FOUND"); ASSERT_EQ(Get("b"), "b"); + + VerifyInputFilesInternalStatsForOutputLevel( + /*output_level*/ 6, + kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel, + kCompactionNumInputFilesAtOutputLevel, + kCompactionNumFilteredInputFiles - + kCompactionNumFilteredInputFilesAtOutputLevel, + kCompactionNumFilteredInputFilesAtOutputLevel, + /*bytes_skipped_non_output_levels*/ 0, + /*bytes_skipped_output_level*/ 0); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } diff --git a/db/internal_stats.h b/db/internal_stats.h index 7bc3202f95..c1695308eb 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -182,6 +182,14 @@ class InternalStats { // The number of bytes read from the compaction output level (table files) uint64_t bytes_read_output_level; + // The number of bytes skipped from all non-output levels because the input + // files are filtered by compaction optimizations. + uint64_t bytes_skipped_non_output_levels; + + // The number of bytes skipped from the compaction output level because the + // input files are filtered by compaction optimizations. + uint64_t bytes_skipped_output_level; + // The number of bytes read from blob files uint64_t bytes_read_blob; @@ -201,6 +209,14 @@ class InternalStats { // The number of compaction input files in the output level (table files) int num_input_files_in_output_level; + // The number of non output level compaction input files that are filtered + // by compaction optimizations. + int num_filtered_input_files_in_non_output_levels; + + // The number of output level compaction input files that are filtered by + // compaction optimizations. + int num_filtered_input_files_in_output_level; + // The number of compaction output files (table files) int num_output_files; @@ -228,12 +244,16 @@ class InternalStats { cpu_micros(0), bytes_read_non_output_levels(0), bytes_read_output_level(0), + bytes_skipped_non_output_levels(0), + bytes_skipped_output_level(0), bytes_read_blob(0), bytes_written(0), bytes_written_blob(0), bytes_moved(0), num_input_files_in_non_output_levels(0), num_input_files_in_output_level(0), + num_filtered_input_files_in_non_output_levels(0), + num_filtered_input_files_in_output_level(0), num_output_files(0), num_output_files_blob(0), num_input_records(0), @@ -251,12 +271,16 @@ class InternalStats { cpu_micros(0), bytes_read_non_output_levels(0), bytes_read_output_level(0), + bytes_skipped_non_output_levels(0), + bytes_skipped_output_level(0), bytes_read_blob(0), bytes_written(0), bytes_written_blob(0), bytes_moved(0), num_input_files_in_non_output_levels(0), num_input_files_in_output_level(0), + num_filtered_input_files_in_non_output_levels(0), + num_filtered_input_files_in_output_level(0), num_output_files(0), num_output_files_blob(0), num_input_records(0), @@ -280,6 +304,8 @@ class InternalStats { cpu_micros(c.cpu_micros), bytes_read_non_output_levels(c.bytes_read_non_output_levels), bytes_read_output_level(c.bytes_read_output_level), + bytes_skipped_non_output_levels(c.bytes_skipped_non_output_levels), + bytes_skipped_output_level(c.bytes_skipped_output_level), bytes_read_blob(c.bytes_read_blob), bytes_written(c.bytes_written), bytes_written_blob(c.bytes_written_blob), @@ -287,6 +313,10 @@ class InternalStats { num_input_files_in_non_output_levels( c.num_input_files_in_non_output_levels), num_input_files_in_output_level(c.num_input_files_in_output_level), + num_filtered_input_files_in_non_output_levels( + c.num_filtered_input_files_in_non_output_levels), + num_filtered_input_files_in_output_level( + c.num_filtered_input_files_in_output_level), num_output_files(c.num_output_files), num_output_files_blob(c.num_output_files_blob), num_input_records(c.num_input_records), @@ -304,6 +334,8 @@ class InternalStats { cpu_micros = c.cpu_micros; bytes_read_non_output_levels = c.bytes_read_non_output_levels; bytes_read_output_level = c.bytes_read_output_level; + bytes_skipped_non_output_levels = c.bytes_skipped_non_output_levels; + bytes_skipped_output_level = c.bytes_skipped_output_level; bytes_read_blob = c.bytes_read_blob; bytes_written = c.bytes_written; bytes_written_blob = c.bytes_written_blob; @@ -311,6 +343,10 @@ class InternalStats { num_input_files_in_non_output_levels = c.num_input_files_in_non_output_levels; num_input_files_in_output_level = c.num_input_files_in_output_level; + num_filtered_input_files_in_non_output_levels = + c.num_filtered_input_files_in_non_output_levels; + num_filtered_input_files_in_output_level = + c.num_filtered_input_files_in_output_level; num_output_files = c.num_output_files; num_output_files_blob = c.num_output_files_blob; num_input_records = c.num_input_records; @@ -330,12 +366,16 @@ class InternalStats { this->cpu_micros = 0; this->bytes_read_non_output_levels = 0; this->bytes_read_output_level = 0; + this->bytes_skipped_non_output_levels = 0; + this->bytes_skipped_output_level = 0; this->bytes_read_blob = 0; this->bytes_written = 0; this->bytes_written_blob = 0; this->bytes_moved = 0; this->num_input_files_in_non_output_levels = 0; this->num_input_files_in_output_level = 0; + this->num_filtered_input_files_in_non_output_levels = 0; + this->num_filtered_input_files_in_output_level = 0; this->num_output_files = 0; this->num_output_files_blob = 0; this->num_input_records = 0; @@ -353,6 +393,9 @@ class InternalStats { this->cpu_micros += c.cpu_micros; this->bytes_read_non_output_levels += c.bytes_read_non_output_levels; this->bytes_read_output_level += c.bytes_read_output_level; + this->bytes_skipped_non_output_levels += + c.bytes_skipped_non_output_levels; + this->bytes_skipped_output_level += c.bytes_skipped_output_level; this->bytes_read_blob += c.bytes_read_blob; this->bytes_written += c.bytes_written; this->bytes_written_blob += c.bytes_written_blob; @@ -361,6 +404,10 @@ class InternalStats { c.num_input_files_in_non_output_levels; this->num_input_files_in_output_level += c.num_input_files_in_output_level; + this->num_filtered_input_files_in_non_output_levels += + c.num_filtered_input_files_in_non_output_levels; + this->num_filtered_input_files_in_output_level += + c.num_filtered_input_files_in_output_level; this->num_output_files += c.num_output_files; this->num_output_files_blob += c.num_output_files_blob; this->num_input_records += c.num_input_records; @@ -387,6 +434,9 @@ class InternalStats { this->cpu_micros -= c.cpu_micros; this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels; this->bytes_read_output_level -= c.bytes_read_output_level; + this->bytes_skipped_non_output_levels -= + c.bytes_skipped_non_output_levels; + this->bytes_skipped_output_level -= c.bytes_skipped_output_level; this->bytes_read_blob -= c.bytes_read_blob; this->bytes_written -= c.bytes_written; this->bytes_written_blob -= c.bytes_written_blob; @@ -395,6 +445,10 @@ class InternalStats { c.num_input_files_in_non_output_levels; this->num_input_files_in_output_level -= c.num_input_files_in_output_level; + this->num_filtered_input_files_in_non_output_levels -= + c.num_filtered_input_files_in_non_output_levels; + this->num_filtered_input_files_in_output_level -= + c.num_filtered_input_files_in_output_level; this->num_output_files -= c.num_output_files; this->num_output_files_blob -= c.num_output_files_blob; this->num_input_records -= c.num_input_records; diff --git a/include/rocksdb/compaction_job_stats.h b/include/rocksdb/compaction_job_stats.h index f9c1bb3ec2..91709795a1 100644 --- a/include/rocksdb/compaction_job_stats.h +++ b/include/rocksdb/compaction_job_stats.h @@ -35,6 +35,12 @@ struct CompactionJobStats { size_t num_input_files = 0; // the number of compaction input files at the output level (table files) size_t num_input_files_at_output_level = 0; + // the number of compaction input files that are filtered out by compaction + // optimizations + size_t num_filtered_input_files = 0; + // the number of compaction input files at the output level that are filtered + // out by compaction optimizations + size_t num_filtered_input_files_at_output_level = 0; // the number of compaction output records. uint64_t num_output_records = 0; @@ -58,6 +64,9 @@ struct CompactionJobStats { uint64_t total_output_bytes = 0; // the total size of blob files in the compaction output uint64_t total_output_bytes_blob = 0; + // the total size of table files for compaction input files that are skipped + // because input files are filtered out by compaction optimizations. + uint64_t total_skipped_input_bytes = 0; // number of records being replaced by newer record associated with same key. // this could be a new value or a deletion entry for that key so this field diff --git a/util/compaction_job_stats_impl.cc b/util/compaction_job_stats_impl.cc index 11ab63b620..70551f5b62 100644 --- a/util/compaction_job_stats_impl.cc +++ b/util/compaction_job_stats_impl.cc @@ -17,6 +17,8 @@ void CompactionJobStats::Reset() { num_blobs_read = 0; num_input_files = 0; num_input_files_at_output_level = 0; + num_filtered_input_files = 0; + num_filtered_input_files_at_output_level = 0; num_output_records = 0; num_output_files = 0; @@ -30,6 +32,7 @@ void CompactionJobStats::Reset() { total_blob_bytes_read = 0; total_output_bytes = 0; total_output_bytes_blob = 0; + total_skipped_input_bytes = 0; num_records_replaced = 0; @@ -62,6 +65,9 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) { num_blobs_read += stats.num_blobs_read; num_input_files += stats.num_input_files; num_input_files_at_output_level += stats.num_input_files_at_output_level; + num_filtered_input_files += stats.num_filtered_input_files; + num_filtered_input_files_at_output_level += + stats.num_filtered_input_files_at_output_level; num_output_records += stats.num_output_records; num_output_files += stats.num_output_files; @@ -71,6 +77,7 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) { total_blob_bytes_read += stats.total_blob_bytes_read; total_output_bytes += stats.total_output_bytes; total_output_bytes_blob += stats.total_output_bytes_blob; + total_skipped_input_bytes += stats.total_skipped_input_bytes; num_records_replaced += stats.num_records_replaced;