Add compaction stats for filtered files (#13136)

Summary:
As titled. This PR adds some compaction job stats, internal stats and some logging for filtered files.

Example logging:
[default] compacted to: files[0 0 0 0 2 0 0] max score 0.25, estimated pending compaction bytes 0, MB/sec: 0.3 rd, 0.2 wr, level 6, files in(1, 0) filtered(0, 2) out(1 +0 blob) MB in(0.0, 0.0 +0.0 blob) filtered(0.0, 0.0) out(0.0 +0.0 blob), read-write-amplify(2.0) write-amplify(1.0) OK, records in: 1, records dropped: 1 output_compression: Snappy

Pull Request resolved: https://github.com/facebook/rocksdb/pull/13136

Test Plan: Added unit tests

Reviewed By: cbi42

Differential Revision: D65855380

Pulled By: jowlyzhang

fbshipit-source-id: a4d8eef66f8d999ca5c3d9472aeeae98d7bb03ab

db/compaction/compaction.cc

@@ -368,9 +368,10 @@ Compaction::Compaction(
   }
 #endif
 
-  // setup input_levels_
+  // setup input_levels_ and filtered_input_levels_
   {
     input_levels_.resize(num_input_levels());
+    filtered_input_levels_.resize(num_input_levels());
     if (earliest_snapshot_.has_value()) {
       FilterInputsForCompactionIterator();
     } else {
@@ -1085,6 +1086,7 @@ void Compaction::FilterInputsForCompactionIterator() {
           ucmp->CompareWithoutTimestamp(rangedel_end_ukey,
                                         file->largest.user_key()) > 0) {
         non_start_level_input_files_filtered_.back().back() = true;
+        filtered_input_levels_[level].push_back(file);
       } else {
         non_start_level_input_files.back().push_back(file);
       }

db/compaction/compaction.h

@@ -184,6 +184,16 @@ class Compaction {
     return &input_levels_[compaction_input_level];
   }
 
+  // Returns the filtered input files of the specified compaction input level.
+  // For now, only non start level is filtered.
+  const std::vector<FileMetaData*>& filtered_input_levels(
+      size_t compaction_input_level) const {
+    const std::vector<FileMetaData*>& filtered_input_level =
+        filtered_input_levels_[compaction_input_level];
+    assert(compaction_input_level != 0 || filtered_input_level.size() == 0);
+    return filtered_input_level;
+  }
+
   // Maximum size of files to build during this compaction.
   uint64_t max_output_file_size() const { return max_output_file_size_; }
 
@@ -545,10 +555,13 @@ class Compaction {
 
   // Markers for which non start level input files are filtered out if
   // applicable. Only applicable if earliest_snapshot_ is provided and input
-  // start level has a standalone range deletion file.
+  // start level has a standalone range deletion file. Filtered files are
+  // tracked in `filtered_input_levels_`.
   std::vector<std::vector<bool>> non_start_level_input_files_filtered_;
 
-  //  bool standalone_range_tombstones_used_for_filtering_inputs_;
+  // All files from inputs_ that are filtered.
+  std::vector<std::vector<FileMetaData*>> filtered_input_levels_;
+
   const double score_;  // score that was used to pick this compaction.
 
   // Is this compaction creating a file in the bottom most level?

db/compaction/compaction_job.cc

@@ -911,19 +911,23 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
   ROCKS_LOG_BUFFER(
       log_buffer_,
       "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
-      "files in(%d, %d) out(%d +%d blob) "
+      "files in(%d, %d) filtered(%d, %d) out(%d +%d blob) "
-      "MB in(%.1f, %.1f +%.1f blob) out(%.1f +%.1f blob), "
+      "MB in(%.1f, %.1f +%.1f blob) filtered(%.1f, %.1f) out(%.1f +%.1f blob), "
       "read-write-amplify(%.1f) write-amplify(%.1f) %s, records in: %" PRIu64
       ", records dropped: %" PRIu64 " output_compression: %s\n",
       column_family_name.c_str(), vstorage->LevelSummary(&tmp),
       bytes_read_per_sec, bytes_written_per_sec,
       compact_->compaction->output_level(),
       stats.num_input_files_in_non_output_levels,
-      stats.num_input_files_in_output_level, stats.num_output_files,
+      stats.num_input_files_in_output_level,
+      stats.num_filtered_input_files_in_non_output_levels,
+      stats.num_filtered_input_files_in_output_level, stats.num_output_files,
       stats.num_output_files_blob, stats.bytes_read_non_output_levels / kMB,
       stats.bytes_read_output_level / kMB, stats.bytes_read_blob / kMB,
-      stats.bytes_written / kMB, stats.bytes_written_blob / kMB, read_write_amp,
+      stats.bytes_skipped_non_output_levels / kMB,
-      write_amp, status.ToString().c_str(), stats.num_input_records,
+      stats.bytes_skipped_output_level / kMB, stats.bytes_written / kMB,
+      stats.bytes_written_blob / kMB, read_write_amp, write_amp,
+      status.ToString().c_str(), stats.num_input_records,
       stats.num_dropped_records,
       CompressionTypeToString(compact_->compaction->output_compression())
           .c_str());
@@ -2007,7 +2011,6 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
   bool has_error = false;
   const ReadOptions read_options(Env::IOActivity::kCompaction);
   const auto& input_table_properties = compaction->GetInputTableProperties();
-  // TODO(yuzhangyu): add dedicated stats for filtered files.
   for (int input_level = 0;
        input_level < static_cast<int>(compaction->num_input_levels());
        ++input_level) {
@@ -2047,6 +2050,23 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
         *num_input_range_del += file_num_range_del;
       }
     }
+
+    const std::vector<FileMetaData*>& filtered_flevel =
+        compaction->filtered_input_levels(input_level);
+    size_t num_filtered_input_files = filtered_flevel.size();
+    uint64_t* bytes_skipped;
+    if (compaction->level(input_level) != compaction->output_level()) {
+      compaction_stats_.stats.num_filtered_input_files_in_non_output_levels +=
+          static_cast<int>(num_filtered_input_files);
+      bytes_skipped = &compaction_stats_.stats.bytes_skipped_non_output_levels;
+    } else {
+      compaction_stats_.stats.num_filtered_input_files_in_output_level +=
+          static_cast<int>(num_filtered_input_files);
+      bytes_skipped = &compaction_stats_.stats.bytes_skipped_output_level;
+    }
+    for (const FileMetaData* filtered_file_meta : filtered_flevel) {
+      *bytes_skipped += filtered_file_meta->fd.GetFileSize();
+    }
   }
 
   assert(compaction_job_stats_);
@@ -2071,6 +2091,13 @@ void CompactionJob::UpdateCompactionJobStats(
       stats.num_input_files_in_output_level;
   compaction_job_stats_->num_input_files_at_output_level =
       stats.num_input_files_in_output_level;
+  compaction_job_stats_->num_filtered_input_files =
+      stats.num_filtered_input_files_in_non_output_levels +
+      stats.num_filtered_input_files_in_output_level;
+  compaction_job_stats_->num_filtered_input_files_at_output_level =
+      stats.num_filtered_input_files_in_output_level;
+  compaction_job_stats_->total_skipped_input_bytes =
+      stats.bytes_skipped_non_output_levels + stats.bytes_skipped_output_level;
 
   // output information
   compaction_job_stats_->total_output_bytes = stats.bytes_written;

db/compaction/compaction_job.h

@@ -215,8 +215,7 @@ class CompactionJob {
   virtual void RecordCompactionIOStats();
   void CleanupCompaction();
 
-  // Call compaction filter. Then iterate through input and compact the
+  // Iterate through input and compact the kv-pairs.
-  // kv-pairs
   void ProcessKeyValueCompaction(SubcompactionState* sub_compact);
 
   CompactionState* compact_;

db/external_sst_file_basic_test.cc

@@ -182,6 +182,36 @@ class ExternalSSTFileBasicTest
         write_global_seqno, verify_checksums_before_ingest, true_data);
   }
 
+  void VerifyInputFilesInternalStatsForOutputLevel(
+      int output_level, int num_input_files_in_non_output_levels,
+      int num_input_files_in_output_level,
+      int num_filtered_input_files_in_non_output_levels,
+      int num_filtered_input_files_in_output_level,
+      uint64_t bytes_skipped_non_output_levels,
+      uint64_t bytes_skipped_output_level) {
+    ColumnFamilyHandleImpl* cfh =
+        static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily());
+    ColumnFamilyData* cfd = cfh->cfd();
+    const InternalStats* internal_stats_ptr = cfd->internal_stats();
+    const std::vector<InternalStats::CompactionStats>& comp_stats =
+        internal_stats_ptr->TEST_GetCompactionStats();
+
+    EXPECT_EQ(num_input_files_in_non_output_levels,
+              comp_stats[output_level].num_input_files_in_non_output_levels);
+    EXPECT_EQ(num_input_files_in_output_level,
+              comp_stats[output_level].num_input_files_in_output_level);
+    EXPECT_EQ(
+        num_filtered_input_files_in_non_output_levels,
+        comp_stats[output_level].num_filtered_input_files_in_non_output_levels);
+    EXPECT_EQ(
+        num_filtered_input_files_in_output_level,
+        comp_stats[output_level].num_filtered_input_files_in_output_level);
+    EXPECT_EQ(bytes_skipped_non_output_levels,
+              comp_stats[output_level].bytes_skipped_non_output_levels);
+    EXPECT_EQ(bytes_skipped_output_level,
+              comp_stats[output_level].bytes_skipped_output_level);
+  }
+
   ~ExternalSSTFileBasicTest() override {
     DestroyDir(env_, sst_files_dir_).PermitUncheckedError();
   }
@@ -1841,11 +1871,60 @@ TEST_F(ExternalSSTFileBasicTest, OverlappingFiles) {
   ASSERT_EQ(2, NumTableFilesAtLevel(5));
 }
 
+class CompactionJobStatsCheckerForFilteredFiles : public EventListener {
+ public:
+  CompactionJobStatsCheckerForFilteredFiles(
+      int num_input_files, int num_input_files_at_output_level,
+      int num_filtered_input_files,
+      int num_filtered_input_files_at_output_level)
+      : num_input_files_(num_input_files),
+        num_input_files_at_output_level_(num_input_files_at_output_level),
+        num_filtered_input_files_(num_filtered_input_files),
+        num_filtered_input_files_at_output_level_(
+            num_filtered_input_files_at_output_level) {}
+
+  void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ASSERT_EQ(num_input_files_, ci.stats.num_input_files);
+    ASSERT_EQ(num_input_files_at_output_level_,
+              ci.stats.num_input_files_at_output_level);
+    ASSERT_EQ(num_filtered_input_files_, ci.stats.num_filtered_input_files);
+    ASSERT_EQ(num_filtered_input_files_at_output_level_,
+              ci.stats.num_filtered_input_files_at_output_level);
+    ASSERT_EQ(ci.stats.total_skipped_input_bytes,
+              expected_compaction_skipped_file_size_);
+  }
+
+  void SetExpectedCompactionSkippedFileSize(uint64_t expected_size) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    expected_compaction_skipped_file_size_ = expected_size;
+  }
+
+ private:
+  int num_input_files_ = 0;
+  int num_input_files_at_output_level_ = 0;
+  int num_filtered_input_files_ = 0;
+  int num_filtered_input_files_at_output_level_ = 0;
+  std::mutex mutex_;
+  uint64_t expected_compaction_skipped_file_size_ = 0;
+};
+
 TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
   Options options = CurrentOptions();
   options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 1;
+  int kCompactionNumInputFilesAtOutputLevel = 0;
+  int kCompactionNumFilteredInputFiles = 2;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 2;
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
+          kCompactionNumFilteredInputFiles,
+          kCompactionNumFilteredInputFilesAtOutputLevel);
+  options.listeners.push_back(compaction_listener);
   DestroyAndReopen(options);
 
+  size_t compaction_skipped_file_size = 0;
   std::vector<std::string> files;
   {
     // Writes first version of data in range partitioned files.
@@ -1856,6 +1935,7 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
     ASSERT_OK(sst_file_writer.Put("b", "b1"));
     ExternalSstFileInfo file1_info;
     ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    compaction_skipped_file_size += file1_info.file_size;
     files.push_back(std::move(file1));
 
     std::string file2 = sst_files_dir_ + "file2.sst";
@@ -1864,7 +1944,10 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
     ASSERT_OK(sst_file_writer.Put("y", "y1"));
     ExternalSstFileInfo file2_info;
     ASSERT_OK(sst_file_writer.Finish(&file2_info));
+    compaction_skipped_file_size += file2_info.file_size;
     files.push_back(std::move(file2));
+    compaction_listener->SetExpectedCompactionSkippedFileSize(
+        compaction_skipped_file_size);
   }
 
   IngestExternalFileOptions ifo;
@@ -1944,6 +2027,16 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
   ASSERT_EQ(Get("x"), "x2");
   ASSERT_EQ(Get("y"), "y2");
 
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ compaction_skipped_file_size);
+
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
@@ -1951,9 +2044,20 @@ TEST_F(ExternalSSTFileBasicTest,
        PartiallyReplaceDataWithOneStandaloneRangeDeletion) {
   Options options = CurrentOptions();
   options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 2;
+  int kCompactionNumInputFilesAtOutputLevel = 1;
+  int kCompactionNumFilteredInputFiles = 1;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 1;
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
+          kCompactionNumFilteredInputFiles,
+          kCompactionNumFilteredInputFilesAtOutputLevel);
+  options.listeners.push_back(compaction_listener);
   DestroyAndReopen(options);
 
   std::vector<std::string> files;
+  size_t compaction_skipped_file_size = 0;
   {
     // Writes first version of data in range partitioned files.
     SstFileWriter sst_file_writer(EnvOptions(), options);
@@ -1963,7 +2067,10 @@ TEST_F(ExternalSSTFileBasicTest,
     ASSERT_OK(sst_file_writer.Put("b", "b1"));
     ExternalSstFileInfo file1_info;
     ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    compaction_skipped_file_size += file1_info.file_size;
     files.push_back(std::move(file1));
+    compaction_listener->SetExpectedCompactionSkippedFileSize(
+        compaction_skipped_file_size);
 
     std::string file2 = sst_files_dir_ + "file2.sst";
     ASSERT_OK(sst_file_writer.Open(file2));
@@ -2032,6 +2139,17 @@ TEST_F(ExternalSSTFileBasicTest,
   ASSERT_EQ(Get("h"), "h1");
   ASSERT_EQ(Get("x"), "x2");
   ASSERT_EQ(Get("y"), "y");
+
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ compaction_skipped_file_size);
+
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
@@ -2039,15 +2157,29 @@ TEST_F(ExternalSSTFileBasicTest,
        PartiallyReplaceDataWithMultipleStandaloneRangeDeletions) {
   Options options = CurrentOptions();
   options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 2;
+  int kCompactionNumInputFilesAtOutputLevel = 0;
+  int kCompactionNumFilteredInputFiles = 2;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 2;
+  // Two compactions each included on standalone range deletion file that
+  // filters input file on the non start level.
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles / 2,
+          kCompactionNumInputFilesAtOutputLevel / 2,
+          kCompactionNumFilteredInputFiles / 2,
+          kCompactionNumFilteredInputFilesAtOutputLevel / 2);
+  options.listeners.push_back(compaction_listener);
   DestroyAndReopen(options);
 
   std::vector<std::string> files;
+  ExternalSstFileInfo file1_info;
+  ExternalSstFileInfo file3_info;
   {
     SstFileWriter sst_file_writer(EnvOptions(), options);
     std::string file1 = sst_files_dir_ + "file1.sst";
     ASSERT_OK(sst_file_writer.Open(file1));
     ASSERT_OK(sst_file_writer.Put("a", "a1"));
-    ExternalSstFileInfo file1_info;
     ASSERT_OK(sst_file_writer.Finish(&file1_info));
     files.push_back(std::move(file1));
     std::string file2 = sst_files_dir_ + "file2.sst";
@@ -2059,7 +2191,6 @@ TEST_F(ExternalSSTFileBasicTest,
     std::string file3 = sst_files_dir_ + "file3.sst";
     ASSERT_OK(sst_file_writer.Open(file3));
     ASSERT_OK(sst_file_writer.Put("x", "x1"));
-    ExternalSstFileInfo file3_info;
     ASSERT_OK(sst_file_writer.Finish(&file3_info));
     files.push_back(std::move(file3));
   }
@@ -2107,9 +2238,15 @@ TEST_F(ExternalSSTFileBasicTest,
         size_t* num_input_files = static_cast<size_t*>(arg);
         EXPECT_EQ(1, *num_input_files);
         num_compactions += 1;
+        if (num_compactions == 2) {
+          compaction_listener->SetExpectedCompactionSkippedFileSize(
+              file3_info.file_size);
+        }
       });
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
+  compaction_listener->SetExpectedCompactionSkippedFileSize(
+      file1_info.file_size);
   ASSERT_OK(db_->IngestExternalFile(files, ifo));
 
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
@@ -2121,12 +2258,35 @@ TEST_F(ExternalSSTFileBasicTest,
   ASSERT_EQ(Get("a"), "a2");
   ASSERT_EQ(Get("h"), "h");
   ASSERT_EQ(Get("x"), "x2");
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ file1_info.file_size +
+          file3_info.file_size);
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
 TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) {
   Options options = CurrentOptions();
   options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 2;
+  int kCompactionNumInputFilesAtOutputLevel = 1;
+  int kCompactionNumFilteredInputFiles = 0;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 0;
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
+          kCompactionNumFilteredInputFiles,
+          kCompactionNumFilteredInputFilesAtOutputLevel);
+  options.listeners.push_back(compaction_listener);
+  // No compaction input files are filtered because the range deletion file's
+  // end is exclusive, so it cannot cover the whole file.
+  compaction_listener->SetExpectedCompactionSkippedFileSize(0);
   DestroyAndReopen(options);
 
   std::vector<std::string> files;
@@ -2181,6 +2341,16 @@ TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) {
 
   ASSERT_EQ(Get("a"), "NOT_FOUND");
   ASSERT_EQ(Get("b"), "b");
+
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ 0);
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 

db/internal_stats.h

@@ -182,6 +182,14 @@ class InternalStats {
     // The number of bytes read from the compaction output level (table files)
     uint64_t bytes_read_output_level;
 
+    // The number of bytes skipped from all non-output levels because the input
+    // files are filtered by compaction optimizations.
+    uint64_t bytes_skipped_non_output_levels;
+
+    // The number of bytes skipped from the compaction output level because the
+    // input files are filtered by compaction optimizations.
+    uint64_t bytes_skipped_output_level;
+
     // The number of bytes read from blob files
     uint64_t bytes_read_blob;
 
@@ -201,6 +209,14 @@ class InternalStats {
     // The number of compaction input files in the output level (table files)
     int num_input_files_in_output_level;
 
+    // The number of non output level compaction input files that are filtered
+    // by compaction optimizations.
+    int num_filtered_input_files_in_non_output_levels;
+
+    // The number of output level compaction input files that are filtered by
+    // compaction optimizations.
+    int num_filtered_input_files_in_output_level;
+
     // The number of compaction output files (table files)
     int num_output_files;
 
@@ -228,12 +244,16 @@ class InternalStats {
           cpu_micros(0),
           bytes_read_non_output_levels(0),
           bytes_read_output_level(0),
+          bytes_skipped_non_output_levels(0),
+          bytes_skipped_output_level(0),
           bytes_read_blob(0),
           bytes_written(0),
           bytes_written_blob(0),
           bytes_moved(0),
           num_input_files_in_non_output_levels(0),
           num_input_files_in_output_level(0),
+          num_filtered_input_files_in_non_output_levels(0),
+          num_filtered_input_files_in_output_level(0),
           num_output_files(0),
           num_output_files_blob(0),
           num_input_records(0),
@@ -251,12 +271,16 @@ class InternalStats {
           cpu_micros(0),
           bytes_read_non_output_levels(0),
           bytes_read_output_level(0),
+          bytes_skipped_non_output_levels(0),
+          bytes_skipped_output_level(0),
           bytes_read_blob(0),
           bytes_written(0),
           bytes_written_blob(0),
           bytes_moved(0),
           num_input_files_in_non_output_levels(0),
           num_input_files_in_output_level(0),
+          num_filtered_input_files_in_non_output_levels(0),
+          num_filtered_input_files_in_output_level(0),
           num_output_files(0),
           num_output_files_blob(0),
           num_input_records(0),
@@ -280,6 +304,8 @@ class InternalStats {
           cpu_micros(c.cpu_micros),
           bytes_read_non_output_levels(c.bytes_read_non_output_levels),
           bytes_read_output_level(c.bytes_read_output_level),
+          bytes_skipped_non_output_levels(c.bytes_skipped_non_output_levels),
+          bytes_skipped_output_level(c.bytes_skipped_output_level),
           bytes_read_blob(c.bytes_read_blob),
           bytes_written(c.bytes_written),
           bytes_written_blob(c.bytes_written_blob),
@@ -287,6 +313,10 @@ class InternalStats {
           num_input_files_in_non_output_levels(
               c.num_input_files_in_non_output_levels),
           num_input_files_in_output_level(c.num_input_files_in_output_level),
+          num_filtered_input_files_in_non_output_levels(
+              c.num_filtered_input_files_in_non_output_levels),
+          num_filtered_input_files_in_output_level(
+              c.num_filtered_input_files_in_output_level),
           num_output_files(c.num_output_files),
           num_output_files_blob(c.num_output_files_blob),
           num_input_records(c.num_input_records),
@@ -304,6 +334,8 @@ class InternalStats {
       cpu_micros = c.cpu_micros;
       bytes_read_non_output_levels = c.bytes_read_non_output_levels;
       bytes_read_output_level = c.bytes_read_output_level;
+      bytes_skipped_non_output_levels = c.bytes_skipped_non_output_levels;
+      bytes_skipped_output_level = c.bytes_skipped_output_level;
       bytes_read_blob = c.bytes_read_blob;
       bytes_written = c.bytes_written;
       bytes_written_blob = c.bytes_written_blob;
@@ -311,6 +343,10 @@ class InternalStats {
       num_input_files_in_non_output_levels =
           c.num_input_files_in_non_output_levels;
       num_input_files_in_output_level = c.num_input_files_in_output_level;
+      num_filtered_input_files_in_non_output_levels =
+          c.num_filtered_input_files_in_non_output_levels;
+      num_filtered_input_files_in_output_level =
+          c.num_filtered_input_files_in_output_level;
       num_output_files = c.num_output_files;
       num_output_files_blob = c.num_output_files_blob;
       num_input_records = c.num_input_records;
@@ -330,12 +366,16 @@ class InternalStats {
       this->cpu_micros = 0;
       this->bytes_read_non_output_levels = 0;
       this->bytes_read_output_level = 0;
+      this->bytes_skipped_non_output_levels = 0;
+      this->bytes_skipped_output_level = 0;
       this->bytes_read_blob = 0;
       this->bytes_written = 0;
       this->bytes_written_blob = 0;
       this->bytes_moved = 0;
       this->num_input_files_in_non_output_levels = 0;
       this->num_input_files_in_output_level = 0;
+      this->num_filtered_input_files_in_non_output_levels = 0;
+      this->num_filtered_input_files_in_output_level = 0;
       this->num_output_files = 0;
       this->num_output_files_blob = 0;
       this->num_input_records = 0;
@@ -353,6 +393,9 @@ class InternalStats {
       this->cpu_micros += c.cpu_micros;
       this->bytes_read_non_output_levels += c.bytes_read_non_output_levels;
       this->bytes_read_output_level += c.bytes_read_output_level;
+      this->bytes_skipped_non_output_levels +=
+          c.bytes_skipped_non_output_levels;
+      this->bytes_skipped_output_level += c.bytes_skipped_output_level;
       this->bytes_read_blob += c.bytes_read_blob;
       this->bytes_written += c.bytes_written;
       this->bytes_written_blob += c.bytes_written_blob;
@@ -361,6 +404,10 @@ class InternalStats {
           c.num_input_files_in_non_output_levels;
       this->num_input_files_in_output_level +=
           c.num_input_files_in_output_level;
+      this->num_filtered_input_files_in_non_output_levels +=
+          c.num_filtered_input_files_in_non_output_levels;
+      this->num_filtered_input_files_in_output_level +=
+          c.num_filtered_input_files_in_output_level;
       this->num_output_files += c.num_output_files;
       this->num_output_files_blob += c.num_output_files_blob;
       this->num_input_records += c.num_input_records;
@@ -387,6 +434,9 @@ class InternalStats {
       this->cpu_micros -= c.cpu_micros;
       this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels;
       this->bytes_read_output_level -= c.bytes_read_output_level;
+      this->bytes_skipped_non_output_levels -=
+          c.bytes_skipped_non_output_levels;
+      this->bytes_skipped_output_level -= c.bytes_skipped_output_level;
       this->bytes_read_blob -= c.bytes_read_blob;
       this->bytes_written -= c.bytes_written;
       this->bytes_written_blob -= c.bytes_written_blob;
@@ -395,6 +445,10 @@ class InternalStats {
           c.num_input_files_in_non_output_levels;
       this->num_input_files_in_output_level -=
           c.num_input_files_in_output_level;
+      this->num_filtered_input_files_in_non_output_levels -=
+          c.num_filtered_input_files_in_non_output_levels;
+      this->num_filtered_input_files_in_output_level -=
+          c.num_filtered_input_files_in_output_level;
       this->num_output_files -= c.num_output_files;
       this->num_output_files_blob -= c.num_output_files_blob;
       this->num_input_records -= c.num_input_records;

include/rocksdb/compaction_job_stats.h

@@ -35,6 +35,12 @@ struct CompactionJobStats {
   size_t num_input_files = 0;
   // the number of compaction input files at the output level (table files)
   size_t num_input_files_at_output_level = 0;
+  // the number of compaction input files that are filtered out by compaction
+  // optimizations
+  size_t num_filtered_input_files = 0;
+  // the number of compaction input files at the output level that are filtered
+  // out by compaction optimizations
+  size_t num_filtered_input_files_at_output_level = 0;
 
   // the number of compaction output records.
   uint64_t num_output_records = 0;
@@ -58,6 +64,9 @@ struct CompactionJobStats {
   uint64_t total_output_bytes = 0;
   // the total size of blob files in the compaction output
   uint64_t total_output_bytes_blob = 0;
+  // the total size of table files for compaction input files that are skipped
+  // because input files are filtered out by compaction optimizations.
+  uint64_t total_skipped_input_bytes = 0;
 
   // number of records being replaced by newer record associated with same key.
   // this could be a new value or a deletion entry for that key so this field

util/compaction_job_stats_impl.cc

@@ -17,6 +17,8 @@ void CompactionJobStats::Reset() {
   num_blobs_read = 0;
   num_input_files = 0;
   num_input_files_at_output_level = 0;
+  num_filtered_input_files = 0;
+  num_filtered_input_files_at_output_level = 0;
 
   num_output_records = 0;
   num_output_files = 0;
@@ -30,6 +32,7 @@ void CompactionJobStats::Reset() {
   total_blob_bytes_read = 0;
   total_output_bytes = 0;
   total_output_bytes_blob = 0;
+  total_skipped_input_bytes = 0;
 
   num_records_replaced = 0;
 
@@ -62,6 +65,9 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
   num_blobs_read += stats.num_blobs_read;
   num_input_files += stats.num_input_files;
   num_input_files_at_output_level += stats.num_input_files_at_output_level;
+  num_filtered_input_files += stats.num_filtered_input_files;
+  num_filtered_input_files_at_output_level +=
+      stats.num_filtered_input_files_at_output_level;
 
   num_output_records += stats.num_output_records;
   num_output_files += stats.num_output_files;
@@ -71,6 +77,7 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
   total_blob_bytes_read += stats.total_blob_bytes_read;
   total_output_bytes += stats.total_output_bytes;
   total_output_bytes_blob += stats.total_output_bytes_blob;
+  total_skipped_input_bytes += stats.total_skipped_input_bytes;
 
   num_records_replaced += stats.num_records_replaced;