Support options.max_open_files != -1 with FIFO compaction (#6503)

Summary:
Allow user to specify options.max_open_files != -1 with FIFO compaction.
If max_open_files != -1, not all table files are kept open.
In the past, FIFO style compaction requires all table files to be open in order
to read file creation time from table properties. Later, we added file creation
time to MANIFEST, making it possible to read file creation time without opening
file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6503

Test Plan: make check

Differential Revision: D20353758

Pulled By: riversand963

fbshipit-source-id: ba5c61a648419e47e9ef6d74e0e280e3ee24f296

db/compaction/compaction_picker_fifo.cc

@@ -70,18 +70,14 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   // avoid underflow
   if (current_time > mutable_cf_options.ttl) {
     for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
-      auto f = *ritr;
+      FileMetaData* f = *ritr;
-      if (f->fd.table_reader != nullptr &&
+      uint64_t creation_time = f->TryGetFileCreationTime();
-          f->fd.table_reader->GetTableProperties() != nullptr) {
+      if (creation_time == kUnknownFileCreationTime ||
-        auto creation_time =
+          creation_time >= (current_time - mutable_cf_options.ttl)) {
-            f->fd.table_reader->GetTableProperties()->creation_time;
+        break;
-        if (creation_time == 0 ||
-            creation_time >= (current_time - mutable_cf_options.ttl)) {
-          break;
-        }
-        total_size -= f->compensated_file_size;
-        inputs[0].files.push_back(f);
       }
+      total_size -= f->compensated_file_size;
+      inputs[0].files.push_back(f);
     }
   }
 
@@ -100,7 +96,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
                      "[%s] FIFO compaction: picking file %" PRIu64
                      " with creation time %" PRIu64 " for deletion",
                      cf_name.c_str(), f->fd.GetNumber(),
-                     f->fd.table_reader->GetTableProperties()->creation_time);
+                     f->TryGetFileCreationTime());
   }
 
   Compaction* c = new Compaction(

db/db_compaction_test.cc

@@ -5151,6 +5151,31 @@ TEST_P(DBCompactionTestWithParam,
   }
 }
 
+TEST_F(DBCompactionTest, FifoCompactionGetFileCreationTime) {
+  MockEnv mock_env(env_);
+  do {
+    Options options = CurrentOptions();
+    options.table_factory.reset(new BlockBasedTableFactory());
+    options.env = &mock_env;
+    options.ttl = static_cast<uint64_t>(24) * 3600;
+    options.compaction_style = kCompactionStyleFIFO;
+    constexpr size_t kNumFiles = 24;
+    options.max_open_files = 20;
+    constexpr size_t kNumKeysPerFile = 10;
+    DestroyAndReopen(options);
+    for (size_t i = 0; i < kNumFiles; ++i) {
+      for (size_t j = 0; j < kNumKeysPerFile; ++j) {
+        ASSERT_OK(Put(std::to_string(j), "value_" + std::to_string(i)));
+      }
+      ASSERT_OK(Flush());
+    }
+    mock_env.FakeSleepForMicroseconds(
+        static_cast<uint64_t>(1000 * 1000 * (1 + options.ttl)));
+    ASSERT_OK(Put("foo", "value"));
+    ASSERT_OK(Flush());
+  } while (ChangeOptions());
+}
+
 #endif // !defined(ROCKSDB_LITE)
 }  // namespace ROCKSDB_NAMESPACE
 

include/rocksdb/advanced_options.h

@@ -645,7 +645,6 @@ struct AdvancedColumnFamilyOptions {
   bool report_bg_io_stats = false;
 
   // Files older than TTL will go through the compaction process.
-  // Pre-req: This needs max_open_files to be set to -1.
   // In Level: Non-bottom-level files older than TTL will go through the
   //           compation process.
   // In FIFO: Files older than TTL will be deleted.
@@ -673,7 +672,6 @@ struct AdvancedColumnFamilyOptions {
   // Supported in Level and FIFO compaction.
   // In FIFO compaction, this option has the same meaning as TTL and whichever
   // stricter will be used.
-  // Pre-req: max_open_file == -1.
   // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60
   //
   // Values: