Renamed 'hybrid_compaction' tp be "Universal Compaction'.

Summary:
All the universal compaction parameters are encapsulated in
a new file universal_compaction.h

Test Plan:
make check

db/db_bench.cc

@@ -9,6 +9,7 @@
 #include "db/db_impl.h"
 #include "db/version_set.h"
 #include "db/db_statistics.h"
+#include "leveldb/options.h"
 #include "leveldb/cache.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
@@ -125,7 +126,7 @@ static int FLAGS_max_write_buffer_number = 0;
 // The minimum number of write buffers that will be merged together
 // before writing to storage. This is cheap because it is an
 // in-memory merge. If this feature is not enabled, then all these
-// write buffers are fushed to L0 as seperate files and this increases
+// write buffers are fushed to L0 as separate files and this increases
 // read amplification because a get request has to check in all of these
 // files. Also, an in-memory merge may result in writing lesser
 // data to storage if there are duplicate records in each of these
@@ -137,14 +138,14 @@ static int FLAGS_min_write_buffer_number_to_merge = 0;
 // This is initialized to default value of 1 in "main" function.
 static int FLAGS_max_background_compactions = 0;
 
-// Run database in hybrid mode where all data resides in L0.
+// style of compaction: level-based vs universal
-static bool FLAGS_hybrid_mode = false;
+static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel;
 
 // Percentage flexibilty while comparing file size.
-static int FLAGS_hybrid_size_ratio = 1;
+static int FLAGS_universal_size_ratio = 1;
 
 // The minimum number of files in a single compaction run.
-static int FLAGS_hybrid_min_numfiles_in_single_compaction = 2;
+static int FLAGS_compaction_universal_min_merge_width = 2;
 
 // Number of bytes to use as a cache of uncompressed data.
 // Negative means use default settings.
@@ -1113,10 +1114,10 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
     options.min_write_buffer_number_to_merge =
       FLAGS_min_write_buffer_number_to_merge;
     options.max_background_compactions = FLAGS_max_background_compactions;
-    options.hybrid_mode = FLAGS_hybrid_mode;
+    options.compaction_style = FLAGS_compaction_style;
-    options.hybrid_size_ratio = FLAGS_hybrid_size_ratio;
+    options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio;
-    options.hybrid_min_numfiles_in_single_compaction = 
+    options.compaction_options_universal.min_merge_width =
-      FLAGS_hybrid_min_numfiles_in_single_compaction;
+      FLAGS_compaction_universal_min_merge_width;
     options.block_size = FLAGS_block_size;
     options.filter_policy = filter_policy_;
     options.max_open_files = FLAGS_open_files;
@@ -1999,12 +2000,11 @@ int main(int argc, char** argv) {
   FLAGS_open_files = leveldb::Options().max_open_files;
   FLAGS_max_background_compactions =
     leveldb::Options().max_background_compactions;
-  FLAGS_hybrid_mode =
+  FLAGS_compaction_style = leveldb::Options().compaction_style;
-    leveldb::Options().hybrid_mode;
+  FLAGS_universal_size_ratio =
-  FLAGS_hybrid_size_ratio =
+    leveldb::Options().compaction_options_universal.size_ratio;
-    leveldb::Options().hybrid_size_ratio;
+  FLAGS_compaction_universal_min_merge_width =
-  FLAGS_hybrid_min_numfiles_in_single_compaction =
+    leveldb::Options().compaction_options_universal.min_merge_width;
-    leveldb::Options().hybrid_min_numfiles_in_single_compaction;
   // Compression test code above refers to FLAGS_block_size
   FLAGS_block_size = leveldb::Options().block_size;
   FLAGS_use_os_buffer = leveldb::EnvOptions().use_os_buffer;
@@ -2063,13 +2063,13 @@ int main(int argc, char** argv) {
       FLAGS_min_write_buffer_number_to_merge = n;
     } else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) {
       FLAGS_max_background_compactions = n;
-    } else if (sscanf(argv[i], "--hybrid_mode=%d%c", &n, &junk) == 1) {
+    } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
-      FLAGS_hybrid_mode = n;
+      FLAGS_compaction_style = (leveldb::CompactionStyle)n;
-    } else if (sscanf(argv[i], "--hybrid_size_ratio=%d%c", &n, &junk) == 1) {
+    } else if (sscanf(argv[i], "--universal_size_ratio=%d%c", &n, &junk) == 1) {
-      FLAGS_hybrid_size_ratio = n;
+      FLAGS_universal_size_ratio = n;
-    } else if (sscanf(argv[i], "--hybrid_min_numfiles_in_single_compaction=%d%c",
+    } else if (sscanf(argv[i], "--universal_min_merge_width=%d%c",
                &n, &junk) == 1) {
-      FLAGS_hybrid_min_numfiles_in_single_compaction = n;
+      FLAGS_compaction_universal_min_merge_width = n;
     } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
       FLAGS_cache_size = l;
     } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {

db/db_impl.cc

@@ -836,7 +836,7 @@ Status DBImpl::WriteLevel0Table(std::vector<MemTable*> &mems, VersionEdit* edit,
     // threads could be concurrently producing compacted files for
     // that key range.
     if (base != nullptr && options_.max_background_compactions <= 1 &&
-        !options_.hybrid_mode) {
+        options_.compaction_style == kCompactionStyleLevel) {
       level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
     }
     edit->AddFile(level, meta.number, meta.file_size,
@@ -1578,7 +1578,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
   for (size_t i = 0; i < compact->outputs.size(); i++) {
     const CompactionState::Output& out = compact->outputs[i];
     compact->compaction->edit()->AddFile(
-        options_.hybrid_mode? level : level + 1,
+        (options_.compaction_style == kCompactionStyleUniversal) ?
+          level : level + 1,
         out.number, out.file_size, out.smallest, out.largest,
         out.smallest_seqno, out.largest_seqno);
   }
@@ -1828,9 +1829,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
       // If this is the bottommost level (no files in lower levels)
       // and the earliest snapshot is larger than this seqno
       // then we can squash the seqno to zero.
-      // Hybrid mode depends on the sequence number to determine
+      // Universal mode depends on the sequence number to determine
       // time-order of files that is needed for compactions.
-      if (!options_.hybrid_mode &&
+      if (options_.compaction_style == kCompactionStyleLevel &&
           bottommost_level && ikey.sequence < earliest_snapshot &&
          ikey.type != kTypeMerge) {
         assert(ikey.type != kTypeDeletion);

db/version_set.cc

@@ -384,7 +384,7 @@ void Version::Get(const ReadOptions& options,
       }
       if (tmp.empty()) continue;
 
-      if (vset_->options_->hybrid_mode) {
+      if (vset_->options_->compaction_style == kCompactionStyleUniversal) {
         std::sort(tmp.begin(), tmp.end(), NewestFirstBySeqNo);
       } else {
         std::sort(tmp.begin(), tmp.end(), NewestFirst);
@@ -1581,7 +1581,7 @@ static bool compareSizeDescending(const VersionSet::Fsize& first,
   return (first.file->file_size > second.file->file_size);
 }
 // A static compator used to sort files based on their seqno
-// In hybrid mode: descending seqno
+// In universal style : descending seqno
 static bool compareSeqnoDescending(const VersionSet::Fsize& first,
   const VersionSet::Fsize& second) {
   if (first.file->smallest_seqno > second.file->smallest_seqno) {
@@ -1596,8 +1596,8 @@ static bool compareSeqnoDescending(const VersionSet::Fsize& first,
 void VersionSet::UpdateFilesBySize(Version* v) {
 
   // No need to sort the highest level because it is never compacted.
-  int max_level = options_->hybrid_mode? NumberLevels() :
+  int max_level = (options_->compaction_style == kCompactionStyleUniversal) ?
-                  NumberLevels() - 1;
+                  NumberLevels() : NumberLevels() - 1;
 
   for (int level = 0; level < max_level; level++) {
 
@@ -1613,7 +1613,7 @@ void VersionSet::UpdateFilesBySize(Version* v) {
     }
 
     // sort the top number_of_files_to_sort_ based on file size
-    if (options_->hybrid_mode) {
+    if (options_->compaction_style == kCompactionStyleUniversal) {
       int num = temp.size();
       std::partial_sort(temp.begin(),  temp.begin() + num,
                         temp.end(), compareSeqnoDescending);
@@ -2020,7 +2020,11 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
   assert (level == 0);
 
   // percentage flexibilty while comparing file sizes
-  uint64_t ratio = (uint64_t)options_->hybrid_size_ratio;
+  uint64_t ratio = options_->compaction_options_universal.size_ratio;
+  unsigned int min_merge_width =
+    options_->compaction_options_universal.min_merge_width;
+  unsigned int max_merge_width =
+    options_->compaction_options_universal.max_merge_width;
 
   if ((current_->files_[level].size() <=
       (unsigned int)options_->level0_file_num_compaction_trigger)) {
@@ -2043,7 +2047,7 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
   bool done = false;
   assert(file_by_time.size() == current_->files_[level].size());
 
-  unsigned int max_files_to_compact = UINT_MAX;
+  unsigned int max_files_to_compact = std::min(max_merge_width, UINT_MAX);
 
   // Make two pass. The first pass considers a candidate file
   // only if it is smaller than the total size accumulated so far.
@@ -2098,8 +2102,7 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
       }
 
       // Found a series of consecutive files that need compaction.
-      if (candidate_count >= (unsigned int)
+      if (candidate_count >= (unsigned int)min_merge_width) {
-          options_->hybrid_min_numfiles_in_single_compaction) {
         for (unsigned int i = loop; i < loop + candidate_count; i++) {
           int index = file_by_time[i];
           FileMetaData* f = current_->files_[level][index];
@@ -2141,8 +2144,8 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
           options_->level0_file_num_compaction_trigger + 1) {
         done = true;     // nothing more to do
       } else {
-        max_files_to_compact = expected_num_files -
+        max_files_to_compact = std::min((int)max_merge_width,
-                               options_->level0_file_num_compaction_trigger;
+          expected_num_files - options_->level0_file_num_compaction_trigger);
         Log(options_->info_log, "Hybrid: second loop with maxfiles %d",
             max_files_to_compact);
       }
@@ -2264,8 +2267,8 @@ Compaction* VersionSet::PickCompaction() {
   current_->vset_->SizeBeingCompacted(size_being_compacted);
   Finalize(current_, size_being_compacted);
 
-  // In hybrid mode compact L0 files back into L0.
+  // In universal style of compaction, compact L0 files back into L0.
-  if (options_->hybrid_mode) {
+  if (options_->compaction_style ==  kCompactionStyleUniversal) {
     int level = 0;
     c = PickCompactionHybrid(level, current_->compaction_score_[level]);
     return c;
@@ -2473,7 +2476,8 @@ Compaction* VersionSet::CompactRange(
       }
     }
   }
-  int out_level = options_->hybrid_mode ? level : level+1;
+  int out_level = (options_->compaction_style == kCompactionStyleUniversal) ?
+                  level : level+1;
 
   Compaction* c = new Compaction(level, out_level, MaxFileSizeForLevel(level),
     MaxGrandParentOverlapBytes(level), NumberLevels());

include/leveldb/options.h

@@ -12,6 +12,7 @@
 #include <stdint.h>
 #include "leveldb/slice.h"
 #include "leveldb/statistics.h"
+#include "leveldb/universal_compaction.h"
 
 namespace leveldb {
 
@@ -39,6 +40,11 @@ enum CompressionType {
   kBZip2Compression = 0x3
 };
 
+enum CompactionStyle {
+  kCompactionStyleLevel       = 0x0, // level based compaction style
+  kCompactionStyleUniversal   = 0x1  // Universal compaction style
+};
+
 // Compression options for different compression algorithms like Zlib
 struct CompressionOptions {
   int window_bits;
@@ -132,7 +138,7 @@ struct Options {
   int max_write_buffer_number;
 
   // The minimum number of write buffers that will be merged together
-  // before writing to storage.  If set to 1, then 
+  // before writing to storage.  If set to 1, then
   // all write buffers are fushed to L0 as individual files and this increases
   // read amplification because a get request has to check in all of these
   // files. Also, an in-memory merge may result in writing lesser
@@ -476,17 +482,12 @@ struct Options {
   // Default: 0
   uint64_t bytes_per_sync;
 
-  // Hybrid Mode. There is only a single level and files in L0 are
+  // The compaction style
   // compacted back into L0. Default: false
-  bool hybrid_mode;
+  CompactionStyle compaction_style;
 
-  // Percentage flexibilty while comparing file size. If the candidate file(s)
+  // The options needed to support Universal Style compactions
-  // size is 1% smaller than the next file's size, then include next file into
+  CompactionOptionsUniversal compaction_options_universal;
-  // this candidate set. // Default: 1
-  int hybrid_size_ratio;
-
-  // The minimum number of files in a single compaction run. Default: 2
-  int hybrid_min_numfiles_in_single_compaction;
 };
 
 // Options that control read operations

include/leveldb/universal_compaction.h

@@ -0,0 +1,57 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
+#define STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
+
+#include <stddef.h>
+#include <string>
+#include <memory>
+#include <vector>
+#include <stdint.h>
+#include <climits>
+#include "leveldb/slice.h"
+#include "leveldb/statistics.h"
+
+namespace leveldb {
+
+//
+// Algorithm used to make a compaction request stop picking new files
+// into a single compaction run
+//
+enum CompactionStopStyle {
+  kCompactionStopStyleSimilarSize, // pick files of similar size
+  kCompactionStopStyleTotalSize    // total size of picked files > next file
+};
+
+class CompactionOptionsUniversal {
+ public:
+
+  // Percentage flexibilty while comparing file size. If the candidate file(s)
+  // size is 1% smaller than the next file's size, then include next file into
+  // this candidate set. // Default: 1
+  unsigned int size_ratio;
+
+  // The minimum number of files in a single compaction run. Default: 2
+  unsigned int min_merge_width;
+
+  // The maximum number of files in a single compaction run. Default: INT_MAX
+  unsigned int max_merge_width;
+
+  // The algorithm used to stop picking files into a single compaction run
+  // Default: kCompactionStopStyleTotalSize
+  CompactionStopStyle stop_style;
+
+  // Default set of parameters
+  CompactionOptionsUniversal() :
+    size_ratio(1),
+    min_merge_width(2),
+    max_merge_width(UINT_MAX),
+    stop_style(kCompactionStopStyleTotalSize) {
+  }
+};
+
+}  // namespace leveldb
+
+#endif  // STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H

tools/db_stress.cc

@@ -89,7 +89,7 @@ static int FLAGS_max_write_buffer_number = 0;
 static int FLAGS_max_background_compactions = 0;
 
 // This is initialized to default value of false
-static bool FLAGS_hybrid_mode = false;
+static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel;
 
 // Number of bytes to use as a cache of uncompressed data.
 static long FLAGS_cache_size = 2 * KB * KB * KB;
@@ -933,7 +933,7 @@ class StressTest {
     options.write_buffer_size = FLAGS_write_buffer_size;
     options.max_write_buffer_number = FLAGS_max_write_buffer_number;
     options.max_background_compactions = FLAGS_max_background_compactions;
-    options.hybrid_mode = FLAGS_hybrid_mode;
+    options.compaction_style = FLAGS_compaction_style;
     options.block_size = FLAGS_block_size;
     options.filter_policy = filter_policy_;
     options.max_open_files = FLAGS_open_files;
@@ -1020,8 +1020,8 @@ int main(int argc, char** argv) {
   FLAGS_open_files = leveldb::Options().max_open_files;
   FLAGS_max_background_compactions =
     leveldb::Options().max_background_compactions;
-  FLAGS_hybrid_mode =
+  FLAGS_compaction_style =
-    leveldb::Options().hybrid_mode;
+    leveldb::Options().compaction_style;
   FLAGS_level0_file_num_compaction_trigger =
     leveldb::Options().level0_file_num_compaction_trigger;
   FLAGS_level0_slowdown_writes_trigger =
@@ -1074,8 +1074,8 @@ int main(int argc, char** argv) {
       FLAGS_max_write_buffer_number = n;
     } else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) {
       FLAGS_max_background_compactions = n;
-    } else if (sscanf(argv[i], "--hybrid_mode=%d%c", &n, &junk) == 1) {
+    } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
-      FLAGS_hybrid_mode = n;
+      FLAGS_compaction_style = (leveldb::CompactionStyle)n;
     } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
       FLAGS_cache_size = l;
     } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {

util/ldb_cmd.cc

@@ -1070,7 +1070,7 @@ void ApproxSizeCommand::DoCommand() {
   uint64_t sizes[1];
   db_->GetApproximateSizes(ranges, 1, sizes);
   fprintf(stdout, "%ld\n", sizes[0]);
-  /* Wierd that GetApproximateSizes() returns void, although documentation
+  /* Weird that GetApproximateSizes() returns void, although documentation
    * says that it returns a Status object.
   if (!st.ok()) {
     exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString());

util/options.cc

@@ -77,9 +77,7 @@ Options::Options()
       access_hint_on_compaction_start(NORMAL),
       use_adaptive_mutex(false),
       bytes_per_sync(0),
-      hybrid_mode(false),
+      compaction_style(kCompactionStyleLevel) {
-      hybrid_size_ratio(1),
-      hybrid_min_numfiles_in_single_compaction(2) {
 }
 
 static const char* const access_hints[] = {
@@ -220,12 +218,14 @@ Options::Dump(Logger* log) const
         use_adaptive_mutex);
     Log(log,"                          Options.bytes_per_sync: %ld",
         bytes_per_sync);
-    Log(log,"                             Options.hybrid_mode: %d",
+    Log(log,"                        Options.compaction_style: %d",
-        hybrid_mode);
+        compaction_style);
-    Log(log,"                       Options.hybrid_size_ratio: %d",
+    Log(log,"        Options.compaction_options_universal.size_ratio: %d",
-        hybrid_size_ratio);
+        compaction_options_universal.size_ratio);
-    Log(log,"Options.hybrid_min_numfiles_in_single_compaction: %d",
+    Log(log,"   Options.compaction_options_universal.min_merge_width: %d",
-        hybrid_min_numfiles_in_single_compaction);
+        compaction_options_universal.min_merge_width);
+    Log(log,"   Options.compaction_options_universal.max_merge_width: %d",
+        compaction_options_universal.max_merge_width);
 }   // Options::Dump
 
 //