log_and_apply_bench on a new benchmark framework

Summary:
db_test includes Benchmark for LogAndApply. This diff removes it from db_test and puts it into a separate log_and_apply bench. I just wanted to play around with our new benchmark framework and figure out how it works.

I would also like to show you how great it is! I believe right set of microbenchmarks can speed up our productivity a lot and help catch early regressions.

Test Plan: no

Reviewers: dhruba, haobo, sdong, ljin, yhchiang

Reviewed By: yhchiang

CC: leveldb

Differential Revision: https://reviews.facebook.net/D18261

Makefile

@@ -123,7 +123,7 @@ TOOLS = \
 	db_repl_stress \
 	blob_store_bench
 
-PROGRAMS = db_bench signal_test table_reader_bench $(TOOLS)
+PROGRAMS = db_bench signal_test table_reader_bench log_and_apply_bench $(TOOLS)
 
 # The library name is configurable since we are maintaining libraries of both
 # debug/release mode.
@@ -337,6 +337,9 @@ simple_table_db_test: db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 table_reader_bench: table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -pg
 
+log_and_apply_bench: db/log_and_apply_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(BENCHHARNESS)
+	$(CXX) db/log_and_apply_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(BENCHHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -pg
+
 perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 

db/db_test.cc

@@ -6612,68 +6612,6 @@ TEST(DBTest, PrefixScan) {
   delete options.filter_policy;
 }
 
-namespace {
-std::string MakeKey(unsigned int num) {
-  char buf[30];
-  snprintf(buf, sizeof(buf), "%016u", num);
-  return std::string(buf);
-}
-
-void BM_LogAndApply(int iters, int num_base_files) {
-  std::string dbname = test::TmpDir() + "/rocksdb_test_benchmark";
-  ASSERT_OK(DestroyDB(dbname, Options()));
-
-  DB* db = nullptr;
-  Options opts;
-  opts.create_if_missing = true;
-  Status s = DB::Open(opts, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  delete db;
-  db = nullptr;
-
-  Env* env = Env::Default();
-
-  port::Mutex mu;
-  MutexLock l(&mu);
-
-  Options options;
-  EnvOptions sopt;
-  VersionSet vset(dbname, &options, sopt, nullptr);
-  std::vector<ColumnFamilyDescriptor> dummy;
-  dummy.push_back(ColumnFamilyDescriptor());
-  ASSERT_OK(vset.Recover(dummy));
-  auto default_cfd = vset.GetColumnFamilySet()->GetDefault();
-  VersionEdit vbase;
-  uint64_t fnum = 1;
-  for (int i = 0; i < num_base_files; i++) {
-    InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
-    InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
-    vbase.AddFile(2, fnum++, 1 /* file size */, start, limit, 1, 1);
-  }
-  ASSERT_OK(vset.LogAndApply(default_cfd, &vbase, &mu));
-
-  uint64_t start_micros = env->NowMicros();
-
-  for (int i = 0; i < iters; i++) {
-    VersionEdit vedit;
-    vedit.DeleteFile(2, fnum);
-    InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
-    InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
-    vedit.AddFile(2, fnum++, 1 /* file size */, start, limit, 1, 1);
-    vset.LogAndApply(default_cfd, &vedit, &mu);
-  }
-  uint64_t stop_micros = env->NowMicros();
-  unsigned int us = stop_micros - start_micros;
-  char buf[16];
-  snprintf(buf, sizeof(buf), "%d", num_base_files);
-  fprintf(stderr,
-          "BM_LogAndApply/%-6s   %8d iters : %9u us (%7.0f us / iter)\n",
-          buf, iters, us, ((float)us) / iters);
-}
-}  // namespace
-
 TEST(DBTest, TailingIteratorSingle) {
   ReadOptions read_options;
   read_options.tailing = true;
@@ -6822,13 +6760,5 @@ TEST(DBTest, ChecksumTest) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
-  if (argc > 1 && std::string(argv[1]) == "--benchmark") {
-    rocksdb::BM_LogAndApply(1000, 1);
-    rocksdb::BM_LogAndApply(1000, 100);
-    rocksdb::BM_LogAndApply(1000, 10000);
-    rocksdb::BM_LogAndApply(100, 100000);
-    return 0;
-  }
-
   return rocksdb::test::RunAllTests();
 }

db/log_and_apply_bench.cc

@@ -0,0 +1,79 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+
+#include <vector>
+
+#include "util/testharness.h"
+#include "util/benchharness.h"
+#include "db/version_set.h"
+#include "util/mutexlock.h"
+
+namespace rocksdb {
+
+std::string MakeKey(unsigned int num) {
+  char buf[30];
+  snprintf(buf, sizeof(buf), "%016u", num);
+  return std::string(buf);
+}
+
+void BM_LogAndApply(int iters, int num_base_files) {
+  VersionSet* vset;
+  ColumnFamilyData* default_cfd;
+  uint64_t fnum = 1;
+  port::Mutex mu;
+  MutexLock l(&mu);
+
+  BENCHMARK_SUSPEND {
+    std::string dbname = test::TmpDir() + "/rocksdb_test_benchmark";
+    ASSERT_OK(DestroyDB(dbname, Options()));
+
+    DB* db = nullptr;
+    Options opts;
+    opts.create_if_missing = true;
+    Status s = DB::Open(opts, dbname, &db);
+    ASSERT_OK(s);
+    ASSERT_TRUE(db != nullptr);
+
+    delete db;
+    db = nullptr;
+
+    Options options;
+    EnvOptions sopt;
+    vset = new VersionSet(dbname, &options, sopt, nullptr);
+    std::vector<ColumnFamilyDescriptor> dummy;
+    dummy.push_back(ColumnFamilyDescriptor());
+    ASSERT_OK(vset->Recover(dummy));
+    default_cfd = vset->GetColumnFamilySet()->GetDefault();
+    VersionEdit vbase;
+    for (int i = 0; i < num_base_files; i++) {
+      InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
+      InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
+      vbase.AddFile(2, ++fnum, 1 /* file size */, start, limit, 1, 1);
+    }
+    ASSERT_OK(vset->LogAndApply(default_cfd, &vbase, &mu));
+  }
+
+  for (int i = 0; i < iters; i++) {
+    VersionEdit vedit;
+    vedit.DeleteFile(2, fnum);
+    InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
+    InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
+    vedit.AddFile(2, ++fnum, 1 /* file size */, start, limit, 1, 1);
+    vset->LogAndApply(default_cfd, &vedit, &mu);
+  }
+}
+
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 1000_iters_1_file, 1000, 1)
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 1000_iters_100_files, 1000, 100)
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 1000_iters_10000_files, 1000, 10000)
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 100_iters_100000_files, 100, 100000)
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::benchmark::RunBenchmarks();
+  return 0;
+}

util/benchharness.h

@@ -261,6 +261,13 @@ AddBenchmark(const char* file, const char* name, Lambda&& lambda) {
  */
 #define BENCHMARK_NAMED_PARAM(name, param_name, ...)                    \
   BENCHMARK_IMPL(                                                       \
+      FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),              \
+      FB_STRINGIZE(name) "(" FB_STRINGIZE(param_name) ")") {            \
+    name(__VA_ARGS__);                                                  \
+  }
+
+#define BENCHMARK_NAMED_PARAM_N(name, param_name, ...)                  \
+  BENCHMARK_IMPL_N(                                                     \
       FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),              \
       FB_STRINGIZE(name) "(" FB_STRINGIZE(param_name) ")",              \
       unsigned,                                                         \
@@ -307,14 +314,14 @@ AddBenchmark(const char* file, const char* name, Lambda&& lambda) {
 /**
  * A combination of BENCHMARK_RELATIVE and BENCHMARK_PARAM.
  */
-#define BENCHMARK_RELATIVE_PARAM(name, param)                           \
+#define BENCHMARK_RELATIVE_PARAM(name, param)                   \
   BENCHMARK_RELATIVE_NAMED_PARAM(name, param, param)
 
 /**
  * A combination of BENCHMARK_RELATIVE and BENCHMARK_NAMED_PARAM.
  */
 #define BENCHMARK_RELATIVE_NAMED_PARAM(name, param_name, ...)           \
-  BENCHMARK_IMPL(                                                       \
+  BENCHMARK_IMPL_N(                                                     \
       FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),              \
       "%" FB_STRINGIZE(name) "(" FB_STRINGIZE(param_name) ")",          \
       unsigned,                                                         \
@@ -327,7 +334,7 @@ AddBenchmark(const char* file, const char* name, Lambda&& lambda) {
  */
 #define BENCHMARK_DRAW_LINE()                                       \
   static bool FB_ANONYMOUS_VARIABLE(rocksdbBenchmarkUnused) = (     \
-    ::rocksdb::benchmark::AddBenchmark(__FILE__, "-", []() { }),               \
+    ::rocksdb::benchmark::AddBenchmark(__FILE__, "-", []() { }),    \
     true);
 
 /**
@@ -346,5 +353,5 @@ AddBenchmark(const char* file, const char* name, Lambda&& lambda) {
  */
 #define BENCHMARK_SUSPEND                               \
   if (auto FB_ANONYMOUS_VARIABLE(BENCHMARK_SUSPEND) =   \
-      ::rocksdb::benchmark::BenchmarkSuspender()) {}               \
+      ::rocksdb::benchmark::BenchmarkSuspender()) {}    \
   else