Merge pull request #120 from google/benchmark-fixtures

Add ability to use benchmark fixtures

README.md

@@ -175,6 +175,36 @@ static void BM_test(benchmark::State& state) {
 }
 ```
 
+Benchmark Fixtures
+------------------
+Fixture tests are created by
+first defining a type that derives from ::benchmark::Fixture and then
+creating/registering the tests using the following macros:
+
+* `BENCHMARK_F(ClassName, Method)`
+* `BENCHMARK_DEFINE_F(ClassName, Method)`
+* `BENCHMARK_REGISTER_F(ClassName, Method)`
+
+For Example:
+
+```c++
+class MyFixture : public benchmark::Fixture {};
+
+BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {
+   while (st.KeepRunning()) {
+     ...
+  }
+}
+
+BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {
+   while (st.KeepRunning()) {
+     ...
+  }
+}
+/* BarTest is NOT registered */
+BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
+/* BarTest is now registered */
+```
 
 Output Formats
 --------------

include/benchmark/benchmark_api.h

@@ -169,6 +169,7 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter);
 namespace internal {
 class Benchmark;
 class BenchmarkImp;
+class BenchmarkFamilies;
 
 template <class T> struct Voider {
     typedef void type;
@@ -184,8 +185,13 @@ struct EnableIfString<T, typename Voider<typename T::basic_string>::type> {
 
 void UseCharPointer(char const volatile*);
 
+// Take ownership of the pointer and register the benchmark. Return the
+// registered benchmark.
+Benchmark* RegisterBenchmarkInternal(Benchmark*);
+
 } // end namespace internal
 
+
 // The DoNotOptimize(...) function can be used to prevent a value or
 // expression from being optimized away by the compiler. This function is
 // intented to add little to no overhead.
@@ -371,10 +377,8 @@ typedef void(Function)(State&);
 // Each method returns "this" so that multiple method calls can
 // chained into one expression.
 class Benchmark {
- public:
+public:
-  Benchmark(const char* name, Function* f);
+  virtual ~Benchmark();
-
-  ~Benchmark();
 
   // Note: the following methods all return "this" so that multiple
   // method calls can be chained together in one expression.
@@ -445,15 +449,56 @@ class Benchmark {
   // Equivalent to ThreadRange(NumCPUs(), NumCPUs())
   Benchmark* ThreadPerCpu();
 
+  virtual void Run(State& state) = 0;
+
   // Used inside the benchmark implementation
   struct Instance;
 
- private:
+protected:
-   BenchmarkImp* imp_;
+  explicit Benchmark(const char* name);
-   BENCHMARK_DISALLOW_COPY_AND_ASSIGN(Benchmark);
+  Benchmark(Benchmark const&);
+  void SetName(const char* name);
+
+private:
+  friend class BenchmarkFamilies;
+  BenchmarkImp* imp_;
+
+  Benchmark& operator=(Benchmark const&);
+};
+
+// The class used to hold all Benchmarks created from static function.
+// (ie those created using the BENCHMARK(...) macros.
+class FunctionBenchmark : public Benchmark {
+public:
+    FunctionBenchmark(const char* name, Function* func)
+        : Benchmark(name), func_(func)
+    {}
+
+    virtual void Run(State& st);
+private:
+    Function* func_;
 };
 
 }  // end namespace internal
+
+// The base class for all fixture tests. 
+class Fixture: public internal::Benchmark {
+public:
+    Fixture() : internal::Benchmark("") {}
+
+    virtual void Run(State& st) {
+      this->SetUp();
+      this->BenchmarkCase(st);
+      this->TearDown();
+    }
+
+    virtual void SetUp() {}
+    virtual void TearDown() {}
+
+protected:
+    virtual void BenchmarkCase(State&) = 0;
+};
+
 }  // end namespace benchmark
 
 
@@ -480,7 +525,9 @@ class Benchmark {
   BENCHMARK_PRIVATE_NAME(n) BENCHMARK_UNUSED
 
 #define BENCHMARK(n) \
-    BENCHMARK_PRIVATE_DECLARE(n) = (new ::benchmark::internal::Benchmark(#n, n))
+    BENCHMARK_PRIVATE_DECLARE(n) =                               \
+        (::benchmark::internal::RegisterBenchmarkInternal(       \
+            new ::benchmark::internal::FunctionBenchmark(#n, n)))
 
 // Old-style macros
 #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a))
@@ -499,21 +546,53 @@ class Benchmark {
 // will register BM_Foo<1> as a benchmark.
 #define BENCHMARK_TEMPLATE1(n, a) \
   BENCHMARK_PRIVATE_DECLARE(n) =  \
-      (new ::benchmark::internal::Benchmark(#n "<" #a ">", n<a>))
+      (::benchmark::internal::RegisterBenchmarkInternal( \
+        new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n<a>)))
 
-#define BENCHMARK_TEMPLATE2(n, a, b) \
+#define BENCHMARK_TEMPLATE2(n, a, b)                     \
-  BENCHMARK_PRIVATE_DECLARE(n) =     \
+  BENCHMARK_PRIVATE_DECLARE(n) =                         \
-      (new ::benchmark::internal::Benchmark(#n "<" #a "," #b ">", n<a, b>))
+      (::benchmark::internal::RegisterBenchmarkInternal( \
+        new ::benchmark::internal::FunctionBenchmark(    \
+            #n "<" #a "," #b ">", n<a, b>)))
 
 #if __cplusplus >= 201103L
 #define BENCHMARK_TEMPLATE(n, ...)           \
   BENCHMARK_PRIVATE_DECLARE(n) =             \
-      (new ::benchmark::internal::Benchmark( \
+      (::benchmark::internal::RegisterBenchmarkInternal( \
-        #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))
+        new ::benchmark::internal::FunctionBenchmark( \
+        #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>)))
 #else
 #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a)
 #endif
 
+
+#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method)      \
+class BaseClass##_##Method##_Benchmark : public BaseClass { \
+public:                                                     \
+    BaseClass##_##Method##_Benchmark() : BaseClass() {      \
+        this->SetName(#BaseClass "/" #Method);}             \
+protected:                                                  \
+    virtual void BenchmarkCase(::benchmark::State&);        \
+};
+
+#define BENCHMARK_DEFINE_F(BaseClass, Method) \
+    BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \
+    void BaseClass##_##Method##_Benchmark::BenchmarkCase
+
+#define BENCHMARK_REGISTER_F(BaseClass, Method) \
+    BENCHMARK_PRIVATE_REGISTER_F(BaseClass##_##Method##_Benchmark)
+
+#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \
+    BENCHMARK_PRIVATE_DECLARE(TestName) = \
+        (::benchmark::internal::RegisterBenchmarkInternal(new TestName()))
+
+// This macro will define and register a benchmark within a fixture class.
+#define BENCHMARK_F(BaseClass, Method) \
+    BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \
+    BENCHMARK_REGISTER_F(BaseClass, Method); \
+    void BaseClass##_##Method##_Benchmark::BenchmarkCase
+
+
 // Helper macro to create a main routine in a test that runs the benchmarks
 #define BENCHMARK_MAIN()                             \
   int main(int argc, const char** argv) {            \

src/benchmark.cc

@@ -254,16 +254,16 @@ namespace internal {
 
 // Information kept per benchmark we may want to run
 struct Benchmark::Instance {
-  std::string   name;
+  std::string    name;
-  Function*     function;
+  Benchmark*     benchmark;
-  bool          has_arg1;
+  bool           has_arg1;
-  int           arg1;
+  int            arg1;
-  bool          has_arg2;
+  bool           has_arg2;
-  int           arg2;
+  int            arg2;
-  bool          use_real_time;
+  bool           use_real_time;
-  double        min_time;
+  double         min_time;
-  int           threads;    // Number of concurrent threads to use
+  int            threads;    // Number of concurrent threads to use
-  bool          multithreaded;  // Is benchmark multi-threaded?
+  bool           multithreaded;  // Is benchmark multi-threaded?
 };
 
 // Class for managing registered benchmarks.  Note that each registered
@@ -273,27 +273,23 @@ class BenchmarkFamilies {
   static BenchmarkFamilies* GetInstance();
 
   // Registers a benchmark family and returns the index assigned to it.
-  size_t AddBenchmark(BenchmarkImp* family);
+  size_t AddBenchmark(std::unique_ptr<Benchmark> family);
-
-  // Unregisters a family at the given index.
-  void RemoveBenchmark(size_t index);
 
   // Extract the list of benchmark instances that match the specified
   // regular expression.
   bool FindBenchmarks(const std::string& re,
                       std::vector<Benchmark::Instance>* benchmarks);
  private:
-  BenchmarkFamilies();
+  BenchmarkFamilies() {}
-  ~BenchmarkFamilies();
 
-  std::vector<BenchmarkImp*> families_;
+  std::vector<std::unique_ptr<Benchmark>> families_;
   Mutex mutex_;
 };
 
 
 class BenchmarkImp {
 public:
-  BenchmarkImp(const char* name, Function* func);
+  explicit BenchmarkImp(const char* name);
   ~BenchmarkImp();
 
   void Arg(int x);
@@ -306,6 +302,7 @@ public:
   void Threads(int t);
   void ThreadRange(int min_threads, int max_threads);
   void ThreadPerCpu();
+  void SetName(const char* name);
 
   static void AddRange(std::vector<int>* dst, int lo, int hi, int mult);
 
@@ -313,15 +310,13 @@ private:
   friend class BenchmarkFamilies;
 
   std::string name_;
-  Function* function_;
   int arg_count_;
   std::vector< std::pair<int, int> > args_;  // Args for all benchmark runs
   double min_time_;
   bool use_real_time_;
   std::vector<int> thread_counts_;
-  std::size_t registration_index_;
 
-  BENCHMARK_DISALLOW_COPY_AND_ASSIGN(BenchmarkImp);
+  BenchmarkImp& operator=(BenchmarkImp const&);
 };
 
 BenchmarkFamilies* BenchmarkFamilies::GetInstance() {
@@ -329,36 +324,14 @@ BenchmarkFamilies* BenchmarkFamilies::GetInstance() {
   return &instance;
 }
 
-BenchmarkFamilies::BenchmarkFamilies() { }
 
-BenchmarkFamilies::~BenchmarkFamilies() {
+size_t BenchmarkFamilies::AddBenchmark(std::unique_ptr<Benchmark> family) {
-  for (BenchmarkImp* family : families_) {
-    delete family;
-  }
-}
-
-size_t BenchmarkFamilies::AddBenchmark(BenchmarkImp* family) {
   MutexLock l(mutex_);
-  // This loop attempts to reuse an entry that was previously removed to avoid
-  // unncessary growth of the vector.
-  for (size_t index = 0; index < families_.size(); ++index) {
-    if (families_[index] == nullptr) {
-      families_[index] = family;
-      return index;
-    }
-  }
   size_t index = families_.size();
-  families_.push_back(family);
+  families_.push_back(std::move(family));
   return index;
 }
 
-void BenchmarkFamilies::RemoveBenchmark(size_t index) {
-  MutexLock l(mutex_);
-  families_[index] = nullptr;
-  // Don't shrink families_ here, we might be called by the destructor of
-  // BenchmarkFamilies which iterates over the vector.
-}
-
 bool BenchmarkFamilies::FindBenchmarks(
     const std::string& spec,
     std::vector<Benchmark::Instance>* benchmarks) {
@@ -375,9 +348,10 @@ bool BenchmarkFamilies::FindBenchmarks(
   one_thread.push_back(1);
 
   MutexLock l(mutex_);
-  for (BenchmarkImp* family : families_) {
+  for (std::unique_ptr<Benchmark>& bench_family : families_) {
     // Family was deleted or benchmark doesn't match
-    if (family == nullptr) continue;
+    if (!bench_family) continue;
+    BenchmarkImp* family = bench_family->imp_;
 
     if (family->arg_count_ == -1) {
       family->arg_count_ = 0;
@@ -392,7 +366,7 @@ bool BenchmarkFamilies::FindBenchmarks(
 
         Benchmark::Instance instance;
         instance.name = family->name_;
-        instance.function = family->function_;
+        instance.benchmark = bench_family.get();
         instance.has_arg1 = family->arg_count_ >= 1;
         instance.arg1 = args.first;
         instance.has_arg2 = family->arg_count_ == 2;
@@ -430,14 +404,12 @@ bool BenchmarkFamilies::FindBenchmarks(
   return true;
 }
 
-BenchmarkImp::BenchmarkImp(const char* name, Function* func)
+BenchmarkImp::BenchmarkImp(const char* name)
-    : name_(name), function_(func), arg_count_(-1),
+    : name_(name), arg_count_(-1),
       min_time_(0.0), use_real_time_(false) {
-    registration_index_ = BenchmarkFamilies::GetInstance()->AddBenchmark(this);
 }
 
 BenchmarkImp::~BenchmarkImp() {
-  BenchmarkFamilies::GetInstance()->RemoveBenchmark(registration_index_);
 }
 
 void BenchmarkImp::Arg(int x) {
@@ -513,6 +485,10 @@ void BenchmarkImp::ThreadPerCpu() {
   thread_counts_.push_back(num_cpus);
 }
 
+void BenchmarkImp::SetName(const char* name) {
+  name_ = name;
+}
+
 void BenchmarkImp::AddRange(std::vector<int>* dst, int lo, int hi, int mult) {
   CHECK_GE(lo, 0);
   CHECK_GE(hi, lo);
@@ -535,8 +511,8 @@ void BenchmarkImp::AddRange(std::vector<int>* dst, int lo, int hi, int mult) {
   }
 }
 
-Benchmark::Benchmark(const char* name, Function* f)
+Benchmark::Benchmark(const char* name)
-    : imp_(new BenchmarkImp(name, f))
+    : imp_(new BenchmarkImp(name))
 {
 }
 
@@ -544,6 +520,11 @@ Benchmark::~Benchmark()  {
   delete imp_;
 }
 
+Benchmark::Benchmark(Benchmark const& other)
+  : imp_(new BenchmarkImp(*other.imp_))
+{
+}
+
 Benchmark* Benchmark::Arg(int x) {
   imp_->Arg(x);
   return this;
@@ -599,6 +580,14 @@ Benchmark* Benchmark::ThreadPerCpu() {
   return this;
 }
 
+void Benchmark::SetName(const char* name) {
+  imp_->SetName(name);
+}
+
+void FunctionBenchmark::Run(State& st) {
+  func_(st);
+}
+
 } // end namespace internal
 
 namespace {
@@ -610,7 +599,7 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
                  int iters, int thread_id,
                  ThreadStats* total) EXCLUDES(GetBenchmarkLock()) {
   State st(iters, b->has_arg1, b->arg1, b->has_arg2, b->arg2, thread_id);
-  b->function(st);
+  b->benchmark->Run(st);
   CHECK(st.iterations() == st.max_iterations) <<
     "Benchmark returned before State::KeepRunning() returned false!";
   {
@@ -906,6 +895,13 @@ void ParseCommandLineFlags(int* argc, const char** argv) {
   }
 }
 
+Benchmark* RegisterBenchmarkInternal(Benchmark* bench) {
+    std::unique_ptr<Benchmark> bench_ptr(bench);
+    BenchmarkFamilies* families = BenchmarkFamilies::GetInstance();
+    families->AddBenchmark(std::move(bench_ptr));
+    return bench;
+}
+
 } // end namespace internal
 
 void Initialize(int* argc, const char** argv) {

test/CMakeLists.txt

@@ -36,6 +36,9 @@ add_test(options_benchmarks options_test --benchmark_min_time=0.01)
 compile_benchmark_test(basic_test)
 add_test(basic_benchmark basic_test --benchmark_min_time=0.01)
 
+compile_benchmark_test(fixture_test)
+add_test(fixture_test fixture_test --benchmark_min_time=0.01)
+
 compile_benchmark_test(cxx03_test)
 set_target_properties(cxx03_test
     PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}")

test/fixture_test.cc

@@ -0,0 +1,42 @@
+
+#include "benchmark/benchmark.h"
+
+#include <cassert>
+
+class MyFixture : public ::benchmark::Fixture
+{
+public:
+    void SetUp() {
+        data = new int(42);
+    }
+
+    void TearDown() {
+        assert(data != nullptr);
+        delete data;
+        data = nullptr;
+    }
+
+    ~MyFixture() {
+      assert(data == nullptr);
+    }
+
+    int* data;
+};
+
+
+BENCHMARK_F(MyFixture, Foo)(benchmark::State& st) {
+    assert(data != nullptr);
+    assert(*data == 42);
+    while (st.KeepRunning()) {
+    }
+}
+
+BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) {
+  while (st.KeepRunning()) {
+  }
+  st.SetItemsProcessed(st.range_x());
+}
+BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42);
+
+
+BENCHMARK_MAIN()