mirror of https://github.com/google/benchmark.git
Merge pull request #120 from google/benchmark-fixtures
Add ability to use benchmark fixtures
This commit is contained in:
Eric
commit
d51ba32791
30
README.md
30
README.md
|
|
@ -175,6 +175,36 @@ static void BM_test(benchmark::State& state) {
|
|||
}
|
||||
```
|
||||
|
||||
Benchmark Fixtures
|
||||
------------------
|
||||
Fixture tests are created by
|
||||
first defining a type that derives from ::benchmark::Fixture and then
|
||||
creating/registering the tests using the following macros:
|
||||
|
||||
* `BENCHMARK_F(ClassName, Method)`
|
||||
* `BENCHMARK_DEFINE_F(ClassName, Method)`
|
||||
* `BENCHMARK_REGISTER_F(ClassName, Method)`
|
||||
|
||||
For Example:
|
||||
|
||||
```c++
|
||||
class MyFixture : public benchmark::Fixture {};
|
||||
|
||||
BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {
|
||||
while (st.KeepRunning()) {
|
||||
...
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {
|
||||
while (st.KeepRunning()) {
|
||||
...
|
||||
}
|
||||
}
|
||||
/* BarTest is NOT registered */
|
||||
BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
|
||||
/* BarTest is now registered */
|
||||
```
|
||||
|
||||
Output Formats
|
||||
--------------
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ void RunSpecifiedBenchmarks(BenchmarkReporter* reporter);
|
|||
namespace internal {
|
||||
class Benchmark;
|
||||
class BenchmarkImp;
|
||||
class BenchmarkFamilies;
|
||||
|
||||
template <class T> struct Voider {
|
||||
typedef void type;
|
||||
|
|
@ -184,8 +185,13 @@ struct EnableIfString<T, typename Voider<typename T::basic_string>::type> {
|
|||
|
||||
void UseCharPointer(char const volatile*);
|
||||
|
||||
// Take ownership of the pointer and register the benchmark. Return the
|
||||
// registered benchmark.
|
||||
Benchmark* RegisterBenchmarkInternal(Benchmark*);
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
|
||||
// The DoNotOptimize(...) function can be used to prevent a value or
|
||||
// expression from being optimized away by the compiler. This function is
|
||||
// intented to add little to no overhead.
|
||||
|
|
@ -371,10 +377,8 @@ typedef void(Function)(State&);
|
|||
// Each method returns "this" so that multiple method calls can
|
||||
// chained into one expression.
|
||||
class Benchmark {
|
||||
public:
|
||||
Benchmark(const char* name, Function* f);
|
||||
|
||||
~Benchmark();
|
||||
public:
|
||||
virtual ~Benchmark();
|
||||
|
||||
// Note: the following methods all return "this" so that multiple
|
||||
// method calls can be chained together in one expression.
|
||||
|
|
@ -445,15 +449,56 @@ class Benchmark {
|
|||
// Equivalent to ThreadRange(NumCPUs(), NumCPUs())
|
||||
Benchmark* ThreadPerCpu();
|
||||
|
||||
virtual void Run(State& state) = 0;
|
||||
|
||||
// Used inside the benchmark implementation
|
||||
struct Instance;
|
||||
|
||||
private:
|
||||
BenchmarkImp* imp_;
|
||||
BENCHMARK_DISALLOW_COPY_AND_ASSIGN(Benchmark);
|
||||
protected:
|
||||
explicit Benchmark(const char* name);
|
||||
Benchmark(Benchmark const&);
|
||||
void SetName(const char* name);
|
||||
|
||||
private:
|
||||
friend class BenchmarkFamilies;
|
||||
BenchmarkImp* imp_;
|
||||
|
||||
Benchmark& operator=(Benchmark const&);
|
||||
};
|
||||
|
||||
// The class used to hold all Benchmarks created from static function.
|
||||
// (ie those created using the BENCHMARK(...) macros.
|
||||
class FunctionBenchmark : public Benchmark {
|
||||
public:
|
||||
FunctionBenchmark(const char* name, Function* func)
|
||||
: Benchmark(name), func_(func)
|
||||
{}
|
||||
|
||||
virtual void Run(State& st);
|
||||
private:
|
||||
Function* func_;
|
||||
};
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
// The base class for all fixture tests.
|
||||
class Fixture: public internal::Benchmark {
|
||||
public:
|
||||
Fixture() : internal::Benchmark("") {}
|
||||
|
||||
virtual void Run(State& st) {
|
||||
this->SetUp();
|
||||
this->BenchmarkCase(st);
|
||||
this->TearDown();
|
||||
}
|
||||
|
||||
virtual void SetUp() {}
|
||||
virtual void TearDown() {}
|
||||
|
||||
protected:
|
||||
virtual void BenchmarkCase(State&) = 0;
|
||||
};
|
||||
|
||||
} // end namespace benchmark
|
||||
|
||||
|
||||
|
|
@ -480,7 +525,9 @@ class Benchmark {
|
|||
BENCHMARK_PRIVATE_NAME(n) BENCHMARK_UNUSED
|
||||
|
||||
#define BENCHMARK(n) \
|
||||
BENCHMARK_PRIVATE_DECLARE(n) = (new ::benchmark::internal::Benchmark(#n, n))
|
||||
BENCHMARK_PRIVATE_DECLARE(n) = \
|
||||
(::benchmark::internal::RegisterBenchmarkInternal( \
|
||||
new ::benchmark::internal::FunctionBenchmark(#n, n)))
|
||||
|
||||
// Old-style macros
|
||||
#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a))
|
||||
|
|
@ -499,21 +546,53 @@ class Benchmark {
|
|||
// will register BM_Foo<1> as a benchmark.
|
||||
#define BENCHMARK_TEMPLATE1(n, a) \
|
||||
BENCHMARK_PRIVATE_DECLARE(n) = \
|
||||
(new ::benchmark::internal::Benchmark(#n "<" #a ">", n<a>))
|
||||
(::benchmark::internal::RegisterBenchmarkInternal( \
|
||||
new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n<a>)))
|
||||
|
||||
#define BENCHMARK_TEMPLATE2(n, a, b) \
|
||||
BENCHMARK_PRIVATE_DECLARE(n) = \
|
||||
(new ::benchmark::internal::Benchmark(#n "<" #a "," #b ">", n<a, b>))
|
||||
#define BENCHMARK_TEMPLATE2(n, a, b) \
|
||||
BENCHMARK_PRIVATE_DECLARE(n) = \
|
||||
(::benchmark::internal::RegisterBenchmarkInternal( \
|
||||
new ::benchmark::internal::FunctionBenchmark( \
|
||||
#n "<" #a "," #b ">", n<a, b>)))
|
||||
|
||||
#if __cplusplus >= 201103L
|
||||
#define BENCHMARK_TEMPLATE(n, ...) \
|
||||
BENCHMARK_PRIVATE_DECLARE(n) = \
|
||||
(new ::benchmark::internal::Benchmark( \
|
||||
#n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))
|
||||
(::benchmark::internal::RegisterBenchmarkInternal( \
|
||||
new ::benchmark::internal::FunctionBenchmark( \
|
||||
#n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>)))
|
||||
#else
|
||||
#define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a)
|
||||
#endif
|
||||
|
||||
|
||||
#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \
|
||||
class BaseClass##_##Method##_Benchmark : public BaseClass { \
|
||||
public: \
|
||||
BaseClass##_##Method##_Benchmark() : BaseClass() { \
|
||||
this->SetName(#BaseClass "/" #Method);} \
|
||||
protected: \
|
||||
virtual void BenchmarkCase(::benchmark::State&); \
|
||||
};
|
||||
|
||||
#define BENCHMARK_DEFINE_F(BaseClass, Method) \
|
||||
BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \
|
||||
void BaseClass##_##Method##_Benchmark::BenchmarkCase
|
||||
|
||||
#define BENCHMARK_REGISTER_F(BaseClass, Method) \
|
||||
BENCHMARK_PRIVATE_REGISTER_F(BaseClass##_##Method##_Benchmark)
|
||||
|
||||
#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \
|
||||
BENCHMARK_PRIVATE_DECLARE(TestName) = \
|
||||
(::benchmark::internal::RegisterBenchmarkInternal(new TestName()))
|
||||
|
||||
// This macro will define and register a benchmark within a fixture class.
|
||||
#define BENCHMARK_F(BaseClass, Method) \
|
||||
BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \
|
||||
BENCHMARK_REGISTER_F(BaseClass, Method); \
|
||||
void BaseClass##_##Method##_Benchmark::BenchmarkCase
|
||||
|
||||
|
||||
// Helper macro to create a main routine in a test that runs the benchmarks
|
||||
#define BENCHMARK_MAIN() \
|
||||
int main(int argc, const char** argv) { \
|
||||
|
|
|
|||
106
src/benchmark.cc
106
src/benchmark.cc
|
|
@ -254,16 +254,16 @@ namespace internal {
|
|||
|
||||
// Information kept per benchmark we may want to run
|
||||
struct Benchmark::Instance {
|
||||
std::string name;
|
||||
Function* function;
|
||||
bool has_arg1;
|
||||
int arg1;
|
||||
bool has_arg2;
|
||||
int arg2;
|
||||
bool use_real_time;
|
||||
double min_time;
|
||||
int threads; // Number of concurrent threads to use
|
||||
bool multithreaded; // Is benchmark multi-threaded?
|
||||
std::string name;
|
||||
Benchmark* benchmark;
|
||||
bool has_arg1;
|
||||
int arg1;
|
||||
bool has_arg2;
|
||||
int arg2;
|
||||
bool use_real_time;
|
||||
double min_time;
|
||||
int threads; // Number of concurrent threads to use
|
||||
bool multithreaded; // Is benchmark multi-threaded?
|
||||
};
|
||||
|
||||
// Class for managing registered benchmarks. Note that each registered
|
||||
|
|
@ -273,27 +273,23 @@ class BenchmarkFamilies {
|
|||
static BenchmarkFamilies* GetInstance();
|
||||
|
||||
// Registers a benchmark family and returns the index assigned to it.
|
||||
size_t AddBenchmark(BenchmarkImp* family);
|
||||
|
||||
// Unregisters a family at the given index.
|
||||
void RemoveBenchmark(size_t index);
|
||||
size_t AddBenchmark(std::unique_ptr<Benchmark> family);
|
||||
|
||||
// Extract the list of benchmark instances that match the specified
|
||||
// regular expression.
|
||||
bool FindBenchmarks(const std::string& re,
|
||||
std::vector<Benchmark::Instance>* benchmarks);
|
||||
private:
|
||||
BenchmarkFamilies();
|
||||
~BenchmarkFamilies();
|
||||
BenchmarkFamilies() {}
|
||||
|
||||
std::vector<BenchmarkImp*> families_;
|
||||
std::vector<std::unique_ptr<Benchmark>> families_;
|
||||
Mutex mutex_;
|
||||
};
|
||||
|
||||
|
||||
class BenchmarkImp {
|
||||
public:
|
||||
BenchmarkImp(const char* name, Function* func);
|
||||
explicit BenchmarkImp(const char* name);
|
||||
~BenchmarkImp();
|
||||
|
||||
void Arg(int x);
|
||||
|
|
@ -306,6 +302,7 @@ public:
|
|||
void Threads(int t);
|
||||
void ThreadRange(int min_threads, int max_threads);
|
||||
void ThreadPerCpu();
|
||||
void SetName(const char* name);
|
||||
|
||||
static void AddRange(std::vector<int>* dst, int lo, int hi, int mult);
|
||||
|
||||
|
|
@ -313,15 +310,13 @@ private:
|
|||
friend class BenchmarkFamilies;
|
||||
|
||||
std::string name_;
|
||||
Function* function_;
|
||||
int arg_count_;
|
||||
std::vector< std::pair<int, int> > args_; // Args for all benchmark runs
|
||||
double min_time_;
|
||||
bool use_real_time_;
|
||||
std::vector<int> thread_counts_;
|
||||
std::size_t registration_index_;
|
||||
|
||||
BENCHMARK_DISALLOW_COPY_AND_ASSIGN(BenchmarkImp);
|
||||
BenchmarkImp& operator=(BenchmarkImp const&);
|
||||
};
|
||||
|
||||
BenchmarkFamilies* BenchmarkFamilies::GetInstance() {
|
||||
|
|
@ -329,36 +324,14 @@ BenchmarkFamilies* BenchmarkFamilies::GetInstance() {
|
|||
return &instance;
|
||||
}
|
||||
|
||||
BenchmarkFamilies::BenchmarkFamilies() { }
|
||||
|
||||
BenchmarkFamilies::~BenchmarkFamilies() {
|
||||
for (BenchmarkImp* family : families_) {
|
||||
delete family;
|
||||
}
|
||||
}
|
||||
|
||||
size_t BenchmarkFamilies::AddBenchmark(BenchmarkImp* family) {
|
||||
size_t BenchmarkFamilies::AddBenchmark(std::unique_ptr<Benchmark> family) {
|
||||
MutexLock l(mutex_);
|
||||
// This loop attempts to reuse an entry that was previously removed to avoid
|
||||
// unncessary growth of the vector.
|
||||
for (size_t index = 0; index < families_.size(); ++index) {
|
||||
if (families_[index] == nullptr) {
|
||||
families_[index] = family;
|
||||
return index;
|
||||
}
|
||||
}
|
||||
size_t index = families_.size();
|
||||
families_.push_back(family);
|
||||
families_.push_back(std::move(family));
|
||||
return index;
|
||||
}
|
||||
|
||||
void BenchmarkFamilies::RemoveBenchmark(size_t index) {
|
||||
MutexLock l(mutex_);
|
||||
families_[index] = nullptr;
|
||||
// Don't shrink families_ here, we might be called by the destructor of
|
||||
// BenchmarkFamilies which iterates over the vector.
|
||||
}
|
||||
|
||||
bool BenchmarkFamilies::FindBenchmarks(
|
||||
const std::string& spec,
|
||||
std::vector<Benchmark::Instance>* benchmarks) {
|
||||
|
|
@ -375,9 +348,10 @@ bool BenchmarkFamilies::FindBenchmarks(
|
|||
one_thread.push_back(1);
|
||||
|
||||
MutexLock l(mutex_);
|
||||
for (BenchmarkImp* family : families_) {
|
||||
for (std::unique_ptr<Benchmark>& bench_family : families_) {
|
||||
// Family was deleted or benchmark doesn't match
|
||||
if (family == nullptr) continue;
|
||||
if (!bench_family) continue;
|
||||
BenchmarkImp* family = bench_family->imp_;
|
||||
|
||||
if (family->arg_count_ == -1) {
|
||||
family->arg_count_ = 0;
|
||||
|
|
@ -392,7 +366,7 @@ bool BenchmarkFamilies::FindBenchmarks(
|
|||
|
||||
Benchmark::Instance instance;
|
||||
instance.name = family->name_;
|
||||
instance.function = family->function_;
|
||||
instance.benchmark = bench_family.get();
|
||||
instance.has_arg1 = family->arg_count_ >= 1;
|
||||
instance.arg1 = args.first;
|
||||
instance.has_arg2 = family->arg_count_ == 2;
|
||||
|
|
@ -430,14 +404,12 @@ bool BenchmarkFamilies::FindBenchmarks(
|
|||
return true;
|
||||
}
|
||||
|
||||
BenchmarkImp::BenchmarkImp(const char* name, Function* func)
|
||||
: name_(name), function_(func), arg_count_(-1),
|
||||
BenchmarkImp::BenchmarkImp(const char* name)
|
||||
: name_(name), arg_count_(-1),
|
||||
min_time_(0.0), use_real_time_(false) {
|
||||
registration_index_ = BenchmarkFamilies::GetInstance()->AddBenchmark(this);
|
||||
}
|
||||
|
||||
BenchmarkImp::~BenchmarkImp() {
|
||||
BenchmarkFamilies::GetInstance()->RemoveBenchmark(registration_index_);
|
||||
}
|
||||
|
||||
void BenchmarkImp::Arg(int x) {
|
||||
|
|
@ -513,6 +485,10 @@ void BenchmarkImp::ThreadPerCpu() {
|
|||
thread_counts_.push_back(num_cpus);
|
||||
}
|
||||
|
||||
void BenchmarkImp::SetName(const char* name) {
|
||||
name_ = name;
|
||||
}
|
||||
|
||||
void BenchmarkImp::AddRange(std::vector<int>* dst, int lo, int hi, int mult) {
|
||||
CHECK_GE(lo, 0);
|
||||
CHECK_GE(hi, lo);
|
||||
|
|
@ -535,8 +511,8 @@ void BenchmarkImp::AddRange(std::vector<int>* dst, int lo, int hi, int mult) {
|
|||
}
|
||||
}
|
||||
|
||||
Benchmark::Benchmark(const char* name, Function* f)
|
||||
: imp_(new BenchmarkImp(name, f))
|
||||
Benchmark::Benchmark(const char* name)
|
||||
: imp_(new BenchmarkImp(name))
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -544,6 +520,11 @@ Benchmark::~Benchmark() {
|
|||
delete imp_;
|
||||
}
|
||||
|
||||
Benchmark::Benchmark(Benchmark const& other)
|
||||
: imp_(new BenchmarkImp(*other.imp_))
|
||||
{
|
||||
}
|
||||
|
||||
Benchmark* Benchmark::Arg(int x) {
|
||||
imp_->Arg(x);
|
||||
return this;
|
||||
|
|
@ -599,6 +580,14 @@ Benchmark* Benchmark::ThreadPerCpu() {
|
|||
return this;
|
||||
}
|
||||
|
||||
void Benchmark::SetName(const char* name) {
|
||||
imp_->SetName(name);
|
||||
}
|
||||
|
||||
void FunctionBenchmark::Run(State& st) {
|
||||
func_(st);
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
namespace {
|
||||
|
|
@ -610,7 +599,7 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
|
|||
int iters, int thread_id,
|
||||
ThreadStats* total) EXCLUDES(GetBenchmarkLock()) {
|
||||
State st(iters, b->has_arg1, b->arg1, b->has_arg2, b->arg2, thread_id);
|
||||
b->function(st);
|
||||
b->benchmark->Run(st);
|
||||
CHECK(st.iterations() == st.max_iterations) <<
|
||||
"Benchmark returned before State::KeepRunning() returned false!";
|
||||
{
|
||||
|
|
@ -906,6 +895,13 @@ void ParseCommandLineFlags(int* argc, const char** argv) {
|
|||
}
|
||||
}
|
||||
|
||||
Benchmark* RegisterBenchmarkInternal(Benchmark* bench) {
|
||||
std::unique_ptr<Benchmark> bench_ptr(bench);
|
||||
BenchmarkFamilies* families = BenchmarkFamilies::GetInstance();
|
||||
families->AddBenchmark(std::move(bench_ptr));
|
||||
return bench;
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
void Initialize(int* argc, const char** argv) {
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ add_test(options_benchmarks options_test --benchmark_min_time=0.01)
|
|||
compile_benchmark_test(basic_test)
|
||||
add_test(basic_benchmark basic_test --benchmark_min_time=0.01)
|
||||
|
||||
compile_benchmark_test(fixture_test)
|
||||
add_test(fixture_test fixture_test --benchmark_min_time=0.01)
|
||||
|
||||
compile_benchmark_test(cxx03_test)
|
||||
set_target_properties(cxx03_test
|
||||
PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
|
||||
#include "benchmark/benchmark.h"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
class MyFixture : public ::benchmark::Fixture
|
||||
{
|
||||
public:
|
||||
void SetUp() {
|
||||
data = new int(42);
|
||||
}
|
||||
|
||||
void TearDown() {
|
||||
assert(data != nullptr);
|
||||
delete data;
|
||||
data = nullptr;
|
||||
}
|
||||
|
||||
~MyFixture() {
|
||||
assert(data == nullptr);
|
||||
}
|
||||
|
||||
int* data;
|
||||
};
|
||||
|
||||
|
||||
BENCHMARK_F(MyFixture, Foo)(benchmark::State& st) {
|
||||
assert(data != nullptr);
|
||||
assert(*data == 42);
|
||||
while (st.KeepRunning()) {
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) {
|
||||
while (st.KeepRunning()) {
|
||||
}
|
||||
st.SetItemsProcessed(st.range_x());
|
||||
}
|
||||
BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42);
|
||||
|
||||
|
||||
BENCHMARK_MAIN()
|
||||
Loading…
Reference in New Issue