From 5f5ca31ce0f9e5fa33e622aa98f9feee31b0c099 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 2 Aug 2016 17:22:46 -0600 Subject: [PATCH] Add `RegisterBenchmark(name, func, args...)` for creating/registering benchmarks. (#259) * Add RegisterBenchmark * fix test inputs * fix UB caused by unitialized value * Add RegisterBenchmark * fix test inputs * fix UB caused by unitialized value * Work around GCC 4.6/4.7/4.8 bug --- README.md | 28 ++++++ include/benchmark/benchmark_api.h | 66 ++++++++++++++ include/benchmark/macros.h | 10 +- test/CMakeLists.txt | 3 + test/register_benchmark_test.cc | 147 ++++++++++++++++++++++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 test/register_benchmark_test.cc diff --git a/README.md b/README.md index 95326bd1..9df94a3a 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,34 @@ BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); Note that elements of `...args` may refer to global variables. Users should avoid modifying global state inside of a benchmark. +## Using RegisterBenchmark(name, fn, args...) + +The `RegisterBenchmark(name, func, args...)` function provides an alternative +way to create and register benchmarks. +`RegisterBenchmark(name, func, args...)` creates, registers, and returns a +pointer to a new benchmark with the specified `name` that invokes +`func(st, args...)` where `st` is a `benchmark::State` object. + +Unlike the `BENCHMARK` registration macros, which can only be used at the global +scope, the `RegisterBenchmark` can be called anywhere. This allows for +benchmark tests to be registered programmatically. + +Additionally `RegisterBenchmark` allows any callable object to be registered +as a benchmark. Including capturing lambdas and function objects. This +allows the creation + +For Example: +```c++ +auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; + +int main(int argc, char** argv) { + for (auto& test_input : { /* ... */ }) + benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} +``` + ### Multithreaded benchmarks In a multithreaded test (benchmark invoked by multiple threads simultaneously), it is guaranteed that none of the threads will start until all have called diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 4bcf35a9..5961c9bb 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -155,6 +155,11 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include "macros.h" +#if defined(BENCHMARK_HAS_CXX11) +#include +#include +#endif + namespace benchmark { class BenchmarkReporter; @@ -592,6 +597,20 @@ private: Benchmark& operator=(Benchmark const&); }; +} // namespace internal + +// Create and register a benchmark with the specified 'name' that invokes +// the specified functor 'fn'. +// +// RETURNS: A pointer to the registered benchmark. +internal::Benchmark* RegisterBenchmark(const char* name, internal::Function* fn); + +#if defined(BENCHMARK_HAS_CXX11) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); +#endif + +namespace internal { // The class used to hold all Benchmarks created from static function. // (ie those created using the BENCHMARK(...) macros. class FunctionBenchmark : public Benchmark { @@ -605,8 +624,55 @@ private: Function* func_; }; +#ifdef BENCHMARK_HAS_CXX11 +template +class LambdaBenchmark : public Benchmark { +public: + virtual void Run(State& st) { lambda_(st); } + +private: + template + LambdaBenchmark(const char* name, OLambda&& lam) + : Benchmark(name), lambda_(std::forward(lam)) {} + + LambdaBenchmark(LambdaBenchmark const&) = delete; + +private: + template + friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); + + Lambda lambda_; +}; +#endif + } // end namespace internal +inline internal::Benchmark* +RegisterBenchmark(const char* name, internal::Function* fn) { + return internal::RegisterBenchmarkInternal( + ::new internal::FunctionBenchmark(name, fn)); +} + +#ifdef BENCHMARK_HAS_CXX11 +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { + using BenchType = internal::LambdaBenchmark::type>; + return internal::RegisterBenchmarkInternal( + ::new BenchType(name, std::forward(fn))); +} + +#if !defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409 +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, + Args&&... args) { + return benchmark::RegisterBenchmark(name, + [=](benchmark::State& st) { fn(st, args...); }); +} +#else +#define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK +#endif +#endif // BENCHMARK_HAS_CXX11 + // The base class for all fixture tests. class Fixture: public internal::Benchmark { public: diff --git a/include/benchmark/macros.h b/include/benchmark/macros.h index 09d13c19..5892ac42 100644 --- a/include/benchmark/macros.h +++ b/include/benchmark/macros.h @@ -14,7 +14,11 @@ #ifndef BENCHMARK_MACROS_H_ #define BENCHMARK_MACROS_H_ -#if __cplusplus < 201103L +#if __cplusplus >= 201103L +#define BENCHMARK_HAS_CXX11 +#endif + +#ifndef BENCHMARK_HAS_CXX11 # define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ TypeName(const TypeName&); \ TypeName& operator=(const TypeName&) @@ -53,4 +57,8 @@ # define BENCHMARK_BUILTIN_EXPECT(x, y) x #endif +#if defined(__GNUC__) && !defined(__clang__) +#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif + #endif // BENCHMARK_MACROS_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index aeb720a3..3b8f7c91 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,6 +45,9 @@ add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01) compile_benchmark_test(fixture_test) add_test(fixture_test fixture_test --benchmark_min_time=0.01) +compile_benchmark_test(register_benchmark_test) +add_test(register_benchmark_test register_benchmark_test --benchmark_min_time=0.01) + compile_benchmark_test(map_test) add_test(map_test map_test --benchmark_min_time=0.01) diff --git a/test/register_benchmark_test.cc b/test/register_benchmark_test.cc new file mode 100644 index 00000000..f11c5eaa --- /dev/null +++ b/test/register_benchmark_test.cc @@ -0,0 +1,147 @@ + +#undef NDEBUG +#include "benchmark/benchmark.h" +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include +#include + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { +public: + virtual void ReportRuns(const std::vector& report) { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + std::vector all_runs_; +}; + +struct TestCase { + std::string name; + const char* label; + TestCase(const char* xname) : name(xname), label(nullptr) {} + TestCase(const char* xname, const char* xlabel) + : name(xname), label(xlabel) {} + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + CHECK(name == run.benchmark_name) << "expected " << name + << " got " << run.benchmark_name; + if (label) { + CHECK(run.report_label == label) << "expected " << label + << " got " << run.report_label; + } else { + CHECK(run.report_label == ""); + } + } +}; + +std::vector ExpectedResults; + +int AddCases(std::initializer_list const& v) { + for (auto N : v) { + ExpectedResults.push_back(N); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) \ +int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__}) + +} // end namespace + +typedef benchmark::internal::Benchmark* ReturnVal; + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with no additional arguments +//----------------------------------------------------------------------------// +void BM_function(benchmark::State& state) { while (state.KeepRunning()) {} } +BENCHMARK(BM_function); +ReturnVal dummy = benchmark::RegisterBenchmark( + "BM_function_manual_registration", + BM_function); +ADD_CASES({"BM_function"}, {"BM_function_manual_registration"}); + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with additional arguments +// Note: GCC <= 4.8 do not support this form of RegisterBenchmark because they +// reject the variadic pack expansion of lambda captures. +//----------------------------------------------------------------------------// +#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + +void BM_extra_args(benchmark::State& st, const char* label) { + while (st.KeepRunning()) {} + st.SetLabel(label); +} +int RegisterFromFunction() { + std::pair cases[] = { + {"test1", "One"}, + {"test2", "Two"}, + {"test3", "Three"} + }; + for (auto& c : cases) + benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second); + return 0; +} +int dummy2 = RegisterFromFunction(); +ADD_CASES( + {"test1", "One"}, + {"test2", "Two"}, + {"test3", "Three"} +); + +#endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with different callable types +//----------------------------------------------------------------------------// + +struct CustomFixture { + void operator()(benchmark::State& st) { + while (st.KeepRunning()) {} + } +}; + +void TestRegistrationAtRuntime() { + { + CustomFixture fx; + benchmark::RegisterBenchmark("custom_fixture", fx); + AddCases({"custom_fixture"}); + } +#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + { + int x = 42; + auto capturing_lam = [=](benchmark::State& st) { + while (st.KeepRunning()) {} + st.SetLabel(std::to_string(x)); + }; + benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam); + AddCases({{"lambda_benchmark", "42"}}); + } +#endif +} + +int main(int argc, char* argv[]) { + TestRegistrationAtRuntime(); + + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); + + return 0; +}