mirror of
https://github.com/google/benchmark.git
synced 2024-11-28 05:44:40 +00:00
32a1e39720
* Update AUTHORS and CONTRIBUTORS * Fix WSL self-test failures Some of the benchmark self-tests expect and check for a particular output format from the benchmark library. The numerical values must not be infinity or not-a-number, or the test will report an error. Some of the values are computed bytes-per-second or items-per-second values, so these require that the measured CPU time for the test to be non-zero. But the loop that is being measured was empty, so the measured CPU time for the loop was extremely small. On systems like Windows Subsystem for Linux (WSL) the timer doesn't have enough resolution to measure this, so the measured CPU time was zero. This fix just makes sure that these tests have something within the timing loop, so that the benchmark library will not decide that the loop takes zero CPU time. This makes these tests more robust, and in particular makes them pass on WSL.
286 lines
14 KiB
C++
286 lines
14 KiB
C++
|
|
#undef NDEBUG
|
|
|
|
#include "benchmark/benchmark.h"
|
|
#include "output_test.h"
|
|
|
|
// @todo: <jpmag> this checks the full output at once; the rule for
|
|
// CounterSet1 was failing because it was not matching "^[-]+$".
|
|
// @todo: <jpmag> check that the counters are vertically aligned.
|
|
ADD_CASES(
|
|
TC_ConsoleOut,
|
|
{
|
|
// keeping these lines long improves readability, so:
|
|
// clang-format off
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
|
|
// clang-format on
|
|
});
|
|
ADD_CASES(TC_CSVOut, {{"%csv_header,"
|
|
"\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
void BM_Counters_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {1, bm::Counter::kAvgThreads}},
|
|
{"Bar", {2, bm::Counter::kAvgThreads}},
|
|
{"Baz", {4, bm::Counter::kAvgThreads}},
|
|
{"Bat", {8, bm::Counter::kAvgThreads}},
|
|
{"Frob", {16, bm::Counter::kAvgThreads}},
|
|
{"Lob", {32, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
|
|
{"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 0,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckTabular(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
|
|
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
|
|
CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
|
|
CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
|
|
|
|
// ========================================================================= //
|
|
// -------------------- Tabular+Rate Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
void BM_CounterRates_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
// This test requires a non-zero CPU time to avoid divide-by-zero
|
|
benchmark::DoNotOptimize(state.iterations());
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {1, bm::Counter::kAvgThreadsRate}},
|
|
{"Bar", {2, bm::Counter::kAvgThreadsRate}},
|
|
{"Baz", {4, bm::Counter::kAvgThreadsRate}},
|
|
{"Bat", {8, bm::Counter::kAvgThreadsRate}},
|
|
{"Frob", {16, bm::Counter::kAvgThreadsRate}},
|
|
{"Lob", {32, bm::Counter::kAvgThreadsRate}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
|
|
{"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 0,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckTabularRate(Results const& e) {
|
|
double t = e.DurationCPUTime();
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
|
|
&CheckTabularRate);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
// set only some of the counters
|
|
void BM_CounterSet0_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {10, bm::Counter::kAvgThreads}},
|
|
{"Bar", {20, bm::Counter::kAvgThreads}},
|
|
{"Baz", {40, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
|
|
{"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 0,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
|
|
"%float,,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet0(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
|
|
|
|
// again.
|
|
void BM_CounterSet1_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {15, bm::Counter::kAvgThreads}},
|
|
{"Bar", {25, bm::Counter::kAvgThreads}},
|
|
{"Baz", {45, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
|
|
{"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 0,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
|
|
"%float,,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet1(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
// set only some of the counters, different set now.
|
|
void BM_CounterSet2_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {10, bm::Counter::kAvgThreads}},
|
|
{"Bat", {30, bm::Counter::kAvgThreads}},
|
|
{"Baz", {40, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
|
|
{"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 0,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
|
|
",%float,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet2(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
|
|
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
|
|
|
|
// ========================================================================= //
|
|
// --------------------------- TEST CASES END ------------------------------ //
|
|
// ========================================================================= //
|
|
|
|
int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
|