dolysis
/
benchmark
mirror of https://github.com/google/benchmark.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Statistics: add support for percentage unit in addition to time (#1219) 

* Statistics: add support for percentage unit in addition to time

I think, `stddev` statistic is useful, but confusing.

What does it mean if `stddev` of `1ms` is reported?
Is that good or bad? If the `median` is `1s`,
then that means that the measurements are pretty noise-less.

And what about `stddev` of `100ms` is reported?
If the `median` is `1s` - awful, if the `median` is `10s` - good.

And hurray, there is just the statistic that we need:
https://en.wikipedia.org/wiki/Coefficient_of_variation

But, naturally, that produces a value in percents,
but the statistics are currently hardcoded to produce time.

So this refactors thinkgs a bit, and allows a percentage unit for statistics.

I'm not sure whether or not `benchmark` would be okay
with adding this `RSD` statistic by default,
but regales, that is a separate patch.

Refs. https://github.com/google/benchmark/issues/1146

* Address review notes
This commit is contained in:
Roman Lebedev 2021-09-03 17:36:56 +03:00 committed by GitHub
parent 67b77da3c0
commit 12dc5eeafc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 365 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
docs/user_guide.md
View File

@ -140,8 +140,8 @@ Write benchmark results to a file with the `--benchmark_out=<filename>` option
(or set `BENCHMARK_OUT`). Specify the output format with
`--benchmark_out_format={json|console|csv}` (or set
`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is
deprecated and the saved `.csv` file
[is not parsable](https://github.com/google/benchmark/issues/794) by csv
deprecated and the saved `.csv` file
[is not parsable](https://github.com/google/benchmark/issues/794) by csv
parsers.
Specifying `--benchmark_out` does not suppress the console output.
@ -983,6 +983,25 @@ BENCHMARK(BM_spin_empty)
->Arg(512);
```
While usually the statistics produce values in time units,
you can also produce percentages:
```c++
void BM_spin_empty(benchmark::State& state) {
for (auto _ : state) {
for (int x = 0; x < state.range(0); ++x) {
benchmark::DoNotOptimize(x);
}
}
}
BENCHMARK(BM_spin_empty)
->ComputeStatistics("ratio", [](const std::vector<double>& v) -> double {
return std::begin(v) / std::end(v);
}, benchmark::StatisticUnit::Percentage)
->Arg(512);
```
<a name="using-register-benchmark" />
## Using RegisterBenchmark(name, fn, args...)

13
include/benchmark/benchmark.h
View File

@ -450,6 +450,8 @@ enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda };
typedef uint64_t IterationCount;
enum StatisticUnit { kTime, kPercentage };
// BigOFunc is passed to a benchmark in order to specify the asymptotic
// computational complexity for the benchmark.
typedef double(BigOFunc)(IterationCount);
@ -462,9 +464,11 @@ namespace internal {
struct Statistics {
std::string name_;
StatisticsFunc* compute_;
StatisticUnit unit_;
Statistics(const std::string& name, StatisticsFunc* compute)
: name_(name), compute_(compute) {}
Statistics(const std::string& name, StatisticsFunc* compute,
StatisticUnit unit = kTime)
: name_(name), compute_(compute), unit_(unit) {}
};
class BenchmarkInstance;
@ -965,7 +969,8 @@ class Benchmark {
Benchmark* Complexity(BigOFunc* complexity);
// Add this statistics to be computed over all the values of benchmark run
Benchmark* ComputeStatistics(std::string name, StatisticsFunc* statistics);
Benchmark* ComputeStatistics(std::string name, StatisticsFunc* statistics,
StatisticUnit unit = kTime);
// Support for running multiple copies of the same benchmark concurrently
// in multiple threads. This may be useful when measuring the scaling
@ -1421,6 +1426,7 @@ class BenchmarkReporter {
Run()
: run_type(RT_Iteration),
aggregate_unit(kTime),
error_occurred(false),
iterations(1),
threads(1),
@ -1444,6 +1450,7 @@ class BenchmarkReporter {
int64_t per_family_instance_index;
RunType run_type;
std::string aggregate_name;
StatisticUnit aggregate_unit;
std::string report_label; // Empty if not set by benchmark.
bool error_occurred;
std::string error_message;

5
src/benchmark_register.cc
View File

@ -399,8 +399,9 @@ Benchmark* Benchmark::Complexity(BigOFunc* complexity) {
}
Benchmark* Benchmark::ComputeStatistics(std::string name,
StatisticsFunc* statistics) {
statistics_.emplace_back(name, statistics);
StatisticsFunc* statistics,
StatisticUnit unit) {
statistics_.emplace_back(name, statistics, unit);
return this;
}

2
src/complexity.cc
View File

@ -199,6 +199,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
big_o.repetition_index = Run::no_repetition_index;
big_o.threads = reports[0].threads;
big_o.aggregate_name = "BigO";
big_o.aggregate_unit = StatisticUnit::kTime;
big_o.report_label = reports[0].report_label;
big_o.iterations = 0;
big_o.real_accumulated_time = result_real.coef;
@ -220,6 +221,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
rms.per_family_instance_index = reports[0].per_family_instance_index;
rms.run_type = BenchmarkReporter::Run::RT_Aggregate;
rms.aggregate_name = "RMS";
rms.aggregate_unit = StatisticUnit::kPercentage;
rms.report_label = big_o.report_label;
rms.iterations = 0;
rms.repetition_index = Run::no_repetition_index;

8
src/console_reporter.cc
View File

@ -142,10 +142,16 @@ void ConsoleReporter::PrintRunData(const Run& result) {
} else if (result.report_rms) {
printer(Out, COLOR_YELLOW, "%10.0f %-4s %10.0f %-4s ", real_time * 100, "%",
cpu_time * 100, "%");
} else {
} else if (result.run_type != Run::RT_Aggregate ||
result.aggregate_unit == StatisticUnit::kTime) {
const char* timeLabel = GetTimeUnitString(result.time_unit);
printer(Out, COLOR_YELLOW, "%s %-4s %s %-4s ", real_time_str.c_str(), timeLabel,
cpu_time_str.c_str(), timeLabel);
} else {
assert(result.aggregate_unit == StatisticUnit::kPercentage);
printer(Out, COLOR_YELLOW, "%10.2f %-4s %10.2f %-4s ",
(100. * result.real_accumulated_time), "%",
(100. * result.cpu_accumulated_time), "%");
}
if (!result.report_big_o && !result.report_rms) {

22
src/json_reporter.cc
View File

@ -251,6 +251,15 @@ void JSONReporter::PrintRunData(Run const& run) {
out << indent << FormatKV("threads", run.threads) << ",\n";
if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) {
out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n";
out << indent << FormatKV("aggregate_unit", [&run]() -> const char* {
switch (run.aggregate_unit) {
case StatisticUnit::kTime:
return "time";
case StatisticUnit::kPercentage:
return "percentage";
}
BENCHMARK_UNREACHABLE();
}()) << ",\n";
}
if (run.error_occurred) {
out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n";
@ -258,8 +267,17 @@ void JSONReporter::PrintRunData(Run const& run) {
}
if (!run.report_big_o && !run.report_rms) {
out << indent << FormatKV("iterations", run.iterations) << ",\n";
out << indent << FormatKV("real_time", run.GetAdjustedRealTime()) << ",\n";
out << indent << FormatKV("cpu_time", run.GetAdjustedCPUTime());
if (run.run_type != Run::RT_Aggregate ||
run.aggregate_unit == StatisticUnit::kTime) {
out << indent << FormatKV("real_time", run.GetAdjustedRealTime())
<< ",\n";
out << indent << FormatKV("cpu_time", run.GetAdjustedCPUTime());
} else {
assert(run.aggregate_unit == StatisticUnit::kPercentage);
out << indent << FormatKV("real_time", run.real_accumulated_time)
<< ",\n";
out << indent << FormatKV("cpu_time", run.cpu_accumulated_time);
}
out << ",\n"
<< indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
} else if (run.report_big_o) {

17
src/statistics.cc
View File

@ -155,6 +155,7 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
data.repetitions = reports[0].repetitions;
data.repetition_index = Run::no_repetition_index;
data.aggregate_name = Stat.name_;
data.aggregate_unit = Stat.unit_;
data.report_label = report_label;
// It is incorrect to say that an aggregate is computed over
@ -167,13 +168,15 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat);
data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat);
// We will divide these times by data.iterations when reporting, but the
// data.iterations is not nessesairly the scale of these measurements,
// because in each repetition, these timers are sum over all the iterations.
// And if we want to say that the stats are over N repetitions and not
// M iterations, we need to multiply these by (N/M).
data.real_accumulated_time *= iteration_rescale_factor;
data.cpu_accumulated_time *= iteration_rescale_factor;
if (data.aggregate_unit == StatisticUnit::kTime) {
// We will divide these times by data.iterations when reporting, but the
// data.iterations is not necessarily the scale of these measurements,
// because in each repetition, these timers are sum over all the iters.
// And if we want to say that the stats are over N repetitions and not
// M iterations, we need to multiply these by (N/M).
data.real_accumulated_time *= iteration_rescale_factor;
data.cpu_accumulated_time *= iteration_rescale_factor;
}
data.time_unit = reports[0].time_unit;

2
test/complexity_test.cc
View File

@ -36,6 +36,7 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name,
{"\"repetitions\": %int,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"BigO\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next},
@ -49,6 +50,7 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name,
{"\"repetitions\": %int,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"RMS\",$", MR_Next},
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
{"\"rms\": %float$", MR_Next},
{"}", MR_Next}});
AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},

6
test/repetitions_test.cc
View File

@ -59,6 +59,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -73,6 +74,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -87,6 +89,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -164,6 +167,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -177,6 +181,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -190,6 +195,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},

171
test/reporter_output_test.cc
View File

@ -1,5 +1,6 @@
#undef NDEBUG
#include <numeric>
#include <utility>
#include "benchmark/benchmark.h"
@ -454,6 +455,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_median\",$"},
{"\"family_index\": 15,$", MR_Next},
@ -463,6 +465,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
{"\"family_index\": 15,$", MR_Next},
@ -472,6 +475,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2\",%csv_report$"},
@ -519,6 +523,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_median\",$"},
{"\"family_index\": 16,$", MR_Next},
@ -528,6 +533,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
{"\"family_index\": 16,$", MR_Next},
@ -537,6 +543,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
@ -594,6 +601,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"repetitions\": 4,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_median\",$"},
{"\"family_index\": 17,$", MR_Next},
@ -603,6 +611,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"repetitions\": 4,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
{"\"family_index\": 17,$", MR_Next},
@ -612,6 +621,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"repetitions\": 4,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"},
@ -661,6 +671,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
{"\"family_index\": 19,$", MR_Next},
@ -670,6 +681,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
{"\"family_index\": 19,$", MR_Next},
@ -679,6 +691,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
@ -709,6 +722,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
{"\"family_index\": 20,$", MR_Next},
@ -718,6 +732,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
{"\"family_index\": 20,$", MR_Next},
@ -727,6 +742,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut,
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
@ -761,6 +777,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
@ -771,6 +788,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
@ -781,6 +799,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}});
ADD_CASES(TC_CSVOut,
@ -869,6 +888,7 @@ ADD_CASES(
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
@ -880,6 +900,7 @@ ADD_CASES(
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
@ -891,6 +912,7 @@ ADD_CASES(
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
@ -902,6 +924,7 @@ ADD_CASES(
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
ADD_CASES(
@ -916,6 +939,154 @@ ADD_CASES(
"manual_time_stddev\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
// ========================================================================= //
// ------------- Testing relative standard deviation statistics ------------ //
// ========================================================================= //
const auto UserPercentStatistics = [](const std::vector<double>& v) {
return 1. / 100.;
};
void BM_UserPercentStats(benchmark::State& state) {
for (auto _ : state) {
state.SetIterationTime(150 / 10e8);
}
}
// clang-format off
BENCHMARK(BM_UserPercentStats)
->Repetitions(3)
->Iterations(5)
->UseManualTime()
->Unit(benchmark::TimeUnit::kNanosecond)
->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
// clang-format on
// check that UserPercent-provided stats is calculated, and is after the
// default-ones empty string as name is intentional, it would sort before
// anything else
ADD_CASES(TC_ConsoleOut,
{{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
"]* 150 ns %time [ ]*5$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
"]* 150 ns %time [ ]*5$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
"]* 150 ns %time [ ]*5$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_mean [ ]* 150 ns %time [ ]*3$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_median [ ]* 150 ns %time [ ]*3$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
"[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
ADD_CASES(
TC_JSONOut,
{{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"repetition_index\": 0,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"repetition_index\": 1,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"repetition_index\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"\",$", MR_Next},
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_mean\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_median\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_stddev\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_\",%csv_report$"}});
// ========================================================================= //
// ------------------------- Testing StrEscape JSON ------------------------ //
// ========================================================================= //

5
test/user_counters_tabular_test.cc
View File

@ -125,6 +125,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -146,6 +147,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -167,6 +169,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -231,6 +234,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 2,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -252,6 +256,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 2,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},

3
test/user_counters_thousands_test.cc
View File

@ -96,6 +96,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -115,6 +116,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@ -134,6 +136,7 @@ ADD_CASES(TC_JSONOut,
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},

21
tools/gbench/Inputs/test4_run0.json Normal file
View File

@ -0,0 +1,21 @@
{
"context": {
"date": "2016-08-02 17:44:46",
"num_cpus": 4,
"mhz_per_cpu": 4228,
"cpu_scaling_enabled": false,
"library_build_type": "release"
},
"benchmarks": [
{
"name": "whocares",
"run_type": "aggregate",
"aggregate_name": "zz",
"aggregate_unit": "percentage",
"iterations": 1000,
"real_time": 0.01,
"cpu_time": 0.10,
"time_unit": "ns"
}
]
}

21
tools/gbench/Inputs/test4_run1.json Normal file
View File

@ -0,0 +1,21 @@
{
"context": {
"date": "2016-08-02 17:44:46",
"num_cpus": 4,
"mhz_per_cpu": 4228,
"cpu_scaling_enabled": false,
"library_build_type": "release"
},
"benchmarks": [
{
"name": "whocares",
"run_type": "aggregate",
"aggregate_name": "zz",
"aggregate_unit": "percentage",
"iterations": 1000,
"real_time": 0.005,
"cpu_time": 0.15,
"time_unit": "ns"
}
]
}

63
tools/gbench/report.py
View File

@ -914,6 +914,69 @@ class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(
assert_measurements(self, out, expected)
class TestReportDifferenceForPercentageAggregates(
unittest.TestCase):
@classmethod
def setUpClass(cls):
def load_results():
import json
testInputs = os.path.join(
os.path.dirname(
os.path.realpath(__file__)),
'Inputs')
testOutput1 = os.path.join(testInputs, 'test4_run0.json')
testOutput2 = os.path.join(testInputs, 'test4_run1.json')
with open(testOutput1, 'r') as f:
json1 = json.load(f)
with open(testOutput2, 'r') as f:
json2 = json.load(f)
return json1, json2
json1, json2 = load_results()
cls.json_diff_report = get_difference_report(
json1, json2, utest=True)
def test_json_diff_report_pretty_printing(self):
expect_lines = [
['whocares', '-0.5000', '+0.5000', '0', '0', '0', '0']
]
output_lines_with_header = print_difference_report(
self.json_diff_report,
utest=True, utest_alpha=0.05, use_color=False)
output_lines = output_lines_with_header[2:]
print("\n")
print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines))
for i in range(0, len(output_lines)):
parts = [x for x in output_lines[i].split(' ') if x]
self.assertEqual(expect_lines[i], parts)
def test_json_diff_report(self):
expected_output = [
{
'name': u'whocares',
'measurements': [
{'time': -0.5,
'cpu': 0.5,
'real_time': 0.01,
'real_time_other': 0.005,
'cpu_time': 0.10,
'cpu_time_other': 0.15}
],
'time_unit': 'ns',
'utest': {}
}
]
self.assertEqual(len(self.json_diff_report), len(expected_output))
for out, expected in zip(
self.json_diff_report, expected_output):
self.assertEqual(out['name'], expected['name'])
self.assertEqual(out['time_unit'], expected['time_unit'])
assert_utest(self, out, expected)
assert_measurements(self, out, expected)
class TestReportSorting(unittest.TestCase):
@classmethod
def setUpClass(cls):