2017-12-13 23:26:47 +00:00
|
|
|
//===---------------------------------------------------------------------===//
|
|
|
|
// statistics_test - Unit tests for src/statistics.cc
|
|
|
|
//===---------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "../src/statistics.h"
|
|
|
|
#include "gtest/gtest.h"
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
TEST(StatisticsTest, Mean) {
|
2018-06-01 10:14:19 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({42, 42, 42, 42}), 42.0);
|
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 3, 4}), 2.5);
|
2018-05-29 12:13:06 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 5, 10, 10, 14}), 7.0);
|
2017-12-13 23:26:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(StatisticsTest, Median) {
|
2018-06-01 10:14:19 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({42, 42, 42, 42}), 42.0);
|
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 3, 4}), 2.5);
|
2018-05-29 12:13:06 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 5, 10, 10}), 5.0);
|
2017-12-13 23:26:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(StatisticsTest, StdDev) {
|
2018-05-29 12:13:06 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0);
|
2018-06-01 10:14:19 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0);
|
2020-04-06 12:52:09 +00:00
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({2.5, 2.4, 3.3, 4.2, 5.1}),
|
|
|
|
1.151086443322134);
|
2017-12-13 23:26:47 +00:00
|
|
|
}
|
|
|
|
|
Introduce Coefficient of variation aggregate (#1220)
* Introduce Coefficient of variation aggregate
I believe, it is much more useful / use to understand,
because it is already normalized by the mean,
so it is not affected by the duration of the benchmark,
unlike the standard deviation.
Example of real-world output:
```
raw.pixls.us-unique/GoPro/HERO6 Black$ ~/rawspeed/build-old/src/utilities/rsbench/rsbench GOPR9172.GPR --benchmark_repetitions=27 --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true
2021-09-03T18:05:56+03:00
Running /home/lebedevri/rawspeed/build-old/src/utilities/rsbench/rsbench
Run on (32 X 3596.16 MHz CPU s)
CPU Caches:
L1 Data 32 KiB (x16)
L1 Instruction 32 KiB (x16)
L2 Unified 512 KiB (x16)
L3 Unified 32768 KiB (x2)
Load Average: 7.00, 2.99, 1.85
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations CPUTime,s CPUTime/WallTime Pixels Pixels/CPUTime Pixels/WallTime Raws/CPUTime Raws/WallTime WallTime,s
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
GOPR9172.GPR/threads:32/process_time/real_time_mean 11.1 ms 353 ms 27 0.353122 31.9473 12M 33.9879M 1085.84M 2.83232 90.4864 0.0110535
GOPR9172.GPR/threads:32/process_time/real_time_median 11.0 ms 352 ms 27 0.351696 31.9599 12M 34.1203M 1090.11M 2.84336 90.8425 0.0110081
GOPR9172.GPR/threads:32/process_time/real_time_stddev 0.159 ms 4.60 ms 27 4.59539m 0.0462064 0 426.371k 14.9631M 0.0355309 1.24692 158.944u
GOPR9172.GPR/threads:32/process_time/real_time_cv 1.44 % 1.30 % 27 0.0130136 1.44633m 0 0.0125448 0.0137802 0.0125448 0.0137802 0.0143795
```
Fixes https://github.com/google/benchmark/issues/1146
* Be consistent, it's CV, not 'rel std dev'
2021-09-03 17:44:10 +00:00
|
|
|
TEST(StatisticsTest, CV) {
|
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0);
|
|
|
|
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.);
|
Support windows MSYS2 environments (#1704)
* [CI] Attempt to add windows MSYS2-based coverage
* Mark decl of `State::KeepRunningInternal()` as `inline`
Maybe helps with
```
D:\a\_temp\msys64\ucrt64\bin\g++.exe -DHAVE_STD_REGEX -DHAVE_STEADY_CLOCK -DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS -ID:/a/benchmark/benchmark/include -Wall -Wextra -Wshadow -Wfloat-equal -Wold-style-cast -Werror -pedantic -pedantic-errors -fstrict-aliasing -Wno-deprecated-declarations -Wno-deprecated -Wstrict-aliasing -Wno-unused-variable -std=c++11 -fvisibility=hidden -fno-keep-inline-dllexport -UNDEBUG -MD -MT test/CMakeFiles/benchmark_test.dir/benchmark_test.cc.obj -MF test\CMakeFiles\benchmark_test.dir\benchmark_test.cc.obj.d -o test/CMakeFiles/benchmark_test.dir/benchmark_test.cc.obj -c D:/a/benchmark/benchmark/test/benchmark_test.cc
In file included from D:/a/benchmark/benchmark/test/benchmark_test.cc:1:
D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1007:37: error: 'bool benchmark::State::KeepRunningInternal(benchmark::IterationCount, bool)' redeclared without dllimport attribute after being referenced with dll linkage [-Werror]
1007 | inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n,
| ^~~~~
```
* Mark more `State`'s member function decls as `inline`
```
[27/110] Building CXX object test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj
FAILED: test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj
D:\a\_temp\msys64\clang32\bin\clang++.exe -DHAVE_STD_REGEX -DHAVE_STEADY_CLOCK -DHAVE_THREAD_SAFETY_ATTRIBUTES -DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS -ID:/a/benchmark/benchmark/include -Wall -Wextra -Wshadow -Wfloat-equal -Wold-style-cast -Werror -pedantic -pedantic-errors -Wshorten-64-to-32 -fstrict-aliasing -Wno-deprecated-declarations -Wno-deprecated -Wstrict-aliasing -Wthread-safety -Wno-unused-variable -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden -UNDEBUG -MD -MT test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj -MF test\CMakeFiles\spec_arg_verbosity_test.dir\spec_arg_verbosity_test.cc.obj.d -o test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj -c D:/a/benchmark/benchmark/test/spec_arg_verbosity_test.cc
In file included from D:/a/benchmark/benchmark/test/spec_arg_verbosity_test.cc:5:
D:/a/benchmark/benchmark/include/benchmark/benchmark.h:999:44: error: 'benchmark::State::KeepRunning' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes]
999 | inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() {
| ^
D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1003:44: error: 'benchmark::State::KeepRunningBatch' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes]
1003 | inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) {
| ^
D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1075:60: error: 'benchmark::State::begin' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes]
1075 | inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() {
| ^
D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1078:60: error: 'benchmark::State::end' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes]
1078 | inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() {
| ^
```
* StatisticsTest.CV: don't require precise FP match, tolerate some abs error
We get ever so slightly different results on windows with GCC.
```
71: Test command: D:\a\benchmark\benchmark\_build\test\statistics_gtest.exe
71: Working Directory: D:/a/benchmark/benchmark/_build/test
71: Test timeout computed to be: 10000000
71: Running main() from gmock_main.cc
71: [==========] Running 4 tests from 1 test suite.
71: [----------] Global test environment set-up.
71: [----------] 4 tests from StatisticsTest
71: [ RUN ] StatisticsTest.Mean
71: [ OK ] StatisticsTest.Mean (0 ms)
71: [ RUN ] StatisticsTest.Median
71: [ OK ] StatisticsTest.Median (0 ms)
71: [ RUN ] StatisticsTest.StdDev
71: [ OK ] StatisticsTest.StdDev (0 ms)
71: [ RUN ] StatisticsTest.CV
71: D:/a/benchmark/benchmark/test/statistics_gtest.cc:31: Failure
71: Expected equality of these values:
71: benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1})
71: Which is: 0.32888184094918088
71: 0.32888184094918121
71: [ FAILED ] StatisticsTest.CV (0 ms)
71: [----------] 4 tests from StatisticsTest (0 ms total)
```
* Fix DLL path discovery for tests
2023-11-23 14:47:04 +00:00
|
|
|
ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}),
|
|
|
|
0.32888184094918121, 1e-15);
|
Introduce Coefficient of variation aggregate (#1220)
* Introduce Coefficient of variation aggregate
I believe, it is much more useful / use to understand,
because it is already normalized by the mean,
so it is not affected by the duration of the benchmark,
unlike the standard deviation.
Example of real-world output:
```
raw.pixls.us-unique/GoPro/HERO6 Black$ ~/rawspeed/build-old/src/utilities/rsbench/rsbench GOPR9172.GPR --benchmark_repetitions=27 --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true
2021-09-03T18:05:56+03:00
Running /home/lebedevri/rawspeed/build-old/src/utilities/rsbench/rsbench
Run on (32 X 3596.16 MHz CPU s)
CPU Caches:
L1 Data 32 KiB (x16)
L1 Instruction 32 KiB (x16)
L2 Unified 512 KiB (x16)
L3 Unified 32768 KiB (x2)
Load Average: 7.00, 2.99, 1.85
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations CPUTime,s CPUTime/WallTime Pixels Pixels/CPUTime Pixels/WallTime Raws/CPUTime Raws/WallTime WallTime,s
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
GOPR9172.GPR/threads:32/process_time/real_time_mean 11.1 ms 353 ms 27 0.353122 31.9473 12M 33.9879M 1085.84M 2.83232 90.4864 0.0110535
GOPR9172.GPR/threads:32/process_time/real_time_median 11.0 ms 352 ms 27 0.351696 31.9599 12M 34.1203M 1090.11M 2.84336 90.8425 0.0110081
GOPR9172.GPR/threads:32/process_time/real_time_stddev 0.159 ms 4.60 ms 27 4.59539m 0.0462064 0 426.371k 14.9631M 0.0355309 1.24692 158.944u
GOPR9172.GPR/threads:32/process_time/real_time_cv 1.44 % 1.30 % 27 0.0130136 1.44633m 0 0.0125448 0.0137802 0.0125448 0.0137802 0.0143795
```
Fixes https://github.com/google/benchmark/issues/1146
* Be consistent, it's CV, not 'rel std dev'
2021-09-03 17:44:10 +00:00
|
|
|
}
|
|
|
|
|
2017-12-13 23:26:47 +00:00
|
|
|
} // end namespace
|