benchmark/test/statistics_gtest.cc

36 lines
1.3 KiB
C++
Raw Permalink Normal View History

//===---------------------------------------------------------------------===//
// statistics_test - Unit tests for src/statistics.cc
//===---------------------------------------------------------------------===//
#include "../src/statistics.h"
#include "gtest/gtest.h"
namespace {
TEST(StatisticsTest, Mean) {
EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({42, 42, 42, 42}), 42.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 3, 4}), 2.5);
EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 5, 10, 10, 14}), 7.0);
}
TEST(StatisticsTest, Median) {
EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({42, 42, 42, 42}), 42.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 3, 4}), 2.5);
EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 5, 10, 10}), 5.0);
}
TEST(StatisticsTest, StdDev) {
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({2.5, 2.4, 3.3, 4.2, 5.1}),
1.151086443322134);
}
Introduce Coefficient of variation aggregate (#1220) * Introduce Coefficient of variation aggregate I believe, it is much more useful / use to understand, because it is already normalized by the mean, so it is not affected by the duration of the benchmark, unlike the standard deviation. Example of real-world output: ``` raw.pixls.us-unique/GoPro/HERO6 Black$ ~/rawspeed/build-old/src/utilities/rsbench/rsbench GOPR9172.GPR --benchmark_repetitions=27 --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true 2021-09-03T18:05:56+03:00 Running /home/lebedevri/rawspeed/build-old/src/utilities/rsbench/rsbench Run on (32 X 3596.16 MHz CPU s) CPU Caches: L1 Data 32 KiB (x16) L1 Instruction 32 KiB (x16) L2 Unified 512 KiB (x16) L3 Unified 32768 KiB (x2) Load Average: 7.00, 2.99, 1.85 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations CPUTime,s CPUTime/WallTime Pixels Pixels/CPUTime Pixels/WallTime Raws/CPUTime Raws/WallTime WallTime,s ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ GOPR9172.GPR/threads:32/process_time/real_time_mean 11.1 ms 353 ms 27 0.353122 31.9473 12M 33.9879M 1085.84M 2.83232 90.4864 0.0110535 GOPR9172.GPR/threads:32/process_time/real_time_median 11.0 ms 352 ms 27 0.351696 31.9599 12M 34.1203M 1090.11M 2.84336 90.8425 0.0110081 GOPR9172.GPR/threads:32/process_time/real_time_stddev 0.159 ms 4.60 ms 27 4.59539m 0.0462064 0 426.371k 14.9631M 0.0355309 1.24692 158.944u GOPR9172.GPR/threads:32/process_time/real_time_cv 1.44 % 1.30 % 27 0.0130136 1.44633m 0 0.0125448 0.0137802 0.0125448 0.0137802 0.0143795 ``` Fixes https://github.com/google/benchmark/issues/1146 * Be consistent, it's CV, not 'rel std dev'
2021-09-03 17:44:10 +00:00
TEST(StatisticsTest, CV) {
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.);
Support windows MSYS2 environments (#1704) * [CI] Attempt to add windows MSYS2-based coverage * Mark decl of `State::KeepRunningInternal()` as `inline` Maybe helps with ``` D:\a\_temp\msys64\ucrt64\bin\g++.exe -DHAVE_STD_REGEX -DHAVE_STEADY_CLOCK -DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS -ID:/a/benchmark/benchmark/include -Wall -Wextra -Wshadow -Wfloat-equal -Wold-style-cast -Werror -pedantic -pedantic-errors -fstrict-aliasing -Wno-deprecated-declarations -Wno-deprecated -Wstrict-aliasing -Wno-unused-variable -std=c++11 -fvisibility=hidden -fno-keep-inline-dllexport -UNDEBUG -MD -MT test/CMakeFiles/benchmark_test.dir/benchmark_test.cc.obj -MF test\CMakeFiles\benchmark_test.dir\benchmark_test.cc.obj.d -o test/CMakeFiles/benchmark_test.dir/benchmark_test.cc.obj -c D:/a/benchmark/benchmark/test/benchmark_test.cc In file included from D:/a/benchmark/benchmark/test/benchmark_test.cc:1: D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1007:37: error: 'bool benchmark::State::KeepRunningInternal(benchmark::IterationCount, bool)' redeclared without dllimport attribute after being referenced with dll linkage [-Werror] 1007 | inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, | ^~~~~ ``` * Mark more `State`'s member function decls as `inline` ``` [27/110] Building CXX object test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj FAILED: test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj D:\a\_temp\msys64\clang32\bin\clang++.exe -DHAVE_STD_REGEX -DHAVE_STEADY_CLOCK -DHAVE_THREAD_SAFETY_ATTRIBUTES -DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS -ID:/a/benchmark/benchmark/include -Wall -Wextra -Wshadow -Wfloat-equal -Wold-style-cast -Werror -pedantic -pedantic-errors -Wshorten-64-to-32 -fstrict-aliasing -Wno-deprecated-declarations -Wno-deprecated -Wstrict-aliasing -Wthread-safety -Wno-unused-variable -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden -UNDEBUG -MD -MT test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj -MF test\CMakeFiles\spec_arg_verbosity_test.dir\spec_arg_verbosity_test.cc.obj.d -o test/CMakeFiles/spec_arg_verbosity_test.dir/spec_arg_verbosity_test.cc.obj -c D:/a/benchmark/benchmark/test/spec_arg_verbosity_test.cc In file included from D:/a/benchmark/benchmark/test/spec_arg_verbosity_test.cc:5: D:/a/benchmark/benchmark/include/benchmark/benchmark.h:999:44: error: 'benchmark::State::KeepRunning' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes] 999 | inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { | ^ D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1003:44: error: 'benchmark::State::KeepRunningBatch' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes] 1003 | inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { | ^ D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1075:60: error: 'benchmark::State::begin' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes] 1075 | inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { | ^ D:/a/benchmark/benchmark/include/benchmark/benchmark.h:1078:60: error: 'benchmark::State::end' redeclared inline; 'dllimport' attribute ignored [-Werror,-Wignored-attributes] 1078 | inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { | ^ ``` * StatisticsTest.CV: don't require precise FP match, tolerate some abs error We get ever so slightly different results on windows with GCC. ``` 71: Test command: D:\a\benchmark\benchmark\_build\test\statistics_gtest.exe 71: Working Directory: D:/a/benchmark/benchmark/_build/test 71: Test timeout computed to be: 10000000 71: Running main() from gmock_main.cc 71: [==========] Running 4 tests from 1 test suite. 71: [----------] Global test environment set-up. 71: [----------] 4 tests from StatisticsTest 71: [ RUN ] StatisticsTest.Mean 71: [ OK ] StatisticsTest.Mean (0 ms) 71: [ RUN ] StatisticsTest.Median 71: [ OK ] StatisticsTest.Median (0 ms) 71: [ RUN ] StatisticsTest.StdDev 71: [ OK ] StatisticsTest.StdDev (0 ms) 71: [ RUN ] StatisticsTest.CV 71: D:/a/benchmark/benchmark/test/statistics_gtest.cc:31: Failure 71: Expected equality of these values: 71: benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}) 71: Which is: 0.32888184094918088 71: 0.32888184094918121 71: [ FAILED ] StatisticsTest.CV (0 ms) 71: [----------] 4 tests from StatisticsTest (0 ms total) ``` * Fix DLL path discovery for tests
2023-11-23 14:47:04 +00:00
ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}),
0.32888184094918121, 1e-15);
Introduce Coefficient of variation aggregate (#1220) * Introduce Coefficient of variation aggregate I believe, it is much more useful / use to understand, because it is already normalized by the mean, so it is not affected by the duration of the benchmark, unlike the standard deviation. Example of real-world output: ``` raw.pixls.us-unique/GoPro/HERO6 Black$ ~/rawspeed/build-old/src/utilities/rsbench/rsbench GOPR9172.GPR --benchmark_repetitions=27 --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true 2021-09-03T18:05:56+03:00 Running /home/lebedevri/rawspeed/build-old/src/utilities/rsbench/rsbench Run on (32 X 3596.16 MHz CPU s) CPU Caches: L1 Data 32 KiB (x16) L1 Instruction 32 KiB (x16) L2 Unified 512 KiB (x16) L3 Unified 32768 KiB (x2) Load Average: 7.00, 2.99, 1.85 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations CPUTime,s CPUTime/WallTime Pixels Pixels/CPUTime Pixels/WallTime Raws/CPUTime Raws/WallTime WallTime,s ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ GOPR9172.GPR/threads:32/process_time/real_time_mean 11.1 ms 353 ms 27 0.353122 31.9473 12M 33.9879M 1085.84M 2.83232 90.4864 0.0110535 GOPR9172.GPR/threads:32/process_time/real_time_median 11.0 ms 352 ms 27 0.351696 31.9599 12M 34.1203M 1090.11M 2.84336 90.8425 0.0110081 GOPR9172.GPR/threads:32/process_time/real_time_stddev 0.159 ms 4.60 ms 27 4.59539m 0.0462064 0 426.371k 14.9631M 0.0355309 1.24692 158.944u GOPR9172.GPR/threads:32/process_time/real_time_cv 1.44 % 1.30 % 27 0.0130136 1.44633m 0 0.0125448 0.0137802 0.0125448 0.0137802 0.0143795 ``` Fixes https://github.com/google/benchmark/issues/1146 * Be consistent, it's CV, not 'rel std dev'
2021-09-03 17:44:10 +00:00
}
} // end namespace