dolysis
/
benchmark
mirror of https://github.com/google/benchmark.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplify clang-format and apply to tests (#302)

This commit is contained in:
Dominic Hamon 2016-10-07 11:04:50 -07:00 committed by GitHub
parent d1daeee4e9
commit 1100e91907
17 changed files with 478 additions and 599 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
.clang-format
View File

@ -1,96 +1,5 @@
--- ---
Language: Cpp Language: Cpp
# BasedOnStyle: Google BasedOnStyle: Google
AccessModifierOffset: -1
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: false
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: false
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: true
ColumnLimit: 80
CommentPragmas: '^ IWYU pragma:'
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: true
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]
IncludeCategories:
- Regex: '^<.*\.h>'
Priority: 1
- Regex: '^<.*'
Priority: 2
- Regex: '.*'
Priority: 3
IncludeIsMainRegex: '([-_](test|unittest))?$'
IndentCaseLabels: true
IndentWidth: 2
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: false
PenaltyBreakBeforeFirstCallParameter: 1
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Left
ReflowComments: true
SortIncludes: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Auto
TabWidth: 8
UseTab: Never
... ...

19
test/basic_test.cc
View File

@ -1,8 +1,7 @@
#include "benchmark/benchmark_api.h" #include "benchmark/benchmark_api.h"
#define BASIC_BENCHMARK_TEST(x) \ #define BASIC_BENCHMARK_TEST(x) BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192)
BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192)
void BM_empty(benchmark::State& state) { void BM_empty(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
@ -26,7 +25,7 @@ void BM_spin_pause_before(benchmark::State& state) {
for (int i = 0; i < state.range(0); ++i) { for (int i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
while(state.KeepRunning()) { while (state.KeepRunning()) {
for (int i = 0; i < state.range(0); ++i) { for (int i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
@ -35,9 +34,8 @@ void BM_spin_pause_before(benchmark::State& state) {
BASIC_BENCHMARK_TEST(BM_spin_pause_before); BASIC_BENCHMARK_TEST(BM_spin_pause_before);
BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu();
void BM_spin_pause_during(benchmark::State& state) { void BM_spin_pause_during(benchmark::State& state) {
while(state.KeepRunning()) { while (state.KeepRunning()) {
state.PauseTiming(); state.PauseTiming();
for (int i = 0; i < state.range(0); ++i) { for (int i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
@ -52,7 +50,7 @@ BASIC_BENCHMARK_TEST(BM_spin_pause_during);
BASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu();
void BM_pause_during(benchmark::State& state) { void BM_pause_during(benchmark::State& state) {
while(state.KeepRunning()) { while (state.KeepRunning()) {
state.PauseTiming(); state.PauseTiming();
state.ResumeTiming(); state.ResumeTiming();
} }
@ -63,7 +61,7 @@ BENCHMARK(BM_pause_during)->UseRealTime();
BENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu(); BENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu();
void BM_spin_pause_after(benchmark::State& state) { void BM_spin_pause_after(benchmark::State& state) {
while(state.KeepRunning()) { while (state.KeepRunning()) {
for (int i = 0; i < state.range(0); ++i) { for (int i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
@ -75,12 +73,11 @@ void BM_spin_pause_after(benchmark::State& state) {
BASIC_BENCHMARK_TEST(BM_spin_pause_after); BASIC_BENCHMARK_TEST(BM_spin_pause_after);
BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu();
void BM_spin_pause_before_and_after(benchmark::State& state) { void BM_spin_pause_before_and_after(benchmark::State& state) {
for (int i = 0; i < state.range(0); ++i) { for (int i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
while(state.KeepRunning()) { while (state.KeepRunning()) {
for (int i = 0; i < state.range(0); ++i) { for (int i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
@ -92,9 +89,9 @@ void BM_spin_pause_before_and_after(benchmark::State& state) {
BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after); BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after);
BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu();
void BM_empty_stop_start(benchmark::State& state) { void BM_empty_stop_start(benchmark::State& state) {
while (state.KeepRunning()) { } while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_empty_stop_start); BENCHMARK(BM_empty_stop_start);
BENCHMARK(BM_empty_stop_start)->ThreadPerCpu(); BENCHMARK(BM_empty_stop_start)->ThreadPerCpu();

96
test/benchmark_test.cc
View File

@ -4,6 +4,7 @@
#include <math.h> #include <math.h>
#include <stdint.h> #include <stdint.h>
#include <chrono>
#include <cstdlib> #include <cstdlib>
#include <iostream> #include <iostream>
#include <limits> #include <limits>
@ -13,15 +14,14 @@
#include <set> #include <set>
#include <sstream> #include <sstream>
#include <string> #include <string>
#include <vector>
#include <chrono>
#include <thread> #include <thread>
#include <utility> #include <utility>
#include <vector>
#if defined(__GNUC__) #if defined(__GNUC__)
# define BENCHMARK_NOINLINE __attribute__((noinline)) #define BENCHMARK_NOINLINE __attribute__((noinline))
#else #else
# define BENCHMARK_NOINLINE #define BENCHMARK_NOINLINE
#endif #endif
namespace { namespace {
@ -42,8 +42,7 @@ double CalculatePi(int depth) {
std::set<int> ConstructRandomSet(int size) { std::set<int> ConstructRandomSet(int size) {
std::set<int> s; std::set<int> s;
for (int i = 0; i < size; ++i) for (int i = 0; i < size; ++i) s.insert(i);
s.insert(i);
return s; return s;
} }
@ -54,8 +53,7 @@ std::vector<int>* test_vector = nullptr;
static void BM_Factorial(benchmark::State& state) { static void BM_Factorial(benchmark::State& state) {
int fac_42 = 0; int fac_42 = 0;
while (state.KeepRunning()) while (state.KeepRunning()) fac_42 = Factorial(8);
fac_42 = Factorial(8);
// Prevent compiler optimizations // Prevent compiler optimizations
std::stringstream ss; std::stringstream ss;
ss << fac_42; ss << fac_42;
@ -66,8 +64,7 @@ BENCHMARK(BM_Factorial)->UseRealTime();
static void BM_CalculatePiRange(benchmark::State& state) { static void BM_CalculatePiRange(benchmark::State& state) {
double pi = 0.0; double pi = 0.0;
while (state.KeepRunning()) while (state.KeepRunning()) pi = CalculatePi(state.range(0));
pi = CalculatePi(state.range(0));
std::stringstream ss; std::stringstream ss;
ss << pi; ss << pi;
state.SetLabel(ss.str()); state.SetLabel(ss.str());
@ -89,27 +86,27 @@ static void BM_SetInsert(benchmark::State& state) {
state.PauseTiming(); state.PauseTiming();
std::set<int> data = ConstructRandomSet(state.range(0)); std::set<int> data = ConstructRandomSet(state.range(0));
state.ResumeTiming(); state.ResumeTiming();
for (int j = 0; j < state.range(1); ++j) for (int j = 0; j < state.range(1); ++j) data.insert(rand());
data.insert(rand());
} }
state.SetItemsProcessed(state.iterations() * state.range(1)); state.SetItemsProcessed(state.iterations() * state.range(1));
state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int));
} }
BENCHMARK(BM_SetInsert)->Ranges({{1<<10,8<<10}, {1,10}}); BENCHMARK(BM_SetInsert)->Ranges({{1 << 10, 8 << 10}, {1, 10}});
template<typename Container, typename ValueType = typename Container::value_type> template <typename Container,
typename ValueType = typename Container::value_type>
static void BM_Sequential(benchmark::State& state) { static void BM_Sequential(benchmark::State& state) {
ValueType v = 42; ValueType v = 42;
while (state.KeepRunning()) { while (state.KeepRunning()) {
Container c; Container c;
for (int i = state.range(0); --i; ) for (int i = state.range(0); --i;) c.push_back(v);
c.push_back(v);
} }
const size_t items_processed = state.iterations() * state.range(0); const size_t items_processed = state.iterations() * state.range(0);
state.SetItemsProcessed(items_processed); state.SetItemsProcessed(items_processed);
state.SetBytesProcessed(items_processed * sizeof(v)); state.SetBytesProcessed(items_processed * sizeof(v));
} }
BENCHMARK_TEMPLATE2(BM_Sequential, std::vector<int>, int)->Range(1 << 0, 1 << 10); BENCHMARK_TEMPLATE2(BM_Sequential, std::vector<int>, int)
->Range(1 << 0, 1 << 10);
BENCHMARK_TEMPLATE(BM_Sequential, std::list<int>)->Range(1 << 0, 1 << 10); BENCHMARK_TEMPLATE(BM_Sequential, std::list<int>)->Range(1 << 0, 1 << 10);
// Test the variadic version of BENCHMARK_TEMPLATE in C++11 and beyond. // Test the variadic version of BENCHMARK_TEMPLATE in C++11 and beyond.
#if __cplusplus >= 201103L #if __cplusplus >= 201103L
@ -119,10 +116,9 @@ BENCHMARK_TEMPLATE(BM_Sequential, std::vector<int>, int)->Arg(512);
static void BM_StringCompare(benchmark::State& state) { static void BM_StringCompare(benchmark::State& state) {
std::string s1(state.range(0), '-'); std::string s1(state.range(0), '-');
std::string s2(state.range(0), '-'); std::string s2(state.range(0), '-');
while (state.KeepRunning()) while (state.KeepRunning()) benchmark::DoNotOptimize(s1.compare(s2));
benchmark::DoNotOptimize(s1.compare(s2));
} }
BENCHMARK(BM_StringCompare)->Range(1, 1<<20); BENCHMARK(BM_StringCompare)->Range(1, 1 << 20);
static void BM_SetupTeardown(benchmark::State& state) { static void BM_SetupTeardown(benchmark::State& state) {
if (state.thread_index == 0) { if (state.thread_index == 0) {
@ -132,7 +128,7 @@ static void BM_SetupTeardown(benchmark::State& state) {
int i = 0; int i = 0;
while (state.KeepRunning()) { while (state.KeepRunning()) {
std::lock_guard<std::mutex> l(test_vector_mu); std::lock_guard<std::mutex> l(test_vector_mu);
if (i%2 == 0) if (i % 2 == 0)
test_vector->push_back(i); test_vector->push_back(i);
else else
test_vector->pop_back(); test_vector->pop_back();
@ -151,7 +147,7 @@ static void BM_LongTest(benchmark::State& state) {
benchmark::DoNotOptimize(tracker += i); benchmark::DoNotOptimize(tracker += i);
} }
} }
BENCHMARK(BM_LongTest)->Range(1<<16,1<<28); BENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28);
static void BM_ParallelMemset(benchmark::State& state) { static void BM_ParallelMemset(benchmark::State& state) {
int size = state.range(0) / sizeof(int); int size = state.range(0) / sizeof(int);
@ -180,20 +176,18 @@ BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4);
static void BM_ManualTiming(benchmark::State& state) { static void BM_ManualTiming(benchmark::State& state) {
size_t slept_for = 0; size_t slept_for = 0;
int microseconds = state.range(0); int microseconds = state.range(0);
std::chrono::duration<double, std::micro> sleep_duration { std::chrono::duration<double, std::micro> sleep_duration{
static_cast<double>(microseconds) static_cast<double>(microseconds)};
};
while (state.KeepRunning()) { while (state.KeepRunning()) {
auto start = std::chrono::high_resolution_clock::now(); auto start = std::chrono::high_resolution_clock::now();
// Simulate some useful workload with a sleep // Simulate some useful workload with a sleep
std::this_thread::sleep_for(std::chrono::duration_cast< std::this_thread::sleep_for(
std::chrono::nanoseconds>(sleep_duration)); std::chrono::duration_cast<std::chrono::nanoseconds>(sleep_duration));
auto end = std::chrono::high_resolution_clock::now(); auto end = std::chrono::high_resolution_clock::now();
auto elapsed = auto elapsed =
std::chrono::duration_cast<std::chrono::duration<double>>( std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
end - start);
state.SetIterationTime(elapsed.count()); state.SetIterationTime(elapsed.count());
slept_for += microseconds; slept_for += microseconds;
@ -205,35 +199,37 @@ BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime();
#if __cplusplus >= 201103L #if __cplusplus >= 201103L
template <class ...Args> template <class... Args>
void BM_with_args(benchmark::State& state, Args&&...) { void BM_with_args(benchmark::State& state, Args&&...) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44); BENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44);
BENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, BENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, std::string("abc"),
std::string("abc"), std::pair<int, double>(42, 3.8)); std::pair<int, double>(42, 3.8));
void BM_non_template_args(benchmark::State& state, int, double) { void BM_non_template_args(benchmark::State& state, int, double) {
while(state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0);
#endif // __cplusplus >= 201103L #endif // __cplusplus >= 201103L
static void BM_DenseThreadRanges(benchmark::State &st) { static void BM_DenseThreadRanges(benchmark::State& st) {
switch (st.range(0)) { switch (st.range(0)) {
case 1: case 1:
assert(st.threads == 1 || st.threads == 2 || st.threads == 3); assert(st.threads == 1 || st.threads == 2 || st.threads == 3);
break; break;
case 2: case 2:
assert(st.threads == 1 || st.threads == 3 || st.threads == 4); assert(st.threads == 1 || st.threads == 3 || st.threads == 4);
break; break;
case 3: case 3:
assert(st.threads == 5 || st.threads == 8 || st.threads == 11 || assert(st.threads == 5 || st.threads == 8 || st.threads == 11 ||
st.threads == 14); st.threads == 14);
break; break;
default: default:
assert(false && "Invalid test case number"); assert(false && "Invalid test case number");
} }
while (st.KeepRunning()) { while (st.KeepRunning()) {
} }

158
test/complexity_test.cc
View File

@ -1,47 +1,41 @@
#undef NDEBUG #undef NDEBUG
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
#include <vector>
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
#include <cassert>
#include <vector>
#include <algorithm>
#include <cstdlib>
#include <cmath>
namespace { namespace {
#define ADD_COMPLEXITY_CASES(...) \ #define ADD_COMPLEXITY_CASES(...) \
int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)
int AddComplexityTest(std::string big_o_test_name, int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name,
std::string rms_test_name, std::string big_o) { std::string big_o) {
SetSubstitutions({ SetSubstitutions({{"%bigo_name", big_o_test_name},
{"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name},
{"%rms_name", rms_test_name}, {"%bigo_str", "[ ]*" + std::string(dec_re) + " " + big_o},
{"%bigo_str", "[ ]*" + std::string(dec_re) + " " + big_o}, {"%bigo", big_o},
{"%bigo", big_o}, {"%rms", "[ ]*[0-9]+ %"}});
{"%rms", "[ ]*[0-9]+ %"} AddCases(
}); TC_ConsoleOut,
AddCases(TC_ConsoleOut, { {{"^%bigo_name %bigo_str %bigo_str[ ]*$"},
{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
{"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}});
{"^%rms_name %rms %rms[ ]*$", MR_Next} AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"},
}); {"\"cpu_coefficient\": [0-9]+,$", MR_Next},
AddCases(TC_JSONOut, { {"\"real_coefficient\": [0-9]{1,5},$", MR_Next},
{"\"name\": \"%bigo_name\",$"}, {"\"big_o\": \"%bigo\",$", MR_Next},
{"\"cpu_coefficient\": [0-9]+,$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"\"real_coefficient\": [0-9]{1,5},$", MR_Next}, {"}", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next}, {"\"name\": \"%rms_name\",$"},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"rms\": [0-9]+%$", MR_Next},
{"}", MR_Next}, {"}", MR_Next}});
{"\"name\": \"%rms_name\",$"}, AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
{"\"rms\": [0-9]+%$", MR_Next}, {"^\"%bigo_name\"", MR_Not},
{"}", MR_Next} {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}});
});
AddCases(TC_CSVOut, {
{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
{"^\"%bigo_name\"", MR_Not},
{"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}
});
return 0; return 0;
} }
@ -53,23 +47,26 @@ int AddComplexityTest(std::string big_o_test_name,
void BM_Complexity_O1(benchmark::State& state) { void BM_Complexity_O1(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
for (int i=0; i < 1024; ++i) { for (int i = 0; i < 1024; ++i) {
benchmark::DoNotOptimize(&i); benchmark::DoNotOptimize(&i);
} }
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity();
BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](int){return 1.0; }); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int) {
return 1.0;
});
const char* big_o_1_test_name = "BM_Complexity_O1_BigO"; const char *big_o_1_test_name = "BM_Complexity_O1_BigO";
const char* rms_o_1_test_name = "BM_Complexity_O1_RMS"; const char *rms_o_1_test_name = "BM_Complexity_O1_RMS";
const char* enum_big_o_1 = "\\([0-9]+\\)"; const char *enum_big_o_1 = "\\([0-9]+\\)";
// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto deduced. // FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto
// deduced.
// See https://github.com/google/benchmark/issues/272 // See https://github.com/google/benchmark/issues/272
const char* auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)";
const char* lambda_big_o_1 = "f\\(N\\)"; const char *lambda_big_o_1 = "f\\(N\\)";
// Add enum tests // Add enum tests
ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1); ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1);
@ -95,20 +92,30 @@ std::vector<int> ConstructRandomVector(int size) {
void BM_Complexity_O_N(benchmark::State& state) { void BM_Complexity_O_N(benchmark::State& state) {
auto v = ConstructRandomVector(state.range(0)); auto v = ConstructRandomVector(state.range(0));
const int item_not_in_vector = state.range(0)*2; // Test worst case scenario (item not in vector) const int item_not_in_vector =
state.range(0) * 2; // Test worst case scenario (item not in vector)
while (state.KeepRunning()) { while (state.KeepRunning()) {
benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector));
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N)
BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) -> double{return n; }); ->RangeMultiplier(2)
BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); ->Range(1 << 10, 1 << 16)
->Complexity(benchmark::oN);
BENCHMARK(BM_Complexity_O_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
->Complexity([](int n) -> double { return n; });
BENCHMARK(BM_Complexity_O_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
->Complexity();
const char* big_o_n_test_name = "BM_Complexity_O_N_BigO"; const char *big_o_n_test_name = "BM_Complexity_O_N_BigO";
const char* rms_o_n_test_name = "BM_Complexity_O_N_RMS"; const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS";
const char* enum_auto_big_o_n = "N"; const char *enum_auto_big_o_n = "N";
const char* lambda_big_o_n = "f\\(N\\)"; const char *lambda_big_o_n = "f\\(N\\)";
// Add enum tests // Add enum tests
ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
@ -123,31 +130,38 @@ ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
static void BM_Complexity_O_N_log_N(benchmark::State& state) { static void BM_Complexity_O_N_log_N(benchmark::State& state) {
auto v = ConstructRandomVector(state.range(0)); auto v = ConstructRandomVector(state.range(0));
while (state.KeepRunning()) { while (state.KeepRunning()) {
std::sort(v.begin(), v.end()); std::sort(v.begin(), v.end());
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N)
BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) {return n * std::log2(n); }); ->RangeMultiplier(2)
BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(); ->Range(1 << 10, 1 << 16)
->Complexity(benchmark::oNLogN);
BENCHMARK(BM_Complexity_O_N_log_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
->Complexity([](int n) { return n * std::log2(n); });
BENCHMARK(BM_Complexity_O_N_log_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
->Complexity();
const char* big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO";
const char* rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS";
const char* enum_auto_big_o_n_lg_n = "NlgN"; const char *enum_auto_big_o_n_lg_n = "NlgN";
const char* lambda_big_o_n_lg_n = "f\\(N\\)"; const char *lambda_big_o_n_lg_n = "f\\(N\\)";
// Add enum tests // Add enum tests
ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name,
enum_auto_big_o_n_lg_n);
// Add lambda tests // Add lambda tests
ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name,
lambda_big_o_n_lg_n);
// ========================================================================= // // ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ // // --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= // // ========================================================================= //
int main(int argc, char* argv[]) { int main(int argc, char *argv[]) { RunOutputTests(argc, argv); }
RunOutputTests(argc, argv);
}

22
test/cxx03_test.cc
View File

@ -1,6 +1,6 @@
#undef NDEBUG #undef NDEBUG
#include <cstddef>
#include <cassert> #include <cassert>
#include <cstddef>
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
@ -9,32 +9,32 @@
#endif #endif
void BM_empty(benchmark::State& state) { void BM_empty(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
volatile std::size_t x = state.iterations(); volatile std::size_t x = state.iterations();
((void)x); ((void)x);
} }
} }
BENCHMARK(BM_empty); BENCHMARK(BM_empty);
// The new C++11 interface for args/ranges requires initializer list support. // The new C++11 interface for args/ranges requires initializer list support.
// Therefore we provide the old interface to support C++03. // Therefore we provide the old interface to support C++03.
void BM_old_arg_range_interface(benchmark::State& state) { void BM_old_arg_range_interface(benchmark::State& state) {
assert((state.range(0) == 1 && state.range(1) == 2) || assert((state.range(0) == 1 && state.range(1) == 2) ||
(state.range(0) == 5 && state.range(1) == 6)); (state.range(0) == 5 && state.range(1) == 6));
while (state.KeepRunning()) { while (state.KeepRunning()) {
} }
} }
BENCHMARK(BM_old_arg_range_interface)->ArgPair(1, 2)->RangePair(5, 5, 6, 6); BENCHMARK(BM_old_arg_range_interface)->ArgPair(1, 2)->RangePair(5, 5, 6, 6);
template <class T, class U> template <class T, class U>
void BM_template2(benchmark::State& state) { void BM_template2(benchmark::State& state) {
BM_empty(state); BM_empty(state);
} }
BENCHMARK_TEMPLATE2(BM_template2, int, long); BENCHMARK_TEMPLATE2(BM_template2, int, long);
template <class T> template <class T>
void BM_template1(benchmark::State& state) { void BM_template1(benchmark::State& state) {
BM_empty(state); BM_empty(state);
} }
BENCHMARK_TEMPLATE(BM_template1, long); BENCHMARK_TEMPLATE(BM_template1, long);
BENCHMARK_TEMPLATE1(BM_template1, int); BENCHMARK_TEMPLATE1(BM_template1, int);

17
test/diagnostics_test.cc
View File

@ -7,10 +7,11 @@
// NOTE: Users should NOT include or use src/check.h. This is only done in // NOTE: Users should NOT include or use src/check.h. This is only done in
// order to test library internals. // order to test library internals.
#include "benchmark/benchmark_api.h"
#include "../src/check.h"
#include <stdexcept>
#include <cstdlib> #include <cstdlib>
#include <stdexcept>
#include "../src/check.h"
#include "benchmark/benchmark_api.h"
#if defined(__GNUC__) && !defined(__EXCEPTIONS) #if defined(__GNUC__) && !defined(__EXCEPTIONS)
#define TEST_HAS_NO_EXCEPTIONS #define TEST_HAS_NO_EXCEPTIONS
@ -29,13 +30,15 @@ void try_invalid_pause_resume(benchmark::State& state) {
try { try {
state.PauseTiming(); state.PauseTiming();
std::abort(); std::abort();
} catch (std::logic_error const&) {} } catch (std::logic_error const&) {
}
try { try {
state.ResumeTiming(); state.ResumeTiming();
std::abort(); std::abort();
} catch (std::logic_error const&) {} } catch (std::logic_error const&) {
}
#else #else
(void)state; // avoid unused warning (void)state; // avoid unused warning
#endif #endif
} }
@ -54,7 +57,7 @@ void BM_diagnostic_test(benchmark::State& state) {
} }
BENCHMARK(BM_diagnostic_test); BENCHMARK(BM_diagnostic_test);
int main(int argc, char** argv) { int main(int argc, char* argv[]) {
benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::internal::GetAbortHandler() = &TestHandler;
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks(); benchmark::RunSpecifiedBenchmarks();

7
test/donotoptimize_test.cc
View File

@ -4,15 +4,12 @@
namespace { namespace {
#if defined(__GNUC__) #if defined(__GNUC__)
std::uint64_t double_up(const std::uint64_t x) __attribute__ ((const)); std::uint64_t double_up(const std::uint64_t x) __attribute__((const));
#endif #endif
std::uint64_t double_up(const std::uint64_t x) { std::uint64_t double_up(const std::uint64_t x) { return x * 2; }
return x * 2;
}
} }
int main(int, char*[]) { int main(int, char*[]) {
// this test verifies compilation of DoNotOptimize() for some types // this test verifies compilation of DoNotOptimize() for some types
char buffer8[8]; char buffer8[8];

35
test/filter_test.cc
View File

@ -27,9 +27,7 @@ class TestReporter : public benchmark::ConsoleReporter {
virtual ~TestReporter() {} virtual ~TestReporter() {}
size_t GetCount() const { size_t GetCount() const { return count_; }
return count_;
}
private: private:
mutable size_t count_; mutable size_t count_;
@ -37,46 +35,47 @@ class TestReporter : public benchmark::ConsoleReporter {
} // end namespace } // end namespace
static void NoPrefix(benchmark::State& state) { static void NoPrefix(benchmark::State& state) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(NoPrefix); BENCHMARK(NoPrefix);
static void BM_Foo(benchmark::State& state) { static void BM_Foo(benchmark::State& state) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_Foo); BENCHMARK(BM_Foo);
static void BM_Bar(benchmark::State& state) { static void BM_Bar(benchmark::State& state) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_Bar); BENCHMARK(BM_Bar);
static void BM_FooBar(benchmark::State& state) { static void BM_FooBar(benchmark::State& state) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_FooBar); BENCHMARK(BM_FooBar);
static void BM_FooBa(benchmark::State& state) { static void BM_FooBa(benchmark::State& state) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_FooBa); BENCHMARK(BM_FooBa);
int main(int argc, char **argv) {
int main(int argc, char** argv) {
bool list_only = false; bool list_only = false;
for (int i=0; i < argc; ++i) for (int i = 0; i < argc; ++i)
list_only |= std::string(argv[i]).find("--benchmark_list_tests") != std::string::npos; list_only |= std::string(argv[i]).find("--benchmark_list_tests") !=
std::string::npos;
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
TestReporter test_reporter; TestReporter test_reporter;
const size_t returned_count = benchmark::RunSpecifiedBenchmarks(&test_reporter); const size_t returned_count =
benchmark::RunSpecifiedBenchmarks(&test_reporter);
if (argc == 2) { if (argc == 2) {
// Make sure we ran all of the tests // Make sure we ran all of the tests

7
test/fixture_test.cc
View File

@ -20,15 +20,12 @@ class MyFixture : public ::benchmark::Fixture {
} }
} }
~MyFixture() { ~MyFixture() { assert(data == nullptr); }
assert(data == nullptr);
}
std::unique_ptr<int> data; std::unique_ptr<int> data;
}; };
BENCHMARK_F(MyFixture, Foo)(benchmark::State &st) {
BENCHMARK_F(MyFixture, Foo)(benchmark::State& st) {
assert(data.get() != nullptr); assert(data.get() != nullptr);
assert(*data == 42); assert(*data == 42);
while (st.KeepRunning()) { while (st.KeepRunning()) {

6
test/map_test.cc
View File

@ -37,9 +37,7 @@ class MapFixture : public ::benchmark::Fixture {
m = ConstructRandomMap(st.range(0)); m = ConstructRandomMap(st.range(0));
} }
void TearDown(const ::benchmark::State&) { void TearDown(const ::benchmark::State&) { m.clear(); }
m.clear();
}
std::map<int, int> m; std::map<int, int> m;
}; };
@ -53,6 +51,6 @@ BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) {
} }
state.SetItemsProcessed(state.iterations() * size); state.SetItemsProcessed(state.iterations() * size);
} }
BENCHMARK_REGISTER_F(MapFixture, Lookup)->Range(1<<3, 1<<12); BENCHMARK_REGISTER_F(MapFixture, Lookup)->Range(1 << 3, 1 << 12);
BENCHMARK_MAIN() BENCHMARK_MAIN()

43
test/multiple_ranges_test.cc
View File

@ -1,19 +1,30 @@
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include <set>
#include <cassert> #include <cassert>
#include <set>
class MultipleRangesFixture : public ::benchmark::Fixture { class MultipleRangesFixture : public ::benchmark::Fixture {
public: public:
MultipleRangesFixture() MultipleRangesFixture()
: expectedValues({ : expectedValues({{1, 3, 5},
{1, 3, 5}, {1, 3, 8}, {1, 3, 15}, {2, 3, 5}, {2, 3, 8}, {2, 3, 15}, {1, 3, 8},
{1, 4, 5}, {1, 4, 8}, {1, 4, 15}, {2, 4, 5}, {2, 4, 8}, {2, 4, 15}, {1, 3, 15},
{1, 7, 5}, {1, 7, 8}, {1, 7, 15}, {2, 7, 5}, {2, 7, 8}, {2, 7, 15}, {2, 3, 5},
{7, 6, 3} {2, 3, 8},
}) {2, 3, 15},
{ {1, 4, 5},
} {1, 4, 8},
{1, 4, 15},
{2, 4, 5},
{2, 4, 8},
{2, 4, 15},
{1, 7, 5},
{1, 7, 8},
{1, 7, 15},
{2, 7, 5},
{2, 7, 8},
{2, 7, 15},
{7, 6, 3}}) {}
void SetUp(const ::benchmark::State& state) { void SetUp(const ::benchmark::State& state) {
std::vector<int> ranges = {state.range(0), state.range(1), state.range(2)}; std::vector<int> ranges = {state.range(0), state.range(1), state.range(2)};
@ -31,7 +42,6 @@ class MultipleRangesFixture : public ::benchmark::Fixture {
std::set<std::vector<int>> actualValues; std::set<std::vector<int>> actualValues;
}; };
BENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) { BENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
int product = state.range(0) * state.range(1) * state.range(2); int product = state.range(0) * state.range(1) * state.range(2);
@ -41,21 +51,24 @@ BENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) {
} }
} }
BENCHMARK_REGISTER_F(MultipleRangesFixture, Empty)->RangeMultiplier(2) BENCHMARK_REGISTER_F(MultipleRangesFixture, Empty)
->Ranges({{1, 2}, {3, 7}, {5, 15}})->Args({7, 6, 3}); ->RangeMultiplier(2)
->Ranges({{1, 2}, {3, 7}, {5, 15}})
->Args({7, 6, 3});
void BM_CheckDefaultArgument(benchmark::State& state) { void BM_CheckDefaultArgument(benchmark::State& state) {
// Test that the 'range()' without an argument is the same as 'range(0)'. // Test that the 'range()' without an argument is the same as 'range(0)'.
assert(state.range() == state.range(0)); assert(state.range() == state.range(0));
assert(state.range() != state.range(1)); assert(state.range() != state.range(1));
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_CheckDefaultArgument)->Ranges({{1, 5}, {6, 10}}); BENCHMARK(BM_CheckDefaultArgument)->Ranges({{1, 5}, {6, 10}});
static void BM_MultipleRanges(benchmark::State& st) { static void BM_MultipleRanges(benchmark::State& st) {
while (st.KeepRunning()) {} while (st.KeepRunning()) {
}
} }
BENCHMARK(BM_MultipleRanges)->Ranges({{5, 5}, {6, 6}}); BENCHMARK(BM_MultipleRanges)->Ranges({{5, 5}, {6, 6}});
BENCHMARK_MAIN() BENCHMARK_MAIN()

3
test/options_test.cc
View File

@ -12,8 +12,7 @@ void BM_basic_slow(benchmark::State& state) {
std::chrono::milliseconds sleep_duration(state.range(0)); std::chrono::milliseconds sleep_duration(state.range(0));
while (state.KeepRunning()) { while (state.KeepRunning()) {
std::this_thread::sleep_for( std::this_thread::sleep_for(
std::chrono::duration_cast<std::chrono::nanoseconds>(sleep_duration) std::chrono::duration_cast<std::chrono::nanoseconds>(sleep_duration));
);
} }
} }

29
test/output_test.h
View File

@ -2,28 +2,28 @@
#define TEST_OUTPUT_TEST_H #define TEST_OUTPUT_TEST_H
#undef NDEBUG #undef NDEBUG
#include "benchmark/benchmark.h"
#include "../src/re.h"
#include <vector>
#include <string>
#include <initializer_list> #include <initializer_list>
#include <memory> #include <memory>
#include <string>
#include <utility> #include <utility>
#include <vector>
#include "../src/re.h"
#include "benchmark/benchmark.h"
#define CONCAT2(x, y) x##y #define CONCAT2(x, y) x##y
#define CONCAT(x, y) CONCAT2(x, y) #define CONCAT(x, y) CONCAT2(x, y)
#define ADD_CASES(...) \ #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
#define SET_SUBSTITUTIONS(...) \ #define SET_SUBSTITUTIONS(...) \
int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
enum MatchRules { enum MatchRules {
MR_Default, // Skip non-matching lines until a match is found. MR_Default, // Skip non-matching lines until a match is found.
MR_Next, // Match must occur on the next line. MR_Next, // Match must occur on the next line.
MR_Not // No line between the current position and the next match matches MR_Not // No line between the current position and the next match matches
// the regex // the regex
}; };
struct TestCase { struct TestCase {
@ -43,7 +43,7 @@ enum TestCaseID {
TC_CSVOut, TC_CSVOut,
TC_CSVErr, TC_CSVErr,
TC_NumID // PRIVATE TC_NumID // PRIVATE
}; };
// Add a list of test cases to be run against the output specified by // Add a list of test cases to be run against the output specified by
@ -66,7 +66,6 @@ namespace {
const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
} // end namespace } // end namespace
#endif // TEST_OUTPUT_TEST_H
#endif // TEST_OUTPUT_TEST_H

243
test/output_test_helper.cc
View File

@ -1,16 +1,17 @@
#include "output_test.h"
#include "../src/check.h" // NOTE: check.h is for internal use only!
#include "../src/re.h" // NOTE: re.h is for internal use only
#include <memory>
#include <map>
#include <iostream> #include <iostream>
#include <map>
#include <memory>
#include <sstream> #include <sstream>
#include "../src/check.h" // NOTE: check.h is for internal use only!
#include "../src/re.h" // NOTE: re.h is for internal use only
#include "output_test.h"
// ========================================================================= // // ========================================================================= //
// ------------------------------ Internals -------------------------------- // // ------------------------------ Internals -------------------------------- //
// ========================================================================= // // ========================================================================= //
namespace internal { namespace { namespace internal {
namespace {
using TestCaseList = std::vector<TestCase>; using TestCaseList = std::vector<TestCase>;
@ -22,91 +23,89 @@ using TestCaseList = std::vector<TestCase>;
using SubMap = std::vector<std::pair<std::string, std::string>>; using SubMap = std::vector<std::pair<std::string, std::string>>;
TestCaseList& GetTestCaseList(TestCaseID ID) { TestCaseList& GetTestCaseList(TestCaseID ID) {
// Uses function-local statics to ensure initialization occurs // Uses function-local statics to ensure initialization occurs
// before first use. // before first use.
static TestCaseList lists[TC_NumID]; static TestCaseList lists[TC_NumID];
return lists[ID]; return lists[ID];
} }
SubMap& GetSubstitutions() { SubMap& GetSubstitutions() {
// Don't use 'dec_re' from header because it may not yet be initialized. // Don't use 'dec_re' from header because it may not yet be initialized.
static std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; static std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
static SubMap map = { static SubMap map = {
{"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
{"%int", "[ ]*[0-9]+"}, {"%int", "[ ]*[0-9]+"},
{" %s ", "[ ]+"}, {" %s ", "[ ]+"},
{"%time", "[ ]*[0-9]{1,5} ns"}, {"%time", "[ ]*[0-9]{1,5} ns"},
{"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"}, {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
{"%csv_report", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,"} {"%csv_report", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,"}};
}; return map;
return map;
} }
std::string PerformSubstitutions(std::string source) { std::string PerformSubstitutions(std::string source) {
SubMap const& subs = GetSubstitutions(); SubMap const& subs = GetSubstitutions();
using SizeT = std::string::size_type; using SizeT = std::string::size_type;
for (auto const& KV : subs) { for (auto const& KV : subs) {
SizeT pos; SizeT pos;
SizeT next_start = 0; SizeT next_start = 0;
while ((pos = source.find(KV.first, next_start)) != std::string::npos) { while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
next_start = pos + KV.second.size(); next_start = pos + KV.second.size();
source.replace(pos, KV.first.size(), KV.second); source.replace(pos, KV.first.size(), KV.second);
}
} }
return source; }
return source;
} }
void CheckCase(std::stringstream& remaining_output, TestCase const& TC, void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
TestCaseList const& not_checks) TestCaseList const& not_checks) {
{ std::string first_line;
std::string first_line; bool on_first = true;
bool on_first = true; std::string line;
std::string line; while (remaining_output.eof() == false) {
while (remaining_output.eof() == false) { CHECK(remaining_output.good());
CHECK(remaining_output.good()); std::getline(remaining_output, line);
std::getline(remaining_output, line); if (on_first) {
if (on_first) { first_line = line;
first_line = line; on_first = false;
on_first = false;
}
for (auto& NC : not_checks) {
CHECK(!NC.regex->Match(line))
<< "Unexpected match for line \"" << line
<< "\" for MR_Not regex \"" << NC.regex_str << "\""
<< "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line;
}
if (TC.regex->Match(line)) return;
CHECK(TC.match_rule != MR_Next)
<< "Expected line \"" << line << "\" to match regex \"" << TC.regex_str << "\""
<< "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line;
} }
CHECK(remaining_output.eof() == false) for (const auto& NC : not_checks) {
<< "End of output reached before match for regex \"" << TC.regex_str CHECK(!NC.regex->Match(line))
<< "\" was found" << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
<< NC.regex_str << "\""
<< "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line;
}
if (TC.regex->Match(line)) return;
CHECK(TC.match_rule != MR_Next)
<< "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
<< "\""
<< "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line; << "\n started matching near: " << first_line;
}
CHECK(remaining_output.eof() == false)
<< "End of output reached before match for regex \"" << TC.regex_str
<< "\" was found"
<< "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line;
} }
void CheckCases(TestCaseList const& checks, std::stringstream& output) { void CheckCases(TestCaseList const& checks, std::stringstream& output) {
std::vector<TestCase> not_checks; std::vector<TestCase> not_checks;
for (size_t i=0; i < checks.size(); ++i) { for (size_t i = 0; i < checks.size(); ++i) {
const auto& TC = checks[i]; const auto& TC = checks[i];
if (TC.match_rule == MR_Not) { if (TC.match_rule == MR_Not) {
not_checks.push_back(TC); not_checks.push_back(TC);
continue; continue;
}
CheckCase(output, TC, not_checks);
not_checks.clear();
} }
CheckCase(output, TC, not_checks);
not_checks.clear();
}
} }
class TestReporter : public benchmark::BenchmarkReporter { class TestReporter : public benchmark::BenchmarkReporter {
public: public:
TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
: reporters_(reps) {} : reporters_(reps) {}
virtual bool ReportContext(const Context& context) { virtual bool ReportContext(const Context& context) {
bool last_ret = false; bool last_ret = false;
@ -118,56 +117,61 @@ public:
first = false; first = false;
last_ret = new_ret; last_ret = new_ret;
} }
(void)first;
return last_ret; return last_ret;
} }
void ReportRuns(const std::vector<Run>& report) void ReportRuns(const std::vector<Run>& report) {
{ for (auto rep : reporters_) rep->ReportRuns(report); } for (auto rep : reporters_) rep->ReportRuns(report);
void Finalize() { for (auto rep : reporters_) rep->Finalize(); } }
void Finalize() {
for (auto rep : reporters_) rep->Finalize();
}
private: private:
std::vector<benchmark::BenchmarkReporter*> reporters_; std::vector<benchmark::BenchmarkReporter *> reporters_;
}; };
}
}} // end namespace internal } // end namespace internal
// ========================================================================= // // ========================================================================= //
// -------------------------- Public API Definitions------------------------ // // -------------------------- Public API Definitions------------------------ //
// ========================================================================= // // ========================================================================= //
TestCase::TestCase(std::string re, int rule) TestCase::TestCase(std::string re, int rule)
: regex_str(std::move(re)), match_rule(rule), : regex_str(std::move(re)),
match_rule(rule),
substituted_regex(internal::PerformSubstitutions(regex_str)), substituted_regex(internal::PerformSubstitutions(regex_str)),
regex(std::make_shared<benchmark::Regex>()) regex(std::make_shared<benchmark::Regex>()) {
{ std::string err_str;
std::string err_str; regex->Init(substituted_regex,& err_str);
regex->Init(substituted_regex, &err_str); CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
CHECK(err_str.empty()) << "\""
<< "Could not construct regex \"" << substituted_regex << "\"" << "\n originally \"" << regex_str << "\""
<< "\n originally \"" << regex_str << "\"" << "\n got error: " << err_str;
<< "\n got error: " << err_str;
} }
int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
auto& L = internal::GetTestCaseList(ID); auto& L = internal::GetTestCaseList(ID);
L.insert(L.end(), il); L.insert(L.end(), il);
return 0; return 0;
} }
int SetSubstitutions(std::initializer_list<std::pair<std::string, std::string>> il) { int SetSubstitutions(
auto& subs = internal::GetSubstitutions(); std::initializer_list<std::pair<std::string, std::string>> il) {
for (auto const& KV : il) { auto& subs = internal::GetSubstitutions();
bool exists = false; for (auto const& KV : il) {
for (auto& EKV : subs) { bool exists = false;
if (EKV.first == KV.first) { for (auto& EKV : subs) {
EKV.second = KV.second; if (EKV.first == KV.first) {
exists = true; EKV.second = KV.second;
break; exists = true;
} break;
} }
if (!exists) subs.push_back(KV);
} }
return 0; if (!exists) subs.push_back(KV);
}
return 0;
} }
void RunOutputTests(int argc, char* argv[]) { void RunOutputTests(int argc, char* argv[]) {
@ -184,21 +188,20 @@ void RunOutputTests(int argc, char* argv[]) {
std::stringstream out_stream; std::stringstream out_stream;
std::stringstream err_stream; std::stringstream err_stream;
ReporterTest(const char* n, ReporterTest(const char* n, std::vector<TestCase>& out_tc,
std::vector<TestCase>& out_tc,
std::vector<TestCase>& err_tc, std::vector<TestCase>& err_tc,
benchmark::BenchmarkReporter& br) benchmark::BenchmarkReporter& br)
: name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
reporter.SetOutputStream(&out_stream); reporter.SetOutputStream(&out_stream);
reporter.SetErrorStream(&err_stream); reporter.SetErrorStream(&err_stream);
} }
} TestCases[] = { } TestCases[] = {
{"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
GetTestCaseList(TC_ConsoleErr), CR}, GetTestCaseList(TC_ConsoleErr), CR},
{"JSONReporter", GetTestCaseList(TC_JSONOut), {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
GetTestCaseList(TC_JSONErr), JR}, JR},
{"CSVReporter", GetTestCaseList(TC_CSVOut), {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
GetTestCaseList(TC_CSVErr), CSVR}, CSVR},
}; };
// Create the test reporter and run the benchmarks. // Create the test reporter and run the benchmarks.
@ -207,18 +210,16 @@ void RunOutputTests(int argc, char* argv[]) {
benchmark::RunSpecifiedBenchmarks(&test_rep); benchmark::RunSpecifiedBenchmarks(&test_rep);
for (auto& rep_test : TestCases) { for (auto& rep_test : TestCases) {
std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
std::string banner(msg.size() - 1, '-'); std::string banner(msg.size() - 1, '-');
std::cout << banner << msg << banner << "\n"; std::cout << banner << msg << banner << "\n";
std::cerr << rep_test.err_stream.str(); std::cerr << rep_test.err_stream.str();
std::cout << rep_test.out_stream.str(); std::cout << rep_test.out_stream.str();
internal::CheckCases(rep_test.error_cases,rep_test.err_stream); internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
internal::CheckCases(rep_test.output_cases, rep_test.out_stream); internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
std::cout << "\n"; std::cout << "\n";
} }
} }

55
test/register_benchmark_test.cc
View File

@ -1,14 +1,15 @@
#undef NDEBUG #undef NDEBUG
#include "benchmark/benchmark.h"
#include "../src/check.h" // NOTE: check.h is for internal use only!
#include <cassert> #include <cassert>
#include <vector> #include <vector>
#include "../src/check.h" // NOTE: check.h is for internal use only!
#include "benchmark/benchmark.h"
namespace { namespace {
class TestReporter : public benchmark::ConsoleReporter { class TestReporter : public benchmark::ConsoleReporter {
public: public:
virtual void ReportRuns(const std::vector<Run>& report) { virtual void ReportRuns(const std::vector<Run>& report) {
all_runs_.insert(all_runs_.end(), begin(report), end(report)); all_runs_.insert(all_runs_.end(), begin(report), end(report));
ConsoleReporter::ReportRuns(report); ConsoleReporter::ReportRuns(report);
@ -20,18 +21,19 @@ public:
struct TestCase { struct TestCase {
std::string name; std::string name;
const char* label; const char* label;
TestCase(const char* xname) : name(xname), label(nullptr) {} // Note: not explicit as we rely on it being converted through ADD_CASES.
TestCase(const char* xname) : TestCase(xname, nullptr) {}
TestCase(const char* xname, const char* xlabel) TestCase(const char* xname, const char* xlabel)
: name(xname), label(xlabel) {} : name(xname), label(xlabel) {}
typedef benchmark::BenchmarkReporter::Run Run; typedef benchmark::BenchmarkReporter::Run Run;
void CheckRun(Run const& run) const { void CheckRun(Run const& run) const {
CHECK(name == run.benchmark_name) << "expected " << name CHECK(name == run.benchmark_name) << "expected " << name << " got "
<< " got " << run.benchmark_name; << run.benchmark_name;
if (label) { if (label) {
CHECK(run.report_label == label) << "expected " << label CHECK(run.report_label == label) << "expected " << label << " got "
<< " got " << run.report_label; << run.report_label;
} else { } else {
CHECK(run.report_label == ""); CHECK(run.report_label == "");
} }
@ -49,8 +51,7 @@ int AddCases(std::initializer_list<TestCase> const& v) {
#define CONCAT(x, y) CONCAT2(x, y) #define CONCAT(x, y) CONCAT2(x, y)
#define CONCAT2(x, y) x##y #define CONCAT2(x, y) x##y
#define ADD_CASES(...) \ #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__})
int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__})
} // end namespace } // end namespace
@ -59,11 +60,13 @@ typedef benchmark::internal::Benchmark* ReturnVal;
//----------------------------------------------------------------------------// //----------------------------------------------------------------------------//
// Test RegisterBenchmark with no additional arguments // Test RegisterBenchmark with no additional arguments
//----------------------------------------------------------------------------// //----------------------------------------------------------------------------//
void BM_function(benchmark::State& state) { while (state.KeepRunning()) {} } void BM_function(benchmark::State& state) {
while (state.KeepRunning()) {
}
}
BENCHMARK(BM_function); BENCHMARK(BM_function);
ReturnVal dummy = benchmark::RegisterBenchmark( ReturnVal dummy = benchmark::RegisterBenchmark(
"BM_function_manual_registration", "BM_function_manual_registration", BM_function);
BM_function);
ADD_CASES({"BM_function"}, {"BM_function_manual_registration"}); ADD_CASES({"BM_function"}, {"BM_function_manual_registration"});
//----------------------------------------------------------------------------// //----------------------------------------------------------------------------//
@ -74,27 +77,21 @@ ADD_CASES({"BM_function"}, {"BM_function_manual_registration"});
#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK #ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK
void BM_extra_args(benchmark::State& st, const char* label) { void BM_extra_args(benchmark::State& st, const char* label) {
while (st.KeepRunning()) {} while (st.KeepRunning()) {
}
st.SetLabel(label); st.SetLabel(label);
} }
int RegisterFromFunction() { int RegisterFromFunction() {
std::pair<const char*, const char*> cases[] = { std::pair<const char*, const char*> cases[] = {
{"test1", "One"}, {"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}};
{"test2", "Two"}, for (auto const& c : cases)
{"test3", "Three"}
};
for (auto& c : cases)
benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second); benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second);
return 0; return 0;
} }
int dummy2 = RegisterFromFunction(); int dummy2 = RegisterFromFunction();
ADD_CASES( ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"});
{"test1", "One"},
{"test2", "Two"},
{"test3", "Three"}
);
#endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK #endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK
//----------------------------------------------------------------------------// //----------------------------------------------------------------------------//
// Test RegisterBenchmark with different callable types // Test RegisterBenchmark with different callable types
@ -102,7 +99,8 @@ ADD_CASES(
struct CustomFixture { struct CustomFixture {
void operator()(benchmark::State& st) { void operator()(benchmark::State& st) {
while (st.KeepRunning()) {} while (st.KeepRunning()) {
}
} }
}; };
@ -118,7 +116,8 @@ void TestRegistrationAtRuntime() {
{ {
int x = 42; int x = 42;
auto capturing_lam = [=](benchmark::State& st) { auto capturing_lam = [=](benchmark::State& st) {
while (st.KeepRunning()) {} while (st.KeepRunning()) {
}
st.SetLabel(std::to_string(x)); st.SetLabel(std::to_string(x));
}; };
benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam); benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam);

173
test/reporter_output_test.cc
View File

@ -1,69 +1,55 @@
#undef NDEBUG #undef NDEBUG
#include "benchmark/benchmark.h"
#include "output_test.h"
#include <utility> #include <utility>
#include "benchmark/benchmark.h"
#include "output_test.h"
// ========================================================================= // // ========================================================================= //
// ---------------------- Testing Prologue Output -------------------------- // // ---------------------- Testing Prologue Output -------------------------- //
// ========================================================================= // // ========================================================================= //
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_ConsoleOut, {{"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
{"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, {"^[-]+$", MR_Next}});
{"^[-]+$", MR_Next} ADD_CASES(TC_CSVOut, {{"name,iterations,real_time,cpu_time,time_unit,bytes_per_"
}); "second,items_per_second,"
ADD_CASES(TC_CSVOut, { "label,error_occurred,error_message"}});
{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,items_per_second,"
"label,error_occurred,error_message"}
});
// ========================================================================= // // ========================================================================= //
// ------------------------ Testing Basic Output --------------------------- // // ------------------------ Testing Basic Output --------------------------- //
// ========================================================================= // // ========================================================================= //
void BM_basic(benchmark::State& state) { void BM_basic(benchmark::State& state) {
while (state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_basic); BENCHMARK(BM_basic);
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
{"^BM_basic %console_report$"} ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
}); {"\"iterations\": %int,$", MR_Next},
ADD_CASES(TC_JSONOut, { {"\"real_time\": %int,$", MR_Next},
{"\"name\": \"BM_basic\",$"}, {"\"cpu_time\": %int,$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"}", MR_Next}});
{"\"cpu_time\": %int,$", MR_Next}, ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
{"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}
});
ADD_CASES(TC_CSVOut, {
{"^\"BM_basic\",%csv_report$"}
});
// ========================================================================= // // ========================================================================= //
// ------------------------ Testing Error Output --------------------------- // // ------------------------ Testing Error Output --------------------------- //
// ========================================================================= // // ========================================================================= //
void BM_error(benchmark::State& state) { void BM_error(benchmark::State& state) {
state.SkipWithError("message"); state.SkipWithError("message");
while(state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK(BM_error); BENCHMARK(BM_error);
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
{"^BM_error[ ]+ERROR OCCURRED: 'message'$"} ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
}); {"\"error_occurred\": true,$", MR_Next},
ADD_CASES(TC_JSONOut, { {"\"error_message\": \"message\",$", MR_Next}});
{"\"name\": \"BM_error\",$"},
{"\"error_occurred\": true,$", MR_Next},
{"\"error_message\": \"message\",$", MR_Next}
});
ADD_CASES(TC_CSVOut, {
{"^\"BM_error\",,,,,,,,true,\"message\"$"}
});
ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
// ========================================================================= // // ========================================================================= //
// ----------------------- Testing Complexity Output ----------------------- // // ----------------------- Testing Complexity Output ----------------------- //
@ -74,85 +60,68 @@ void BM_Complexity_O1(benchmark::State& state) {
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
SET_SUBSTITUTIONS({ SET_SUBSTITUTIONS({{"%bigOStr", "[ ]*[0-9]+\\.[0-9]+ \\([0-9]+\\)"},
{"%bigOStr", "[ ]*[0-9]+\\.[0-9]+ \\([0-9]+\\)"}, {"%RMS", "[ ]*[0-9]+ %"}});
{"%RMS", "[ ]*[0-9]+ %"} ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
}); {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
ADD_CASES(TC_ConsoleOut, {
{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
{"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}
});
// ========================================================================= // // ========================================================================= //
// ----------------------- Testing Aggregate Output ------------------------ // // ----------------------- Testing Aggregate Output ------------------------ //
// ========================================================================= // // ========================================================================= //
// Test that non-aggregate data is printed by default // Test that non-aggregate data is printed by default
void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} } void BM_Repeat(benchmark::State& state) {
while (state.KeepRunning()) {
}
}
BENCHMARK(BM_Repeat)->Repetitions(3); BENCHMARK(BM_Repeat)->Repetitions(3);
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3_mean %console_report$"},
{"^BM_Repeat/repeats:3_mean %console_report$"}, {"^BM_Repeat/repeats:3_stddev %console_report$"}});
{"^BM_Repeat/repeats:3_stddev %console_report$"} ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
}); {"\"name\": \"BM_Repeat/repeats:3\",$"},
ADD_CASES(TC_JSONOut, { {"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
{"\"name\": \"BM_Repeat/repeats:3\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"} {"^\"BM_Repeat/repeats:3\",%csv_report$"},
}); {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
ADD_CASES(TC_CSVOut, { {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}
});
// Test that a non-repeated test still prints non-aggregate results even when // Test that a non-repeated test still prints non-aggregate results even when
// only-aggregate reports have been requested // only-aggregate reports have been requested
void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} } void BM_RepeatOnce(benchmark::State& state) {
while (state.KeepRunning()) {
}
}
BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
{"^BM_RepeatOnce/repeats:1 %console_report$"} ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
ADD_CASES(TC_JSONOut, {
{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}
});
ADD_CASES(TC_CSVOut, {
{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}
});
// Test that non-aggregate data is not reported // Test that non-aggregate data is not reported
void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} } void BM_SummaryRepeat(benchmark::State& state) {
while (state.KeepRunning()) {
}
}
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
ADD_CASES(TC_ConsoleOut, { ADD_CASES(TC_ConsoleOut,
{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
{"^BM_SummaryRepeat/repeats:3_stddev %console_report$"} {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
}); ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
ADD_CASES(TC_JSONOut, { {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"} {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
}); {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}
});
// ========================================================================= // // ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ // // --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= // // ========================================================================= //
int main(int argc, char *argv[]) { RunOutputTests(argc, argv); }
int main(int argc, char* argv[]) {
RunOutputTests(argc, argv);
}

69
test/skip_with_error_test.cc
View File

@ -1,10 +1,11 @@
#undef NDEBUG #undef NDEBUG
#include "benchmark/benchmark.h"
#include "../src/check.h" // NOTE: check.h is for internal use only!
#include <cassert> #include <cassert>
#include <vector> #include <vector>
#include "../src/check.h" // NOTE: check.h is for internal use only!
#include "benchmark/benchmark.h"
namespace { namespace {
class TestReporter : public benchmark::ConsoleReporter { class TestReporter : public benchmark::ConsoleReporter {
@ -18,7 +19,7 @@ class TestReporter : public benchmark::ConsoleReporter {
ConsoleReporter::ReportRuns(report); ConsoleReporter::ReportRuns(report);
} }
TestReporter() {} TestReporter() {}
virtual ~TestReporter() {} virtual ~TestReporter() {}
mutable std::vector<Run> all_runs_; mutable std::vector<Run> all_runs_;
@ -32,11 +33,12 @@ struct TestCase {
typedef benchmark::BenchmarkReporter::Run Run; typedef benchmark::BenchmarkReporter::Run Run;
void CheckRun(Run const& run) const { void CheckRun(Run const& run) const {
CHECK(name == run.benchmark_name) << "expected " << name << " got " << run.benchmark_name; CHECK(name == run.benchmark_name) << "expected " << name << " got "
<< run.benchmark_name;
CHECK(error_occurred == run.error_occurred); CHECK(error_occurred == run.error_occurred);
CHECK(error_message == run.error_message); CHECK(error_message == run.error_message);
if (error_occurred) { if (error_occurred) {
//CHECK(run.iterations == 0); // CHECK(run.iterations == 0);
} else { } else {
CHECK(run.iterations != 0); CHECK(run.iterations != 0);
} }
@ -55,12 +57,10 @@ int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
#define CONCAT(x, y) CONCAT2(x, y) #define CONCAT(x, y) CONCAT2(x, y)
#define CONCAT2(x, y) x##y #define CONCAT2(x, y) x##y
#define ADD_CASES(...) \ #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
} // end namespace } // end namespace
void BM_error_before_running(benchmark::State& state) { void BM_error_before_running(benchmark::State& state) {
state.SkipWithError("error message"); state.SkipWithError("error message");
while (state.KeepRunning()) { while (state.KeepRunning()) {
@ -68,8 +68,7 @@ void BM_error_before_running(benchmark::State& state) {
} }
} }
BENCHMARK(BM_error_before_running); BENCHMARK(BM_error_before_running);
ADD_CASES("BM_error_before_running", ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
{{"", true, "error message"}});
void BM_error_during_running(benchmark::State& state) { void BM_error_during_running(benchmark::State& state) {
int first_iter = true; int first_iter = true;
@ -85,17 +84,14 @@ void BM_error_during_running(benchmark::State& state) {
} }
} }
BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8); BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
ADD_CASES( ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
"BM_error_during_running", {"/1/threads:2", true, "error message"},
{{"/1/threads:1", true, "error message"}, {"/1/threads:4", true, "error message"},
{"/1/threads:2", true, "error message"}, {"/1/threads:8", true, "error message"},
{"/1/threads:4", true, "error message"}, {"/2/threads:1", false, ""},
{"/1/threads:8", true, "error message"}, {"/2/threads:2", false, ""},
{"/2/threads:1", false, ""}, {"/2/threads:4", false, ""},
{"/2/threads:2", false, ""}, {"/2/threads:8", false, ""}});
{"/2/threads:4", false, ""},
{"/2/threads:8", false, ""}}
);
void BM_error_after_running(benchmark::State& state) { void BM_error_after_running(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
@ -105,13 +101,10 @@ void BM_error_after_running(benchmark::State& state) {
state.SkipWithError("error message"); state.SkipWithError("error message");
} }
BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
ADD_CASES( ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
"BM_error_after_running", {"/threads:2", true, "error message"},
{{"/threads:1", true, "error message"}, {"/threads:4", true, "error message"},
{"/threads:2", true, "error message"}, {"/threads:8", true, "error message"}});
{"/threads:4", true, "error message"},
{"/threads:8", true, "error message"}}
);
void BM_error_while_paused(benchmark::State& state) { void BM_error_while_paused(benchmark::State& state) {
bool first_iter = true; bool first_iter = true;
@ -128,18 +121,14 @@ void BM_error_while_paused(benchmark::State& state) {
} }
} }
BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8); BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
ADD_CASES( ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
"BM_error_while_paused", {"/1/threads:2", true, "error message"},
{{"/1/threads:1", true, "error message"}, {"/1/threads:4", true, "error message"},
{"/1/threads:2", true, "error message"}, {"/1/threads:8", true, "error message"},
{"/1/threads:4", true, "error message"}, {"/2/threads:1", false, ""},
{"/1/threads:8", true, "error message"}, {"/2/threads:2", false, ""},
{"/2/threads:1", false, ""}, {"/2/threads:4", false, ""},
{"/2/threads:2", false, ""}, {"/2/threads:8", false, ""}});
{"/2/threads:4", false, ""},
{"/2/threads:8", false, ""}}
);
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);