mirror of https://github.com/google/benchmark.git
Unit testing: add facilities to check benchmark results.
This is needed for examining the values of user counters (needed for #348). It is also needed for checking the values of standard benchmark results like items_processed or complexities (for example, checking the standard deviation is needed for unit testing #357 as discussed in #362).
This commit is contained in:
Joao Paulo Magalhaes
parent
693a43013d
commit
6452883027
|
|
@ -58,6 +58,79 @@ int SetSubstitutions(
|
|||
// Run all output tests.
|
||||
void RunOutputTests(int argc, char* argv[]);
|
||||
|
||||
// ========================================================================= //
|
||||
// ------------------------- Results checking ------------------------------ //
|
||||
// ========================================================================= //
|
||||
|
||||
struct ResultsCheckerEntry;
|
||||
typedef std::function< void(ResultsCheckerEntry const&) > ResultsCheckFn;
|
||||
|
||||
// Class to test the results of a benchmark.
|
||||
// It inspects the results by looking at the CSV output of a subscribed
|
||||
// benchmark.
|
||||
struct ResultsCheckerEntry {
|
||||
std::string name;
|
||||
std::map< std::string, std::string > values;
|
||||
ResultsCheckFn check_fn;
|
||||
|
||||
// int NumThreads() const; // TODO
|
||||
// double duration_real_time() const {} // TODO
|
||||
// double duration_cpu_time() const {} // TODO
|
||||
|
||||
// get the string for a result by name, or nullptr if the name
|
||||
// is not found
|
||||
const std::string* Get(std::string const& entry_name) const {
|
||||
auto it = values.find(entry_name);
|
||||
if(it == values.end()) return nullptr;
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
// get a result by name, parsed as a specific type.
|
||||
// For counters, use GetCounterAs instead.
|
||||
template< class T > T GetAs(std::string const& entry_name) const {
|
||||
auto *sv = Get(entry_name);
|
||||
CHECK(sv != nullptr && !sv->empty());
|
||||
std::stringstream ss;
|
||||
ss << *sv;
|
||||
T out;
|
||||
ss >> out;
|
||||
CHECK(!ss.fail());
|
||||
return out;
|
||||
}
|
||||
|
||||
// counters are written as doubles, so they have to be read first
|
||||
// as a double, and only then converted to the asked type.
|
||||
template< class T > T GetCounterAs(std::string const& entry_name) const {
|
||||
double dval = GetAs< double >(entry_name);
|
||||
T tval = static_cast< T >(dval);
|
||||
return tval;
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
||||
#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
|
||||
CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value)) \
|
||||
<< "\n" << __FILE__ << ":" << __LINE__ << ": " \
|
||||
<< entry.name << ": expected (" << #var_type << ")" \
|
||||
<< var_name << "=" << entry.GetAs< var_type >(var_name) \
|
||||
<< " to be " #relationship " to " << (value);
|
||||
|
||||
#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
|
||||
_CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
|
||||
|
||||
#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
|
||||
_CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
|
||||
|
||||
#define CHECK_BENCHMARK_RESULTS(bm_name, checker_function) \
|
||||
size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function)
|
||||
|
||||
// Add a function to check the (CSV) results of a benchmark. These
|
||||
// functions will be called only after the output was successfully
|
||||
// checked.
|
||||
size_t AddChecker(const char* bm_name, ResultsCheckFn fn);
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------------- Misc Utilities ------------------------------ //
|
||||
// ========================================================================= //
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ SubMap& GetSubstitutions() {
|
|||
static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
||||
static SubMap map = {
|
||||
{"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
|
||||
// human-readable float
|
||||
{"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
|
||||
{"%int", "[ ]*[0-9]+"},
|
||||
{" %s ", "[ ]+"},
|
||||
{"%time", "[ ]*[0-9]{1,5} ns"},
|
||||
|
|
@ -146,8 +148,139 @@ class TestReporter : public benchmark::BenchmarkReporter {
|
|||
std::vector<benchmark::BenchmarkReporter *> reporters_;
|
||||
};
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Results checking ----------------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
namespace internal {
|
||||
|
||||
// Utility class to manage subscribers for checking benchmark results.
|
||||
// It works by parsing the CSV output to read the results.
|
||||
class ResultsChecker {
|
||||
public:
|
||||
|
||||
std::map< std::string, ResultsCheckerEntry > results;
|
||||
std::vector< std::string > result_names;
|
||||
|
||||
void Add(const std::string& entry_name, ResultsCheckFn fn);
|
||||
|
||||
void CheckResults(std::stringstream& output);
|
||||
|
||||
private:
|
||||
|
||||
ResultsCheckerEntry* Find_(const std::string& entry_name);
|
||||
|
||||
void SetHeader_(const std::string& csv_header);
|
||||
void SetValues_(const std::string& entry_csv_line);
|
||||
|
||||
std::vector< std::string > SplitCsv_(std::string const& line);
|
||||
|
||||
};
|
||||
|
||||
// store the static ResultsChecker in a function to prevent initialization
|
||||
// order problems
|
||||
ResultsChecker& GetResultsChecker() {
|
||||
static ResultsChecker rc;
|
||||
return rc;
|
||||
}
|
||||
|
||||
// add a results checker for a benchmark
|
||||
void ResultsChecker::Add(const std::string& entry_name, ResultsCheckFn fn) {
|
||||
results[entry_name] = {entry_name, {}, fn};
|
||||
}
|
||||
|
||||
// check the results of all subscribed benchmarks
|
||||
void ResultsChecker::CheckResults(std::stringstream& output)
|
||||
{
|
||||
// first reset the stream to the start
|
||||
{
|
||||
auto start = std::ios::streampos(0);
|
||||
// clear before calling tellg()
|
||||
output.clear();
|
||||
// seek to zero only when needed
|
||||
if(output.tellg() > start) output.seekg(start);
|
||||
// and just in case
|
||||
output.clear();
|
||||
}
|
||||
// now go over every line and publish it to the ResultsChecker
|
||||
std::string line;
|
||||
bool on_first = true;
|
||||
while (output.eof() == false) {
|
||||
CHECK(output.good());
|
||||
std::getline(output, line);
|
||||
if (on_first) {
|
||||
SetHeader_(line); // this is important
|
||||
on_first = false;
|
||||
continue;
|
||||
}
|
||||
SetValues_(line);
|
||||
}
|
||||
// finally we can call the subscribed check functions
|
||||
for(const auto& p : results) {
|
||||
CHECK(p.second.check_fn);
|
||||
p.second.check_fn(p.second);
|
||||
}
|
||||
}
|
||||
|
||||
// prepare for the names in this header
|
||||
void ResultsChecker::SetHeader_(const std::string& csv_header) {
|
||||
result_names = SplitCsv_(csv_header);
|
||||
}
|
||||
|
||||
// set the values for subscribed benchmarks, and silently ignore all others
|
||||
void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
|
||||
CHECK(!result_names.empty());
|
||||
auto vals = SplitCsv_(entry_csv_line);
|
||||
if(vals.empty()) return;
|
||||
CHECK_EQ(vals.size(), result_names.size());
|
||||
ResultsCheckerEntry* entry = Find_(vals[0]);
|
||||
if(!entry) return;
|
||||
for (size_t i = 1, e = vals.size(); i < e; ++i) {
|
||||
entry->values[result_names[i]] = vals[i];
|
||||
}
|
||||
}
|
||||
|
||||
// find a subscribed benchmark, or return null
|
||||
ResultsCheckerEntry* ResultsChecker::Find_(const std::string& entry_name) {
|
||||
auto it = results.find(entry_name);
|
||||
if(it == results.end()) return nullptr;
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
// a quick'n'dirty csv splitter (eliminating quotes)
|
||||
std::vector< std::string > ResultsChecker::SplitCsv_(std::string const& line) {
|
||||
std::vector< std::string > out;
|
||||
if(line.empty()) return out;
|
||||
if(!result_names.empty()) out.reserve(result_names.size());
|
||||
size_t prev = 0, pos = line.find_first_of(','), curr = pos;
|
||||
while(pos != line.npos) {
|
||||
CHECK(curr > 0);
|
||||
if(line[prev] == '"') ++prev;
|
||||
if(line[curr-1] == '"') --curr;
|
||||
out.push_back(line.substr(prev, curr-prev));
|
||||
prev = pos + 1;
|
||||
pos = line.find_first_of(',', pos + 1);
|
||||
curr = pos;
|
||||
}
|
||||
curr = line.size();
|
||||
if(line[prev] == '"') ++prev;
|
||||
if(line[curr-1] == '"') --curr;
|
||||
out.push_back(line.substr(prev, curr-prev));
|
||||
return out;
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
|
||||
{
|
||||
auto &rc = internal::GetResultsChecker();
|
||||
rc.Add(bm_name, fn);
|
||||
return rc.results.size();
|
||||
}
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Public API Definitions------------------------ //
|
||||
// ========================================================================= //
|
||||
|
|
@ -237,4 +370,10 @@ void RunOutputTests(int argc, char* argv[]) {
|
|||
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
// now that we know the output is as expected, we can dispatch
|
||||
// the checks to subscribees.
|
||||
auto &csv = TestCases[2];
|
||||
CHECK(strcmp(csv.name, "CSVReporter") == 0); // would use == but gcc spits a warning
|
||||
internal::GetResultsChecker().CheckResults(csv.out_stream);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ void BM_Counters_Simple(benchmark::State& state) {
|
|||
state.counters["bar"] = 2;
|
||||
}
|
||||
BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%float foo=%float$"}});
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
|
|
@ -35,23 +35,28 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
|
|||
{"\"foo\": %float$", MR_Next},
|
||||
{"}", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}});
|
||||
CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) {
|
||||
CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
|
||||
CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2);
|
||||
});
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------- Counters+Items+Bytes/s Output --------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
namespace { int num_calls1 = 0; }
|
||||
void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
|
||||
while (state.KeepRunning()) {
|
||||
}
|
||||
state.counters["foo"] = 1;
|
||||
state.counters["bar"] = 2;
|
||||
state.SetItemsProcessed(150);
|
||||
state.counters["bar"] = ++num_calls1;
|
||||
state.SetBytesProcessed(364);
|
||||
state.SetItemsProcessed(150);
|
||||
}
|
||||
BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32);
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{"^BM_Counters_WithBytesAndItemsPSec %console_report "
|
||||
"bar=%float foo=%float +%floatB/s +%float items/s$"}});
|
||||
"bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
|
|
@ -64,6 +69,13 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
|
|||
{"}", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
|
||||
"%csv_bytes_items_report,%float,%float$"}});
|
||||
CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
|
||||
[](ResultsCheckerEntry const& e) {
|
||||
CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
|
||||
CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
|
||||
//TODO CHECK_RESULT_VALUE(e, int, "bytes_per_second", EQ, 364 / e.duration_cpu_time());
|
||||
//TODO CHECK_RESULT_VALUE(e, int, "items_per_second", EQ, 150 / e.duration_cpu_time());
|
||||
});
|
||||
|
||||
// ========================================================================= //
|
||||
// ------------------------- Rate Counters Output -------------------------- //
|
||||
|
|
@ -77,7 +89,7 @@ void BM_Counters_Rate(benchmark::State& state) {
|
|||
state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
|
||||
}
|
||||
BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%float foo=%float$"}});
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
|
|
@ -99,7 +111,7 @@ void BM_Counters_Threads(benchmark::State& state) {
|
|||
state.counters["bar"] = 2;
|
||||
}
|
||||
BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%float foo=%float$"}});
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
|
|
@ -122,7 +134,7 @@ void BM_Counters_AvgThreads(benchmark::State& state) {
|
|||
state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads};
|
||||
}
|
||||
BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%float foo=%float$"}});
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
|
|
@ -145,7 +157,7 @@ void BM_Counters_AvgThreadsRate(benchmark::State& state) {
|
|||
state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};
|
||||
}
|
||||
BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%float foo=%float$"}});
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
|
|
|
|||
Loading…
Reference in New Issue