mirror of https://github.com/google/benchmark.git
Refactor output test runner into standalone module. (#277)
* refactor * Move default substitutions into library * Move default substitutions to the *right* place in the library * Fix init order issues that caused test failures * improve diagnostics * add missing include * general cleanup * Address review comments
This commit is contained in:
Eric
GitHub
parent
577baa0dbd
commit
0ed4456097
|
|
@ -2,11 +2,20 @@
|
|||
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
add_library(output_test_helper STATIC output_test_helper.cc)
|
||||
|
||||
macro(compile_benchmark_test name)
|
||||
add_executable(${name} "${name}.cc")
|
||||
target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT})
|
||||
endmacro(compile_benchmark_test)
|
||||
|
||||
|
||||
macro(compile_output_test name)
|
||||
add_executable(${name} "${name}.cc")
|
||||
target_link_libraries(${name} output_test_helper benchmark ${CMAKE_THREAD_LIBS_INIT})
|
||||
endmacro(compile_output_test)
|
||||
|
||||
|
||||
# Demonstration executable
|
||||
compile_benchmark_test(benchmark_test)
|
||||
add_test(benchmark benchmark_test --benchmark_min_time=0.01)
|
||||
|
|
@ -54,7 +63,7 @@ add_test(map_test map_test --benchmark_min_time=0.01)
|
|||
compile_benchmark_test(multiple_ranges_test)
|
||||
add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01)
|
||||
|
||||
compile_benchmark_test(reporter_output_test)
|
||||
compile_output_test(reporter_output_test)
|
||||
add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01)
|
||||
|
||||
check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
|
||||
|
|
@ -69,7 +78,7 @@ if (BENCHMARK_HAS_CXX03_FLAG)
|
|||
add_test(cxx03 cxx03_test --benchmark_min_time=0.01)
|
||||
endif()
|
||||
|
||||
compile_benchmark_test(complexity_test)
|
||||
compile_output_test(complexity_test)
|
||||
add_test(complexity_benchmark complexity_test --benchmark_min_time=0.01)
|
||||
|
||||
# Add the coverage command(s)
|
||||
|
|
|
|||
|
|
@ -1,153 +1,46 @@
|
|||
|
||||
#undef NDEBUG
|
||||
#include "benchmark/benchmark.h"
|
||||
#include "../src/check.h" // NOTE: check.h is for internal use only!
|
||||
#include "../src/re.h" // NOTE: re.h is for internal use only
|
||||
#include "output_test.h"
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
|
||||
namespace {
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Testing Case --------------------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
enum MatchRules {
|
||||
MR_Default, // Skip non-matching lines until a match is found.
|
||||
MR_Next // Match must occur on the next line.
|
||||
};
|
||||
|
||||
struct TestCase {
|
||||
std::string regex;
|
||||
int match_rule;
|
||||
|
||||
TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {}
|
||||
|
||||
void Check(std::stringstream& remaining_output) const {
|
||||
benchmark::Regex r;
|
||||
std::string err_str;
|
||||
r.Init(regex, &err_str);
|
||||
CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\""
|
||||
<< " got Error: " << err_str;
|
||||
|
||||
std::string near = "<EOF>";
|
||||
std::string line;
|
||||
bool first = true;
|
||||
while (remaining_output.eof() == false) {
|
||||
CHECK(remaining_output.good());
|
||||
std::getline(remaining_output, line);
|
||||
// Keep the first line as context.
|
||||
if (first) {
|
||||
near = line;
|
||||
first = false;
|
||||
}
|
||||
if (r.Match(line)) return;
|
||||
CHECK(match_rule != MR_Next) << "Expected line \"" << line
|
||||
<< "\" to match regex \"" << regex << "\""
|
||||
<< "\nstarted matching at line: \"" << near << "\"";
|
||||
}
|
||||
|
||||
CHECK(remaining_output.eof() == false)
|
||||
<< "End of output reached before match for regex \"" << regex
|
||||
<< "\" was found"
|
||||
<< "\nstarted matching at line: \"" << near << "\"";
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<TestCase> ConsoleOutputTests;
|
||||
std::vector<TestCase> JSONOutputTests;
|
||||
std::vector<TestCase> CSVOutputTests;
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Test Helpers --------------------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
class TestReporter : public benchmark::BenchmarkReporter {
|
||||
public:
|
||||
TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
|
||||
: reporters_(reps) {}
|
||||
|
||||
virtual bool ReportContext(const Context& context) {
|
||||
bool last_ret = false;
|
||||
bool first = true;
|
||||
for (auto rep : reporters_) {
|
||||
bool new_ret = rep->ReportContext(context);
|
||||
CHECK(first || new_ret == last_ret)
|
||||
<< "Reports return different values for ReportContext";
|
||||
first = false;
|
||||
last_ret = new_ret;
|
||||
}
|
||||
return last_ret;
|
||||
}
|
||||
|
||||
virtual void ReportRuns(const std::vector<Run>& report) {
|
||||
for (auto rep : reporters_)
|
||||
rep->ReportRuns(report);
|
||||
}
|
||||
|
||||
virtual void Finalize() {
|
||||
for (auto rep : reporters_)
|
||||
rep->Finalize();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<benchmark::BenchmarkReporter*> reporters_;
|
||||
};
|
||||
|
||||
|
||||
#define CONCAT2(x, y) x##y
|
||||
#define CONCAT(x, y) CONCAT2(x, y)
|
||||
|
||||
#define ADD_CASES(...) \
|
||||
int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
|
||||
|
||||
int AddCases(std::vector<TestCase>* out, std::initializer_list<TestCase> const& v) {
|
||||
for (auto const& TC : v)
|
||||
out->push_back(TC);
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <class First>
|
||||
std::string join(First f) { return f; }
|
||||
|
||||
template <class First, class ...Args>
|
||||
std::string join(First f, Args&&... args) {
|
||||
return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
||||
|
||||
#define ADD_COMPLEXITY_CASES(...) \
|
||||
int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)
|
||||
|
||||
int AddComplexityTest(std::vector<TestCase>* console_out, std::vector<TestCase>* json_out,
|
||||
std::vector<TestCase>* csv_out, std::string big_o_test_name,
|
||||
int AddComplexityTest(std::string big_o_test_name,
|
||||
std::string rms_test_name, std::string big_o) {
|
||||
std::string big_o_str = dec_re + " " + big_o;
|
||||
AddCases(console_out, {
|
||||
{join("^" + big_o_test_name + "", big_o_str, big_o_str) + "[ ]*$"},
|
||||
{join("^" + rms_test_name + "", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"}
|
||||
SetSubstitutions({
|
||||
{"%bigo_name", big_o_test_name},
|
||||
{"%rms_name", rms_test_name},
|
||||
{"%bigo_str", "[ ]*" + std::string(dec_re) + " " + big_o},
|
||||
{"%bigo", big_o},
|
||||
{"%rms", "[ ]*[0-9]+ %"}
|
||||
});
|
||||
AddCases(json_out, {
|
||||
{"\"name\": \"" + big_o_test_name + "\",$"},
|
||||
AddCases(TC_ConsoleOut, {
|
||||
{"^%bigo_name %bigo_str %bigo_str[ ]*$"},
|
||||
{"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
|
||||
{"^%rms_name %rms %rms[ ]*$", MR_Next}
|
||||
});
|
||||
AddCases(TC_JSONOut, {
|
||||
{"\"name\": \"%bigo_name\",$"},
|
||||
{"\"cpu_coefficient\": [0-9]+,$", MR_Next},
|
||||
{"\"real_coefficient\": [0-9]{1,5},$", MR_Next},
|
||||
{"\"big_o\": \"" + big_o + "\",$", MR_Next},
|
||||
{"\"big_o\": \"%bigo\",$", MR_Next},
|
||||
{"\"time_unit\": \"ns\"$", MR_Next},
|
||||
{"}", MR_Next},
|
||||
{"\"name\": \"" + rms_test_name + "\",$"},
|
||||
{"\"name\": \"%rms_name\",$"},
|
||||
{"\"rms\": [0-9]+%$", MR_Next},
|
||||
{"}", MR_Next}
|
||||
});
|
||||
AddCases(csv_out, {
|
||||
{"^\"" + big_o_test_name + "\",," + dec_re + "," + dec_re + "," + big_o + ",,,,,$"},
|
||||
{"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$", MR_Next}
|
||||
AddCases(TC_CSVOut, {
|
||||
{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
|
||||
{"^\"%bigo_name\"", MR_Not},
|
||||
{"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}
|
||||
});
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -179,16 +72,13 @@ const char* auto_big_o_1 = "(\\([0-9]+\\))|(lgN)";
|
|||
const char* lambda_big_o_1 = "f\\(N\\)";
|
||||
|
||||
// Add enum tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_1_test_name, rms_o_1_test_name, enum_big_o_1);
|
||||
ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1);
|
||||
|
||||
// Add auto enum tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_1_test_name, rms_o_1_test_name, auto_big_o_1);
|
||||
ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1);
|
||||
|
||||
// Add lambda tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1);
|
||||
ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1);
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------------- Testing BigO O(N) --------------------------- //
|
||||
|
|
@ -198,7 +88,7 @@ std::vector<int> ConstructRandomVector(int size) {
|
|||
std::vector<int> v;
|
||||
v.reserve(size);
|
||||
for (int i = 0; i < size; ++i) {
|
||||
v.push_back(rand() % size);
|
||||
v.push_back(std::rand() % size);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
|
@ -221,12 +111,10 @@ const char* enum_auto_big_o_n = "N";
|
|||
const char* lambda_big_o_n = "f\\(N\\)";
|
||||
|
||||
// Add enum tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
|
||||
ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
|
||||
|
||||
// Add lambda tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
|
||||
ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
|
||||
|
||||
// ========================================================================= //
|
||||
// ------------------------- Testing BigO O(N*lgN) ------------------------- //
|
||||
|
|
@ -249,62 +137,17 @@ const char* enum_auto_big_o_n_lg_n = "NlgN";
|
|||
const char* lambda_big_o_n_lg_n = "f\\(N\\)";
|
||||
|
||||
// Add enum tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
|
||||
ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
|
||||
|
||||
// Add lambda tests
|
||||
ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
|
||||
big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
|
||||
ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
|
||||
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------------- TEST CASES END ------------------------------ //
|
||||
// ========================================================================= //
|
||||
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
benchmark::Initialize(&argc, argv);
|
||||
benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
|
||||
benchmark::JSONReporter JR;
|
||||
benchmark::CSVReporter CSVR;
|
||||
struct ReporterTest {
|
||||
const char* name;
|
||||
std::vector<TestCase>& output_cases;
|
||||
benchmark::BenchmarkReporter& reporter;
|
||||
std::stringstream out_stream;
|
||||
std::stringstream err_stream;
|
||||
|
||||
ReporterTest(const char* n,
|
||||
std::vector<TestCase>& out_tc,
|
||||
benchmark::BenchmarkReporter& br)
|
||||
: name(n), output_cases(out_tc), reporter(br) {
|
||||
reporter.SetOutputStream(&out_stream);
|
||||
reporter.SetErrorStream(&err_stream);
|
||||
}
|
||||
} TestCases[] = {
|
||||
{"ConsoleReporter", ConsoleOutputTests, CR},
|
||||
{"JSONReporter", JSONOutputTests, JR},
|
||||
{"CSVReporter", CSVOutputTests, CSVR}
|
||||
};
|
||||
|
||||
// Create the test reporter and run the benchmarks.
|
||||
std::cout << "Running benchmarks...\n";
|
||||
TestReporter test_rep({&CR, &JR, &CSVR});
|
||||
benchmark::RunSpecifiedBenchmarks(&test_rep);
|
||||
|
||||
for (auto& rep_test : TestCases) {
|
||||
std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
|
||||
std::string banner(msg.size() - 1, '-');
|
||||
std::cout << banner << msg << banner << "\n";
|
||||
|
||||
std::cerr << rep_test.err_stream.str();
|
||||
std::cout << rep_test.out_stream.str();
|
||||
|
||||
for (const auto& TC : rep_test.output_cases)
|
||||
TC.Check(rep_test.out_stream);
|
||||
|
||||
std::cout << "\n";
|
||||
}
|
||||
return 0;
|
||||
RunOutputTests(argc, argv);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
#ifndef TEST_OUTPUT_TEST_H
|
||||
#define TEST_OUTPUT_TEST_H
|
||||
|
||||
#undef NDEBUG
|
||||
#include "benchmark/benchmark.h"
|
||||
#include "../src/re.h"
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <initializer_list>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
#define CONCAT2(x, y) x##y
|
||||
#define CONCAT(x, y) CONCAT2(x, y)
|
||||
|
||||
#define ADD_CASES(...) \
|
||||
int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
|
||||
|
||||
#define SET_SUBSTITUTIONS(...) \
|
||||
int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
|
||||
|
||||
enum MatchRules {
|
||||
MR_Default, // Skip non-matching lines until a match is found.
|
||||
MR_Next, // Match must occur on the next line.
|
||||
MR_Not // No line between the current position and the next match matches
|
||||
// the regex
|
||||
};
|
||||
|
||||
struct TestCase {
|
||||
TestCase(std::string re, int rule = MR_Default);
|
||||
|
||||
std::string regex_str;
|
||||
int match_rule;
|
||||
std::string substituted_regex;
|
||||
std::shared_ptr<benchmark::Regex> regex;
|
||||
};
|
||||
|
||||
enum TestCaseID {
|
||||
TC_ConsoleOut,
|
||||
TC_ConsoleErr,
|
||||
TC_JSONOut,
|
||||
TC_JSONErr,
|
||||
TC_CSVOut,
|
||||
TC_CSVErr,
|
||||
|
||||
TC_NumID // PRIVATE
|
||||
};
|
||||
|
||||
// Add a list of test cases to be run against the output specified by
|
||||
// 'ID'
|
||||
int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
|
||||
|
||||
// Add or set a list of substitutions to be performed on constructed regex's
|
||||
// See 'output_test_helper.cc' for a list of default substitutions.
|
||||
int SetSubstitutions(
|
||||
std::initializer_list<std::pair<std::string, std::string>> il);
|
||||
|
||||
// Run all output tests.
|
||||
void RunOutputTests(int argc, char* argv[]);
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------------- Misc Utilities ------------------------------ //
|
||||
// ========================================================================= //
|
||||
|
||||
namespace {
|
||||
|
||||
const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
||||
|
||||
} // end namespace
|
||||
|
||||
|
||||
#endif // TEST_OUTPUT_TEST_H
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
#include "output_test.h"
|
||||
#include "../src/check.h" // NOTE: check.h is for internal use only!
|
||||
#include "../src/re.h" // NOTE: re.h is for internal use only
|
||||
#include <memory>
|
||||
#include <map>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
|
||||
// ========================================================================= //
|
||||
// ------------------------------ Internals -------------------------------- //
|
||||
// ========================================================================= //
|
||||
namespace internal { namespace {
|
||||
|
||||
using TestCaseList = std::vector<TestCase>;
|
||||
|
||||
// Use a vector because the order elements are added matters during iteration.
|
||||
// std::map/unordered_map don't guarantee that.
|
||||
// For example:
|
||||
// SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
|
||||
// Substitute("%HelloWorld") // Always expands to Hello.
|
||||
using SubMap = std::vector<std::pair<std::string, std::string>>;
|
||||
|
||||
TestCaseList& GetTestCaseList(TestCaseID ID) {
|
||||
// Uses function-local statics to ensure initialization occurs
|
||||
// before first use.
|
||||
static TestCaseList lists[TC_NumID];
|
||||
return lists[ID];
|
||||
}
|
||||
|
||||
SubMap& GetSubstitutions() {
|
||||
// Don't use 'dec_re' from header because it may not yet be initialized.
|
||||
static std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
||||
static SubMap map = {
|
||||
{"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
|
||||
{"%int", "[ ]*[0-9]+"},
|
||||
{" %s ", "[ ]+"},
|
||||
{"%time", "[ ]*[0-9]{1,5} ns"},
|
||||
{"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
|
||||
{"%csv_report", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,"}
|
||||
};
|
||||
return map;
|
||||
}
|
||||
|
||||
std::string PerformSubstitutions(std::string source) {
|
||||
SubMap const& subs = GetSubstitutions();
|
||||
using SizeT = std::string::size_type;
|
||||
for (auto const& KV : subs) {
|
||||
SizeT pos;
|
||||
SizeT next_start = 0;
|
||||
while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
|
||||
next_start = pos + KV.second.size();
|
||||
source.replace(pos, KV.first.size(), KV.second);
|
||||
}
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
|
||||
TestCaseList const& not_checks)
|
||||
{
|
||||
std::string first_line;
|
||||
bool on_first = true;
|
||||
std::string line;
|
||||
while (remaining_output.eof() == false) {
|
||||
CHECK(remaining_output.good());
|
||||
std::getline(remaining_output, line);
|
||||
if (on_first) {
|
||||
first_line = line;
|
||||
on_first = false;
|
||||
}
|
||||
for (auto& NC : not_checks) {
|
||||
CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
|
||||
<< line << "\" for MR_Not regex \""
|
||||
<< NC.regex_str << "\"";
|
||||
}
|
||||
if (TC.regex->Match(line)) return;
|
||||
CHECK(TC.match_rule != MR_Next) << "Expected line \"" << line
|
||||
<< "\" to match regex \"" << TC.regex_str << "\"";
|
||||
}
|
||||
CHECK(remaining_output.eof() == false)
|
||||
<< "End of output reached before match for regex \"" << TC.regex_str
|
||||
<< "\" was found"
|
||||
<< "\n actual regex string \"" << TC.substituted_regex << "\""
|
||||
<< "\n started matching near: " << first_line;
|
||||
}
|
||||
|
||||
|
||||
void CheckCases(TestCaseList const& checks, std::stringstream& output) {
|
||||
std::vector<TestCase> not_checks;
|
||||
for (size_t i=0; i < checks.size(); ++i) {
|
||||
const auto& TC = checks[i];
|
||||
if (TC.match_rule == MR_Not) {
|
||||
not_checks.push_back(TC);
|
||||
continue;
|
||||
}
|
||||
CheckCase(output, TC, not_checks);
|
||||
not_checks.clear();
|
||||
}
|
||||
}
|
||||
|
||||
class TestReporter : public benchmark::BenchmarkReporter {
|
||||
public:
|
||||
TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
|
||||
: reporters_(reps) {}
|
||||
|
||||
virtual bool ReportContext(const Context& context) {
|
||||
bool last_ret = false;
|
||||
bool first = true;
|
||||
for (auto rep : reporters_) {
|
||||
bool new_ret = rep->ReportContext(context);
|
||||
CHECK(first || new_ret == last_ret)
|
||||
<< "Reports return different values for ReportContext";
|
||||
first = false;
|
||||
last_ret = new_ret;
|
||||
}
|
||||
return last_ret;
|
||||
}
|
||||
|
||||
void ReportRuns(const std::vector<Run>& report)
|
||||
{ for (auto rep : reporters_) rep->ReportRuns(report); }
|
||||
void Finalize() { for (auto rep : reporters_) rep->Finalize(); }
|
||||
|
||||
private:
|
||||
std::vector<benchmark::BenchmarkReporter*> reporters_;
|
||||
};
|
||||
|
||||
}} // end namespace internal
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Public API Definitions------------------------ //
|
||||
// ========================================================================= //
|
||||
|
||||
TestCase::TestCase(std::string re, int rule)
|
||||
: regex_str(std::move(re)), match_rule(rule),
|
||||
substituted_regex(internal::PerformSubstitutions(regex_str)),
|
||||
regex(std::make_shared<benchmark::Regex>())
|
||||
{
|
||||
std::string err_str;
|
||||
regex->Init(substituted_regex, &err_str);
|
||||
CHECK(err_str.empty())
|
||||
<< "Could not construct regex \"" << substituted_regex << "\""
|
||||
<< "\n originally \"" << regex_str << "\""
|
||||
<< "\n got error: " << err_str;
|
||||
}
|
||||
|
||||
int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
|
||||
auto& L = internal::GetTestCaseList(ID);
|
||||
L.insert(L.end(), il);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int SetSubstitutions(std::initializer_list<std::pair<std::string, std::string>> il) {
|
||||
auto& subs = internal::GetSubstitutions();
|
||||
for (auto const& KV : il) {
|
||||
bool exists = false;
|
||||
for (auto& EKV : subs) {
|
||||
if (EKV.first == KV.first) {
|
||||
EKV.second = KV.second;
|
||||
exists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!exists) subs.push_back(KV);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void RunOutputTests(int argc, char* argv[]) {
|
||||
using internal::GetTestCaseList;
|
||||
benchmark::Initialize(&argc, argv);
|
||||
benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
|
||||
benchmark::JSONReporter JR;
|
||||
benchmark::CSVReporter CSVR;
|
||||
struct ReporterTest {
|
||||
const char* name;
|
||||
std::vector<TestCase>& output_cases;
|
||||
std::vector<TestCase>& error_cases;
|
||||
benchmark::BenchmarkReporter& reporter;
|
||||
std::stringstream out_stream;
|
||||
std::stringstream err_stream;
|
||||
|
||||
ReporterTest(const char* n,
|
||||
std::vector<TestCase>& out_tc,
|
||||
std::vector<TestCase>& err_tc,
|
||||
benchmark::BenchmarkReporter& br)
|
||||
: name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
|
||||
reporter.SetOutputStream(&out_stream);
|
||||
reporter.SetErrorStream(&err_stream);
|
||||
}
|
||||
} TestCases[] = {
|
||||
{"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
|
||||
GetTestCaseList(TC_ConsoleErr), CR},
|
||||
{"JSONReporter", GetTestCaseList(TC_JSONOut),
|
||||
GetTestCaseList(TC_JSONErr), JR},
|
||||
{"CSVReporter", GetTestCaseList(TC_CSVOut),
|
||||
GetTestCaseList(TC_CSVErr), CSVR},
|
||||
};
|
||||
|
||||
// Create the test reporter and run the benchmarks.
|
||||
std::cout << "Running benchmarks...\n";
|
||||
internal::TestReporter test_rep({&CR, &JR, &CSVR});
|
||||
benchmark::RunSpecifiedBenchmarks(&test_rep);
|
||||
|
||||
for (auto& rep_test : TestCases) {
|
||||
std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
|
||||
std::string banner(msg.size() - 1, '-');
|
||||
std::cout << banner << msg << banner << "\n";
|
||||
|
||||
std::cerr << rep_test.err_stream.str();
|
||||
std::cout << rep_test.out_stream.str();
|
||||
|
||||
internal::CheckCases(rep_test.error_cases,rep_test.err_stream);
|
||||
internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
|
||||
|
||||
std::cout << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1,156 +1,19 @@
|
|||
|
||||
#undef NDEBUG
|
||||
#include "benchmark/benchmark.h"
|
||||
#include "../src/check.h" // NOTE: check.h is for internal use only!
|
||||
#include "../src/re.h" // NOTE: re.h is for internal use only
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
#include "output_test.h"
|
||||
#include <utility>
|
||||
|
||||
namespace {
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Testing Case --------------------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
enum MatchRules {
|
||||
MR_Default, // Skip non-matching lines until a match is found.
|
||||
MR_Next, // Match must occur on the next line.
|
||||
MR_Not // No line between the current position and the next match matches
|
||||
// the regex
|
||||
};
|
||||
|
||||
struct TestCase {
|
||||
std::string regex_str;
|
||||
int match_rule;
|
||||
std::shared_ptr<benchmark::Regex> regex;
|
||||
|
||||
TestCase(std::string re, int rule = MR_Default)
|
||||
: regex_str(re), match_rule(rule), regex(std::make_shared<benchmark::Regex>()) {
|
||||
std::string err_str;
|
||||
regex->Init(regex_str, &err_str);
|
||||
CHECK(err_str.empty()) << "Could not construct regex \"" << regex_str << "\""
|
||||
<< " got Error: " << err_str;
|
||||
}
|
||||
|
||||
void Check(std::stringstream& remaining_output,
|
||||
std::vector<TestCase>& not_checks) const {
|
||||
std::string line;
|
||||
while (remaining_output.eof() == false) {
|
||||
CHECK(remaining_output.good());
|
||||
std::getline(remaining_output, line);
|
||||
for (auto& NC : not_checks) {
|
||||
CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
|
||||
<< line << "\" for MR_Not regex \""
|
||||
<< NC.regex_str << "\"";
|
||||
}
|
||||
if (regex->Match(line)) return;
|
||||
CHECK(match_rule != MR_Next) << "Expected line \"" << line
|
||||
<< "\" to match regex \"" << regex_str << "\"";
|
||||
}
|
||||
|
||||
CHECK(remaining_output.eof() == false)
|
||||
<< "End of output reached before match for regex \"" << regex_str
|
||||
<< "\" was found";
|
||||
}
|
||||
|
||||
static void CheckCases(std::vector<TestCase> const& checks,
|
||||
std::stringstream& output) {
|
||||
std::vector<TestCase> not_checks;
|
||||
for (size_t i=0; i < checks.size(); ++i) {
|
||||
const auto& TC = checks[i];
|
||||
if (TC.match_rule == MR_Not) {
|
||||
not_checks.push_back(TC);
|
||||
continue;
|
||||
}
|
||||
TC.Check(output, not_checks);
|
||||
not_checks.clear();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<TestCase> ConsoleOutputTests;
|
||||
std::vector<TestCase> JSONOutputTests;
|
||||
std::vector<TestCase> CSVOutputTests;
|
||||
|
||||
std::vector<TestCase> ConsoleErrorTests;
|
||||
std::vector<TestCase> JSONErrorTests;
|
||||
std::vector<TestCase> CSVErrorTests;
|
||||
|
||||
// ========================================================================= //
|
||||
// -------------------------- Test Helpers --------------------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
class TestReporter : public benchmark::BenchmarkReporter {
|
||||
public:
|
||||
TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
|
||||
: reporters_(reps) {}
|
||||
|
||||
virtual bool ReportContext(const Context& context) {
|
||||
bool last_ret = false;
|
||||
bool first = true;
|
||||
for (auto rep : reporters_) {
|
||||
bool new_ret = rep->ReportContext(context);
|
||||
CHECK(first || new_ret == last_ret)
|
||||
<< "Reports return different values for ReportContext";
|
||||
first = false;
|
||||
last_ret = new_ret;
|
||||
}
|
||||
return last_ret;
|
||||
}
|
||||
|
||||
virtual void ReportRuns(const std::vector<Run>& report) {
|
||||
for (auto rep : reporters_)
|
||||
rep->ReportRuns(report);
|
||||
}
|
||||
|
||||
virtual void Finalize() {
|
||||
for (auto rep : reporters_)
|
||||
rep->Finalize();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<benchmark::BenchmarkReporter*> reporters_;
|
||||
};
|
||||
|
||||
|
||||
#define CONCAT2(x, y) x##y
|
||||
#define CONCAT(x, y) CONCAT2(x, y)
|
||||
|
||||
#define ADD_CASES(...) \
|
||||
int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
|
||||
|
||||
int AddCases(std::vector<TestCase>* out, std::initializer_list<TestCase> const& v) {
|
||||
for (auto const& TC : v)
|
||||
out->push_back(TC);
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <class First>
|
||||
std::string join(First f) { return f; }
|
||||
|
||||
template <class First, class ...Args>
|
||||
std::string join(First f, Args&&... args) {
|
||||
return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
||||
|
||||
} // end namespace
|
||||
|
||||
// ========================================================================= //
|
||||
// ---------------------- Testing Prologue Output -------------------------- //
|
||||
// ========================================================================= //
|
||||
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{join("^Benchmark", "Time", "CPU", "Iterations$"), MR_Next},
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
|
||||
{"^[-]+$", MR_Next}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
ADD_CASES(TC_CSVOut, {
|
||||
{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,items_per_second,"
|
||||
"label,error_occurred,error_message"}
|
||||
});
|
||||
|
|
@ -164,19 +27,19 @@ void BM_basic(benchmark::State& state) {
|
|||
}
|
||||
BENCHMARK(BM_basic);
|
||||
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{"^BM_basic[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{"^BM_basic %console_report$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
ADD_CASES(TC_JSONOut, {
|
||||
{"\"name\": \"BM_basic\",$"},
|
||||
{"\"iterations\": [0-9]+,$", MR_Next},
|
||||
{"\"real_time\": [0-9]{1,5},$", MR_Next},
|
||||
{"\"cpu_time\": [0-9]{1,5},$", MR_Next},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"real_time\": %int,$", MR_Next},
|
||||
{"\"cpu_time\": %int,$", MR_Next},
|
||||
{"\"time_unit\": \"ns\"$", MR_Next},
|
||||
{"}", MR_Next}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
{"^\"BM_basic\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
ADD_CASES(TC_CSVOut, {
|
||||
{"^\"BM_basic\",%csv_report$"}
|
||||
});
|
||||
|
||||
// ========================================================================= //
|
||||
|
|
@ -188,16 +51,16 @@ void BM_error(benchmark::State& state) {
|
|||
while(state.KeepRunning()) {}
|
||||
}
|
||||
BENCHMARK(BM_error);
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
ADD_CASES(TC_JSONOut, {
|
||||
{"\"name\": \"BM_error\",$"},
|
||||
{"\"error_occurred\": true,$", MR_Next},
|
||||
{"\"error_message\": \"message\",$", MR_Next}
|
||||
});
|
||||
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
ADD_CASES(TC_CSVOut, {
|
||||
{"^\"BM_error\",,,,,,,,true,\"message\"$"}
|
||||
});
|
||||
|
||||
|
|
@ -212,12 +75,13 @@ void BM_Complexity_O1(benchmark::State& state) {
|
|||
state.SetComplexityN(state.range(0));
|
||||
}
|
||||
BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1);
|
||||
|
||||
std::string bigOStr = "[0-9]+\\.[0-9]+ \\([0-9]+\\)";
|
||||
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{join("^BM_Complexity_O1_BigO", bigOStr, bigOStr) + "[ ]*$"},
|
||||
{join("^BM_Complexity_O1_RMS", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"}
|
||||
SET_SUBSTITUTIONS({
|
||||
{"%bigOStr", "[ ]*[0-9]+\\.[0-9]+ \\([0-9]+\\)"},
|
||||
{"%RMS", "[ ]*[0-9]+ %"}
|
||||
});
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
|
||||
{"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}
|
||||
});
|
||||
|
||||
|
||||
|
|
@ -228,59 +92,60 @@ ADD_CASES(&ConsoleOutputTests, {
|
|||
// Test that non-aggregate data is printed by default
|
||||
void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} }
|
||||
BENCHMARK(BM_Repeat)->Repetitions(3);
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3_mean %console_report$"},
|
||||
{"^BM_Repeat/repeats:3_stddev %console_report$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
ADD_CASES(TC_JSONOut, {
|
||||
{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
{"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
ADD_CASES(TC_CSVOut, {
|
||||
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}
|
||||
});
|
||||
|
||||
// Test that a non-repeated test still prints non-aggregate results even when
|
||||
// only-aggregate reports have been requested
|
||||
void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} }
|
||||
BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{"^BM_RepeatOnce/repeats:1[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{"^BM_RepeatOnce/repeats:1 %console_report$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
ADD_CASES(TC_JSONOut, {
|
||||
{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
{"^\"BM_RepeatOnce/repeats:1\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
ADD_CASES(TC_CSVOut, {
|
||||
{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}
|
||||
});
|
||||
|
||||
|
||||
// Test that non-aggregate data is not reported
|
||||
void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} }
|
||||
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
ADD_CASES(TC_ConsoleOut, {
|
||||
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^BM_SummaryRepeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
{"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
ADD_CASES(TC_JSONOut, {
|
||||
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
ADD_CASES(TC_CSVOut, {
|
||||
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}
|
||||
});
|
||||
|
||||
// ========================================================================= //
|
||||
|
|
@ -289,49 +154,5 @@ ADD_CASES(&CSVOutputTests, {
|
|||
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
benchmark::Initialize(&argc, argv);
|
||||
benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
|
||||
benchmark::JSONReporter JR;
|
||||
benchmark::CSVReporter CSVR;
|
||||
struct ReporterTest {
|
||||
const char* name;
|
||||
std::vector<TestCase>& output_cases;
|
||||
std::vector<TestCase>& error_cases;
|
||||
benchmark::BenchmarkReporter& reporter;
|
||||
std::stringstream out_stream;
|
||||
std::stringstream err_stream;
|
||||
|
||||
ReporterTest(const char* n,
|
||||
std::vector<TestCase>& out_tc,
|
||||
std::vector<TestCase>& err_tc,
|
||||
benchmark::BenchmarkReporter& br)
|
||||
: name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
|
||||
reporter.SetOutputStream(&out_stream);
|
||||
reporter.SetErrorStream(&err_stream);
|
||||
}
|
||||
} TestCases[] = {
|
||||
{"ConsoleReporter", ConsoleOutputTests, ConsoleErrorTests, CR},
|
||||
{"JSONReporter", JSONOutputTests, JSONErrorTests, JR},
|
||||
{"CSVReporter", CSVOutputTests, CSVErrorTests, CSVR}
|
||||
};
|
||||
|
||||
// Create the test reporter and run the benchmarks.
|
||||
std::cout << "Running benchmarks...\n";
|
||||
TestReporter test_rep({&CR, &JR, &CSVR});
|
||||
benchmark::RunSpecifiedBenchmarks(&test_rep);
|
||||
|
||||
for (auto& rep_test : TestCases) {
|
||||
std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
|
||||
std::string banner(msg.size() - 1, '-');
|
||||
std::cout << banner << msg << banner << "\n";
|
||||
|
||||
std::cerr << rep_test.err_stream.str();
|
||||
std::cout << rep_test.out_stream.str();
|
||||
|
||||
TestCase::CheckCases(rep_test.error_cases, rep_test.err_stream);
|
||||
TestCase::CheckCases(rep_test.output_cases, rep_test.out_stream);
|
||||
|
||||
std::cout << "\n";
|
||||
}
|
||||
return 0;
|
||||
RunOutputTests(argc, argv);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue