From 5686bf1b38f8aa713267097d7c1944140f71b5d3 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 27 May 2016 13:34:37 -0600 Subject: [PATCH] Change reporters to use a specified output and error stream. Add tests for output. (#219) * Add test for reporter output. * setup err_stream tests * Fix warnings in tests * whitespace * Fix build errors caused by super pedantic compilers * Pass streams by pointer not non-const reference --- include/benchmark/reporter.h | 32 +++++ src/colorprint.cc | 52 ++++++- src/colorprint.h | 10 +- src/console_reporter.cc | 52 ++++--- src/csv_reporter.cc | 53 +++---- src/json_reporter.cc | 10 +- src/reporter.cc | 7 + test/CMakeLists.txt | 2 + test/reporter_output_test.cc | 264 +++++++++++++++++++++++++++++++++++ 9 files changed, 427 insertions(+), 55 deletions(-) create mode 100644 test/reporter_output_test.cc diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 56fe1f99..ded01d21 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -14,6 +14,8 @@ #ifndef BENCHMARK_REPORTER_H_ #define BENCHMARK_REPORTER_H_ +#include +#include #include #include #include @@ -81,6 +83,10 @@ class BenchmarkReporter { bool report_rms; }; + // Construct a BenchmarkReporter with the output stream set to 'std::cout' + // and the error stream set to 'std::cerr' + BenchmarkReporter(); + // Called once for every suite of benchmarks run. // The parameter "context" contains information that the // reporter may wish to use when generating its report, for example the @@ -105,12 +111,38 @@ class BenchmarkReporter { // reported. virtual void Finalize(); + // REQUIRES: The object referenced by 'out' is valid for the lifetime + // of the reporter. + void SetOutputStream(std::ostream* out) { + assert(out); + output_stream_ = out; + } + + // REQUIRES: The object referenced by 'err' is valid for the lifetime + // of the reporter. + void SetErrorStream(std::ostream* err) { + assert(err); + error_stream_ = err; + } + + std::ostream& GetOutputStream() const { + return *output_stream_; + } + + std::ostream& GetErrorStream() const { + return *error_stream_; + } + virtual ~BenchmarkReporter(); protected: static void ComputeStats(const std::vector& reports, Run* mean, Run* stddev); static void ComputeBigO(const std::vector& reports, Run* bigO, Run* rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); + +private: + std::ostream* output_stream_; + std::ostream* error_stream_; }; // Simple reporter that outputs benchmark data to the console. This is the diff --git a/src/colorprint.cc b/src/colorprint.cc index 81f917b2..efb8626a 100644 --- a/src/colorprint.cc +++ b/src/colorprint.cc @@ -16,8 +16,12 @@ #include #include +#include +#include +#include #include "commandlineflags.h" +#include "check.h" #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS @@ -74,14 +78,51 @@ PlatformColorCode GetPlatformColorCode(LogColor color) { }; #endif } + } // end namespace -void ColorPrintf(LogColor color, const char* fmt, ...) { +std::string FormatString(const char *msg, va_list args) { + // we might need a second shot at this, so pre-emptivly make a copy + va_list args_cp; + va_copy(args_cp, args); + + std::size_t size = 256; + char local_buff[256]; + auto ret = std::vsnprintf(local_buff, size, msg, args_cp); + + va_end(args_cp); + + // currently there is no error handling for failure, so this is hack. + CHECK(ret >= 0); + + if (ret == 0) // handle empty expansion + return {}; + else if (static_cast(ret) < size) + return local_buff; + else { + // we did not provide a long enough buffer on our first attempt. + size = (size_t)ret + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + ret = std::vsnprintf(buff.get(), size, msg, args); + CHECK(ret > 0 && ((size_t)ret) < size); + return buff.get(); + } +} + +std::string FormatString(const char *msg, ...) { + va_list args; + va_start(args, msg); + auto tmp = FormatString(msg, args); + va_end(args); + return tmp; +} + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...) { va_list args; va_start(args, fmt); if (!FLAGS_color_print) { - vprintf(fmt, args); + out << FormatString(fmt, args); va_end(args); return; } @@ -107,10 +148,11 @@ void ColorPrintf(LogColor color, const char* fmt, ...) { SetConsoleTextAttribute(stdout_handle, old_color_attrs); #else const char* color_code = GetPlatformColorCode(color); - if (color_code) fprintf(stdout, "\033[0;3%sm", color_code); - vprintf(fmt, args); - printf("\033[m"); // Resets the terminal to default. + if (color_code) out << FormatString("\033[0;3%sm", color_code); + out << FormatString(fmt, args) << "\033[m"; #endif + va_end(args); } + } // end namespace benchmark diff --git a/src/colorprint.h b/src/colorprint.h index 54d1f664..2b3c0825 100644 --- a/src/colorprint.h +++ b/src/colorprint.h @@ -1,6 +1,10 @@ #ifndef BENCHMARK_COLORPRINT_H_ #define BENCHMARK_COLORPRINT_H_ +#include +#include +#include + namespace benchmark { enum LogColor { COLOR_DEFAULT, @@ -13,7 +17,11 @@ enum LogColor { COLOR_WHITE }; -void ColorPrintf(LogColor color, const char* fmt, ...); +std::string FormatString(const char* msg, va_list args); +std::string FormatString(const char* msg, ...); + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...); + } // end namespace benchmark #endif // BENCHMARK_COLORPRINT_H_ diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 5dd98f65..6e39ad94 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -25,6 +25,8 @@ #include "check.h" #include "colorprint.h" +#include "commandlineflags.h" +#include "internal_macros.h" #include "string_util.h" #include "walltime.h" @@ -33,26 +35,36 @@ namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; - std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu + auto& Out = GetOutputStream(); + auto& Err = GetErrorStream(); + +#ifdef BENCHMARK_OS_WINDOWS + if (FLAGS_color_print && &Out != &std::cout) { + Err << "Color printing is only supported for stdout on windows. " + "Disabling color printing\n"; + FLAGS_color_print = false; + } +#endif + + Err << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; - std::cerr << LocalDateTimeString() << "\n"; + Err << LocalDateTimeString() << "\n"; if (context.cpu_scaling_enabled) { - std::cerr << "***WARNING*** CPU scaling is enabled, the benchmark " + Err << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " "overhead.\n"; } #ifndef NDEBUG - std::cerr << "***WARNING*** Library was built as DEBUG. Timings may be " + Err << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; #endif - - int output_width = fprintf(stdout, "%-*s %13s %13s %10s\n", + std::string str = FormatString("%-*s %13s %13s %10s\n", static_cast(name_field_width_), "Benchmark", "Time", "CPU", "Iterations"); - std::cout << std::string(output_width - 1, '-') << "\n"; + Out << str << std::string(str.length() - 1, '-') << "\n"; return true; } @@ -101,15 +113,17 @@ void ConsoleReporter::ReportComplexity(const std::vector & complexity_repor } void ConsoleReporter::PrintRunData(const Run& result) { + auto& Out = GetOutputStream(); + auto name_color = (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; - ColorPrintf(name_color, "%-*s ", name_field_width_, + ColorPrintf(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if (result.error_occurred) { - ColorPrintf(COLOR_RED, "ERROR OCCURRED: \'%s\'", + ColorPrintf(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", result.error_message.c_str()); - ColorPrintf(COLOR_DEFAULT, "\n"); + ColorPrintf(Out, COLOR_DEFAULT, "\n"); return; } // Format bytes per second @@ -131,24 +145,24 @@ void ConsoleReporter::PrintRunData(const Run& result) { if(result.report_big_o) { std::string big_o = result.report_big_o ? GetBigOString(result.complexity) : ""; - ColorPrintf(COLOR_YELLOW, "%10.4f %s %10.4f %s ", + ColorPrintf(Out, COLOR_YELLOW, "%10.4f %s %10.4f %s ", result.real_accumulated_time * multiplier, big_o.c_str(), result.cpu_accumulated_time * multiplier, big_o.c_str()); } else if(result.report_rms) { - ColorPrintf(COLOR_YELLOW, "%10.0f %% %10.0f %% ", + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", result.real_accumulated_time * multiplier * 100, result.cpu_accumulated_time * multiplier * 100); } else if (result.iterations == 0) { - ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", result.real_accumulated_time * multiplier, timeLabel, result.cpu_accumulated_time * multiplier, timeLabel); } else { - ColorPrintf(COLOR_YELLOW, "%10.0f %s %10.0f %s ", + ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", (result.real_accumulated_time * multiplier) / (static_cast(result.iterations)), timeLabel, @@ -158,22 +172,22 @@ void ConsoleReporter::PrintRunData(const Run& result) { } if(!result.report_big_o && !result.report_rms) { - ColorPrintf(COLOR_CYAN, "%10lld", result.iterations); + ColorPrintf(Out, COLOR_CYAN, "%10lld", result.iterations); } if (!rate.empty()) { - ColorPrintf(COLOR_DEFAULT, " %*s", 13, rate.c_str()); + ColorPrintf(Out, COLOR_DEFAULT, " %*s", 13, rate.c_str()); } if (!items.empty()) { - ColorPrintf(COLOR_DEFAULT, " %*s", 18, items.c_str()); + ColorPrintf(Out, COLOR_DEFAULT, " %*s", 18, items.c_str()); } if (!result.report_label.empty()) { - ColorPrintf(COLOR_DEFAULT, " %s", result.report_label.c_str()); + ColorPrintf(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); } - ColorPrintf(COLOR_DEFAULT, "\n"); + ColorPrintf(Out, COLOR_DEFAULT, "\n"); } } // end namespace benchmark diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index ac4bd3db..36f149b5 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -44,27 +44,30 @@ std::vector elements = { } bool CSVReporter::ReportContext(const Context& context) { - std::cerr << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu + std::ostream& Err = GetErrorStream(); + std::ostream& Out = GetOutputStream(); + + Err << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; - std::cerr << LocalDateTimeString() << "\n"; + Err << LocalDateTimeString() << "\n"; if (context.cpu_scaling_enabled) { - std::cerr << "***WARNING*** CPU scaling is enabled, the benchmark " + Err << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " "overhead.\n"; } #ifndef NDEBUG - std::cerr << "***WARNING*** Library was built as DEBUG. Timings may be " + Err << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; #endif for (auto B = elements.begin(); B != elements.end(); ) { - std::cout << *B++; + Out << *B++; if (B != elements.end()) - std::cout << ","; + Out << ","; } - std::cout << "\n"; + Out << "\n"; return true; } @@ -106,19 +109,19 @@ void CSVReporter::ReportComplexity(const std::vector& complexity_reports) { } void CSVReporter::PrintRunData(const Run & run) { - + std::ostream& Out = GetOutputStream(); // Field with embedded double-quote characters must be doubled and the field // delimited with double-quotes. std::string name = run.benchmark_name; ReplaceAll(&name, "\"", "\"\""); - std::cout << '"' << name << "\","; + Out << '"' << name << "\","; if (run.error_occurred) { - std::cout << std::string(elements.size() - 3, ','); - std::cout << "true,"; + Out << std::string(elements.size() - 3, ','); + Out << "true,"; std::string msg = run.error_message; ReplaceAll(&msg, "\"", "\"\""); - std::cout << '"' << msg << "\"\n"; + Out << '"' << msg << "\"\n"; return; } @@ -135,36 +138,36 @@ void CSVReporter::PrintRunData(const Run & run) { // Do not print iteration on bigO and RMS report if(!run.report_big_o && !run.report_rms) { - std::cout << run.iterations; + Out << run.iterations; } - std::cout << ","; + Out << ","; - std::cout << real_time << ","; - std::cout << cpu_time << ","; + Out << real_time << ","; + Out << cpu_time << ","; // Do not print timeLabel on RMS report if(!run.report_rms) { - std::cout << timeLabel; + Out << timeLabel; } - std::cout << ","; + Out << ","; if (run.bytes_per_second > 0.0) { - std::cout << run.bytes_per_second; + Out << run.bytes_per_second; } - std::cout << ","; + Out << ","; if (run.items_per_second > 0.0) { - std::cout << run.items_per_second; + Out << run.items_per_second; } - std::cout << ","; + Out << ","; if (!run.report_label.empty()) { // Field with embedded double-quote characters must be doubled and the field // delimited with double-quotes. std::string label = run.report_label; ReplaceAll(&label, "\"", "\"\""); - std::cout << "\"" << label << "\""; + Out << "\"" << label << "\""; } - std::cout << ",,"; // for error_occurred and error_message - std::cout << '\n'; + Out << ",,"; // for error_occurred and error_message + Out << '\n'; } } // end namespace benchmark diff --git a/src/json_reporter.cc b/src/json_reporter.cc index cab527c0..d3effe19 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -53,7 +53,7 @@ int64_t RoundDouble(double v) { } // end namespace bool JSONReporter::ReportContext(const Context& context) { - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); out << "{\n"; std::string inner_indent(2, ' '); @@ -92,7 +92,7 @@ void JSONReporter::ReportRuns(std::vector const& reports) { return; } std::string indent(4, ' '); - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); if (!first_report_) { out << ",\n"; } @@ -128,7 +128,7 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) } std::string indent(4, ' '); - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); if (!first_report_) { out << ",\n"; } @@ -148,7 +148,7 @@ void JSONReporter::ReportComplexity(const std::vector & complexity_reports) void JSONReporter::Finalize() { // Close the list of benchmarks and the top level object. - std::cout << "\n ]\n}\n"; + GetOutputStream() << "\n ]\n}\n"; } void JSONReporter::PrintRunData(Run const& run) { @@ -164,7 +164,7 @@ void JSONReporter::PrintRunData(Run const& run) { } std::string indent(6, ' '); - std::ostream& out = std::cout; + std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name) << ",\n"; diff --git a/src/reporter.cc b/src/reporter.cc index d0a80e6f..5b303f73 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -16,6 +16,8 @@ #include "complexity.h" #include + +#include #include #include @@ -24,6 +26,11 @@ namespace benchmark { +BenchmarkReporter::BenchmarkReporter() + : output_stream_(&std::cout), error_stream_(&std::cerr) +{ +} + void BenchmarkReporter::ComputeStats( const std::vector& reports, Run* mean_data, Run* stddev_data) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 247c6300..aeb720a3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,6 +48,8 @@ add_test(fixture_test fixture_test --benchmark_min_time=0.01) compile_benchmark_test(map_test) add_test(map_test map_test --benchmark_min_time=0.01) +compile_benchmark_test(reporter_output_test) +add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc new file mode 100644 index 00000000..acfbf680 --- /dev/null +++ b/test/reporter_output_test.cc @@ -0,0 +1,264 @@ + +#undef NDEBUG +#include "benchmark/benchmark.h" +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "../src/re.h" // NOTE: re.h is for internal use only +#include +#include +#include +#include +#include +#include + +namespace { + +// ========================================================================= // +// -------------------------- Testing Case --------------------------------- // +// ========================================================================= // + +enum MatchRules { + MR_Default, // Skip non-matching lines until a match is found. + MR_Next // Match must occur on the next line. +}; + +struct TestCase { + std::string regex; + int match_rule; + + TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {} + + void Check(std::stringstream& remaining_output) const { + benchmark::Regex r; + std::string err_str; + r.Init(regex, &err_str); + CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\"" + << " got Error: " << err_str; + + std::string line; + while (remaining_output.eof() == false) { + CHECK(remaining_output.good()); + std::getline(remaining_output, line); + if (r.Match(line)) return; + CHECK(match_rule != MR_Next) << "Expected line \"" << line + << "\" to match regex \"" << regex << "\""; + } + + CHECK(remaining_output.eof() == false) + << "End of output reached before match for regex \"" << regex + << "\" was found"; + } +}; + +std::vector ConsoleOutputTests; +std::vector JSONOutputTests; +std::vector CSVOutputTests; + +std::vector ConsoleErrorTests; +std::vector JSONErrorTests; +std::vector CSVErrorTests; + +// ========================================================================= // +// -------------------------- Test Helpers --------------------------------- // +// ========================================================================= // + +class TestReporter : public benchmark::BenchmarkReporter { +public: + TestReporter(std::vector reps) + : reporters_(reps) {} + + virtual bool ReportContext(const Context& context) { + bool last_ret = false; + bool first = true; + for (auto rep : reporters_) { + bool new_ret = rep->ReportContext(context); + CHECK(first || new_ret == last_ret) + << "Reports return different values for ReportContext"; + first = false; + last_ret = new_ret; + } + return last_ret; + } + + virtual void ReportRuns(const std::vector& report) { + for (auto rep : reporters_) + rep->ReportRuns(report); + } + + virtual void ReportComplexity(const std::vector& complexity_reports) { + for (auto rep : reporters_) + rep->ReportComplexity(complexity_reports); + } + + virtual void Finalize() { + for (auto rep : reporters_) + rep->Finalize(); + } + +private: + std::vector reporters_; +}; + + +#define CONCAT2(x, y) x##y +#define CONCAT(x, y) CONCAT2(x, y) + +#define ADD_CASES(...) \ + int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +int AddCases(std::vector* out, std::initializer_list const& v) { + for (auto const& TC : v) + out->push_back(TC); + return 0; +} + +template +std::string join(First f) { return f; } + +template +std::string join(First f, Args&&... args) { + return std::string(std::move(f)) + "[ ]+" + join(std::forward(args)...); +} + +std::string dec_re = "[0-9]+\\.[0-9]+"; + +} // end namespace + +// ========================================================================= // +// ---------------------- Testing Prologue Output -------------------------- // +// ========================================================================= // + +ADD_CASES(&ConsoleOutputTests, { + {join("^Benchmark", "Time", "CPU", "Iterations$"), MR_Next}, + {"^[-]+$", MR_Next} +}); +ADD_CASES(&CSVOutputTests, { + {"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,items_per_second," + "label,error_occurred,error_message"} +}); + +// ========================================================================= // +// ------------------------ Testing Basic Output --------------------------- // +// ========================================================================= // + +void BM_basic(benchmark::State& state) { + while (state.KeepRunning()) {} +} +BENCHMARK(BM_basic); + +ADD_CASES(&ConsoleOutputTests, { + {"^BM_basic[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"} +}); +ADD_CASES(&JSONOutputTests, { + {"\"name\": \"BM_basic\",$"}, + {"\"iterations\": [0-9]+,$", MR_Next}, + {"\"real_time\": [0-9],$", MR_Next}, + {"\"cpu_time\": [0-9],$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next} +}); +ADD_CASES(&CSVOutputTests, { + {"^\"BM_basic\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} +}); + +// ========================================================================= // +// ------------------------ Testing Error Output --------------------------- // +// ========================================================================= // + +void BM_error(benchmark::State& state) { + state.SkipWithError("message"); + while(state.KeepRunning()) {} +} +BENCHMARK(BM_error); +ADD_CASES(&ConsoleOutputTests, { + {"^BM_error[ ]+ERROR OCCURRED: 'message'$"} +}); +ADD_CASES(&JSONOutputTests, { + {"\"name\": \"BM_error\",$"}, + {"\"error_occurred\": true,$", MR_Next}, + {"\"error_message\": \"message\",$", MR_Next} +}); + +ADD_CASES(&CSVOutputTests, { + {"^\"BM_error\",,,,,,,,true,\"message\"$"} +}); + + +// ========================================================================= // +// ----------------------- Testing Complexity Output ----------------------- // +// ========================================================================= // + +void BM_Complexity_O1(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.SetComplexityN(state.range_x()); +} +BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); + +std::string bigOStr = "[0-9]+\\.[0-9]+ \\* [0-9]+"; + +ADD_CASES(&ConsoleOutputTests, { + {join("^BM_Complexity_O1_BigO", bigOStr, bigOStr) + "[ ]*$"}, + {join("^BM_Complexity_O1_RMS", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"} +}); + + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + + +int main(int argc, char* argv[]) { + // Add --color_print=false to argv since we don't want to match color codes. + char new_arg[64]; + char* new_argv[64]; + std::copy(argv, argv + argc, new_argv); + new_argv[argc++] = std::strcpy(new_arg, "--color_print=false"); + benchmark::Initialize(&argc, new_argv); + + benchmark::ConsoleReporter CR; + benchmark::JSONReporter JR; + benchmark::CSVReporter CSVR; + struct ReporterTest { + const char* name; + std::vector& output_cases; + std::vector& error_cases; + benchmark::BenchmarkReporter& reporter; + std::stringstream out_stream; + std::stringstream err_stream; + + ReporterTest(const char* n, + std::vector& out_tc, + std::vector& err_tc, + benchmark::BenchmarkReporter& br) + : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { + reporter.SetOutputStream(&out_stream); + reporter.SetErrorStream(&err_stream); + } + } TestCases[] = { + {"ConsoleReporter", ConsoleOutputTests, ConsoleErrorTests, CR}, + {"JSONReporter", JSONOutputTests, JSONErrorTests, JR}, + {"CSVReporter", CSVOutputTests, CSVErrorTests, CSVR} + }; + + // Create the test reporter and run the benchmarks. + std::cout << "Running benchmarks...\n"; + TestReporter test_rep({&CR, &JR, &CSVR}); + benchmark::RunSpecifiedBenchmarks(&test_rep); + + for (auto& rep_test : TestCases) { + std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string banner(msg.size() - 1, '-'); + std::cout << banner << msg << banner << "\n"; + + std::cerr << rep_test.err_stream.str(); + std::cout << rep_test.out_stream.str(); + + for (const auto& TC : rep_test.error_cases) + TC.Check(rep_test.err_stream); + for (const auto& TC : rep_test.output_cases) + TC.Check(rep_test.out_stream); + + std::cout << "\n"; + } + return 0; +}