Refactor output test runner into standalone module. (#277)

* refactor

* Move default substitutions into library

* Move default substitutions to the *right* place in the library

* Fix init order issues that caused test failures

* improve diagnostics

* add missing include

* general cleanup

* Address review comments

test/CMakeLists.txt

@@ -2,11 +2,20 @@
 
 find_package(Threads REQUIRED)
 
+add_library(output_test_helper STATIC output_test_helper.cc)
+
 macro(compile_benchmark_test name)
   add_executable(${name} "${name}.cc")
   target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT})
 endmacro(compile_benchmark_test)
 
+
+macro(compile_output_test name)
+  add_executable(${name} "${name}.cc")
+  target_link_libraries(${name} output_test_helper benchmark ${CMAKE_THREAD_LIBS_INIT})
+endmacro(compile_output_test)
+
+
 # Demonstration executable
 compile_benchmark_test(benchmark_test)
 add_test(benchmark benchmark_test --benchmark_min_time=0.01)
@@ -54,7 +63,7 @@ add_test(map_test map_test --benchmark_min_time=0.01)
 compile_benchmark_test(multiple_ranges_test)
 add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01)
 
-compile_benchmark_test(reporter_output_test)
+compile_output_test(reporter_output_test)
 add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01)
 
 check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
@@ -69,7 +78,7 @@ if (BENCHMARK_HAS_CXX03_FLAG)
   add_test(cxx03 cxx03_test --benchmark_min_time=0.01)
 endif()
 
-compile_benchmark_test(complexity_test)
+compile_output_test(complexity_test)
 add_test(complexity_benchmark complexity_test --benchmark_min_time=0.01)
 
 # Add the coverage command(s)

test/complexity_test.cc

@@ -1,153 +1,46 @@
-
 #undef NDEBUG
 #include "benchmark/benchmark.h"
-#include "../src/check.h" // NOTE: check.h is for internal use only!
+#include "output_test.h"
-#include "../src/re.h"    // NOTE: re.h is for internal use only
 #include <cassert>
-#include <cstring>
-#include <iostream>
-#include <sstream>
 #include <vector>
-#include <utility>
 #include <algorithm>
+#include <cstdlib>
 #include <cmath>
 
 namespace {
 
-// ========================================================================= //
-// -------------------------- Testing Case --------------------------------- //
-// ========================================================================= //
-
-enum MatchRules {
-  MR_Default, // Skip non-matching lines until a match is found.
-  MR_Next    // Match must occur on the next line.
-};
-
-struct TestCase {
-  std::string regex;
-  int match_rule;
-
-  TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {}
-
-  void Check(std::stringstream& remaining_output) const {
-    benchmark::Regex r;
-    std::string err_str;
-    r.Init(regex, &err_str);
-    CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\""
-                           << " got Error: " << err_str;
-
-    std::string near = "<EOF>";
-    std::string line;
-    bool first = true;
-    while (remaining_output.eof() == false) {
-        CHECK(remaining_output.good());
-        std::getline(remaining_output, line);
-        // Keep the first line as context.
-        if (first) {
-            near = line;
-            first = false;
-        }
-        if (r.Match(line)) return;
-        CHECK(match_rule != MR_Next) << "Expected line \"" << line
-                                     << "\" to match regex \"" << regex << "\""
-                                     << "\nstarted matching at line: \"" << near << "\"";
-    }
-
-    CHECK(remaining_output.eof() == false)
-        << "End of output reached before match for regex \"" << regex
-        << "\" was found"
-        << "\nstarted matching at line: \"" << near << "\"";
-  }
-};
-
-std::vector<TestCase> ConsoleOutputTests;
-std::vector<TestCase> JSONOutputTests;
-std::vector<TestCase> CSVOutputTests;
-
-// ========================================================================= //
-// -------------------------- Test Helpers --------------------------------- //
-// ========================================================================= //
-
-class TestReporter : public benchmark::BenchmarkReporter {
-public:
-  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
-      : reporters_(reps)  {}
-
-  virtual bool ReportContext(const Context& context) {
-    bool last_ret = false;
-    bool first = true;
-    for (auto rep : reporters_) {
-      bool new_ret = rep->ReportContext(context);
-      CHECK(first || new_ret == last_ret)
-          << "Reports return different values for ReportContext";
-      first = false;
-      last_ret = new_ret;
-    }
-    return last_ret;
-  }
-
-  virtual void ReportRuns(const std::vector<Run>& report) {
-    for (auto rep : reporters_)
-      rep->ReportRuns(report);
-  }
-
-  virtual void Finalize() {
-      for (auto rep : reporters_)
-        rep->Finalize();
-  }
-
-private:
-  std::vector<benchmark::BenchmarkReporter*> reporters_;
-};
-
-
-#define CONCAT2(x, y) x##y
-#define CONCAT(x, y) CONCAT2(x, y)
-
-#define ADD_CASES(...) \
-    int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
-
-int AddCases(std::vector<TestCase>* out, std::initializer_list<TestCase> const& v) {
-  for (auto const& TC : v)
-    out->push_back(TC);
-  return 0;
-}
-
-template <class First>
-std::string join(First f) { return f; }
-
-template <class First, class ...Args>
-std::string join(First f, Args&&... args) {
-    return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
-}
-
-std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
-
 #define ADD_COMPLEXITY_CASES(...) \
     int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)
 
-int AddComplexityTest(std::vector<TestCase>* console_out, std::vector<TestCase>* json_out,
+int AddComplexityTest(std::string big_o_test_name,
-                      std::vector<TestCase>* csv_out, std::string big_o_test_name, 
                       std::string rms_test_name, std::string big_o) {
-  std::string big_o_str = dec_re + " " + big_o;
+  SetSubstitutions({
-  AddCases(console_out, {
+        {"%bigo_name", big_o_test_name},
-    {join("^" + big_o_test_name + "", big_o_str, big_o_str) + "[ ]*$"},
+        {"%rms_name", rms_test_name},
-    {join("^" + rms_test_name + "", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"}
+        {"%bigo_str", "[ ]*" + std::string(dec_re) + " " + big_o},
+        {"%bigo", big_o},
+        {"%rms", "[ ]*[0-9]+ %"}
   });
-  AddCases(json_out, {
+  AddCases(TC_ConsoleOut, {
-    {"\"name\": \"" + big_o_test_name + "\",$"},
+    {"^%bigo_name %bigo_str %bigo_str[ ]*$"},
+    {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
+    {"^%rms_name %rms %rms[ ]*$", MR_Next}
+  });
+  AddCases(TC_JSONOut, {
+    {"\"name\": \"%bigo_name\",$"},
     {"\"cpu_coefficient\": [0-9]+,$", MR_Next},
     {"\"real_coefficient\": [0-9]{1,5},$", MR_Next},
-    {"\"big_o\": \"" + big_o + "\",$", MR_Next},
+    {"\"big_o\": \"%bigo\",$", MR_Next},
     {"\"time_unit\": \"ns\"$", MR_Next},
     {"}", MR_Next},
-    {"\"name\": \"" + rms_test_name + "\",$"},
+    {"\"name\": \"%rms_name\",$"},
     {"\"rms\": [0-9]+%$", MR_Next},
     {"}", MR_Next}
   });
-  AddCases(csv_out, {
+  AddCases(TC_CSVOut, {
-    {"^\"" + big_o_test_name + "\",," + dec_re + "," + dec_re + "," + big_o + ",,,,,$"},
+    {"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
-    {"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$", MR_Next}
+    {"^\"%bigo_name\"", MR_Not},
+    {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}
   });
   return 0;
 }
@@ -179,16 +72,13 @@ const char* auto_big_o_1 = "(\\([0-9]+\\))|(lgN)";
 const char* lambda_big_o_1 = "f\\(N\\)";
 
 // Add enum tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1);
-                     big_o_1_test_name, rms_o_1_test_name, enum_big_o_1);
 
 // Add auto enum tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests,
+ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1);
-                     big_o_1_test_name, rms_o_1_test_name, auto_big_o_1);
 
 // Add lambda tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1);
-                     big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1);
 
 // ========================================================================= //
 // --------------------------- Testing BigO O(N) --------------------------- //
@@ -198,7 +88,7 @@ std::vector<int> ConstructRandomVector(int size) {
   std::vector<int> v;
   v.reserve(size);
   for (int i = 0; i < size; ++i) {
-    v.push_back(rand() % size);
+    v.push_back(std::rand() % size);
   }
   return v;
 }
@@ -221,12 +111,10 @@ const char* enum_auto_big_o_n = "N";
 const char* lambda_big_o_n = "f\\(N\\)";
 
 // Add enum tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
-                     big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
 
 // Add lambda tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
-                     big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
 
 // ========================================================================= //
 // ------------------------- Testing BigO O(N*lgN) ------------------------- //
@@ -249,62 +137,17 @@ const char* enum_auto_big_o_n_lg_n = "NlgN";
 const char* lambda_big_o_n_lg_n = "f\\(N\\)";
 
 // Add enum tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
-                     big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
 
 // Add lambda tests
-ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, 
+ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
-                     big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
 
 
 // ========================================================================= //
 // --------------------------- TEST CASES END ------------------------------ //
 // ========================================================================= //
 
-
 int main(int argc, char* argv[]) {
-  benchmark::Initialize(&argc, argv);
+  RunOutputTests(argc, argv);
-  benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
-  benchmark::JSONReporter JR;
-  benchmark::CSVReporter CSVR;
-  struct ReporterTest {
-    const char* name;
-    std::vector<TestCase>& output_cases;
-    benchmark::BenchmarkReporter& reporter;
-    std::stringstream out_stream;
-    std::stringstream err_stream;
-
-    ReporterTest(const char* n,
-                 std::vector<TestCase>& out_tc,
-                 benchmark::BenchmarkReporter& br)
-        : name(n), output_cases(out_tc), reporter(br) {
-        reporter.SetOutputStream(&out_stream);
-        reporter.SetErrorStream(&err_stream);
-    }
-  } TestCases[] = {
-      {"ConsoleReporter", ConsoleOutputTests, CR},
-      {"JSONReporter", JSONOutputTests, JR},
-      {"CSVReporter", CSVOutputTests, CSVR}
-  };
-
-  // Create the test reporter and run the benchmarks.
-  std::cout << "Running benchmarks...\n";
-  TestReporter test_rep({&CR, &JR, &CSVR});
-  benchmark::RunSpecifiedBenchmarks(&test_rep);
-
-  for (auto& rep_test : TestCases) {
-      std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
-      std::string banner(msg.size() - 1, '-');
-      std::cout << banner << msg << banner << "\n";
-
-      std::cerr << rep_test.err_stream.str();
-      std::cout << rep_test.out_stream.str();
-
-      for (const auto& TC : rep_test.output_cases)
-        TC.Check(rep_test.out_stream);
-
-      std::cout << "\n";
-  }
-  return 0;
 }
 

test/output_test.h

@@ -0,0 +1,72 @@
+#ifndef TEST_OUTPUT_TEST_H
+#define TEST_OUTPUT_TEST_H
+
+#undef NDEBUG
+#include "benchmark/benchmark.h"
+#include "../src/re.h"
+#include <vector>
+#include <string>
+#include <initializer_list>
+#include <memory>
+#include <utility>
+
+#define CONCAT2(x, y) x##y
+#define CONCAT(x, y) CONCAT2(x, y)
+
+#define ADD_CASES(...) \
+    int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
+
+#define SET_SUBSTITUTIONS(...) \
+    int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
+
+enum MatchRules {
+  MR_Default, // Skip non-matching lines until a match is found.
+  MR_Next,    // Match must occur on the next line.
+  MR_Not      // No line between the current position and the next match matches
+              // the regex
+};
+
+struct TestCase {
+  TestCase(std::string re, int rule = MR_Default);
+
+  std::string regex_str;
+  int match_rule;
+  std::string substituted_regex;
+  std::shared_ptr<benchmark::Regex> regex;
+};
+
+enum TestCaseID {
+  TC_ConsoleOut,
+  TC_ConsoleErr,
+  TC_JSONOut,
+  TC_JSONErr,
+  TC_CSVOut,
+  TC_CSVErr,
+
+  TC_NumID // PRIVATE
+};
+
+// Add a list of test cases to be run against the output specified by
+// 'ID'
+int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
+
+// Add or set a list of substitutions to be performed on constructed regex's
+// See 'output_test_helper.cc' for a list of default substitutions.
+int SetSubstitutions(
+    std::initializer_list<std::pair<std::string, std::string>> il);
+
+// Run all output tests.
+void RunOutputTests(int argc, char* argv[]);
+
+// ========================================================================= //
+// --------------------------- Misc Utilities ------------------------------ //
+// ========================================================================= //
+
+namespace {
+
+const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
+
+} //  end namespace
+
+
+#endif // TEST_OUTPUT_TEST_H

test/output_test_helper.cc

@@ -0,0 +1,220 @@
+#include "output_test.h"
+#include "../src/check.h" // NOTE: check.h is for internal use only!
+#include "../src/re.h" // NOTE: re.h is for internal use only
+#include <memory>
+#include <map>
+#include <iostream>
+#include <sstream>
+
+
+// ========================================================================= //
+// ------------------------------ Internals -------------------------------- //
+// ========================================================================= //
+namespace internal { namespace {
+
+using TestCaseList = std::vector<TestCase>;
+
+// Use a vector because the order elements are added matters during iteration.
+// std::map/unordered_map don't guarantee that.
+// For example:
+//  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
+//     Substitute("%HelloWorld") // Always expands to Hello.
+using SubMap = std::vector<std::pair<std::string, std::string>>;
+
+TestCaseList& GetTestCaseList(TestCaseID ID) {
+    // Uses function-local statics to ensure initialization occurs
+    // before first use.
+    static TestCaseList lists[TC_NumID];
+    return lists[ID];
+}
+
+SubMap& GetSubstitutions() {
+    // Don't use 'dec_re' from header because it may not yet be initialized.
+    static std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
+    static SubMap map = {
+        {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
+        {"%int", "[ ]*[0-9]+"},
+        {" %s ", "[ ]+"},
+        {"%time", "[ ]*[0-9]{1,5} ns"},
+        {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
+        {"%csv_report", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,"}
+    };
+    return map;
+}
+
+std::string PerformSubstitutions(std::string source) {
+    SubMap const& subs = GetSubstitutions();
+    using SizeT = std::string::size_type;
+    for (auto const& KV : subs) {
+        SizeT pos;
+        SizeT next_start = 0;
+        while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
+            next_start = pos + KV.second.size();
+            source.replace(pos, KV.first.size(), KV.second);
+        }
+    }
+    return source;
+}
+
+void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
+               TestCaseList const& not_checks)
+{
+    std::string first_line;
+    bool on_first = true;
+    std::string line;
+    while (remaining_output.eof() == false) {
+        CHECK(remaining_output.good());
+        std::getline(remaining_output, line);
+        if (on_first) {
+            first_line = line;
+            on_first = false;
+        }
+        for (auto& NC : not_checks) {
+            CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
+                                          << line << "\" for MR_Not regex \""
+                                          << NC.regex_str << "\"";
+        }
+        if (TC.regex->Match(line)) return;
+        CHECK(TC.match_rule != MR_Next) << "Expected line \"" << line
+                                     << "\" to match regex \"" << TC.regex_str << "\"";
+    }
+    CHECK(remaining_output.eof() == false)
+        << "End of output reached before match for regex \"" << TC.regex_str
+        << "\" was found"
+        << "\n    actual regex string \"" << TC.substituted_regex << "\""
+        << "\n    started matching near: " << first_line;
+}
+
+
+void CheckCases(TestCaseList const& checks, std::stringstream& output) {
+    std::vector<TestCase> not_checks;
+    for (size_t i=0; i < checks.size(); ++i) {
+        const auto& TC = checks[i];
+        if (TC.match_rule == MR_Not) {
+            not_checks.push_back(TC);
+            continue;
+        }
+        CheckCase(output, TC, not_checks);
+        not_checks.clear();
+    }
+}
+
+class TestReporter : public benchmark::BenchmarkReporter {
+public:
+  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
+      : reporters_(reps)  {}
+
+  virtual bool ReportContext(const Context& context) {
+    bool last_ret = false;
+    bool first = true;
+    for (auto rep : reporters_) {
+      bool new_ret = rep->ReportContext(context);
+      CHECK(first || new_ret == last_ret)
+          << "Reports return different values for ReportContext";
+      first = false;
+      last_ret = new_ret;
+    }
+    return last_ret;
+  }
+
+  void ReportRuns(const std::vector<Run>& report)
+    { for (auto rep : reporters_) rep->ReportRuns(report); }
+  void Finalize() { for (auto rep : reporters_) rep->Finalize(); }
+
+private:
+  std::vector<benchmark::BenchmarkReporter*> reporters_;
+};
+
+}} // end namespace internal
+
+// ========================================================================= //
+// -------------------------- Public API Definitions------------------------ //
+// ========================================================================= //
+
+TestCase::TestCase(std::string re, int rule)
+    : regex_str(std::move(re)), match_rule(rule),
+      substituted_regex(internal::PerformSubstitutions(regex_str)),
+      regex(std::make_shared<benchmark::Regex>())
+{
+    std::string err_str;
+    regex->Init(substituted_regex, &err_str);
+    CHECK(err_str.empty())
+        << "Could not construct regex \"" << substituted_regex << "\""
+        << "\n    originally \"" << regex_str << "\""
+        << "\n    got error: " << err_str;
+}
+
+int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
+    auto& L = internal::GetTestCaseList(ID);
+    L.insert(L.end(), il);
+    return 0;
+}
+
+int SetSubstitutions(std::initializer_list<std::pair<std::string, std::string>> il) {
+    auto& subs = internal::GetSubstitutions();
+    for (auto const& KV : il) {
+        bool exists = false;
+        for (auto& EKV : subs) {
+            if (EKV.first == KV.first) {
+                EKV.second = KV.second;
+                exists = true;
+                break;
+            }
+        }
+        if (!exists) subs.push_back(KV);
+    }
+    return 0;
+}
+
+void RunOutputTests(int argc, char* argv[]) {
+  using internal::GetTestCaseList;
+  benchmark::Initialize(&argc, argv);
+  benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
+  benchmark::JSONReporter JR;
+  benchmark::CSVReporter CSVR;
+  struct ReporterTest {
+    const char* name;
+    std::vector<TestCase>& output_cases;
+    std::vector<TestCase>& error_cases;
+    benchmark::BenchmarkReporter& reporter;
+    std::stringstream out_stream;
+    std::stringstream err_stream;
+
+    ReporterTest(const char* n,
+                 std::vector<TestCase>& out_tc,
+                 std::vector<TestCase>& err_tc,
+                 benchmark::BenchmarkReporter& br)
+        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
+        reporter.SetOutputStream(&out_stream);
+        reporter.SetErrorStream(&err_stream);
+    }
+  } TestCases[] = {
+      {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
+                          GetTestCaseList(TC_ConsoleErr), CR},
+      {"JSONReporter",    GetTestCaseList(TC_JSONOut),
+                          GetTestCaseList(TC_JSONErr), JR},
+      {"CSVReporter",     GetTestCaseList(TC_CSVOut),
+                          GetTestCaseList(TC_CSVErr), CSVR},
+  };
+
+  // Create the test reporter and run the benchmarks.
+  std::cout << "Running benchmarks...\n";
+  internal::TestReporter test_rep({&CR, &JR, &CSVR});
+  benchmark::RunSpecifiedBenchmarks(&test_rep);
+
+  for (auto& rep_test : TestCases) {
+      std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
+      std::string banner(msg.size() - 1, '-');
+      std::cout << banner << msg << banner << "\n";
+
+      std::cerr << rep_test.err_stream.str();
+      std::cout << rep_test.out_stream.str();
+
+      internal::CheckCases(rep_test.error_cases,rep_test.err_stream);
+      internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
+
+      std::cout << "\n";
+  }
+}
+
+

test/reporter_output_test.cc

@@ -1,156 +1,19 @@
 
 #undef NDEBUG
 #include "benchmark/benchmark.h"
-#include "../src/check.h" // NOTE: check.h is for internal use only!
+#include "output_test.h"
-#include "../src/re.h" // NOTE: re.h is for internal use only
-#include <cassert>
-#include <cstring>
-#include <iostream>
-#include <memory>
-#include <sstream>
-#include <vector>
 #include <utility>
 
-namespace {
-
-// ========================================================================= //
-// -------------------------- Testing Case --------------------------------- //
-// ========================================================================= //
-
-enum MatchRules {
-  MR_Default, // Skip non-matching lines until a match is found.
-  MR_Next,    // Match must occur on the next line.
-  MR_Not      // No line between the current position and the next match matches
-              // the regex
-};
-
-struct TestCase {
-  std::string regex_str;
-  int match_rule;
-  std::shared_ptr<benchmark::Regex> regex;
-
-  TestCase(std::string re, int rule = MR_Default)
-      : regex_str(re), match_rule(rule), regex(std::make_shared<benchmark::Regex>()) {
-    std::string err_str;
-    regex->Init(regex_str, &err_str);
-    CHECK(err_str.empty()) << "Could not construct regex \"" << regex_str << "\""
-                           << " got Error: " << err_str;
-  }
-
-  void Check(std::stringstream& remaining_output,
-             std::vector<TestCase>& not_checks) const {
-    std::string line;
-    while (remaining_output.eof() == false) {
-        CHECK(remaining_output.good());
-        std::getline(remaining_output, line);
-        for (auto& NC : not_checks) {
-            CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
-                                          << line << "\" for MR_Not regex \""
-                                          << NC.regex_str << "\"";
-        }
-        if (regex->Match(line)) return;
-        CHECK(match_rule != MR_Next) << "Expected line \"" << line
-                                     << "\" to match regex \"" << regex_str << "\"";
-    }
-
-    CHECK(remaining_output.eof() == false)
-        << "End of output reached before match for regex \"" << regex_str
-        << "\" was found";
-  }
-
-  static void CheckCases(std::vector<TestCase> const& checks,
-                         std::stringstream& output) {
-      std::vector<TestCase> not_checks;
-      for (size_t i=0; i < checks.size(); ++i) {
-          const auto& TC = checks[i];
-          if (TC.match_rule == MR_Not) {
-              not_checks.push_back(TC);
-              continue;
-          }
-          TC.Check(output, not_checks);
-          not_checks.clear();
-      }
-  }
-};
-
-std::vector<TestCase> ConsoleOutputTests;
-std::vector<TestCase> JSONOutputTests;
-std::vector<TestCase> CSVOutputTests;
-
-std::vector<TestCase> ConsoleErrorTests;
-std::vector<TestCase> JSONErrorTests;
-std::vector<TestCase> CSVErrorTests;
-
-// ========================================================================= //
-// -------------------------- Test Helpers --------------------------------- //
-// ========================================================================= //
-
-class TestReporter : public benchmark::BenchmarkReporter {
-public:
-  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
-      : reporters_(reps)  {}
-
-  virtual bool ReportContext(const Context& context) {
-    bool last_ret = false;
-    bool first = true;
-    for (auto rep : reporters_) {
-      bool new_ret = rep->ReportContext(context);
-      CHECK(first || new_ret == last_ret)
-          << "Reports return different values for ReportContext";
-      first = false;
-      last_ret = new_ret;
-    }
-    return last_ret;
-  }
-
-  virtual void ReportRuns(const std::vector<Run>& report) {
-    for (auto rep : reporters_)
-      rep->ReportRuns(report);
-  }
-
-  virtual void Finalize() {
-      for (auto rep : reporters_)
-        rep->Finalize();
-  }
-
-private:
-  std::vector<benchmark::BenchmarkReporter*> reporters_;
-};
-
-
-#define CONCAT2(x, y) x##y
-#define CONCAT(x, y) CONCAT2(x, y)
-
-#define ADD_CASES(...) \
-    int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
-
-int AddCases(std::vector<TestCase>* out, std::initializer_list<TestCase> const& v) {
-  for (auto const& TC : v)
-    out->push_back(TC);
-  return 0;
-}
-
-template <class First>
-std::string join(First f) { return f; }
-
-template <class First, class ...Args>
-std::string join(First f, Args&&... args) {
-    return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
-}
-
-std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
-
-}  // end namespace
 
 // ========================================================================= //
 // ---------------------- Testing Prologue Output -------------------------- //
 // ========================================================================= //
 
-ADD_CASES(&ConsoleOutputTests, {
+ADD_CASES(TC_ConsoleOut, {
-    {join("^Benchmark", "Time", "CPU", "Iterations$"), MR_Next},
+    {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
     {"^[-]+$", MR_Next}
 });
-ADD_CASES(&CSVOutputTests, {
+ADD_CASES(TC_CSVOut, {
   {"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,items_per_second,"
     "label,error_occurred,error_message"}
 });
@@ -164,19 +27,19 @@ void BM_basic(benchmark::State& state) {
 }
 BENCHMARK(BM_basic);
 
-ADD_CASES(&ConsoleOutputTests, {
+ADD_CASES(TC_ConsoleOut, {
-    {"^BM_basic[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+    {"^BM_basic %console_report$"}
 });
-ADD_CASES(&JSONOutputTests, {
+ADD_CASES(TC_JSONOut, {
     {"\"name\": \"BM_basic\",$"},
-    {"\"iterations\": [0-9]+,$", MR_Next},
+    {"\"iterations\": %int,$", MR_Next},
-    {"\"real_time\": [0-9]{1,5},$", MR_Next},
+    {"\"real_time\": %int,$", MR_Next},
-    {"\"cpu_time\": [0-9]{1,5},$", MR_Next},
+    {"\"cpu_time\": %int,$", MR_Next},
     {"\"time_unit\": \"ns\"$", MR_Next},
     {"}", MR_Next}
 });
-ADD_CASES(&CSVOutputTests, {
+ADD_CASES(TC_CSVOut, {
-    {"^\"BM_basic\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+    {"^\"BM_basic\",%csv_report$"}
 });
 
 // ========================================================================= //
@@ -188,16 +51,16 @@ void BM_error(benchmark::State& state) {
     while(state.KeepRunning()) {}
 }
 BENCHMARK(BM_error);
-ADD_CASES(&ConsoleOutputTests, {
+ADD_CASES(TC_ConsoleOut, {
     {"^BM_error[ ]+ERROR OCCURRED: 'message'$"}
 });
-ADD_CASES(&JSONOutputTests, {
+ADD_CASES(TC_JSONOut, {
     {"\"name\": \"BM_error\",$"},
     {"\"error_occurred\": true,$", MR_Next},
     {"\"error_message\": \"message\",$", MR_Next}
 });
 
-ADD_CASES(&CSVOutputTests, {
+ADD_CASES(TC_CSVOut, {
     {"^\"BM_error\",,,,,,,,true,\"message\"$"}
 });
 
@@ -212,12 +75,13 @@ void BM_Complexity_O1(benchmark::State& state) {
   state.SetComplexityN(state.range(0));
 }
 BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1);
-
+SET_SUBSTITUTIONS({
-std::string bigOStr = "[0-9]+\\.[0-9]+ \\([0-9]+\\)";
+  {"%bigOStr", "[ ]*[0-9]+\\.[0-9]+ \\([0-9]+\\)"},
-
+  {"%RMS", "[ ]*[0-9]+ %"}
-ADD_CASES(&ConsoleOutputTests, {
+});
-   {join("^BM_Complexity_O1_BigO", bigOStr, bigOStr) + "[ ]*$"},
+ADD_CASES(TC_ConsoleOut, {
-   {join("^BM_Complexity_O1_RMS", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"}
+   {"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
+   {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}
 });
 
 
@@ -228,59 +92,60 @@ ADD_CASES(&ConsoleOutputTests, {
 // Test that non-aggregate data is printed by default
 void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} }
 BENCHMARK(BM_Repeat)->Repetitions(3);
-ADD_CASES(&ConsoleOutputTests, {
+ADD_CASES(TC_ConsoleOut, {
-    {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3 %console_report$"},
-    {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3 %console_report$"},
-    {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3 %console_report$"},
-    {"^BM_Repeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3_mean %console_report$"},
-    {"^BM_Repeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+    {"^BM_Repeat/repeats:3_stddev %console_report$"}
 });
-ADD_CASES(&JSONOutputTests, {
+ADD_CASES(TC_JSONOut, {
     {"\"name\": \"BM_Repeat/repeats:3\",$"},
     {"\"name\": \"BM_Repeat/repeats:3\",$"},
     {"\"name\": \"BM_Repeat/repeats:3\",$"},
     {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
     {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}
 });
-ADD_CASES(&CSVOutputTests, {
+ADD_CASES(TC_CSVOut, {
-    {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3\",%csv_report$"},
-    {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3\",%csv_report$"},
-    {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3\",%csv_report$"},
-    {"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
-    {"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+    {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}
 });
 
 // Test that a non-repeated test still prints non-aggregate results even when
 // only-aggregate reports have been requested
 void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} }
 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
-ADD_CASES(&ConsoleOutputTests, {
+ADD_CASES(TC_ConsoleOut, {
-    {"^BM_RepeatOnce/repeats:1[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+    {"^BM_RepeatOnce/repeats:1 %console_report$"}
 });
-ADD_CASES(&JSONOutputTests, {
+ADD_CASES(TC_JSONOut, {
     {"\"name\": \"BM_RepeatOnce/repeats:1\",$"}
 });
-ADD_CASES(&CSVOutputTests, {
+ADD_CASES(TC_CSVOut, {
-    {"^\"BM_RepeatOnce/repeats:1\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+    {"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}
 });
 
+
 // Test that non-aggregate data is not reported
 void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} }
 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
-ADD_CASES(&ConsoleOutputTests, {
+ADD_CASES(TC_ConsoleOut, {
     {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
-    {"^BM_SummaryRepeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
-    {"^BM_SummaryRepeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+    {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}
 });
-ADD_CASES(&JSONOutputTests, {
+ADD_CASES(TC_JSONOut, {
     {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
     {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
     {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}
 });
-ADD_CASES(&CSVOutputTests, {
+ADD_CASES(TC_CSVOut, {
     {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
-    {"^\"BM_SummaryRepeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
-    {"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+    {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}
 });
 
 // ========================================================================= //
@@ -289,49 +154,5 @@ ADD_CASES(&CSVOutputTests, {
 
 
 int main(int argc, char* argv[]) {
-  benchmark::Initialize(&argc, argv);
+  RunOutputTests(argc, argv);
-  benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
-  benchmark::JSONReporter JR;
-  benchmark::CSVReporter CSVR;
-  struct ReporterTest {
-    const char* name;
-    std::vector<TestCase>& output_cases;
-    std::vector<TestCase>& error_cases;
-    benchmark::BenchmarkReporter& reporter;
-    std::stringstream out_stream;
-    std::stringstream err_stream;
-
-    ReporterTest(const char* n,
-                 std::vector<TestCase>& out_tc,
-                 std::vector<TestCase>& err_tc,
-                 benchmark::BenchmarkReporter& br)
-        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
-        reporter.SetOutputStream(&out_stream);
-        reporter.SetErrorStream(&err_stream);
-    }
-  } TestCases[] = {
-      {"ConsoleReporter", ConsoleOutputTests, ConsoleErrorTests, CR},
-      {"JSONReporter", JSONOutputTests, JSONErrorTests, JR},
-      {"CSVReporter", CSVOutputTests, CSVErrorTests, CSVR}
-  };
-
-  // Create the test reporter and run the benchmarks.
-  std::cout << "Running benchmarks...\n";
-  TestReporter test_rep({&CR, &JR, &CSVR});
-  benchmark::RunSpecifiedBenchmarks(&test_rep);
-
-  for (auto& rep_test : TestCases) {
-      std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
-      std::string banner(msg.size() - 1, '-');
-      std::cout << banner << msg << banner << "\n";
-
-      std::cerr << rep_test.err_stream.str();
-      std::cout << rep_test.out_stream.str();
-
-      TestCase::CheckCases(rep_test.error_cases, rep_test.err_stream);
-      TestCase::CheckCases(rep_test.output_cases, rep_test.out_stream);
-
-      std::cout << "\n";
-  }
-  return 0;
 }