benchmark/test/output_test_helper.cc

#include <iostream>
#include <map>
#include <memory>
#include <sstream>
#include <cstring>

#include "../src/check.h"  // NOTE: check.h is for internal use only!
#include "../src/re.h"     // NOTE: re.h is for internal use only
#include "output_test.h"
#include "../src/benchmark_api_internal.h"

// ========================================================================= //
// ------------------------------ Internals -------------------------------- //
// ========================================================================= //
namespace internal {
namespace {

using TestCaseList = std::vector<TestCase>;

// Use a vector because the order elements are added matters during iteration.
// std::map/unordered_map don't guarantee that.
// For example:
//  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
//     Substitute("%HelloWorld") // Always expands to Hello.
using SubMap = std::vector<std::pair<std::string, std::string>>;

TestCaseList& GetTestCaseList(TestCaseID ID) {
  // Uses function-local statics to ensure initialization occurs
  // before first use.
  static TestCaseList lists[TC_NumID];
  return lists[ID];
}

SubMap& GetSubstitutions() {
  // Don't use 'dec_re' from header because it may not yet be initialized.
  static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
  static SubMap map = {
      {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
      // human-readable float
      {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
      {"%int", "[ ]*[0-9]+"},
      {" %s ", "[ ]+"},
      {"%time", "[ ]*[0-9]{1,5} ns"},
      {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
      {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
      {"%csv_header",
       "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
       "items_per_second,label,error_occurred,error_message"},
      {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
      {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
      {"%csv_bytes_report",
       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
      {"%csv_items_report",
       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
      {"%csv_bytes_items_report",
       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
       "," + safe_dec_re + ",,,"},
      {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
      {"%csv_label_report_end", ",,"}};
  return map;
}

std::string PerformSubstitutions(std::string source) {
  SubMap const& subs = GetSubstitutions();
  using SizeT = std::string::size_type;
  for (auto const& KV : subs) {
    SizeT pos;
    SizeT next_start = 0;
    while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
      next_start = pos + KV.second.size();
      source.replace(pos, KV.first.size(), KV.second);
    }
  }
  return source;
}

void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
               TestCaseList const& not_checks) {
  std::string first_line;
  bool on_first = true;
  std::string line;
  while (remaining_output.eof() == false) {
    CHECK(remaining_output.good());
    std::getline(remaining_output, line);
    if (on_first) {
      first_line = line;
      on_first = false;
    }
    for (const auto& NC : not_checks) {
      CHECK(!NC.regex->Match(line))
          << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
          << NC.regex_str << "\""
          << "\n    actual regex string \"" << TC.substituted_regex << "\""
          << "\n    started matching near: " << first_line;
    }
    if (TC.regex->Match(line)) return;
    CHECK(TC.match_rule != MR_Next)
        << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
        << "\""
        << "\n    actual regex string \"" << TC.substituted_regex << "\""
        << "\n    started matching near: " << first_line;
  }
  CHECK(remaining_output.eof() == false)
      << "End of output reached before match for regex \"" << TC.regex_str
      << "\" was found"
      << "\n    actual regex string \"" << TC.substituted_regex << "\""
      << "\n    started matching near: " << first_line;
}

void CheckCases(TestCaseList const& checks, std::stringstream& output) {
  std::vector<TestCase> not_checks;
  for (size_t i = 0; i < checks.size(); ++i) {
    const auto& TC = checks[i];
    if (TC.match_rule == MR_Not) {
      not_checks.push_back(TC);
      continue;
    }
    CheckCase(output, TC, not_checks);
    not_checks.clear();
  }
}

class TestReporter : public benchmark::BenchmarkReporter {
 public:
  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
      : reporters_(reps) {}

  virtual bool ReportContext(const Context& context) {
    bool last_ret = false;
    bool first = true;
    for (auto rep : reporters_) {
      bool new_ret = rep->ReportContext(context);
      CHECK(first || new_ret == last_ret)
          << "Reports return different values for ReportContext";
      first = false;
      last_ret = new_ret;
    }
    (void)first;
    return last_ret;
  }

  void ReportRuns(const std::vector<Run>& report) {
    for (auto rep : reporters_) rep->ReportRuns(report);
  }
  void Finalize() {
    for (auto rep : reporters_) rep->Finalize();
  }

 private:
  std::vector<benchmark::BenchmarkReporter *> reporters_;
};
}

}  // end namespace internal

// ========================================================================= //
// -------------------------- Results checking ----------------------------- //
// ========================================================================= //

namespace internal {

// Utility class to manage subscribers for checking benchmark results.
// It works by parsing the CSV output to read the results.
class ResultsChecker {
 public:

  struct PatternAndFn : public TestCase { // reusing TestCase for its regexes
    PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
    : TestCase(rx), fn(fn_) {}
    ResultsCheckFn fn;
  };

  std::vector< PatternAndFn > check_patterns;
  std::vector< Results > results;
  std::vector< std::string > field_names;

  void Add(const std::string& entry_pattern, ResultsCheckFn fn);

  void CheckResults(std::stringstream& output);

 private:

  void SetHeader_(const std::string& csv_header);
  void SetValues_(const std::string& entry_csv_line);

  std::vector< std::string > SplitCsv_(const std::string& line);

};

// store the static ResultsChecker in a function to prevent initialization
// order problems
ResultsChecker& GetResultsChecker() {
  static ResultsChecker rc;
  return rc;
}

// add a results checker for a benchmark
void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
  check_patterns.emplace_back(entry_pattern, fn);
}

// check the results of all subscribed benchmarks
void ResultsChecker::CheckResults(std::stringstream& output) {
  // first reset the stream to the start
  {
    auto start = std::ios::streampos(0);
    // clear before calling tellg()
    output.clear();
    // seek to zero only when needed
    if(output.tellg() > start) output.seekg(start);
    // and just in case
    output.clear();
  }
  // now go over every line and publish it to the ResultsChecker
  std::string line;
  bool on_first = true;
  while (output.eof() == false) {
    CHECK(output.good());
    std::getline(output, line);
    if (on_first) {
      SetHeader_(line); // this is important
      on_first = false;
      continue;
    }
    SetValues_(line);
  }
  // finally we can call the subscribed check functions
  for(const auto& p : check_patterns) {
    VLOG(2) << "--------------------------------\n";
    VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
    for(const auto& r : results) {
      if(!p.regex->Match(r.name)) {
        VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
        continue;
      } else {
        VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
      }
      VLOG(1) << "Checking results of " << r.name << ": ... \n";
      p.fn(r);
      VLOG(1) << "Checking results of " << r.name << ": OK.\n";
    }
  }
}

// prepare for the names in this header
void ResultsChecker::SetHeader_(const std::string& csv_header) {
  field_names = SplitCsv_(csv_header);
}

// set the values for a benchmark
void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
  if(entry_csv_line.empty()) return; // some lines are empty
  CHECK(!field_names.empty());
  auto vals = SplitCsv_(entry_csv_line);
  CHECK_EQ(vals.size(), field_names.size());
  results.emplace_back(vals[0]); // vals[0] is the benchmark name
  auto &entry = results.back();
  for (size_t i = 1, e = vals.size(); i < e; ++i) {
    entry.values[field_names[i]] = vals[i];
  }
}

// a quick'n'dirty csv splitter (eliminating quotes)
std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) {
  std::vector< std::string > out;
  if(line.empty()) return out;
  if(!field_names.empty()) out.reserve(field_names.size());
  size_t prev = 0, pos = line.find_first_of(','), curr = pos;
  while(pos != line.npos) {
    CHECK(curr > 0);
    if(line[prev] == '"') ++prev;
    if(line[curr-1] == '"') --curr;
    out.push_back(line.substr(prev, curr-prev));
    prev = pos + 1;
    pos = line.find_first_of(',', pos + 1);
    curr = pos;
  }
  curr = line.size();
  if(line[prev] == '"') ++prev;
  if(line[curr-1] == '"') --curr;
  out.push_back(line.substr(prev, curr-prev));
  return out;
}

}  // end namespace internal

size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
{
  auto &rc = internal::GetResultsChecker();
  rc.Add(bm_name, fn);
  return rc.results.size();
}

int Results::NumThreads() const {
  auto pos = name.find("/threads:");
  if(pos == name.npos) return 1;
  auto end = name.find('/', pos + 9);
  std::stringstream ss;
  ss << name.substr(pos + 9, end);
  int num = 1;
  ss >> num;
  CHECK(!ss.fail());
  return num;
}

double Results::GetTime(BenchmarkTime which) const {
  CHECK(which == kCpuTime || which == kRealTime);
  const char *which_str = which == kCpuTime ? "cpu_time" : "real_time";
  double val = GetAs< double >(which_str);
  auto unit = Get("time_unit");
  CHECK(unit);
  if(*unit == "ns") {
    return val * 1.e-9;
  } else if(*unit == "us") {
    return val * 1.e-6;
  } else if(*unit == "ms") {
    return val * 1.e-3;
  } else if(*unit == "s") {
    return val;
  } else {
    CHECK(1 == 0) << "unknown time unit: " << *unit;
    return 0;
  }
}

// ========================================================================= //
// -------------------------- Public API Definitions------------------------ //
// ========================================================================= //

TestCase::TestCase(std::string re, int rule)
    : regex_str(std::move(re)),
      match_rule(rule),
      substituted_regex(internal::PerformSubstitutions(regex_str)),
      regex(std::make_shared<benchmark::Regex>()) {
  std::string err_str;
  regex->Init(substituted_regex,& err_str);
  CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
                         << "\""
                         << "\n    originally \"" << regex_str << "\""
                         << "\n    got error: " << err_str;
}

int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
  auto& L = internal::GetTestCaseList(ID);
  L.insert(L.end(), il);
  return 0;
}

int SetSubstitutions(
    std::initializer_list<std::pair<std::string, std::string>> il) {
  auto& subs = internal::GetSubstitutions();
  for (auto KV : il) {
    bool exists = false;
    KV.second = internal::PerformSubstitutions(KV.second);
    for (auto& EKV : subs) {
      if (EKV.first == KV.first) {
        EKV.second = std::move(KV.second);
        exists = true;
        break;
      }
    }
    if (!exists) subs.push_back(std::move(KV));
  }
  return 0;
}

void RunOutputTests(int argc, char* argv[]) {
  using internal::GetTestCaseList;
  benchmark::Initialize(&argc, argv);
  auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true);
  benchmark::ConsoleReporter CR(options);
  benchmark::JSONReporter JR;
  benchmark::CSVReporter CSVR;
  struct ReporterTest {
    const char* name;
    std::vector<TestCase>& output_cases;
    std::vector<TestCase>& error_cases;
    benchmark::BenchmarkReporter& reporter;
    std::stringstream out_stream;
    std::stringstream err_stream;

    ReporterTest(const char* n, std::vector<TestCase>& out_tc,
                 std::vector<TestCase>& err_tc,
                 benchmark::BenchmarkReporter& br)
        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
      reporter.SetOutputStream(&out_stream);
      reporter.SetErrorStream(&err_stream);
    }
  } TestCases[] = {
      {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
       GetTestCaseList(TC_ConsoleErr), CR},
      {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
       JR},
      {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
       CSVR},
  };

  // Create the test reporter and run the benchmarks.
  std::cout << "Running benchmarks...\n";
  internal::TestReporter test_rep({&CR, &JR, &CSVR});
  benchmark::RunSpecifiedBenchmarks(&test_rep);

  for (auto& rep_test : TestCases) {
    std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
    std::string banner(msg.size() - 1, '-');
    std::cout << banner << msg << banner << "\n";

    std::cerr << rep_test.err_stream.str();
    std::cout << rep_test.out_stream.str();

    internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
    internal::CheckCases(rep_test.output_cases, rep_test.out_stream);

    std::cout << "\n";
  }

  // now that we know the output is as expected, we can dispatch
  // the checks to subscribees.
  auto &csv = TestCases[2];
  // would use == but gcc spits a warning
  CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
  internal::GetResultsChecker().CheckResults(csv.out_stream);
}