dolysis
/
benchmark
mirror of https://github.com/google/benchmark.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow custom reporters

This commit is contained in:
Dominic Hamon 2014-01-07 16:33:40 -08:00
parent 2ff306af30
commit acc65f48d3
2 changed files with 102 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
include/benchmark/benchmark.h
View File

@ -145,11 +145,13 @@ BENCHMARK(BM_MultiThreaded)->Threads(4);
#include "macros.h"
namespace benchmark {
// If the --benchmarks flag is empty, do nothing.
//
// Otherwise, run all benchmarks specified by the --benchmarks flag,
class BenchmarkReporter;
void Initialize(int* argc, const char** argv);
// Otherwise, run all benchmarks specified by the --benchmark_filter flag,
// and exit after running the benchmarks.
extern void RunSpecifiedBenchmarks();
void RunSpecifiedBenchmarks(const BenchmarkReporter* reporter = nullptr);
// ------------------------------------------------------
// Routines that can be called from within a benchmark
@ -290,15 +292,73 @@ class State {
DISALLOW_COPY_AND_ASSIGN(State);
};
// Interface for custom benchmark result printers.
// By default, benchmark reports are printed to stdout. However an application
// can control the destination of the reports by calling
// RunSpecifiedBenchmarks and passing it a custom reporter object.
// The reporter object must implement the following interface.
class BenchmarkReporter {
public:
struct Context {
int num_cpus;
double mhz_per_cpu;
//std::string cpu_info;
bool cpu_scaling_enabled;
// The number of chars in the longest benchmark name.
int name_field_width;
};
struct Run {
Run() :
thread_index(-1),
iterations(1),
real_accumulated_time(0),
cpu_accumulated_time(0),
bytes_per_second(0),
items_per_second(0),
max_heapbytes_used(0) {}
std::string benchmark_name;
std::string report_label;
int thread_index;
int64_t iterations;
double real_accumulated_time;
double cpu_accumulated_time;
// Zero if not set by benchmark.
double bytes_per_second;
double items_per_second;
// This is set to 0.0 if memory tracing is not enabled.
double max_heapbytes_used;
};
// Called once for every suite of benchmarks run.
// The parameter "context" contains information that the
// reporter may wish to use when generating its report, for example the
// platform under which the benchmarks are running. The benchmark run is
// never started if this function returns false, allowing the reporter
// to skip runs based on the context information.
virtual bool ReportContext(const Context& context) const = 0;
// Called once for each group of benchmark runs, gives information about
// cpu-time and heap memory usage during the benchmark run.
// Note that all the grouped benchmark runs should refer to the same
// benchmark, thus have the same name.
virtual void ReportRuns(const std::vector<Run>& report) const = 0;
virtual ~BenchmarkReporter() {}
};
namespace internal {
class BenchmarkReporter;
typedef std::function<void(State&)> BenchmarkFunction;
// Run all benchmarks whose name is a partial match for the regular
// expression in "spec". The results of benchmark runs are fed to "reporter".
void RunMatchingBenchmarks(const std::string& spec,
BenchmarkReporter* reporter);
const BenchmarkReporter* reporter);
// Extract the list of benchmark names that match the specified regular
// expression.
@ -411,101 +471,34 @@ class Benchmark {
static void AddRange(std::vector<int>* dst, int lo, int hi, int mult);
static double MeasurePeakHeapMemory(const Instance& b);
static void RunInstance(const Instance& b, BenchmarkReporter* br);
static void RunInstance(const Instance& b, const BenchmarkReporter* br);
friend class ::benchmark::State;
friend struct ::benchmark::internal::Benchmark::Instance;
friend void ::benchmark::internal::RunMatchingBenchmarks(
const std::string&, BenchmarkReporter*);
const std::string&, const BenchmarkReporter*);
DISALLOW_COPY_AND_ASSIGN(Benchmark);
};
// ------------------------------------------------------
// Benchmarks reporter interface + data containers.
struct BenchmarkContextData {
int num_cpus;
double mhz_per_cpu;
//std::string cpu_info;
bool cpu_scaling_enabled;
// The number of chars in the longest benchmark name.
int name_field_width;
};
struct BenchmarkRunData {
BenchmarkRunData() :
thread_index(-1),
iterations(1),
real_accumulated_time(0),
cpu_accumulated_time(0),
bytes_per_second(0),
items_per_second(0),
max_heapbytes_used(0) {}
std::string benchmark_name;
std::string report_label;
int thread_index;
int64_t iterations;
double real_accumulated_time;
double cpu_accumulated_time;
// Zero if not set by benchmark.
double bytes_per_second;
double items_per_second;
// This is set to 0.0 if memory tracing is not enabled.
double max_heapbytes_used;
};
// Interface for custom benchmark result printers.
// By default, benchmark reports are printed to stdout. However an application
// can control the destination of the reports by calling
// RunMatchingBenchmarks and passing it a custom reporter object.
// The reporter object must implement the following interface.
class BenchmarkReporter {
public:
// Called once for every suite of benchmarks run.
// The parameter "context" contains information that the
// reporter may wish to use when generating its report, for example the
// platform under which the benchmarks are running. The benchmark run is
// never started if this function returns false, allowing the reporter
// to skip runs based on the context information.
virtual bool ReportContext(const BenchmarkContextData& context) = 0;
// Called once for each group of benchmark runs, gives information about
// cpu-time and heap memory usage during the benchmark run.
// Note that all the grouped benchmark runs should refer to the same
// benchmark, thus have the same name.
virtual void ReportRuns(const std::vector<BenchmarkRunData>& report) = 0;
virtual ~BenchmarkReporter();
};
// ------------------------------------------------------
// Internal implementation details follow; please ignore
// Given a collection of reports, computes their mean and stddev.
// REQUIRES: all runs in "reports" must be from the same benchmark.
void ComputeStats(const std::vector<BenchmarkRunData>& reports,
BenchmarkRunData* mean_data,
BenchmarkRunData* stddev_data);
// Simple reporter that outputs benchmark data to the console. This is the
// default reporter used by RunSpecifiedBenchmarks().
class ConsoleReporter : public BenchmarkReporter {
public:
virtual bool ReportContext(const BenchmarkContextData& context);
virtual void ReportRuns(const std::vector<BenchmarkRunData>& reports);
virtual bool ReportContext(const Context& context) const;
virtual void ReportRuns(const std::vector<Run>& reports) const;
private:
std::string PrintMemoryUsage(double bytes);
virtual void PrintRunData(const BenchmarkRunData& report);
int name_field_width_;
std::string PrintMemoryUsage(double bytes) const;
virtual void PrintRunData(const Run& report) const;
mutable int name_field_width_;
};
} // end namespace internal
void Initialize(int* argc, const char** argv);
} // end namespace benchmark
// ------------------------------------------------------

56
src/benchmark.cc
View File

@ -229,15 +229,11 @@ static bool CpuScalingEnabled() {
return false;
}
} // namespace
namespace internal {
BenchmarkReporter::~BenchmarkReporter() {}
void ComputeStats(const std::vector<BenchmarkRunData>& reports,
BenchmarkRunData* mean_data,
BenchmarkRunData* stddev_data) {
// Given a collection of reports, computes their mean and stddev.
// REQUIRES: all runs in "reports" must be from the same benchmark.
void ComputeStats(const std::vector<BenchmarkReporter::Run>& reports,
BenchmarkReporter::Run* mean_data,
BenchmarkReporter::Run* stddev_data) {
// Accumulators.
Stat1_d real_accumulated_time_stat;
Stat1_d cpu_accumulated_time_stat;
@ -247,7 +243,7 @@ void ComputeStats(const std::vector<BenchmarkRunData>& reports,
Stat1MinMax_d max_heapbytes_used_stat;
// Populate the accumulators.
for (std::vector<BenchmarkRunData>::const_iterator it = reports.begin();
for (std::vector<BenchmarkReporter::Run>::const_iterator it = reports.begin();
it != reports.end(); ++it) {
CHECK_EQ(reports[0].benchmark_name, it->benchmark_name);
real_accumulated_time_stat +=
@ -289,7 +285,11 @@ void ComputeStats(const std::vector<BenchmarkRunData>& reports,
stddev_data->max_heapbytes_used = max_heapbytes_used_stat.StdDev();
}
std::string ConsoleReporter::PrintMemoryUsage(double bytes) {
} // namespace
namespace internal {
std::string ConsoleReporter::PrintMemoryUsage(double bytes) const {
if (!get_memory_usage || bytes < 0.0)
return "";
@ -298,7 +298,8 @@ std::string ConsoleReporter::PrintMemoryUsage(double bytes) {
return ss.str();
}
bool ConsoleReporter::ReportContext(const BenchmarkContextData& context) {
bool ConsoleReporter::ReportContext(
const BenchmarkReporter::Context& context) const {
name_field_width_ = context.name_field_width;
std::cout << "Benchmarking on " << context.num_cpus << " X "
@ -326,8 +327,9 @@ bool ConsoleReporter::ReportContext(const BenchmarkContextData& context) {
return true;
}
void ConsoleReporter::ReportRuns(const std::vector<BenchmarkRunData>& reports) {
for (std::vector<BenchmarkRunData>::const_iterator it = reports.begin();
void ConsoleReporter::ReportRuns(
const std::vector<BenchmarkReporter::Run>& reports) const {
for (std::vector<BenchmarkReporter::Run>::const_iterator it = reports.begin();
it != reports.end(); ++it) {
CHECK_EQ(reports[0].benchmark_name, it->benchmark_name);
PrintRunData(*it);
@ -337,15 +339,15 @@ void ConsoleReporter::ReportRuns(const std::vector<BenchmarkRunData>& reports) {
if (reports.size() < 2)
return;
BenchmarkRunData mean_data;
BenchmarkRunData stddev_data;
internal::ComputeStats(reports, &mean_data, &stddev_data);
BenchmarkReporter::Run mean_data;
BenchmarkReporter::Run stddev_data;
ComputeStats(reports, &mean_data, &stddev_data);
PrintRunData(mean_data);
PrintRunData(stddev_data);
}
void ConsoleReporter::PrintRunData(const BenchmarkRunData& result) {
void ConsoleReporter::PrintRunData(const BenchmarkReporter::Run& result) const {
// Format bytes per second
std::string rate;
if (result.bytes_per_second > 0) {
@ -546,7 +548,7 @@ struct State::SharedState {
int stopping; // Number of threads that have entered STOPPING state
int threads; // Number of total threads that are running concurrently
ThreadStats stats;
std::vector<internal::BenchmarkRunData> runs; // accumulated runs
std::vector<BenchmarkReporter::Run> runs; // accumulated runs
std::string label;
explicit SharedState(const internal::Benchmark::Instance* b)
@ -776,7 +778,7 @@ void Benchmark::MeasureOverhead() {
#endif
}
void Benchmark::RunInstance(const Instance& b, BenchmarkReporter* br) {
void Benchmark::RunInstance(const Instance& b, const BenchmarkReporter* br) {
use_real_time = false;
running_benchmark = true;
// get_memory_usage = FLAGS_gbenchmark_memory_usage;
@ -821,7 +823,7 @@ void Benchmark::RunInstance(const Instance& b, BenchmarkReporter* br) {
*/
running_benchmark = false;
for (internal::BenchmarkRunData& report : state.runs) {
for (BenchmarkReporter::Run& report : state.runs) {
double seconds = (use_real_time ? report.real_accumulated_time :
report.cpu_accumulated_time);
report.benchmark_name = b.name;
@ -1014,8 +1016,7 @@ bool State::FinishInterval() {
return true;
}
internal::BenchmarkRunData data;
data.thread_index = thread_index;
BenchmarkReporter::Run data;
data.iterations = iterations_;
data.thread_index = thread_index;
@ -1106,8 +1107,7 @@ void* State::RunWrapper(void* arg) {
namespace internal {
void RunMatchingBenchmarks(const std::string& spec,
BenchmarkReporter* reporter) {
CHECK(reporter != NULL);
const BenchmarkReporter* reporter) {
if (spec.empty()) return;
std::vector<internal::Benchmark::Instance> benchmarks;
@ -1134,7 +1134,7 @@ void RunMatchingBenchmarks(const std::string& spec,
}
// Print header here
BenchmarkContextData context;
BenchmarkReporter::Context context;
context.num_cpus = NumCPUs();
context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f;
// context.cpu_info = base::CompactCPUIDInfoString();
@ -1158,12 +1158,12 @@ void FindMatchingBenchmarkNames(const std::string& spec,
} // end namespace internal
void RunSpecifiedBenchmarks() {
void RunSpecifiedBenchmarks(const BenchmarkReporter* reporter /*= nullptr*/) {
std::string spec = FLAGS_benchmark_filter;
if (spec.empty() || spec == "all")
spec = "."; // Regexp that matches all benchmarks
internal::ConsoleReporter default_reporter;
internal::RunMatchingBenchmarks(spec, &default_reporter);
internal::RunMatchingBenchmarks(spec, reporter == nullptr ? &default_reporter : reporter);
pthread_cond_destroy(&starting_cv);
pthread_mutex_destroy(&starting_mutex);
pthread_mutex_destroy(&benchmark_mutex);