From 4c2e32f1d02a18074c60252aab7bf97e76f32db2 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 2 Jun 2021 18:06:45 +0300 Subject: [PATCH] Introduce "family index" field into JSON output (#1164) It may be useful for those wishing to further post-process JSON results, but it is mainly geared towards better support for run interleaving, where results from the same family may not be close-by in the JSON. While we won't be able to do much about that for outputs, the tools can and perhaps should reorder the results to that at least in their output they are in proper order, not run order. Note that this only counts the families that were filtered-in, so if e.g. there were three families, and we filtered-out the second one, the two families (which were first and third) will have family indexes 0 and 1. --- include/benchmark/benchmark.h | 1 + src/benchmark_api_internal.cc | 3 +- src/benchmark_api_internal.h | 6 ++- src/benchmark_register.cc | 11 ++++- src/benchmark_runner.cc | 1 + src/complexity.cc | 2 + src/json_reporter.cc | 1 + src/statistics.cc | 1 + test/BUILD | 1 + test/complexity_test.cc | 63 +++++++++++++++------------- test/filter_test.cc | 22 ++++++++-- test/memory_manager_test.cc | 1 + test/repetitions_test.cc | 11 +++++ test/reporter_output_test.cc | 49 ++++++++++++++++++++++ test/user_counters_tabular_test.cc | 5 +++ test/user_counters_test.cc | 12 ++++++ test/user_counters_thousands_test.cc | 5 +++ 17 files changed, 159 insertions(+), 36 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 58d92375..693c2b82 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1418,6 +1418,7 @@ class BenchmarkReporter { std::string benchmark_name() const; BenchmarkName run_name; + int64_t family_index; RunType run_type; std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 553ff44e..c4813172 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -7,10 +7,11 @@ namespace benchmark { namespace internal { -BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, +BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, const std::vector& args, int thread_count) : benchmark_(*benchmark), + family_index_(family_idx), aggregation_report_mode_(benchmark_.aggregation_report_mode_), args_(args), time_unit_(benchmark_.time_unit_), diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index b0595e5a..5ee6fadf 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -17,10 +17,11 @@ namespace internal { // Information kept per benchmark we may want to run class BenchmarkInstance { public: - BenchmarkInstance(Benchmark* benchmark, const std::vector& args, - int threads); + BenchmarkInstance(Benchmark* benchmark, int family_index, + const std::vector& args, int threads); const BenchmarkName& name() const { return name_; } + int family_index() const { return family_index_; } AggregationReportMode aggregation_report_mode() const { return aggregation_report_mode_; } @@ -45,6 +46,7 @@ class BenchmarkInstance { private: BenchmarkName name_; Benchmark& benchmark_; + const int family_index_; AggregationReportMode aggregation_report_mode_; const std::vector& args_; TimeUnit time_unit_; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index ebea2d92..bc6c6686 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -129,8 +129,12 @@ bool BenchmarkFamilies::FindBenchmarks( // Special list of thread counts to use when none are specified const std::vector one_thread = {1}; + int next_family_index = 0; + MutexLock l(mutex_); for (std::unique_ptr& family : families_) { + int family_index = next_family_index; + // Family was deleted or benchmark doesn't match if (!family) continue; @@ -154,13 +158,18 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { - BenchmarkInstance instance(family.get(), args, num_threads); + BenchmarkInstance instance(family.get(), family_index, args, + num_threads); const auto full_name = instance.name().str(); if ((re.Match(full_name) && !isNegativeFilter) || (!re.Match(full_name) && isNegativeFilter)) { instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); + + // Only bump the next family index once we've estabilished that + // at least one instance of this family will be run. + if (next_family_index == family_index) ++next_family_index; } } } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index d4eb7d9f..c71fc719 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -72,6 +72,7 @@ BenchmarkReporter::Run CreateRunReport( BenchmarkReporter::Run report; report.run_name = b.name(); + report.family_index = b.family_index(); report.error_occurred = results.has_error_; report.error_message = results.error_message_; report.report_label = results.report_label_; diff --git a/src/complexity.cc b/src/complexity.cc index aeed67f0..22f42636 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -193,6 +193,7 @@ std::vector ComputeBigO( // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; big_o.run_name = run_name; + big_o.family_index = reports[0].family_index; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; big_o.repetitions = reports[0].repetitions; big_o.repetition_index = Run::no_repetition_index; @@ -215,6 +216,7 @@ std::vector ComputeBigO( // Only add label to mean/stddev if it is same for all runs Run rms; rms.run_name = run_name; + rms.family_index = reports[0].family_index; rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.aggregate_name = "RMS"; rms.report_label = big_o.report_label; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index effa6bd1..5a7d843d 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -207,6 +207,7 @@ void JSONReporter::PrintRunData(Run const& run) { std::string indent(6, ' '); std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name()) << ",\n"; + out << indent << FormatKV("family_index", run.family_index) << ",\n"; out << indent << FormatKV("run_name", run.run_name.str()) << ",\n"; out << indent << FormatKV("run_type", [&run]() -> const char* { switch (run.run_type) { diff --git a/src/statistics.cc b/src/statistics.cc index bd5a3d65..c9235e0d 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -148,6 +148,7 @@ std::vector ComputeStats( // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; data.run_name = reports[0].run_name; + data.family_index = reports[0].family_index; data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.threads = reports[0].threads; data.repetitions = reports[0].repetitions; diff --git a/test/BUILD b/test/BUILD index 9bb8cb02..1f27f99e 100644 --- a/test/BUILD +++ b/test/BUILD @@ -20,6 +20,7 @@ TEST_ARGS = ["--benchmark_min_time=0.01"] PER_SRC_TEST_ARGS = ({ "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], + "repetitions_test.cc": [" --benchmark_repetitions=3"], }) load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 5681fdcf..34731e8d 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -13,7 +13,8 @@ namespace { int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) int AddComplexityTest(std::string test_name, std::string big_o_test_name, - std::string rms_test_name, std::string big_o) { + std::string rms_test_name, std::string big_o, + int family_index) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, @@ -25,25 +26,29 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); - AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, - {"\"run_name\": \"%name\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": %int,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"aggregate_name\": \"BigO\",$", MR_Next}, - {"\"cpu_coefficient\": %float,$", MR_Next}, - {"\"real_coefficient\": %float,$", MR_Next}, - {"\"big_o\": \"%bigo\",$", MR_Next}, - {"\"time_unit\": \"ns\"$", MR_Next}, - {"}", MR_Next}, - {"\"name\": \"%rms_name\",$"}, - {"\"run_name\": \"%name\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": %int,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"aggregate_name\": \"RMS\",$", MR_Next}, - {"\"rms\": %float$", MR_Next}, - {"}", MR_Next}}); + AddCases( + TC_JSONOut, + {{"\"name\": \"%bigo_name\",$"}, + {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, + {"\"run_name\": \"%name\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": %int,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"BigO\",$", MR_Next}, + {"\"cpu_coefficient\": %float,$", MR_Next}, + {"\"real_coefficient\": %float,$", MR_Next}, + {"\"big_o\": \"%bigo\",$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}, + {"\"name\": \"%rms_name\",$"}, + {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, + {"\"run_name\": \"%name\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": %int,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"RMS\",$", MR_Next}, + {"\"rms\": %float$", MR_Next}, + {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, {"^\"%bigo_name\"", MR_Not}, {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); @@ -82,15 +87,15 @@ const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_big_o_1); + enum_big_o_1, /*family_index=*/0); // Add auto enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - auto_big_o_1); + auto_big_o_1, /*family_index=*/1); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1); + lambda_big_o_1, /*family_index=*/2); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -137,11 +142,11 @@ const char *lambda_big_o_n = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n); + enum_auto_big_o_n, /*family_index=*/3); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n); + lambda_big_o_n, /*family_index=*/4); // ========================================================================= // // ------------------------- Testing BigO O(N*lgN) ------------------------- // @@ -178,11 +183,13 @@ const char *lambda_big_o_n_lg_n = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, + /*family_index=*/6); // Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + /*family_index=*/7); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -204,7 +211,7 @@ const std::string complexity_capture_name = "BM_ComplexityCaptureArgs/capture_test"; ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N"); + complexity_capture_name + "_RMS", "N", /*family_index=*/9); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/test/filter_test.cc b/test/filter_test.cc index fece6dc2..1c198913 100644 --- a/test/filter_test.cc +++ b/test/filter_test.cc @@ -1,15 +1,15 @@ -#include "benchmark/benchmark.h" - +#include #include #include #include #include - #include #include #include #include +#include "benchmark/benchmark.h" + namespace { class TestReporter : public benchmark::ConsoleReporter { @@ -20,17 +20,22 @@ class TestReporter : public benchmark::ConsoleReporter { virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { ++count_; + max_family_index_ = + std::max(max_family_index_, report[0].family_index); ConsoleReporter::ReportRuns(report); }; - TestReporter() : count_(0) {} + TestReporter() : count_(0), max_family_index_(0) {} virtual ~TestReporter() {} size_t GetCount() const { return count_; } + size_t GetMaxFamilyIndex() const { return max_family_index_; } + private: mutable size_t count_; + mutable size_t max_family_index_; }; } // end namespace @@ -98,6 +103,15 @@ int main(int argc, char **argv) { << std::endl; return -1; } + + const size_t max_family_index = test_reporter.GetMaxFamilyIndex(); + const size_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; + if (num_families != expected_reports) { + std::cerr << "ERROR: Expected " << expected_reports + << " test families to be run but num_families = " + << num_families << std::endl; + return -1; + } } return 0; diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 4e0e6495..0c312744 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -21,6 +21,7 @@ BENCHMARK(BM_empty); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/repetitions_test.cc b/test/repetitions_test.cc index 5980c993..a5ab8759 100644 --- a/test/repetitions_test.cc +++ b/test/repetitions_test.cc @@ -24,6 +24,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_ExplicitRepetitions/repeats:2_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -36,6 +37,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -48,6 +50,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_mean\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -60,6 +63,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -72,6 +76,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -108,6 +113,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_mean %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_median %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -119,6 +125,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -130,6 +137,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -141,6 +149,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -152,6 +161,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -163,6 +173,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 31bedfa7..91c6f497 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -71,6 +71,7 @@ BENCHMARK(BM_basic); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -99,6 +100,7 @@ BENCHMARK(BM_bytes_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " "bytes_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -128,6 +130,7 @@ BENCHMARK(BM_items_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " "items_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, + {"\"family_index\": 2,$", MR_Next}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -154,6 +157,7 @@ BENCHMARK(BM_label); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, + {"\"family_index\": 3,$", MR_Next}, {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -181,6 +185,7 @@ BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_nanosecond\",$"}, + {"\"family_index\": 4,$", MR_Next}, {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -202,6 +207,7 @@ BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_microsecond\",$"}, + {"\"family_index\": 5,$", MR_Next}, {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -223,6 +229,7 @@ BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_millisecond\",$"}, + {"\"family_index\": 6,$", MR_Next}, {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -243,6 +250,7 @@ BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"}, + {"\"family_index\": 7,$", MR_Next}, {"\"run_name\": \"BM_time_label_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -267,6 +275,7 @@ void BM_error(benchmark::State& state) { BENCHMARK(BM_error); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, + {"\"family_index\": 8,$", MR_Next}, {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -289,6 +298,7 @@ void BM_no_arg_name(benchmark::State& state) { BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, + {"\"family_index\": 9,$", MR_Next}, {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -307,6 +317,7 @@ void BM_arg_name(benchmark::State& state) { BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, + {"\"family_index\": 10,$", MR_Next}, {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -327,6 +338,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, + {"\"family_index\": 11,$", MR_Next}, {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -346,6 +358,7 @@ BENCHMARK(BM_name)->Name("BM_custom_name"); ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"}, + {"\"family_index\": 12,$", MR_Next}, {"\"run_name\": \"BM_custom_name\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -405,18 +418,21 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"}, {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, {"\"repetition_index\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -424,6 +440,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -431,6 +448,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -452,24 +470,28 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"}, {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, {"\"repetition_index\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, {"\"repetition_index\": 2,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -477,6 +499,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -484,6 +507,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -507,30 +531,35 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"}, {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 2,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 3,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -538,6 +567,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -545,6 +575,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -568,6 +599,7 @@ void BM_RepeatOnce(benchmark::State& state) { BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, + {"\"family_index\": 18,$", MR_Next}, {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -590,6 +622,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"family_index\": 19,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -597,6 +630,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, + {"\"family_index\": 19,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -604,6 +638,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, + {"\"family_index\": 19,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -632,6 +667,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, + {"\"family_index\": 20,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -639,6 +675,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, + {"\"family_index\": 20,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -646,6 +683,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, + {"\"family_index\": 20,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -678,6 +716,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, + {"\"family_index\": 21,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -686,6 +725,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, + {"\"family_index\": 21,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -694,6 +734,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, + {"\"family_index\": 21,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -746,6 +787,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " ADD_CASES( TC_JSONOut, {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -755,6 +797,7 @@ ADD_CASES( {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -764,6 +807,7 @@ ADD_CASES( {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -773,6 +817,7 @@ ADD_CASES( {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -782,6 +827,7 @@ ADD_CASES( {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -791,6 +837,7 @@ ADD_CASES( {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -800,6 +847,7 @@ ADD_CASES( {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -831,6 +879,7 @@ void BM_JSON_Format(benchmark::State& state) { } BENCHMARK(BM_JSON_Format); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"}, + {"\"family_index\": 23,$", MR_Next}, {"\"run_name\": \"BM_JSON_Format\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index d49ce55f..056923e3 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -71,6 +71,7 @@ void BM_Counters_Tabular(benchmark::State& state) { BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -123,6 +124,7 @@ void BM_CounterRates_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -174,6 +176,7 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, + {"\"family_index\": 2,$", MR_Next}, {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -212,6 +215,7 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, + {"\"family_index\": 3,$", MR_Next}, {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -254,6 +258,7 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, + {"\"family_index\": 4,$", MR_Next}, {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 4d53f5c4..29b33f2d 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -78,6 +79,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " "foo=%hrfloat items_per_second=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -125,6 +127,7 @@ ADD_CASES( TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, + {"\"family_index\": 2,$", MR_Next}, {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -165,6 +168,7 @@ BENCHMARK(BM_Invert); ADD_CASES(TC_ConsoleOut, {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, + {"\"family_index\": 3,$", MR_Next}, {"\"run_name\": \"BM_Invert\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -207,6 +211,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report " "bar=%hrfloats foo=%hrfloats$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_InvertedRate\",$"}, + {"\"family_index\": 4,$", MR_Next}, {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -246,6 +251,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"family_index\": 5,$", MR_Next}, {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -285,6 +291,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " "%console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"family_index\": 6,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -327,6 +334,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"family_index\": 7,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -367,6 +375,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, + {"\"family_index\": 8,$", MR_Next}, {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -412,6 +421,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, + {"\"family_index\": 9,$", MR_Next}, {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -455,6 +465,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, + {"\"family_index\": 10,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -498,6 +509,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, + {"\"family_index\": 11,$", MR_Next}, {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index 21d8285d..842b3668 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -51,6 +51,7 @@ ADD_CASES( }); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -68,6 +69,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -85,6 +87,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -102,6 +105,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -119,6 +123,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},