From 7fad964a94da9364af1b31acae6d092d40591cff Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Mon, 18 Oct 2021 11:29:35 -0400 Subject: [PATCH] Introduce additional memory metrics (#1238) - added total_allocs and net_allocs - updated reporter code to report these, if available. --- include/benchmark/benchmark.h | 75 ++++++++++++++++++++++------------- src/benchmark_runner.cc | 17 +++++--- src/benchmark_runner.h | 2 + src/json_reporter.cc | 17 +++++++- 4 files changed, 75 insertions(+), 36 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index da93558e..3938f765 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -180,6 +180,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include #include +#include #include #include #include @@ -273,7 +274,6 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); namespace benchmark { class BenchmarkReporter; -class MemoryManager; void Initialize(int* argc, char** argv); void Shutdown(); @@ -299,6 +299,49 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); +// If a MemoryManager is registered (via RegisterMemoryManager()), +// it can be used to collect and report allocation metrics for a run of the +// benchmark. +class MemoryManager { + public: + static const int64_t TombstoneValue; + + struct Result { + Result() + : num_allocs(0), + max_bytes_used(0), + total_allocated_bytes(TombstoneValue), + net_heap_growth(TombstoneValue) {} + + // The number of allocations made in total between Start and Stop. + int64_t num_allocs; + + // The peak memory use between Start and Stop. + int64_t max_bytes_used; + + // The total memory allocated, in bytes, between Start and Stop. + // Init'ed to TombstoneValue if metric not available. + int64_t total_allocated_bytes; + + // The net changes in memory, in bytes, between Start and Stop. + // ie., total_allocated_bytes - total_deallocated_bytes. + // Init'ed to TombstoneValue if metric not available. + int64_t net_heap_growth; + }; + + virtual ~MemoryManager() {} + + // Implement this to start recording allocation information. + virtual void Start() = 0; + + // Implement this to stop recording and fill out the given Result structure. + BENCHMARK_DEPRECATED_MSG("Use Stop(Result&) instead") + virtual void Stop(Result* result) = 0; + + // FIXME(vyng): Make this pure virtual once we've migrated current users. + virtual void Stop(Result& result) { Stop(&result); } +}; + // Register a MemoryManager instance that will be used to collect and report // allocation measurements for benchmark runs. void RegisterMemoryManager(MemoryManager* memory_manager); @@ -1440,9 +1483,8 @@ class BenchmarkReporter { report_big_o(false), report_rms(false), counters(), - has_memory_result(false), - allocs_per_iter(0.0), - max_bytes_used(0) {} + memory_result(NULL), + allocs_per_iter(0.0) {} std::string benchmark_name() const; BenchmarkName run_name; @@ -1493,9 +1535,8 @@ class BenchmarkReporter { UserCounters counters; // Memory metrics. - bool has_memory_result; + const MemoryManager::Result* memory_result; double allocs_per_iter; - int64_t max_bytes_used; }; struct PerFamilyRunReports { @@ -1624,28 +1665,6 @@ class BENCHMARK_DEPRECATED_MSG( std::set user_counter_names_; }; -// If a MemoryManager is registered, it can be used to collect and report -// allocation metrics for a run of the benchmark. -class MemoryManager { - public: - struct Result { - Result() : num_allocs(0), max_bytes_used(0) {} - - // The number of allocations made in total between Start and Stop. - int64_t num_allocs; - - // The peak memory use between Start and Stop. - int64_t max_bytes_used; - }; - - virtual ~MemoryManager() {} - - // Implement this to start recording allocation information. - virtual void Start() = 0; - - // Implement this to stop recording and fill out the given Result structure. - virtual void Stop(Result* result) = 0; -}; inline const char* GetTimeUnitString(TimeUnit unit) { switch (unit) { diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index ead5c5a2..84ed3bf8 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -67,7 +67,7 @@ BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, const internal::ThreadManager::Result& results, IterationCount memory_iterations, - const MemoryManager::Result& memory_result, double seconds, + const MemoryManager::Result* memory_result, double seconds, int64_t repetition_index, int64_t repeats) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -99,12 +99,12 @@ BenchmarkReporter::Run CreateRunReport( report.counters = results.counters; if (memory_iterations > 0) { - report.has_memory_result = true; + assert(memory_result != nullptr); + report.memory_result = memory_result; report.allocs_per_iter = - memory_iterations ? static_cast(memory_result.num_allocs) / + memory_iterations ? static_cast(memory_result->num_allocs) / memory_iterations : 0; - report.max_bytes_used = memory_result.max_bytes_used; } internal::Finish(&report.counters, results.iterations, seconds, @@ -302,9 +302,14 @@ void BenchmarkRunner::DoOneRepetition() { } // Oh, one last thing, we need to also produce the 'memory measurements'.. - MemoryManager::Result memory_result; + MemoryManager::Result* memory_result = nullptr; IterationCount memory_iterations = 0; if (memory_manager != nullptr) { + // TODO(vyng): Consider making BenchmarkReporter::Run::memory_result an + // optional so we don't have to own the Result here. + // Can't do it now due to cxx03. + memory_results.push_back(MemoryManager::Result()); + memory_result = &memory_results.back(); // Only run a few iterations to reduce the impact of one-time // allocations in benchmarks that are not properly managed. memory_iterations = std::min(16, iters); @@ -316,7 +321,7 @@ void BenchmarkRunner::DoOneRepetition() { manager->WaitForAllThreads(); manager.reset(); - memory_manager->Stop(&memory_result); + memory_manager->Stop(memory_result); } // Ok, now actually report. diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 8427ce6a..752eefdc 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -76,6 +76,8 @@ class BenchmarkRunner { std::vector pool; + std::vector memory_results; + IterationCount iters; // preserved between repetitions! // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 22d5ce02..4f7912b5 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -295,9 +295,20 @@ void JSONReporter::PrintRunData(Run const& run) { out << ",\n" << indent << FormatKV(c.first, c.second); } - if (run.has_memory_result) { + if (run.memory_result) { + const MemoryManager::Result memory_result = *run.memory_result; out << ",\n" << indent << FormatKV("allocs_per_iter", run.allocs_per_iter); - out << ",\n" << indent << FormatKV("max_bytes_used", run.max_bytes_used); + out << ",\n" + << indent << FormatKV("max_bytes_used", memory_result.max_bytes_used); + + auto report_if_present = [&out, &indent](const char* label, int64_t val) { + if (val != MemoryManager::TombstoneValue) + out << ",\n" << indent << FormatKV(label, val); + }; + + report_if_present("total_allocated_bytes", + memory_result.total_allocated_bytes); + report_if_present("net_heap_growth", memory_result.net_heap_growth); } if (!run.report_label.empty()) { @@ -306,4 +317,6 @@ void JSONReporter::PrintRunData(Run const& run) { out << '\n'; } +const int64_t MemoryManager::TombstoneValue = std::numeric_limits::max(); + } // end namespace benchmark