From 33c133a2067316e462ad684e8a36207610da1bb6 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 4 May 2021 14:36:11 +0100 Subject: [PATCH] Add `benchmark_context` flag that allows per-run custom context. (#1127) * Add `benchmark_context` flag that allows per-run custom context. Add support for key-value flags in general. Added test for key-value flags. Added `benchmark_context` flag. Output content of `benchmark_context` to base reporter. Solves the first part of #525. * Docs and better help --- README.md | 21 +++++++++++ src/benchmark.cc | 12 +++++-- src/commandlineflags.cc | 58 ++++++++++++++++++++++++++++++ src/commandlineflags.h | 65 ++++++++++++++++++++-------------- src/reporter.cc | 7 ++++ test/commandlineflags_gtest.cc | 33 +++++++++++++++-- 6 files changed, 165 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e8c65f8d..478ac896 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,8 @@ too (`-lkstat`). [Result Comparison](#result-comparison) +[Extra Context](#extra-context) + ### Library [Runtime and Reporting Considerations](#runtime-and-reporting-considerations) @@ -442,6 +444,25 @@ BM_memcpy/32k 1834 ns 1837 ns 357143 It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) + + +### Extra Context + +Sometimes it's useful to add extra context to the content printed before the +results. By default this section includes information about the CPU on which +the benchmarks are running. If you do want to add more context, you can use +the `benchmark_context` command line flag: + +```bash +$ ./run_benchmarks --benchmark_context=pwd=`pwd` +Run on (1 x 2300 MHz CPU) +pwd: /home/user/benchmark/ +Benchmark Time CPU Iterations +---------------------------------------------------- +BM_memcpy/32 11 ns 11 ns 79545455 +BM_memcpy/32k 2181 ns 2185 ns 324074 +``` + ### Runtime and Reporting Considerations diff --git a/src/benchmark.cc b/src/benchmark.cc index 1fea654c..930a7567 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/benchmark.h" + #include "benchmark_api_internal.h" #include "benchmark_runner.h" #include "internal_macros.h" @@ -104,6 +105,10 @@ DEFINE_string(benchmark_color, "auto"); // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. DEFINE_bool(benchmark_counters_tabular, false); +// Extra context to include in the output formatted as comma-separated key-value +// pairs. +DEFINE_kvpairs(benchmark_context, {}); + // The level of verbose logging to output DEFINE_int32(v, 0); @@ -446,6 +451,7 @@ void PrintUsageAndExit() { " [--benchmark_out_format=]\n" " [--benchmark_color={auto|true|false}]\n" " [--benchmark_counters_tabular={true|false}]\n" + " [--benchmark_context==,...]\n" " [--v=]\n"); exit(0); } @@ -476,9 +482,11 @@ void ParseCommandLineFlags(int* argc, char** argv) { ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) || ParseBoolFlag(argv[i], "benchmark_counters_tabular", &FLAGS_benchmark_counters_tabular) || - ParseInt32Flag(argv[i], "v", &FLAGS_v) || ParseStringFlag(argv[i], "benchmark_perf_counters", - &FLAGS_benchmark_perf_counters)) { + &FLAGS_benchmark_perf_counters) || + ParseKeyValueFlag(argv[i], "benchmark_context", + &FLAGS_benchmark_context) || + ParseInt32Flag(argv[i], "v", &FLAGS_v)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; --(*argc); diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 0648fe3a..5724aaa2 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -20,6 +20,10 @@ #include #include #include +#include +#include + +#include "../src/string_util.h" namespace benchmark { namespace { @@ -78,6 +82,30 @@ bool ParseDouble(const std::string& src_text, const char* str, double* value) { return true; } +// Parses 'str' into KV pairs. If successful, writes the result to *value and +// returns true; otherwise leaves *value unchanged and returns false. +bool ParseKvPairs(const std::string& src_text, const char* str, + std::map* value) { + std::map kvs; + for (const auto& kvpair : StrSplit(str, ',')) { + const auto kv = StrSplit(kvpair, '='); + if (kv.size() != 2) { + std::cerr << src_text << " is expected to be a comma-separated list of " + << "= strings, but actually has value \"" << str + << "\".\n"; + return false; + } + if (!kvs.emplace(kv[0], kv[1]).second) { + std::cerr << src_text << " is expected to contain unique keys but key \"" + << kv[0] << "\" was repeated.\n"; + return false; + } + } + + *value = kvs; + return true; +} + // Returns the name of the environment variable corresponding to the // given flag. For example, FlagToEnvVar("foo") will return // "BENCHMARK_FOO" in the open-source version. @@ -129,6 +157,20 @@ const char* StringFromEnv(const char* flag, const char* default_val) { return value == nullptr ? default_val : value; } +std::map KvPairsFromEnv( + const char* flag, std::map default_val) { + const std::string env_var = FlagToEnvVar(flag); + const char* const value_str = getenv(env_var.c_str()); + + if (value_str == nullptr) return default_val; + + std::map value; + if (!ParseKvPairs("Environment variable " + env_var, value_str, &value)) { + return default_val; + } + return value; +} + // Parses a string as a command line flag. The string should have // the format "--flag=value". When def_optional is true, the "=value" // part can be omitted. @@ -206,6 +248,22 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value) { return true; } +bool ParseKeyValueFlag( + const char* str, const char* flag, + std::map* value) { + const char* const value_str = ParseFlagValue(str, flag, false); + + if (value_str == nullptr) return false; + + for (const auto& kvpair : StrSplit(value_str, ',')) { + const auto kv = StrSplit(kvpair, '='); + if (kv.size() != 2) return false; + value->emplace(kv[0], kv[1]); + } + + return true; +} + bool IsFlag(const char* str, const char* flag) { return (ParseFlagValue(str, flag, true) != nullptr); } diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 3a1f6a8d..0c988ccc 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -2,6 +2,7 @@ #define BENCHMARK_COMMANDLINEFLAGS_H_ #include +#include #include // Macro for referencing flags. @@ -12,51 +13,59 @@ #define DECLARE_int32(name) extern int32_t FLAG(name) #define DECLARE_double(name) extern double FLAG(name) #define DECLARE_string(name) extern std::string FLAG(name) +#define DECLARE_kvpairs(name) \ + extern std::map FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val) \ - bool FLAG(name) = \ - benchmark::BoolFromEnv(#name, default_val) -#define DEFINE_int32(name, default_val) \ - int32_t FLAG(name) = \ - benchmark::Int32FromEnv(#name, default_val) -#define DEFINE_double(name, default_val) \ - double FLAG(name) = \ - benchmark::DoubleFromEnv(#name, default_val) -#define DEFINE_string(name, default_val) \ - std::string FLAG(name) = \ - benchmark::StringFromEnv(#name, default_val) +#define DEFINE_bool(name, default_val) \ + bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) +#define DEFINE_int32(name, default_val) \ + int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) +#define DEFINE_double(name, default_val) \ + double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) +#define DEFINE_string(name, default_val) \ + std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) +#define DEFINE_kvpairs(name, default_val) \ + std::map FLAG(name) = \ + benchmark::KvPairsFromEnv(#name, default_val) namespace benchmark { -// Parses a bool from the environment variable -// corresponding to the given flag. +// Parses a bool from the environment variable corresponding to the given flag. // // If the variable exists, returns IsTruthyFlagValue() value; if not, // returns the given default value. bool BoolFromEnv(const char* flag, bool default_val); -// Parses an Int32 from the environment variable -// corresponding to the given flag. +// Parses an Int32 from the environment variable corresponding to the given +// flag. // // If the variable exists, returns ParseInt32() value; if not, returns // the given default value. int32_t Int32FromEnv(const char* flag, int32_t default_val); -// Parses an Double from the environment variable -// corresponding to the given flag. +// Parses an Double from the environment variable corresponding to the given +// flag. // // If the variable exists, returns ParseDouble(); if not, returns // the given default value. double DoubleFromEnv(const char* flag, double default_val); -// Parses a string from the environment variable -// corresponding to the given flag. +// Parses a string from the environment variable corresponding to the given +// flag. // // If variable exists, returns its value; if not, returns // the given default value. const char* StringFromEnv(const char* flag, const char* default_val); +// Parses a set of kvpairs from the environment variable corresponding to the +// given flag. +// +// If variable exists, returns its value; if not, returns +// the given default value. +std::map KvPairsFromEnv( + const char* flag, std::map default_val); + // Parses a string for a bool flag, in the form of either // "--flag=value" or "--flag". // @@ -68,27 +77,31 @@ const char* StringFromEnv(const char* flag, const char* default_val); // true. On failure, returns false without changing *value. bool ParseBoolFlag(const char* str, const char* flag, bool* value); -// Parses a string for an Int32 flag, in the form of -// "--flag=value". +// Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseInt32Flag(const char* str, const char* flag, int32_t* value); -// Parses a string for a Double flag, in the form of -// "--flag=value". +// Parses a string for a Double flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseDoubleFlag(const char* str, const char* flag, double* value); -// Parses a string for a string flag, in the form of -// "--flag=value". +// Parses a string for a string flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseStringFlag(const char* str, const char* flag, std::string* value); +// Parses a string for a kvpairs flag in the form "--flag=key=value,key=value" +// +// On success, stores the value of the flag in *value and returns true. On +// failure returns false, though *value may have been mutated. +bool ParseKeyValueFlag(const char* str, const char* flag, + std::map* value); + // Returns true if the string matches the flag. bool IsFlag(const char* str, const char* flag); diff --git a/src/reporter.cc b/src/reporter.cc index 337575a1..b7d64fb1 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -22,8 +22,11 @@ #include #include "check.h" +#include "commandlineflags.h" #include "string_util.h" +DECLARE_kvpairs(benchmark_context); + namespace benchmark { BenchmarkReporter::BenchmarkReporter() @@ -64,6 +67,10 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } + for (const auto& kv: FLAGS_benchmark_context) { + Out << kv.first << ": " << kv.second << "\n"; + } + if (CPUInfo::Scaling::ENABLED == info.scaling) { Out << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc index 656020f2..8412008f 100644 --- a/test/commandlineflags_gtest.cc +++ b/test/commandlineflags_gtest.cc @@ -2,6 +2,7 @@ #include "../src/commandlineflags.h" #include "../src/internal_macros.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace benchmark { @@ -19,9 +20,7 @@ int setenv(const char* name, const char* value, int overwrite) { return _putenv_s(name, value); } -int unsetenv(const char* name) { - return _putenv_s(name, ""); -} +int unsetenv(const char* name) { return _putenv_s(name, ""); } #endif // BENCHMARK_OS_WINDOWS @@ -197,5 +196,33 @@ TEST(StringFromEnv, Valid) { unsetenv("IN_ENV"); } +TEST(KvPairsFromEnv, Default) { + ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0); + EXPECT_THAT(KvPairsFromEnv("not_in_env", {{"foo", "bar"}}), + testing::ElementsAre(testing::Pair("foo", "bar"))); +} + +TEST(KvPairsFromEnv, MalformedReturnsDefault) { + ASSERT_EQ(setenv("IN_ENV", "foo", 1), 0); + EXPECT_THAT(KvPairsFromEnv("in_env", {{"foo", "bar"}}), + testing::ElementsAre(testing::Pair("foo", "bar"))); + unsetenv("IN_ENV"); +} + +TEST(KvPairsFromEnv, Single) { + ASSERT_EQ(setenv("IN_ENV", "foo=bar", 1), 0); + EXPECT_THAT(KvPairsFromEnv("in_env", {}), + testing::ElementsAre(testing::Pair("foo", "bar"))); + unsetenv("IN_ENV"); +} + +TEST(KvPairsFromEnv, Multiple) { + ASSERT_EQ(setenv("IN_ENV", "foo=bar,baz=qux", 1), 0); + EXPECT_THAT(KvPairsFromEnv("in_env", {}), + testing::UnorderedElementsAre(testing::Pair("foo", "bar"), + testing::Pair("baz", "qux"))); + unsetenv("IN_ENV"); +} + } // namespace } // namespace benchmark