Add `benchmark_context` flag that allows per-run custom context. (#1127)

* Add `benchmark_context` flag that allows per-run custom context.

Add support for key-value flags in general.
Added test for key-value flags.
Added `benchmark_context` flag.
Output content of `benchmark_context` to base reporter.

Solves the first part of #525.

* Docs and better help

README.md

@@ -273,6 +273,8 @@ too (`-lkstat`).
 
 [Result Comparison](#result-comparison)
 
+[Extra Context](#extra-context)
+
 ### Library
 
 [Runtime and Reporting Considerations](#runtime-and-reporting-considerations)
@@ -442,6 +444,25 @@ BM_memcpy/32k       1834 ns       1837 ns     357143
 It is possible to compare the benchmarking results.
 See [Additional Tooling Documentation](docs/tools.md)
 
+<a name="extra-context" />
+
+### Extra Context
+
+Sometimes it's useful to add extra context to the content printed before the
+results. By default this section includes information about the CPU on which
+the benchmarks are running. If you do want to add more context, you can use
+the `benchmark_context` command line flag:
+
+```bash
+$ ./run_benchmarks --benchmark_context=pwd=`pwd`
+Run on (1 x 2300 MHz CPU)
+pwd: /home/user/benchmark/
+Benchmark              Time           CPU Iterations
+----------------------------------------------------
+BM_memcpy/32          11 ns         11 ns   79545455
+BM_memcpy/32k       2181 ns       2185 ns     324074
+```
+
 <a name="runtime-and-reporting-considerations" />
 
 ### Runtime and Reporting Considerations

src/benchmark.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "benchmark/benchmark.h"
+
 #include "benchmark_api_internal.h"
 #include "benchmark_runner.h"
 #include "internal_macros.h"
@@ -104,6 +105,10 @@ DEFINE_string(benchmark_color, "auto");
 // Valid values: 'true'/'yes'/1, 'false'/'no'/0.  Defaults to false.
 DEFINE_bool(benchmark_counters_tabular, false);
 
+// Extra context to include in the output formatted as comma-separated key-value
+// pairs.
+DEFINE_kvpairs(benchmark_context, {});
+
 // The level of verbose logging to output
 DEFINE_int32(v, 0);
 
@@ -446,6 +451,7 @@ void PrintUsageAndExit() {
           "          [--benchmark_out_format=<json|console|csv>]\n"
           "          [--benchmark_color={auto|true|false}]\n"
           "          [--benchmark_counters_tabular={true|false}]\n"
+          "          [--benchmark_context=<key>=<value>,...]\n"
           "          [--v=<verbosity>]\n");
   exit(0);
 }
@@ -476,9 +482,11 @@ void ParseCommandLineFlags(int* argc, char** argv) {
         ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) ||
         ParseBoolFlag(argv[i], "benchmark_counters_tabular",
                       &FLAGS_benchmark_counters_tabular) ||
-        ParseInt32Flag(argv[i], "v", &FLAGS_v) ||
         ParseStringFlag(argv[i], "benchmark_perf_counters",
-                        &FLAGS_benchmark_perf_counters)) {
+                        &FLAGS_benchmark_perf_counters) ||
+        ParseKeyValueFlag(argv[i], "benchmark_context",
+                          &FLAGS_benchmark_context) ||
+        ParseInt32Flag(argv[i], "v", &FLAGS_v)) {
       for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1];
 
       --(*argc);

src/commandlineflags.cc

@@ -20,6 +20,10 @@
 #include <cstring>
 #include <iostream>
 #include <limits>
+#include <map>
+#include <utility>
+
+#include "../src/string_util.h"
 
 namespace benchmark {
 namespace {
@@ -78,6 +82,30 @@ bool ParseDouble(const std::string& src_text, const char* str, double* value) {
   return true;
 }
 
+// Parses 'str' into KV pairs. If successful, writes the result to *value and
+// returns true; otherwise leaves *value unchanged and returns false.
+bool ParseKvPairs(const std::string& src_text, const char* str,
+                  std::map<std::string, std::string>* value) {
+  std::map<std::string, std::string> kvs;
+  for (const auto& kvpair : StrSplit(str, ',')) {
+    const auto kv = StrSplit(kvpair, '=');
+    if (kv.size() != 2) {
+      std::cerr << src_text << " is expected to be a comma-separated list of "
+                << "<key>=<value> strings, but actually has value \"" << str
+                << "\".\n";
+      return false;
+    }
+    if (!kvs.emplace(kv[0], kv[1]).second) {
+      std::cerr << src_text << " is expected to contain unique keys but key \""
+                << kv[0] << "\" was repeated.\n";
+      return false;
+    }
+  }
+
+  *value = kvs;
+  return true;
+}
+
 // Returns the name of the environment variable corresponding to the
 // given flag.  For example, FlagToEnvVar("foo") will return
 // "BENCHMARK_FOO" in the open-source version.
@@ -129,6 +157,20 @@ const char* StringFromEnv(const char* flag, const char* default_val) {
   return value == nullptr ? default_val : value;
 }
 
+std::map<std::string, std::string> KvPairsFromEnv(
+    const char* flag, std::map<std::string, std::string> default_val) {
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* const value_str = getenv(env_var.c_str());
+
+  if (value_str == nullptr) return default_val;
+
+  std::map<std::string, std::string> value;
+  if (!ParseKvPairs("Environment variable " + env_var, value_str, &value)) {
+    return default_val;
+  }
+  return value;
+}
+
 // Parses a string as a command line flag.  The string should have
 // the format "--flag=value".  When def_optional is true, the "=value"
 // part can be omitted.
@@ -206,6 +248,22 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value) {
   return true;
 }
 
+bool ParseKeyValueFlag(
+    const char* str, const char* flag,
+    std::map<std::string, std::string>* value) {
+  const char* const value_str = ParseFlagValue(str, flag, false);
+
+  if (value_str == nullptr) return false;
+
+  for (const auto& kvpair : StrSplit(value_str, ',')) {
+    const auto kv = StrSplit(kvpair, '=');
+    if (kv.size() != 2) return false;
+    value->emplace(kv[0], kv[1]);
+  }
+
+  return true;
+}
+
 bool IsFlag(const char* str, const char* flag) {
   return (ParseFlagValue(str, flag, true) != nullptr);
 }

src/commandlineflags.h

@@ -2,6 +2,7 @@
 #define BENCHMARK_COMMANDLINEFLAGS_H_
 
 #include <cstdint>
+#include <map>
 #include <string>
 
 // Macro for referencing flags.
@@ -12,51 +13,59 @@
 #define DECLARE_int32(name) extern int32_t FLAG(name)
 #define DECLARE_double(name) extern double FLAG(name)
 #define DECLARE_string(name) extern std::string FLAG(name)
+#define DECLARE_kvpairs(name) \
+  extern std::map<std::string, std::string> FLAG(name)
 
 // Macros for defining flags.
-#define DEFINE_bool(name, default_val)            \
+#define DEFINE_bool(name, default_val) \
-  bool FLAG(name) =                               \
+  bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val)
-    benchmark::BoolFromEnv(#name, default_val)
+#define DEFINE_int32(name, default_val) \
-#define DEFINE_int32(name, default_val)           \
+  int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val)
-  int32_t FLAG(name) =                            \
+#define DEFINE_double(name, default_val) \
-    benchmark::Int32FromEnv(#name, default_val)
+  double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val)
-#define DEFINE_double(name, default_val)          \
+#define DEFINE_string(name, default_val) \
-  double FLAG(name) =                             \
+  std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val)
-    benchmark::DoubleFromEnv(#name, default_val)
+#define DEFINE_kvpairs(name, default_val)         \
-#define DEFINE_string(name, default_val)          \
+  std::map<std::string, std::string> FLAG(name) = \
-  std::string FLAG(name) =                        \
+      benchmark::KvPairsFromEnv(#name, default_val)
-    benchmark::StringFromEnv(#name, default_val)
 
 namespace benchmark {
 
-// Parses a bool from the environment variable
+// Parses a bool from the environment variable corresponding to the given flag.
-// corresponding to the given flag.
 //
 // If the variable exists, returns IsTruthyFlagValue() value;  if not,
 // returns the given default value.
 bool BoolFromEnv(const char* flag, bool default_val);
 
-// Parses an Int32 from the environment variable
+// Parses an Int32 from the environment variable corresponding to the given
-// corresponding to the given flag.
+// flag.
 //
 // If the variable exists, returns ParseInt32() value;  if not, returns
 // the given default value.
 int32_t Int32FromEnv(const char* flag, int32_t default_val);
 
-// Parses an Double from the environment variable
+// Parses an Double from the environment variable corresponding to the given
-// corresponding to the given flag.
+// flag.
 //
 // If the variable exists, returns ParseDouble();  if not, returns
 // the given default value.
 double DoubleFromEnv(const char* flag, double default_val);
 
-// Parses a string from the environment variable
+// Parses a string from the environment variable corresponding to the given
-// corresponding to the given flag.
+// flag.
 //
 // If variable exists, returns its value;  if not, returns
 // the given default value.
 const char* StringFromEnv(const char* flag, const char* default_val);
 
+// Parses a set of kvpairs from the environment variable corresponding to the
+// given flag.
+//
+// If variable exists, returns its value;  if not, returns
+// the given default value.
+std::map<std::string, std::string> KvPairsFromEnv(
+    const char* flag, std::map<std::string, std::string> default_val);
+
 // Parses a string for a bool flag, in the form of either
 // "--flag=value" or "--flag".
 //
@@ -68,27 +77,31 @@ const char* StringFromEnv(const char* flag, const char* default_val);
 // true.  On failure, returns false without changing *value.
 bool ParseBoolFlag(const char* str, const char* flag, bool* value);
 
-// Parses a string for an Int32 flag, in the form of
+// Parses a string for an Int32 flag, in the form of "--flag=value".
-// "--flag=value".
 //
 // On success, stores the value of the flag in *value, and returns
 // true.  On failure, returns false without changing *value.
 bool ParseInt32Flag(const char* str, const char* flag, int32_t* value);
 
-// Parses a string for a Double flag, in the form of
+// Parses a string for a Double flag, in the form of "--flag=value".
-// "--flag=value".
 //
 // On success, stores the value of the flag in *value, and returns
 // true.  On failure, returns false without changing *value.
 bool ParseDoubleFlag(const char* str, const char* flag, double* value);
 
-// Parses a string for a string flag, in the form of
+// Parses a string for a string flag, in the form of "--flag=value".
-// "--flag=value".
 //
 // On success, stores the value of the flag in *value, and returns
 // true.  On failure, returns false without changing *value.
 bool ParseStringFlag(const char* str, const char* flag, std::string* value);
 
+// Parses a string for a kvpairs flag in the form "--flag=key=value,key=value"
+//
+// On success, stores the value of the flag in *value and returns true. On
+// failure returns false, though *value may have been mutated.
+bool ParseKeyValueFlag(const char* str, const char* flag,
+                       std::map<std::string, std::string>* value);
+
 // Returns true if the string matches the flag.
 bool IsFlag(const char* str, const char* flag);
 

src/reporter.cc

@@ -22,8 +22,11 @@
 #include <vector>
 
 #include "check.h"
+#include "commandlineflags.h"
 #include "string_util.h"
 
+DECLARE_kvpairs(benchmark_context);
+
 namespace benchmark {
 
 BenchmarkReporter::BenchmarkReporter()
@@ -64,6 +67,10 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out,
     Out << "\n";
   }
 
+  for (const auto& kv: FLAGS_benchmark_context) {
+    Out << kv.first << ": " << kv.second << "\n";
+  }
+
   if (CPUInfo::Scaling::ENABLED == info.scaling) {
     Out << "***WARNING*** CPU scaling is enabled, the benchmark "
            "real time measurements may be noisy and will incur extra "

test/commandlineflags_gtest.cc

@@ -2,6 +2,7 @@
 
 #include "../src/commandlineflags.h"
 #include "../src/internal_macros.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 namespace benchmark {
@@ -19,9 +20,7 @@ int setenv(const char* name, const char* value, int overwrite) {
   return _putenv_s(name, value);
 }
 
-int unsetenv(const char* name) {
+int unsetenv(const char* name) { return _putenv_s(name, ""); }
-  return _putenv_s(name, "");
-}
 
 #endif  // BENCHMARK_OS_WINDOWS
 
@@ -197,5 +196,33 @@ TEST(StringFromEnv, Valid) {
   unsetenv("IN_ENV");
 }
 
+TEST(KvPairsFromEnv, Default) {
+  ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0);
+  EXPECT_THAT(KvPairsFromEnv("not_in_env", {{"foo", "bar"}}),
+              testing::ElementsAre(testing::Pair("foo", "bar")));
+}
+
+TEST(KvPairsFromEnv, MalformedReturnsDefault) {
+  ASSERT_EQ(setenv("IN_ENV", "foo", 1), 0);
+  EXPECT_THAT(KvPairsFromEnv("in_env", {{"foo", "bar"}}),
+              testing::ElementsAre(testing::Pair("foo", "bar")));
+  unsetenv("IN_ENV");
+}
+
+TEST(KvPairsFromEnv, Single) {
+  ASSERT_EQ(setenv("IN_ENV", "foo=bar", 1), 0);
+  EXPECT_THAT(KvPairsFromEnv("in_env", {}),
+              testing::ElementsAre(testing::Pair("foo", "bar")));
+  unsetenv("IN_ENV");
+}
+
+TEST(KvPairsFromEnv, Multiple) {
+  ASSERT_EQ(setenv("IN_ENV", "foo=bar,baz=qux", 1), 0);
+  EXPECT_THAT(KvPairsFromEnv("in_env", {}),
+              testing::UnorderedElementsAre(testing::Pair("foo", "bar"),
+                                            testing::Pair("baz", "qux")));
+  unsetenv("IN_ENV");
+}
+
 }  // namespace
 }  // namespace benchmark