diff --git a/tools/compare.py b/tools/compare.py index 8cefdd17..e5eeb247 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -9,25 +9,28 @@ import argparse from argparse import ArgumentParser import json import sys +import os import gbench from gbench import util, report -from gbench.util import * def check_inputs(in1, in2, flags): """ Perform checking on the user provided inputs and diagnose any abnormalities """ - in1_kind, in1_err = classify_input_file(in1) - in2_kind, in2_err = classify_input_file(in2) - output_file = find_benchmark_flag('--benchmark_out=', flags) - output_type = find_benchmark_flag('--benchmark_out_format=', flags) - if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: + in1_kind, in1_err = util.classify_input_file(in1) + in2_kind, in2_err = util.classify_input_file(in2) + output_file = util.find_benchmark_flag('--benchmark_out=', flags) + output_type = util.find_benchmark_flag('--benchmark_out_format=', flags) + if in1_kind == util.IT_Executable and in2_kind == util.IT_Executable and output_file: print(("WARNING: '--benchmark_out=%s' will be passed to both " "benchmarks causing it to be overwritten") % output_file) - if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: - print("WARNING: passing optional flags has no effect since both " - "inputs are JSON") + if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: + # When both sides are JSON the only supported flag is + # --benchmark_filter= + for flag in util.remove_benchmark_flags('--benchmark_filter=', flags): + print("WARNING: passing %s has no effect since both " + "inputs are JSON" % flag) if output_type is not None and output_type != 'json': print(("ERROR: passing '--benchmark_out_format=%s' to 'compare.py`" " is not supported.") % output_type) diff --git a/tools/gbench/util.py b/tools/gbench/util.py index 95d7994b..5e79da8f 100644 --- a/tools/gbench/util.py +++ b/tools/gbench/util.py @@ -2,10 +2,11 @@ """ import json import os -import tempfile +import re import subprocess import sys -import functools +import tempfile + # Input file type enumeration IT_Invalid = 0 @@ -111,13 +112,32 @@ def remove_benchmark_flags(prefix, benchmark_flags): return [f for f in benchmark_flags if not f.startswith(prefix)] -def load_benchmark_results(fname): +def load_benchmark_results(fname, benchmark_filter): """ Read benchmark output from a file and return the JSON object. + + Apply benchmark_filter, a regular expression, with nearly the same + semantics of the --benchmark_filter argument. May be None. + Note: the Python regular expression engine is used instead of the + one used by the C++ code, which may produce different results + in complex cases. + REQUIRES: 'fname' names a file containing JSON benchmark output. """ + def benchmark_wanted(benchmark): + if benchmark_filter is None: + return True + name = benchmark.get('run_name', None) or benchmark['name'] + if re.search(benchmark_filter, name): + return True + return False + with open(fname, 'r') as f: - return json.load(f) + results = json.load(f) + if 'benchmarks' in results: + results['benchmarks'] = list(filter(benchmark_wanted, + results['benchmarks'])) + return results def sort_benchmark_results(result): @@ -160,7 +180,7 @@ def run_benchmark(exe_name, benchmark_flags): if exitCode != 0: print('TEST FAILED...') sys.exit(exitCode) - json_res = load_benchmark_results(output_name) + json_res = load_benchmark_results(output_name, None) if is_temp_output: os.unlink(output_name) return json_res @@ -175,7 +195,9 @@ def run_or_load_benchmark(filename, benchmark_flags): """ ftype = check_input_file(filename) if ftype == IT_JSON: - return load_benchmark_results(filename) + benchmark_filter = find_benchmark_flag('--benchmark_filter=', + benchmark_flags) + return load_benchmark_results(filename, benchmark_filter) if ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) raise ValueError('Unknown file type %s' % ftype)