| #!/usr/bin/env python3 |
| |
| # type: ignore |
| |
| """ |
| compare.py - versatile benchmark output compare tool |
| """ |
| |
| import argparse |
| import json |
| import os |
| import sys |
| import unittest |
| from argparse import ArgumentParser |
| |
| import gbench |
| from gbench import report, util |
| |
| |
| def check_inputs(in1, in2, flags): |
| """ |
| Perform checking on the user provided inputs and diagnose any abnormalities |
| """ |
| in1_kind, in1_err = util.classify_input_file(in1) |
| in2_kind, in2_err = util.classify_input_file(in2) |
| output_file = util.find_benchmark_flag("--benchmark_out=", flags) |
| output_type = util.find_benchmark_flag("--benchmark_out_format=", flags) |
| if ( |
| in1_kind == util.IT_Executable |
| and in2_kind == util.IT_Executable |
| and output_file |
| ): |
| print( |
| ( |
| "WARNING: '--benchmark_out=%s' will be passed to both " |
| "benchmarks causing it to be overwritten" |
| ) |
| % output_file |
| ) |
| if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: |
| # When both sides are JSON the only supported flag is |
| # --benchmark_filter= |
| for flag in util.remove_benchmark_flags("--benchmark_filter=", flags): |
| print( |
| "WARNING: passing %s has no effect since both " |
| "inputs are JSON" % flag |
| ) |
| if output_type is not None and output_type != "json": |
| print( |
| ( |
| "ERROR: passing '--benchmark_out_format=%s' to 'compare.py`" |
| " is not supported." |
| ) |
| % output_type |
| ) |
| sys.exit(1) |
| |
| |
| def create_parser(): |
| parser = ArgumentParser( |
| description="versatile benchmark output compare tool" |
| ) |
| |
| parser.add_argument( |
| "-a", |
| "--display_aggregates_only", |
| dest="display_aggregates_only", |
| action="store_true", |
| help="If there are repetitions, by default, we display everything - the" |
| " actual runs, and the aggregates computed. Sometimes, it is " |
| "desirable to only view the aggregates. E.g. when there are a lot " |
| "of repetitions. Do note that only the display is affected. " |
| "Internally, all the actual runs are still used, e.g. for U test.", |
| ) |
| |
| parser.add_argument( |
| "--no-color", |
| dest="color", |
| default=True, |
| action="store_false", |
| help="Do not use colors in the terminal output", |
| ) |
| |
| parser.add_argument( |
| "-d", |
| "--dump_to_json", |
| dest="dump_to_json", |
| help="Additionally, dump benchmark comparison output to this file in JSON format.", |
| ) |
| |
| utest = parser.add_argument_group() |
| utest.add_argument( |
| "--no-utest", |
| dest="utest", |
| default=True, |
| action="store_false", |
| help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format( |
| report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS |
| ), |
| ) |
| alpha_default = 0.05 |
| utest.add_argument( |
| "--alpha", |
| dest="utest_alpha", |
| default=alpha_default, |
| type=float, |
| help=( |
| "significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)" |
| ) |
| % alpha_default, |
| ) |
| |
| subparsers = parser.add_subparsers( |
| help="This tool has multiple modes of operation:", dest="mode" |
| ) |
| |
| parser_a = subparsers.add_parser( |
| "benchmarks", |
| help="The most simple use-case, compare all the output of these two benchmarks", |
| ) |
| baseline = parser_a.add_argument_group("baseline", "The benchmark baseline") |
| baseline.add_argument( |
| "test_baseline", |
| metavar="test_baseline", |
| type=argparse.FileType("r"), |
| nargs=1, |
| help="A benchmark executable or JSON output file", |
| ) |
| contender = parser_a.add_argument_group( |
| "contender", "The benchmark that will be compared against the baseline" |
| ) |
| contender.add_argument( |
| "test_contender", |
| metavar="test_contender", |
| type=argparse.FileType("r"), |
| nargs=1, |
| help="A benchmark executable or JSON output file", |
| ) |
| parser_a.add_argument( |
| "benchmark_options", |
| metavar="benchmark_options", |
| nargs=argparse.REMAINDER, |
| help="Arguments to pass when running benchmark executables", |
| ) |
| |
| parser_b = subparsers.add_parser( |
| "filters", help="Compare filter one with the filter two of benchmark" |
| ) |
| baseline = parser_b.add_argument_group("baseline", "The benchmark baseline") |
| baseline.add_argument( |
| "test", |
| metavar="test", |
| type=argparse.FileType("r"), |
| nargs=1, |
| help="A benchmark executable or JSON output file", |
| ) |
| baseline.add_argument( |
| "filter_baseline", |
| metavar="filter_baseline", |
| type=str, |
| nargs=1, |
| help="The first filter, that will be used as baseline", |
| ) |
| contender = parser_b.add_argument_group( |
| "contender", "The benchmark that will be compared against the baseline" |
| ) |
| contender.add_argument( |
| "filter_contender", |
| metavar="filter_contender", |
| type=str, |
| nargs=1, |
| help="The second filter, that will be compared against the baseline", |
| ) |
| parser_b.add_argument( |
| "benchmark_options", |
| metavar="benchmark_options", |
| nargs=argparse.REMAINDER, |
| help="Arguments to pass when running benchmark executables", |
| ) |
| |
| parser_c = subparsers.add_parser( |
| "benchmarksfiltered", |
| help="Compare filter one of first benchmark with filter two of the second benchmark", |
| ) |
| baseline = parser_c.add_argument_group("baseline", "The benchmark baseline") |
| baseline.add_argument( |
| "test_baseline", |
| metavar="test_baseline", |
| type=argparse.FileType("r"), |
| nargs=1, |
| help="A benchmark executable or JSON output file", |
| ) |
| baseline.add_argument( |
| "filter_baseline", |
| metavar="filter_baseline", |
| type=str, |
| nargs=1, |
| help="The first filter, that will be used as baseline", |
| ) |
| contender = parser_c.add_argument_group( |
| "contender", "The benchmark that will be compared against the baseline" |
| ) |
| contender.add_argument( |
| "test_contender", |
| metavar="test_contender", |
| type=argparse.FileType("r"), |
| nargs=1, |
| help="The second benchmark executable or JSON output file, that will be compared against the baseline", |
| ) |
| contender.add_argument( |
| "filter_contender", |
| metavar="filter_contender", |
| type=str, |
| nargs=1, |
| help="The second filter, that will be compared against the baseline", |
| ) |
| parser_c.add_argument( |
| "benchmark_options", |
| metavar="benchmark_options", |
| nargs=argparse.REMAINDER, |
| help="Arguments to pass when running benchmark executables", |
| ) |
| |
| return parser |
| |
| |
| def main(): |
| # Parse the command line flags |
| parser = create_parser() |
| args, unknown_args = parser.parse_known_args() |
| if args.mode is None: |
| parser.print_help() |
| exit(1) |
| assert not unknown_args |
| benchmark_options = args.benchmark_options |
| |
| if args.mode == "benchmarks": |
| test_baseline = args.test_baseline[0].name |
| test_contender = args.test_contender[0].name |
| filter_baseline = "" |
| filter_contender = "" |
| |
| # NOTE: if test_baseline == test_contender, you are analyzing the stdev |
| |
| description = "Comparing %s to %s" % (test_baseline, test_contender) |
| elif args.mode == "filters": |
| test_baseline = args.test[0].name |
| test_contender = args.test[0].name |
| filter_baseline = args.filter_baseline[0] |
| filter_contender = args.filter_contender[0] |
| |
| # NOTE: if filter_baseline == filter_contender, you are analyzing the |
| # stdev |
| |
| description = "Comparing %s to %s (from %s)" % ( |
| filter_baseline, |
| filter_contender, |
| args.test[0].name, |
| ) |
| elif args.mode == "benchmarksfiltered": |
| test_baseline = args.test_baseline[0].name |
| test_contender = args.test_contender[0].name |
| filter_baseline = args.filter_baseline[0] |
| filter_contender = args.filter_contender[0] |
| |
| # NOTE: if test_baseline == test_contender and |
| # filter_baseline == filter_contender, you are analyzing the stdev |
| |
| description = "Comparing %s (from %s) to %s (from %s)" % ( |
| filter_baseline, |
| test_baseline, |
| filter_contender, |
| test_contender, |
| ) |
| else: |
| # should never happen |
| print("Unrecognized mode of operation: '%s'" % args.mode) |
| parser.print_help() |
| exit(1) |
| |
| check_inputs(test_baseline, test_contender, benchmark_options) |
| |
| if args.display_aggregates_only: |
| benchmark_options += ["--benchmark_display_aggregates_only=true"] |
| |
| options_baseline = [] |
| options_contender = [] |
| |
| if filter_baseline and filter_contender: |
| options_baseline = ["--benchmark_filter=%s" % filter_baseline] |
| options_contender = ["--benchmark_filter=%s" % filter_contender] |
| |
| # Run the benchmarks and report the results |
| json1 = json1_orig = gbench.util.sort_benchmark_results( |
| gbench.util.run_or_load_benchmark( |
| test_baseline, benchmark_options + options_baseline |
| ) |
| ) |
| json2 = json2_orig = gbench.util.sort_benchmark_results( |
| gbench.util.run_or_load_benchmark( |
| test_contender, benchmark_options + options_contender |
| ) |
| ) |
| |
| # Now, filter the benchmarks so that the difference report can work |
| if filter_baseline and filter_contender: |
| replacement = "[%s vs. %s]" % (filter_baseline, filter_contender) |
| json1 = gbench.report.filter_benchmark( |
| json1_orig, filter_baseline, replacement |
| ) |
| json2 = gbench.report.filter_benchmark( |
| json2_orig, filter_contender, replacement |
| ) |
| |
| diff_report = gbench.report.get_difference_report(json1, json2, args.utest) |
| output_lines = gbench.report.print_difference_report( |
| diff_report, |
| args.display_aggregates_only, |
| args.utest, |
| args.utest_alpha, |
| args.color, |
| ) |
| print(description) |
| for ln in output_lines: |
| print(ln) |
| |
| # Optionally, diff and output to JSON |
| if args.dump_to_json is not None: |
| with open(args.dump_to_json, "w") as f_json: |
| json.dump(diff_report, f_json, indent=1) |
| |
| |
| class TestParser(unittest.TestCase): |
| def setUp(self): |
| self.parser = create_parser() |
| testInputs = os.path.join( |
| os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs" |
| ) |
| self.testInput0 = os.path.join(testInputs, "test1_run1.json") |
| self.testInput1 = os.path.join(testInputs, "test1_run2.json") |
| |
| def test_benchmarks_basic(self): |
| parsed = self.parser.parse_args( |
| ["benchmarks", self.testInput0, self.testInput1] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_benchmarks_basic_without_utest(self): |
| parsed = self.parser.parse_args( |
| ["--no-utest", "benchmarks", self.testInput0, self.testInput1] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertFalse(parsed.utest) |
| self.assertEqual(parsed.utest_alpha, 0.05) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_benchmarks_basic_display_aggregates_only(self): |
| parsed = self.parser.parse_args( |
| ["-a", "benchmarks", self.testInput0, self.testInput1] |
| ) |
| self.assertTrue(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_benchmarks_basic_with_utest_alpha(self): |
| parsed = self.parser.parse_args( |
| ["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.utest_alpha, 0.314) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_benchmarks_basic_without_utest_with_utest_alpha(self): |
| parsed = self.parser.parse_args( |
| [ |
| "--no-utest", |
| "--alpha=0.314", |
| "benchmarks", |
| self.testInput0, |
| self.testInput1, |
| ] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertFalse(parsed.utest) |
| self.assertEqual(parsed.utest_alpha, 0.314) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_benchmarks_with_remainder(self): |
| parsed = self.parser.parse_args( |
| ["benchmarks", self.testInput0, self.testInput1, "d"] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertEqual(parsed.benchmark_options, ["d"]) |
| |
| def test_benchmarks_with_remainder_after_doubleminus(self): |
| parsed = self.parser.parse_args( |
| ["benchmarks", self.testInput0, self.testInput1, "--", "e"] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarks") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertEqual(parsed.benchmark_options, ["e"]) |
| |
| def test_filters_basic(self): |
| parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"]) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "filters") |
| self.assertEqual(parsed.test[0].name, self.testInput0) |
| self.assertEqual(parsed.filter_baseline[0], "c") |
| self.assertEqual(parsed.filter_contender[0], "d") |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_filters_with_remainder(self): |
| parsed = self.parser.parse_args( |
| ["filters", self.testInput0, "c", "d", "e"] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "filters") |
| self.assertEqual(parsed.test[0].name, self.testInput0) |
| self.assertEqual(parsed.filter_baseline[0], "c") |
| self.assertEqual(parsed.filter_contender[0], "d") |
| self.assertEqual(parsed.benchmark_options, ["e"]) |
| |
| def test_filters_with_remainder_after_doubleminus(self): |
| parsed = self.parser.parse_args( |
| ["filters", self.testInput0, "c", "d", "--", "f"] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "filters") |
| self.assertEqual(parsed.test[0].name, self.testInput0) |
| self.assertEqual(parsed.filter_baseline[0], "c") |
| self.assertEqual(parsed.filter_contender[0], "d") |
| self.assertEqual(parsed.benchmark_options, ["f"]) |
| |
| def test_benchmarksfiltered_basic(self): |
| parsed = self.parser.parse_args( |
| ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarksfiltered") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.filter_baseline[0], "c") |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertEqual(parsed.filter_contender[0], "e") |
| self.assertFalse(parsed.benchmark_options) |
| |
| def test_benchmarksfiltered_with_remainder(self): |
| parsed = self.parser.parse_args( |
| [ |
| "benchmarksfiltered", |
| self.testInput0, |
| "c", |
| self.testInput1, |
| "e", |
| "f", |
| ] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarksfiltered") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.filter_baseline[0], "c") |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertEqual(parsed.filter_contender[0], "e") |
| self.assertEqual(parsed.benchmark_options[0], "f") |
| |
| def test_benchmarksfiltered_with_remainder_after_doubleminus(self): |
| parsed = self.parser.parse_args( |
| [ |
| "benchmarksfiltered", |
| self.testInput0, |
| "c", |
| self.testInput1, |
| "e", |
| "--", |
| "g", |
| ] |
| ) |
| self.assertFalse(parsed.display_aggregates_only) |
| self.assertTrue(parsed.utest) |
| self.assertEqual(parsed.mode, "benchmarksfiltered") |
| self.assertEqual(parsed.test_baseline[0].name, self.testInput0) |
| self.assertEqual(parsed.filter_baseline[0], "c") |
| self.assertEqual(parsed.test_contender[0].name, self.testInput1) |
| self.assertEqual(parsed.filter_contender[0], "e") |
| self.assertEqual(parsed.benchmark_options[0], "g") |
| |
| |
| if __name__ == "__main__": |
| # unittest.main() |
| main() |
| |
| # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 |
| # kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off; |
| # kate: indent-mode python; remove-trailing-spaces modified; |
