Refactor mojo_benchmark. This patch extract the logic that interacts with the mojo benchmark app from `mojo_benchmark` into devtoolslib and adds some tests. R=qsr@chromium.org Review URL: https://codereview.chromium.org/1435523002 . Cr-Mirrored-From: https://github.com/domokit/mojo Cr-Mirrored-Commit: 0a40516baad86d171958347cd2bb95199b287110
diff --git a/devtoolslib/benchmark.py b/devtoolslib/benchmark.py new file mode 100644 index 0000000..3383dd4 --- /dev/null +++ b/devtoolslib/benchmark.py
@@ -0,0 +1,103 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Logic that drives runs of the benchmarking mojo app and parses its output.""" + +import os.path +import re + +_BENCHMARK_APP = 'https://core.mojoapps.io/benchmark.mojo' + +# Additional time in seconds allocated per shell run to accommodate start-up. +# The shell should terminate before hitting this time out, it is an error if it +# doesn't. +_EXTRA_TIMEOUT = 20 + +_MEASUREMENT_RESULT_FORMAT = r""" +^ # Beginning of the line. +measurement: # Hard-coded tag. +\s+(\S+) # Match measurement spec. +\s+(\S+) # Match measurement result. +$ # End of the line. +""" + +_MEASUREMENT_REGEX = re.compile(_MEASUREMENT_RESULT_FORMAT, re.VERBOSE) + + +def _parse_measurement_results(output): + """Parses the measurement results present in the benchmark output and returns + the dictionary of correctly recognized and parsed results. + """ + measurement_results = {} + output_lines = [line.strip() for line in output.split('\n')] + for line in output_lines: + match = re.match(_MEASUREMENT_REGEX, line) + if match: + measurement_spec = match.group(1) + measurement_result = match.group(2) + try: + measurement_results[measurement_spec] = float(measurement_result) + except ValueError: + pass + return measurement_results + + +class Results(object): + """Holds results of a benchmark run.""" + + def __init__(self, succeeded, error_str, output): + self.succeeded = succeeded + self.error_str = error_str + self.output = output + self.measurements = None + + +def run(shell, shell_args, app, duration_seconds, measurements, verbose, + android, output_file): + """Runs the given benchmark by running `benchmark.mojo` in mojo shell with + appropriate arguments and returns the produced output. + + Returns: + A tuple of (succeeded, error_msg, output). + """ + timeout = duration_seconds + _EXTRA_TIMEOUT + benchmark_args = [] + benchmark_args.append('--app=' + app) + benchmark_args.append('--duration=' + str(duration_seconds)) + + device_output_file = None + if output_file: + if android: + device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file) + benchmark_args.append('--trace-output=' + device_output_file) + else: + benchmark_args.append('--trace-output=' + output_file) + + for measurement in measurements: + benchmark_args.append(measurement['spec']) + + shell_args = list(shell_args) + shell_args.append(_BENCHMARK_APP) + shell_args.append('--force-offline-by-default') + shell_args.append('--args-for=%s %s' % (_BENCHMARK_APP, + ' '.join(benchmark_args))) + + if verbose: + print 'shell arguments: ' + str(shell_args) + return_code, output, did_time_out = shell.run_and_get_output( + shell_args, timeout=timeout) + + if did_time_out: + return Results(False, 'timed out', output) + if return_code: + return Results(False, 'return code: ' + str(return_code), output) + + # Pull the trace file even if some measurements are missing, as it can be + # useful in debugging. + if device_output_file: + shell.pull_file(device_output_file, output_file, remove_original=True) + + results = Results(True, None, output) + results.measurements = _parse_measurement_results(output) + return results
diff --git a/devtoolslib/benchmark_unittest.py b/devtoolslib/benchmark_unittest.py new file mode 100644 index 0000000..6257041 --- /dev/null +++ b/devtoolslib/benchmark_unittest.py
@@ -0,0 +1,44 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the logic that drives runs of the benchmarking mojo app and parses +its output.""" + +import imp +import os.path +import sys +import unittest + +try: + imp.find_module("devtoolslib") +except ImportError: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from devtoolslib import benchmark + + +class BenchmarkTest(unittest.TestCase): + """Tests the benchmark module.""" + + def test_parse_measurement_results_empty(self): + """Tests parsing empty output.""" + output = """""" + results = benchmark._parse_measurement_results(output) + self.assertEquals({}, results) + + def test_parse_measurement_results_typical(self): + """Tests parsing typical output with unrelated log entries.""" + output = """ +[INFO:network_fetcher.cc(322)] Caching mojo app http://127.0.0.1:31839/benchmark.mojo at /usr/local/google/home/user/.mojo_url_response_disk_cache/cache/4F6FAE752C7958AE122C6A2D778F2014C15578250B3C6746D54B99E4F15A4458/4F6FAE752C7958AE122C6A2D778F2014C15578250B3C6746D54B99E4F15A4458 +[INFO:network_fetcher.cc(322)] Caching mojo app http://127.0.0.1:31839/dart_traced_application.mojo at /usr/local/google/home/user/.mojo_url_response_disk_cache/cache/AB290478907A1DC5434CBCFD053BE2E74254D882644E76B3C28E3E7E1BCDCC3D/AB290478907A1DC5434CBCFD053BE2E74254D882644E76B3C28E3E7E1BCDCC3D +Observatory listening on http://127.0.0.1:38128 +[1109/155613:WARNING:event.cc(234)] Ignoring incorrect complete event (no duration) +measurement: time_until/a/b 42.5 +measurement: time_between/a/b/c/d 21.1 +measurement: time_between/a/b/e/f FAILED +some measurements failed +""" + results = benchmark._parse_measurement_results(output) + self.assertEquals({'time_until/a/b': 42.5, + 'time_between/a/b/c/d': 21.1}, results)
diff --git a/mojo_benchmark b/mojo_benchmark index 6ade1b1..b6c17ae 100755 --- a/mojo_benchmark +++ b/mojo_benchmark
@@ -7,14 +7,13 @@ import argparse import logging -import os.path -import re import sys import time +from devtoolslib import benchmark +from devtoolslib import perf_dashboard from devtoolslib import shell_arguments from devtoolslib import shell_config -from devtoolslib import perf_dashboard _DESCRIPTION = """Runner for Mojo application benchmarks. @@ -26,7 +25,6 @@ _logger = logging.getLogger() -_BENCHMARK_APP = 'https://core.mojoapps.io/benchmark.mojo' _CACHE_SERVICE_URL = 'mojo:url_response_disk_cache' _NETWORK_SERVICE_URL = 'mojo:network_service' @@ -35,21 +33,6 @@ '--args-for=%s %s' % (_NETWORK_SERVICE_URL, '--clear'), ] -# Additional time in seconds allocated per shell run to accommodate start-up. -# The shell should terminate before hitting this time out, it is an error if it -# doesn't. -_EXTRA_TIMEOUT = 20 - -_MEASUREMENT_RESULT_FORMAT = r""" -^ # Beginning of the line. -measurement: # Hard-coded tag. -\s+(\S+) # Match measurement spec. -\s+(\S+) # Match measurement result. -$ # End of the line. -""" - -_MEASUREMENT_REGEX = re.compile(_MEASUREMENT_RESULT_FORMAT, re.VERBOSE) - def _generate_benchmark_variants(benchmark_spec): """Generates benchmark specifications for individual variants of the given @@ -76,72 +59,6 @@ return variants -def _run_benchmark(shell, shell_args, app, duration_seconds, measurements, - verbose, android, output_file): - """Runs the given benchmark by running `benchmark.mojo` in mojo shell with - appropriate arguments and returns the produced output. - - Returns: - A tuple of (succeeded, error_msg, output). - """ - timeout = duration_seconds + _EXTRA_TIMEOUT - benchmark_args = [] - benchmark_args.append('--app=' + app) - benchmark_args.append('--duration=' + str(duration_seconds)) - - device_output_file = None - if output_file: - if android: - device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file) - benchmark_args.append('--trace-output=' + device_output_file) - else: - benchmark_args.append('--trace-output=' + output_file) - - for measurement in measurements: - benchmark_args.append(measurement['spec']) - - shell_args = list(shell_args) - shell_args.append(_BENCHMARK_APP) - shell_args.append('--force-offline-by-default') - shell_args.append('--args-for=%s %s' % (_BENCHMARK_APP, - ' '.join(benchmark_args))) - - if verbose: - print 'shell arguments: ' + str(shell_args) - return_code, output, did_time_out = shell.run_and_get_output( - shell_args, timeout=timeout) - - if did_time_out: - return False, 'timed out', output - if return_code: - return False, 'return code: ' + str(return_code), output - - # Pull the trace file even if some measurements are missing, as it can be - # useful in debugging. - if device_output_file: - shell.pull_file(device_output_file, output_file, remove_original=True) - - return True, None, output - - -def _parse_measurement_results(output): - """Parses the measurement results present in the benchmark output and returns - the dictionary of correctly recognized and parsed results. - """ - measurement_results = {} - output_lines = [line.strip() for line in output.split('\n')] - for line in output_lines: - match = re.match(_MEASUREMENT_REGEX, line) - if match: - measurement_spec = match.group(1) - measurement_result = match.group(2) - try: - measurement_results[measurement_spec] = float(measurement_result) - except ValueError: - pass - return measurement_results - - def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -190,7 +107,7 @@ chart_data_recorder = perf_dashboard.ChartDataRecorder( script_args.test_name) - benchmark_succeeded, benchmark_error, output = _run_benchmark( + results = benchmark.run( shell, shell_args, app, duration, measurements, script_args.verbose, script_args.android, output_file) @@ -198,13 +115,12 @@ some_measurements_failed = False some_measurements_succeeded = False - if benchmark_succeeded: - measurement_results = _parse_measurement_results(output) + if results.succeeded: # Iterate over the list of specs, not the dictionary, to detect missing # results and preserve the required order. for measurement in measurements: - if measurement['spec'] in measurement_results: - result = measurement_results[measurement['spec']] + if measurement['spec'] in results.measurements: + result = results.measurements[measurement['spec']] print '%10.4f %s' % (result, measurement['name']) if chart_data_recorder: @@ -218,14 +134,14 @@ print '? %s' % measurement['name'] some_measurements_failed = True - if not benchmark_succeeded or some_measurements_failed: - if not benchmark_succeeded: - print 'benchmark failed: ' + benchmark_error + if not results.succeeded or some_measurements_failed: + if not results.succeeded: + print 'benchmark failed: ' + results.error_str if some_measurements_failed: print 'some measurements failed' print 'output: ' print '-' * 72 - print output + print results.output print '-' * 72 exit_code = 1