blob: 5ffdbe160c2fe8bd73b7893b2842c337ba1c4e21 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runner for Mojo application benchmarks."""
import argparse
import logging
import sys
import time
import os.path
import re
from devtoolslib import shell_arguments
from devtoolslib import shell_config
from devtoolslib import performance_dashboard
_DESCRIPTION = """Runner for Mojo application benchmarks.
|benchmark_list_file| has to be a valid Python program that sets a |benchmarks|
global variable, containing entries of the following form:
{
'name': '<name of the benchmark>',
'app': '<url of the app to benchmark>',
'shell-args': [],
'duration': <duration in seconds>,
# List of measurements to make.
'measurements': [
'<measurement type>/<event category>/<event name>',
]
}
Available measurement types are:
- 'time_until' - time until the first occurence of the targeted event
- 'avg_duration' - average duration of the targeted event
- 'percentile_duration' - value at XXth percentile of the targeted event where
XX is from the measurement spec, i.e. .../<event name>/0.XX
|benchmark_list_file| may reference the |target_os| global that will be any of
['android', 'linux'], indicating the system on which the benchmarks are to be
run.
"""
_logger = logging.getLogger()
_BENCHMARK_APP = 'https://core.mojoapps.io/benchmark.mojo'
_CACHE_SERVICE_URL = 'mojo:url_response_disk_cache'
_NETWORK_SERVICE_URL = 'mojo:network_service'
_COLD_START_SHELL_ARGS = [
'--args-for=%s %s' % (_CACHE_SERVICE_URL, '--clear'),
'--args-for=%s %s' % (_NETWORK_SERVICE_URL, '--clear'),
]
# Additional time in seconds allocated per shell run to accommodate start-up.
# The shell should terminate before hitting this time out, it is an error if it
# doesn't.
_EXTRA_TIMEOUT = 20
_MEASUREMENT_RESULT_FORMAT = r"""
^ # Beginning of the line.
measurement: # Hard-coded tag.
\s+(\S+) # Match measurement name.
\s+(\S+) # Match measurement result.
$ # End of the line.
"""
_MEASUREMENT_REGEX = re.compile(_MEASUREMENT_RESULT_FORMAT, re.VERBOSE)
def _generate_benchmark_variants(benchmark_spec):
"""Generates benchmark specifications for individual variants of the given
benchmark: cold start and warm start.
Returns:
A list of benchmark specs corresponding to individual variants of the given
benchmark.
"""
variants = []
# Cold start.
variants.append({
'name': benchmark_spec['name'] + ' (cold start)',
'app': benchmark_spec['app'],
'duration': benchmark_spec['duration'],
'measurements': benchmark_spec['measurements'],
'shell-args': benchmark_spec.get('shell-args',
[]) + _COLD_START_SHELL_ARGS})
# Warm start.
variants.append({
'name': benchmark_spec['name'] + ' (warm start)',
'app': benchmark_spec['app'],
'duration': benchmark_spec['duration'],
'measurements': benchmark_spec['measurements'],
'shell-args': benchmark_spec.get('shell-args', [])})
return variants
def _run_benchmark(shell, shell_args, name, app, duration_seconds, measurements,
verbose, android, save_traces):
"""Runs the given benchmark by running `benchmark.mojo` in mojo shell with
appropriate arguments and returns the produced output.
Returns:
A tuple of (succeeded, error_msg, output).
"""
timeout = duration_seconds + _EXTRA_TIMEOUT
benchmark_args = []
benchmark_args.append('--app=' + app)
benchmark_args.append('--duration=' + str(duration_seconds))
output_file = None
device_output_file = None
if save_traces:
output_file = 'benchmark-%s-%s.trace' % (name.replace(' ', '_'),
time.strftime('%Y%m%d%H%M%S'))
if android:
device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file)
benchmark_args.append('--trace-output=' + device_output_file)
else:
benchmark_args.append('--trace-output=' + output_file)
for measurement in measurements:
benchmark_args.append(measurement)
shell_args = list(shell_args)
shell_args.append(_BENCHMARK_APP)
shell_args.append('--force-offline-by-default')
shell_args.append('--args-for=%s %s' % (_BENCHMARK_APP,
' '.join(benchmark_args)))
if verbose:
print 'shell arguments: ' + str(shell_args)
return_code, output, did_time_out = shell.run_and_get_output(
shell_args, timeout=timeout)
if did_time_out:
return False, 'timed out', output
if return_code:
return False, 'return code: ' + str(return_code), output
# Pull the trace file even if some measurements are missing, as it can be
# useful in debugging.
if device_output_file:
shell.pull_file(device_output_file, output_file, remove_original=True)
return True, None, output
def _parse_measurement_results(output):
"""Parses the measurement results present in the benchmark output and returns
the dictionary of correctly recognized and parsed results.
"""
measurement_results = {}
output_lines = [line.strip() for line in output.split('\n')]
for line in output_lines:
match = re.match(_MEASUREMENT_REGEX, line)
if match:
measurement_name = match.group(1)
measurement_result = match.group(2)
try:
measurement_results[measurement_name] = float(measurement_result)
except ValueError:
pass
return measurement_results
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=_DESCRIPTION)
parser.add_argument('benchmark_list_file', type=file,
help='a file listing benchmarks to run')
parser.add_argument('--save-traces', action='store_true',
help='save the traces produced by benchmarks to disk')
parser.add_argument('--chart-data-output-file', type=argparse.FileType('w'),
help='file to write chart data for the performance '
'dashboard to')
# Common shell configuration arguments.
shell_config.add_shell_arguments(parser)
script_args = parser.parse_args()
config = shell_config.get_shell_config(script_args)
try:
shell, common_shell_args = shell_arguments.get_shell(config, [])
except shell_arguments.ShellConfigurationException as e:
print e
return 1
target_os = 'android' if script_args.android else 'linux'
benchmark_list_params = {"target_os": target_os}
exec script_args.benchmark_list_file in benchmark_list_params
chart_data_recorder = None
if script_args.chart_data_output_file:
chart_data_recorder = performance_dashboard.ChartDataRecorder()
exit_code = 0
for benchmark_spec in benchmark_list_params['benchmarks']:
for variant_spec in _generate_benchmark_variants(benchmark_spec):
name = variant_spec['name']
app = variant_spec['app']
duration = variant_spec['duration']
shell_args = variant_spec.get('shell-args', []) + common_shell_args
measurements = variant_spec['measurements']
benchmark_succeeded, benchmark_error, output = _run_benchmark(
shell, shell_args, name, app, duration, measurements,
script_args.verbose, script_args.android,
script_args.save_traces)
print '[ %s ]' % name
some_measurements_failed = False
if benchmark_succeeded:
measurement_results = _parse_measurement_results(output)
# Iterate over the list of specs, not the dictionary, to detect missing
# results and preserve the required order.
for measurement_spec in measurements:
if measurement_spec in measurement_results:
result = measurement_results[measurement_spec]
print '%s %s' % (measurement_spec, result)
if chart_data_recorder:
measurement_name = measurement_spec.replace('/', '-')
chart_data_recorder.record_scalar(name, measurement_name, 'ms',
result)
else:
print '%s ?' % measurement_spec
some_measurements_failed = True
if not benchmark_succeeded or some_measurements_failed:
if not benchmark_succeeded:
print 'benchmark failed: ' + benchmark_error
if some_measurements_failed:
print 'some measurements failed'
print 'output: '
print '-' * 72
print output
print '-' * 72
exit_code = 1
if script_args.chart_data_output_file:
script_args.chart_data_output_file.write(chart_data_recorder.get_json())
return exit_code
if __name__ == '__main__':
sys.exit(main())