mojo_benchmark

#!/usr/bin/env python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runner for Mojo application benchmarks."""

import argparse
import logging
import sys
import time

from devtoolslib import benchmark
from devtoolslib import perf_dashboard
from devtoolslib import shell_arguments
from devtoolslib import shell_config


_DESCRIPTION = """Runner for Mojo application benchmarks.

|benchmark_list_file| has to be a valid Python program that sets a |benchmarks|
dictionary. For description of the required format see
https://github.com/domokit/devtools/blob/master/docs/mojo_benchmark.md .
"""

_logger = logging.getLogger()

_CACHE_SERVICE_URL = 'mojo:url_response_disk_cache'
_NETWORK_SERVICE_URL = 'mojo:network_service'

_COLD_START_SHELL_ARGS = [
    '--args-for=%s %s' % (_CACHE_SERVICE_URL, '--clear'),
    '--args-for=%s %s' % (_NETWORK_SERVICE_URL, '--clear'),
]


def _generate_benchmark_variants(benchmark_spec):
  """Generates benchmark specifications for individual variants of the given
  benchmark: cold start and warm start.

  Returns:
    A list of benchmark specs corresponding to individual variants of the given
        benchmark.
  """
  variants = []
  variants.append({
      'variant_name': 'cold start',
      'app': benchmark_spec['app'],
      'duration': benchmark_spec['duration'],
      'measurements': benchmark_spec['measurements'],
      'shell-args': benchmark_spec.get('shell-args',
                                       []) + _COLD_START_SHELL_ARGS})
  variants.append({
      'variant_name': 'warm start',
      'app': benchmark_spec['app'],
      'duration': benchmark_spec['duration'],
      'measurements': benchmark_spec['measurements'],
      'shell-args': benchmark_spec.get('shell-args', [])})
  return variants


def main():
  parser = argparse.ArgumentParser(
      formatter_class=argparse.RawDescriptionHelpFormatter,
      description=_DESCRIPTION)
  parser.add_argument('benchmark_list_file', type=file,
                      help='a file listing benchmarks to run')
  parser.add_argument('--save-all-traces', action='store_true',
                      help='save the traces produced by benchmarks to disk')
  perf_dashboard.add_argparse_server_arguments(parser)

  # Common shell configuration arguments.
  shell_config.add_shell_arguments(parser)
  script_args = parser.parse_args()
  config = shell_config.get_shell_config(script_args)

  try:
    shell, common_shell_args = shell_arguments.get_shell(config, [])
  except shell_arguments.ShellConfigurationException as e:
    print e
    return 1

  target_os = 'android' if script_args.android else 'linux'
  benchmark_list_params = {"target_os": target_os}
  exec script_args.benchmark_list_file in benchmark_list_params

  exit_code = 0
  for benchmark_spec in benchmark_list_params['benchmarks']:
    benchmark_name = benchmark_spec['name']

    for variant_spec in _generate_benchmark_variants(benchmark_spec):
      variant_name = variant_spec['variant_name']
      app = variant_spec['app']
      duration = variant_spec['duration']
      shell_args = variant_spec.get('shell-args', []) + common_shell_args
      measurements = variant_spec['measurements']

      output_file = None
      if script_args.save_all_traces:
        output_file = 'benchmark-%s-%s-%s.trace' % (
            benchmark_name.replace(' ', '_'),
            variant_name.replace(' ', '_'),
            time.strftime('%Y%m%d%H%M%S'))

      chart_data_recorder = None
      if script_args.upload:
        chart_data_recorder = perf_dashboard.ChartDataRecorder(
            script_args.test_name)

      results = benchmark.run(
          shell, shell_args, app, duration, measurements, script_args.verbose,
          script_args.android, output_file)

      print '[ %s ] %s ' % (benchmark_name, variant_name)

      some_measurements_failed = False
      some_measurements_succeeded = False
      if results.succeeded:
        # Iterate over the list of specs, not the dictionary, to detect missing
        # results and preserve the required order.
        for measurement in measurements:
          if measurement['spec'] in results.measurements:
            result = results.measurements[measurement['spec']]
            print '%10.4f  %s' % (result, measurement['name'])

            if chart_data_recorder:
              chart_name = benchmark_name + '__' + variant_name
              chart_data_recorder.record_scalar(
                  perf_dashboard.normalize_label(chart_name),
                  perf_dashboard.normalize_label(measurement['name']),
                  'ms', result)
            some_measurements_succeeded = True
          else:
            print '?  %s' % measurement['name']
            some_measurements_failed = True

      if not results.succeeded or some_measurements_failed:
        if not results.succeeded:
          print 'benchmark failed: ' + results.error_str
        if some_measurements_failed:
          print 'some measurements failed'
        print 'output: '
        print '-' * 72
        print results.output
        print '-' * 72
        exit_code = 1

      if script_args.upload and some_measurements_succeeded:
        if not perf_dashboard.upload_chart_data(
            script_args.master_name, script_args.bot_name,
            script_args.test_name, script_args.builder_name,
            script_args.build_number, chart_data_recorder.get_chart_data(),
            script_args.server_url, script_args.dry_run):
          exit_code = 1

  return exit_code

if __name__ == '__main__':
  sys.exit(main())