#!/usr/bin/env python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runner for Mojo application benchmarks."""

import argparse
import json
import logging
import os.path
import re
import sys
import time

from devtoolslib import shell_arguments
from devtoolslib import shell_config
from devtoolslib import perf_dashboard


_DESCRIPTION = """Runner for Mojo application benchmarks.

|benchmark_list_file| has to be a valid Python program that sets a |benchmarks|
global variable, containing entries of the following form:

  {
    'name': '<name of the benchmark>',
    'app': '<url of the app to benchmark>',
    'shell-args': [],
    'duration': <duration in seconds>,

    # List of measurements to make.
    'measurements': [
      '<measurement type>/<event category>/<event name>',
    ]
  }

Available measurement types are:

  - 'time_until' - time until the first occurence of the targeted event
  - 'avg_duration' - average duration of the targeted event
  - 'percentile_duration' - value at XXth percentile of the targeted event where
    XX is from the measurement spec, i.e. .../<event name>/0.XX

|benchmark_list_file| may reference the |target_os| global that will be any of
['android', 'linux'], indicating the system on which the benchmarks are to be
run.
"""

_logger = logging.getLogger()

_BENCHMARK_APP = 'https://core.mojoapps.io/benchmark.mojo'
_CACHE_SERVICE_URL = 'mojo:url_response_disk_cache'
_NETWORK_SERVICE_URL = 'mojo:network_service'

_COLD_START_SHELL_ARGS = [
  '--args-for=%s %s' % (_CACHE_SERVICE_URL, '--clear'),
  '--args-for=%s %s' % (_NETWORK_SERVICE_URL, '--clear'),
]

# Additional time in seconds allocated per shell run to accommodate start-up.
# The shell should terminate before hitting this time out, it is an error if it
# doesn't.
_EXTRA_TIMEOUT = 20

_MEASUREMENT_RESULT_FORMAT = r"""
^              # Beginning of the line.
measurement:   # Hard-coded tag.
\s+(\S+)       # Match measurement name.
\s+(\S+)       # Match measurement result.
$              # End of the line.
"""

_MEASUREMENT_REGEX = re.compile(_MEASUREMENT_RESULT_FORMAT, re.VERBOSE)


def _generate_benchmark_variants(benchmark_spec):
  """Generates benchmark specifications for individual variants of the given
  benchmark: cold start and warm start.

  Returns:
    A list of benchmark specs corresponding to individual variants of the given
        benchmark.
  """
  variants = []
  variants.append({
    'variant_name': 'cold start',
    'app': benchmark_spec['app'],
    'duration': benchmark_spec['duration'],
    'measurements': benchmark_spec['measurements'],
    'shell-args': benchmark_spec.get('shell-args',
                                     []) + _COLD_START_SHELL_ARGS})
  variants.append({
    'variant_name': 'warm start',
    'app': benchmark_spec['app'],
    'duration': benchmark_spec['duration'],
    'measurements': benchmark_spec['measurements'],
    'shell-args': benchmark_spec.get('shell-args', [])})
  return variants


def _run_benchmark(shell, shell_args, name, app, duration_seconds, measurements,
                   verbose, android, output_file):
  """Runs the given benchmark by running `benchmark.mojo` in mojo shell with
  appropriate arguments and returns the produced output.

  Returns:
    A tuple of (succeeded, error_msg, output).
  """
  timeout = duration_seconds + _EXTRA_TIMEOUT
  benchmark_args = []
  benchmark_args.append('--app=' + app)
  benchmark_args.append('--duration=' + str(duration_seconds))

  device_output_file = None
  if output_file:
    if android:
      device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file)
      benchmark_args.append('--trace-output=' + device_output_file)
    else:
      benchmark_args.append('--trace-output=' + output_file)

  for measurement in measurements:
    benchmark_args.append(measurement)

  shell_args = list(shell_args)
  shell_args.append(_BENCHMARK_APP)
  shell_args.append('--force-offline-by-default')
  shell_args.append('--args-for=%s %s' % (_BENCHMARK_APP,
                                          ' '.join(benchmark_args)))

  if verbose:
    print 'shell arguments: ' + str(shell_args)
  return_code, output, did_time_out = shell.run_and_get_output(
      shell_args, timeout=timeout)

  if did_time_out:
    return False, 'timed out', output
  if return_code:
    return False, 'return code: ' + str(return_code), output

  # Pull the trace file even if some measurements are missing, as it can be
  # useful in debugging.
  if device_output_file:
    shell.pull_file(device_output_file, output_file, remove_original=True)

  return True, None, output

def _parse_measurement_results(output):
  """Parses the measurement results present in the benchmark output and returns
  the dictionary of correctly recognized and parsed results.
  """
  measurement_results = {}
  output_lines = [line.strip() for line in output.split('\n')]
  for line in output_lines:
    match = re.match(_MEASUREMENT_REGEX, line)
    if match:
      measurement_name = match.group(1)
      measurement_result = match.group(2)
      try:
        measurement_results[measurement_name] = float(measurement_result)
      except ValueError:
        pass
  return measurement_results


def main():
  parser = argparse.ArgumentParser(
      formatter_class=argparse.RawDescriptionHelpFormatter,
      description=_DESCRIPTION)
  parser.add_argument('benchmark_list_file', type=file,
                      help='a file listing benchmarks to run')
  parser.add_argument('--save-traces', action='store_true',
                      help='save the traces produced by benchmarks to disk')
  perf_dashboard.add_argparse_server_arguments(parser)

  # Common shell configuration arguments.
  shell_config.add_shell_arguments(parser)
  script_args = parser.parse_args()
  config = shell_config.get_shell_config(script_args)

  try:
    shell, common_shell_args = shell_arguments.get_shell(config, [])
  except shell_arguments.ShellConfigurationException as e:
    print e
    return 1

  target_os = 'android' if script_args.android else 'linux'
  benchmark_list_params = {"target_os": target_os}
  exec script_args.benchmark_list_file in benchmark_list_params

  exit_code = 0
  for benchmark_spec in benchmark_list_params['benchmarks']:
    benchmark_name = benchmark_spec['name']

    for variant_spec in _generate_benchmark_variants(benchmark_spec):
      variant_name = variant_spec['variant_name']
      app = variant_spec['app']
      duration = variant_spec['duration']
      shell_args = variant_spec.get('shell-args', []) + common_shell_args
      measurements = variant_spec['measurements']

      output_file = None
      if script_args.save_traces:
        output_file = 'benchmark-%s-%s-%s.trace' % (
            benchmark_name.replace(' ', '_'),
            variant_name.replace(' ', '_'),
            time.strftime('%Y%m%d%H%M%S'))

      chart_data_recorder = None
      if script_args.upload:
        chart_data_recorder = perf_dashboard.ChartDataRecorder(
            script_args.test_name)

      benchmark_succeeded, benchmark_error, output = _run_benchmark(
          shell, shell_args, variant_name, app, duration, measurements,
          script_args.verbose, script_args.android, output_file)

      print '[ %s ] %s ' % (benchmark_name, variant_name)

      some_measurements_failed = False
      if benchmark_succeeded:
        measurement_results = _parse_measurement_results(output)
        # Iterate over the list of specs, not the dictionary, to detect missing
        # results and preserve the required order.
        for measurement_spec in measurements:
          if measurement_spec in measurement_results:
            result = measurement_results[measurement_spec]
            print '%s %s' % (measurement_spec, result)

            if chart_data_recorder:
              chart_name = benchmark_name + '__' + variant_name
              chart_data_recorder.record_scalar(
                  perf_dashboard.normalize_label(chart_name),
                  perf_dashboard.normalize_label(measurement_spec),
                  'ms', result)
          else:
            print '%s ?' % measurement_spec
            some_measurements_failed = True

      if not benchmark_succeeded or some_measurements_failed:
        if not benchmark_succeeded:
          print 'benchmark failed: ' + benchmark_error
        if some_measurements_failed:
          print 'some measurements failed'
        print 'output: '
        print '-' * 72
        print output
        print '-' * 72
        exit_code = 1

      if script_args.upload:
        perf_dashboard.upload_chart_data(
            script_args.master_name, script_args.bot_name,
            script_args.test_name, script_args.builder_name,
            script_args.build_number, chart_data_recorder.get_chart_data(),
            script_args.server_url, script_args.dry_run)

  return exit_code

if __name__ == '__main__':
  sys.exit(main())