Teach mojo_benchmark to produce chart_data for the perf dashboard. This patch adds the ability to encode the benchmark results in the 'chart_data' format understood by Chromium Performance Benchmark. Fixes domokit/devtools#52. R=viettrungluu@chromium.org Review URL: https://codereview.chromium.org/1406063002 . Cr-Mirrored-From: https://github.com/domokit/mojo Cr-Mirrored-Commit: 64736fae560422778d97e5e747458e34697cd24e
diff --git a/devtoolslib/perf_dashboard.py b/devtoolslib/perf_dashboard.py new file mode 100644 index 0000000..aa11ff6 --- /dev/null +++ b/devtoolslib/perf_dashboard.py
@@ -0,0 +1,35 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Disable the line-too-long warning. +# pylint: disable=C0301 +"""This module implements the Chromium Performance Dashboard JSON v1.0 data +format. + +See http://www.chromium.org/developers/speed-infra/performance-dashboard/sending-data-to-the-performance-dashboard. +""" + +import json +from collections import defaultdict + +class ChartDataRecorder(object): + """Allows one to record measurement values one by one and then generate the + JSON string that represents them in the 'chart_data' format expected by the + performance dashboard. + """ + + def __init__(self): + self.charts = defaultdict(list) + + def record_scalar(self, chart_name, value_name, units, value): + """Records a single measurement value of a scalar type.""" + self.charts[chart_name].append({ + 'type': 'scalar', + 'name': value_name, + 'units': units, + 'value': value}) + + def get_json(self): + """Returns the JSON string representing the recorded chart data.""" + return json.dumps(self.charts)
diff --git a/devtoolslib/perf_dashboard_unittest.py b/devtoolslib/perf_dashboard_unittest.py new file mode 100644 index 0000000..61809c3 --- /dev/null +++ b/devtoolslib/perf_dashboard_unittest.py
@@ -0,0 +1,68 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the Chromium Performance Dashboard data format implementation.""" + +import imp +import json +import os.path +import sys +import unittest + +try: + imp.find_module("devtoolslib") +except ImportError: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from devtoolslib.perf_dashboard import ChartDataRecorder + +class ChartDataRecorderTest(unittest.TestCase): + """Tests the chart data recorder.""" + + def test_empty(self): + """Tests chart data with no charts.""" + recorder = ChartDataRecorder() + result = json.loads(recorder.get_json()) + self.assertEquals(0, len(result)) + + def test_one_chart(self): + """Tests chart data with two samples in one chart.""" + recorder = ChartDataRecorder() + recorder.record_scalar('chart', 'val1', 'ms', 1) + recorder.record_scalar('chart', 'val2', 'ms', 2) + + result = json.loads(recorder.get_json()) + self.assertEquals(1, len(result)) + self.assertEquals(2, len(result['chart'])) + self.assertEquals({ + 'type': 'scalar', + 'name': 'val1', + 'units': 'ms', + 'value': 1}, result['chart'][0]) + self.assertEquals({ + 'type': 'scalar', + 'name': 'val2', + 'units': 'ms', + 'value': 2}, result['chart'][1]) + + def test_two_charts(self): + """Tests chart data with two samples over two charts.""" + recorder = ChartDataRecorder() + recorder.record_scalar('chart1', 'val1', 'ms', 1) + recorder.record_scalar('chart2', 'val2', 'ms', 2) + + result = json.loads(recorder.get_json()) + self.assertEquals(2, len(result)) + self.assertEquals(1, len(result['chart1'])) + self.assertEquals({ + 'type': 'scalar', + 'name': 'val1', + 'units': 'ms', + 'value': 1}, result['chart1'][0]) + self.assertEquals(1, len(result['chart2'])) + self.assertEquals({ + 'type': 'scalar', + 'name': 'val2', + 'units': 'ms', + 'value': 2}, result['chart2'][0])
diff --git a/mojo_benchmark b/mojo_benchmark index 0f2126b..5ffdbe1 100755 --- a/mojo_benchmark +++ b/mojo_benchmark
@@ -14,6 +14,7 @@ from devtoolslib import shell_arguments from devtoolslib import shell_config +from devtoolslib import performance_dashboard _DESCRIPTION = """Runner for Mojo application benchmarks. @@ -175,6 +176,9 @@ help='a file listing benchmarks to run') parser.add_argument('--save-traces', action='store_true', help='save the traces produced by benchmarks to disk') + parser.add_argument('--chart-data-output-file', type=argparse.FileType('w'), + help='file to write chart data for the performance ' + 'dashboard to') # Common shell configuration arguments. shell_config.add_shell_arguments(parser) @@ -191,6 +195,10 @@ benchmark_list_params = {"target_os": target_os} exec script_args.benchmark_list_file in benchmark_list_params + chart_data_recorder = None + if script_args.chart_data_output_file: + chart_data_recorder = performance_dashboard.ChartDataRecorder() + exit_code = 0 for benchmark_spec in benchmark_list_params['benchmarks']: for variant_spec in _generate_benchmark_variants(benchmark_spec): @@ -213,8 +221,13 @@ # results and preserve the required order. for measurement_spec in measurements: if measurement_spec in measurement_results: - print '%s %s' % (measurement_spec, - measurement_results[measurement_spec]) + result = measurement_results[measurement_spec] + print '%s %s' % (measurement_spec, result) + + if chart_data_recorder: + measurement_name = measurement_spec.replace('/', '-') + chart_data_recorder.record_scalar(name, measurement_name, 'ms', + result) else: print '%s ?' % measurement_spec some_measurements_failed = True @@ -230,6 +243,9 @@ print '-' * 72 exit_code = 1 + if script_args.chart_data_output_file: + script_args.chart_data_output_file.write(chart_data_recorder.get_json()) + return exit_code if __name__ == '__main__':