devtools: download shell binaries and set origin when run w/ --mojo-version. Before, a user of devtools that wanted to use prebuilt mojo at particular version, had to: - set up DEPS hooks and use GN build rules that together downloaded and copied the shell and the network service at the correct version to the output directory - set the --origin to point to deployed mojo services at particular version This patch adds a --mojo-version flag that does all of the above, downloading binaries as needed and setting the origin accordingly. Once consumers switch over to --mojo-version, we can retire the GN rules for downloading prebuilt shells. Fixes https://github.com/domokit/devtools/issues/62. Fixes https://github.com/domokit/devtools/issues/63. R=qsr@chromium.org Review URL: https://codereview.chromium.org/1844943004 . Cr-Mirrored-From: https://github.com/domokit/mojo Cr-Mirrored-Commit: da012592f169e8c531e4a92cdf00d3c39413dd34
diff --git a/.gitignore b/.gitignore index 0d20b64..930730d 100644 --- a/.gitignore +++ b/.gitignore
@@ -1 +1,2 @@ *.pyc +_prebuilt
diff --git a/devtoolslib/download.py b/devtoolslib/download.py new file mode 100644 index 0000000..db6518f --- /dev/null +++ b/devtoolslib/download.py
@@ -0,0 +1,99 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import subprocess +import sys +import tempfile +import urllib2 +import zipfile + +from devtoolslib import paths + + +_SHELL_FILE_NAME = { + 'linux-x64': 'mojo_shell', + 'android-arm': 'MojoShell.apk' +} + + +def _get_artifacts_to_download(mojo_version, platform, verbose): + """Returns a list of tuples of (gs_path, file_name) to be downloaded.""" + artifacts = [] + shell_gs_path = ('gs://mojo/shell/%s/%s.zip' % (mojo_version, platform)) + artifacts.append( + (shell_gs_path, _SHELL_FILE_NAME[platform]) + ) + if platform == 'linux-x64': + network_service_version_url = ( + 'https://raw.githubusercontent.com/domokit/mojo/' + + mojo_version + '/mojo/public/tools/NETWORK_SERVICE_VERSION') + network_service_version = ( + urllib2.urlopen(network_service_version_url).read().strip()) + if verbose: + print('Looked up the network service version for mojo at %s as: %s ' % ( + mojo_version, network_service_version)) + + network_service_gs_path = ( + 'gs://mojo/network_service/%s/%s/network_service.mojo.zip' % + (network_service_version, platform)) + artifacts.append( + (network_service_gs_path, 'network_service.mojo') + ) + return artifacts + + +def _download_from_gs(gs_path, output_path, depot_tools_path, verbose): + """Downloads the file at the given gs_path using depot_tools.""" + # We're downloading from a public bucket which does not need authentication, + # but the user might have busted credential files somewhere such as ~/.boto + # that the gsutil script will try (and fail) to use. Setting these + # environment variables convinces gsutil not to attempt to use these. + env = os.environ.copy() + env['AWS_CREDENTIAL_FILE'] = "" + env['BOTO_CONFIG'] = "" + + gsutil_exe = os.path.join(depot_tools_path, 'third_party', 'gsutil', 'gsutil') + if verbose: + print('Fetching ' + gs_path) + + try: + subprocess.check_output( + [gsutil_exe, + '--bypass_prodaccess', + 'cp', + gs_path, + output_path], + stderr=subprocess.STDOUT, + env=env) + except subprocess.CalledProcessError as e: + print e.output + sys.exit(1) + + +def _extract_file(archive_path, file_name, output_dir): + with zipfile.ZipFile(archive_path) as z: + zi = z.getinfo(file_name) + mode = zi.external_attr >> 16 + z.extract(zi, output_dir) + os.chmod(os.path.join(output_dir, file_name), mode) + + +def download_shell(mojo_version, platform, root_output_dir, verbose): + """Downloads the shell (along with corresponding artifacts if needed) at the + given version. + """ + depot_tools_path = paths.find_depot_tools() + artifacts = _get_artifacts_to_download(mojo_version, platform, verbose) + output_dir = os.path.join(root_output_dir, mojo_version, platform) + + for (gs_path, file_name) in artifacts: + if os.path.isfile(os.path.join(output_dir, file_name)): + continue + + with tempfile.NamedTemporaryFile() as temp_zip_file: + _download_from_gs(gs_path, temp_zip_file.name, depot_tools_path, verbose) + _extract_file(temp_zip_file.name, file_name, output_dir) + + return os.path.join(output_dir, _SHELL_FILE_NAME[platform])
diff --git a/devtoolslib/paths.py b/devtoolslib/paths.py index 870dd0a..a1b1b41 100644 --- a/devtoolslib/paths.py +++ b/devtoolslib/paths.py
@@ -12,6 +12,8 @@ import os.path import sys +DEVTOOLS_ROOT = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + def find_ancestor_with(relpath, start_path=None): """Returns the lowest ancestor of this file that contains |relpath|."""
diff --git a/devtoolslib/shell_arguments.py b/devtoolslib/shell_arguments.py index 616615b..fd012f2 100644 --- a/devtoolslib/shell_arguments.py +++ b/devtoolslib/shell_arguments.py
@@ -11,6 +11,8 @@ import os.path import urlparse +from devtoolslib import download +from devtoolslib import paths from devtoolslib.android_shell import AndroidShell from devtoolslib.linux_shell import LinuxShell from devtoolslib.shell_config import ShellConfigurationException @@ -193,6 +195,20 @@ Throws: ShellConfigurationException if shell abstraction could not be configured. """ + platform = 'android-arm' if shell_config.android else 'linux-x64' + + shell_path = shell_config.shell_path + if not shell_path and shell_config.mojo_version: + download_dir = os.path.join(paths.DEVTOOLS_ROOT, '_prebuilt') + shell_path = download.download_shell(shell_config.mojo_version, platform, + download_dir, shell_config.verbose) + if shell_config.verbose: + if shell_path: + print('Using shell binary at ' + shell_path) + else: + print('No shell path given, only running on Android with pre-installed ' + 'shell will be possible.') + if shell_config.android: shell = AndroidShell(shell_config.adb_path, shell_config.target_device, logcat_tags=shell_config.logcat_tags, @@ -201,13 +217,14 @@ device_status, error = shell.check_device() if not device_status: raise ShellConfigurationException('Device check failed: ' + error) - if shell_config.shell_path: - shell.install_apk(shell_config.shell_path) + if shell_path: + shell.install_apk(shell_path) else: - if not shell_config.shell_path: + if not shell_path: raise ShellConfigurationException('Can not run without a shell binary. ' - 'Please pass --shell-path.') - shell = LinuxShell(shell_config.shell_path) + 'Please pass --mojo-version or ' + '--shell-path.') + shell = LinuxShell(shell_path) if shell_config.use_osmesa: shell_args.append('--args-for=mojo:native_viewport_service --use-osmesa') @@ -215,7 +232,9 @@ shell_config.map_origin_list, shell_config.reuse_servers) + # Configure origin for mojo: urls. if shell_config.origin: + # If origin was set on the command line, this takes precedence. if _is_web_url(shell_config.origin): shell_args.append('--origin=' + shell_config.origin) else: @@ -223,6 +242,13 @@ shell_args.extend(configure_local_origin(shell, shell_config.origin, local_origin_port, shell_config.reuse_servers)) + elif shell_config.mojo_version: + # Otherwise we infer the origin from the mojo_version. + web_origin = "https://storage.googleapis.com/mojo/services/%s/%s" % ( + platform, shell_config.mojo_version) + if shell_config.verbose: + print('Inferring origin from `MOJO_VERSION` as: ' + web_origin) + shell_args.append('--origin=' + web_origin) if shell_config.content_handlers: for (mime_type,
diff --git a/devtoolslib/shell_config.py b/devtoolslib/shell_config.py index e1c04fe..298a54f 100644 --- a/devtoolslib/shell_config.py +++ b/devtoolslib/shell_config.py
@@ -24,6 +24,7 @@ def __init__(self): self.android = None + self.mojo_version = None self.shell_path = None self.origin = None self.map_url_list = [] @@ -59,6 +60,9 @@ # Arguments configuring the shell run. parser.add_argument('--android', help='Run on Android', action='store_true') + parser.add_argument('--mojo-version', help='Version of the Mojo to use. ' + 'This will set the origin and download the shell binary ' + 'at the given version') parser.add_argument('--shell-path', help='Path of the Mojo shell binary.') parser.add_argument('--origin', help='Origin for mojo: URLs. This can be a ' 'web url or a local directory path. If MOJO_VERSION file ' @@ -144,33 +148,33 @@ Returns: An instance of ShellConfig. """ - # Infer paths based on the Chromium configuration options - # (--debug/--release, etc.), if running within a Chromium-like checkout. - inferred_params = paths.infer_params(script_args.android, script_args.debug, - script_args.target_cpu) - inferred_origin = None - if inferred_params['mojo_version']: - inferred_origin = "https://storage.googleapis.com/mojo/services" - if script_args.android: - inferred_origin += "/android-arm" - else: - inferred_origin += "/linux-x64" - # Get the versions that were built against Modular's version of the SDK. - inferred_origin += "/%s" % inferred_params['mojo_version'] - if script_args.verbose: - print('Inferring origin from `MOJO_VERSION` as: ' + - inferred_origin) shell_config = ShellConfig() shell_config.android = script_args.android - shell_config.shell_path = (script_args.shell_path or - inferred_params['shell_path']) - shell_config.origin = script_args.origin or inferred_origin + shell_config.mojo_version = script_args.mojo_version + shell_config.shell_path = script_args.shell_path + shell_config.origin = script_args.origin shell_config.map_url_list = script_args.map_url shell_config.map_origin_list = script_args.map_origin shell_config.reuse_servers = script_args.reuse_servers shell_config.verbose = script_args.verbose + # Infer paths based on the Chromium configuration options + # (--debug/--release, etc.), if running within a Chromium-like checkout. + inferred_params = paths.infer_params(script_args.android, script_args.debug, + script_args.target_cpu) + # Infer |mojo_version| if the client sets it in a MOJO_VERSION file. + shell_config.mojo_version = (shell_config.mojo_version or + inferred_params['mojo_version']) + # Use the shell binary and mojo: apps from the build directory, unless + # |mojo_version| is set. + if not shell_config.mojo_version: + shell_config.shell_path = (shell_config.shell_path or + inferred_params['shell_path']) + if shell_config.android: + shell_config.origin = (shell_config.origin or + inferred_params['build_dir_path']) + # Android-only. shell_config.adb_path = (script_args.adb_path or inferred_params['adb_path'] or 'adb') @@ -180,10 +184,6 @@ # Desktop-only. shell_config.use_osmesa = script_args.use_osmesa - if (shell_config.android and not shell_config.origin and - inferred_params['build_dir_path']): - shell_config.origin = inferred_params['build_dir_path'] - # Read the mojoconfig file. config_file = script_args.config_file if not script_args.no_config_file:
diff --git a/docs/mojo_run.md b/docs/mojo_run.md index aa60937..9279cd6 100644 --- a/docs/mojo_run.md +++ b/docs/mojo_run.md
@@ -4,18 +4,34 @@ Android device. ```sh -mojo_run APP_URL # Run on the host. +mojo_run APP_URL # Run on Linux host. mojo_run APP_URL --android # Run on Android device. mojo_run "APP_URL APP_ARGUMENTS" # Run an app with startup arguments ``` -Unless running within a Mojo checkout, we need to indicate the path to the shell -binary: +## mojo version + +`mojo_run` will download mojo shell and configure it to use `mojo:` apps built +at the corresponding version, if you pass the git commit sha of the +https://github.com/domokit/mojo repository as `--mojo-version`: ```sh -mojo_run --shell-path path/to/shell/binary APP_URL +mojo_run APP_URL --mojo-version SOME_HASH ``` +If your project uses a pinned version of mojo, you can put the pinned hash in +a `MOJO_VERSION` file in any ancestor directory of `mojo_run`. This will make +`mojo_run` infer the parameter automatically. + +If you don't want to use prebuilt binaries at the given version, you can +configure the shell binary and the origin to use manually: + +```sh +mojo_run APP_URL --shell-path path/to/shell/binary --origin ORIGIN_URL +``` + +## Running applications in a view + Some applications implement ViewProvider and are run embedded in a view. To run these, you can pass the app url using the `--embed` flag: @@ -65,12 +81,3 @@ mkdir ~/another_home HOME=~/another_home mojo_run APP_URL --reuse-servers ``` - -## Setting default mojo origin - -When run outside of the `domokit/mojo` repository, `mojo_run` needs `--origin` -parameter to indicate where binaries of the core mojo services come from. If a -`MOJO_VERSION` file is present among ancestors of `mojo_run` and `--origin` -parameter is not set, origin will point to Google Storage location storing -binaries of core mojo services built at the git revision indicated in -`MOJO_VERSION`.