devtools: download shell binaries and set origin when run w/ --mojo-version.
Before, a user of devtools that wanted to use prebuilt mojo at particular
version, had to:
- set up DEPS hooks and use GN build rules that together downloaded and copied
the shell and the network service at the correct version to the output
directory
- set the --origin to point to deployed mojo services at particular version
This patch adds a --mojo-version flag that does all of the above, downloading
binaries as needed and setting the origin accordingly.
Once consumers switch over to --mojo-version, we can retire the GN rules for
downloading prebuilt shells.
Fixes https://github.com/domokit/devtools/issues/62.
Fixes https://github.com/domokit/devtools/issues/63.
R=qsr@chromium.org
Review URL: https://codereview.chromium.org/1844943004 .
Cr-Mirrored-From: https://github.com/domokit/mojo
Cr-Mirrored-Commit: da012592f169e8c531e4a92cdf00d3c39413dd34
diff --git a/.gitignore b/.gitignore
index 0d20b64..930730d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
*.pyc
+_prebuilt
diff --git a/devtoolslib/download.py b/devtoolslib/download.py
new file mode 100644
index 0000000..db6518f
--- /dev/null
+++ b/devtoolslib/download.py
@@ -0,0 +1,99 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import subprocess
+import sys
+import tempfile
+import urllib2
+import zipfile
+
+from devtoolslib import paths
+
+
+_SHELL_FILE_NAME = {
+ 'linux-x64': 'mojo_shell',
+ 'android-arm': 'MojoShell.apk'
+}
+
+
+def _get_artifacts_to_download(mojo_version, platform, verbose):
+ """Returns a list of tuples of (gs_path, file_name) to be downloaded."""
+ artifacts = []
+ shell_gs_path = ('gs://mojo/shell/%s/%s.zip' % (mojo_version, platform))
+ artifacts.append(
+ (shell_gs_path, _SHELL_FILE_NAME[platform])
+ )
+ if platform == 'linux-x64':
+ network_service_version_url = (
+ 'https://raw.githubusercontent.com/domokit/mojo/' +
+ mojo_version + '/mojo/public/tools/NETWORK_SERVICE_VERSION')
+ network_service_version = (
+ urllib2.urlopen(network_service_version_url).read().strip())
+ if verbose:
+ print('Looked up the network service version for mojo at %s as: %s ' % (
+ mojo_version, network_service_version))
+
+ network_service_gs_path = (
+ 'gs://mojo/network_service/%s/%s/network_service.mojo.zip' %
+ (network_service_version, platform))
+ artifacts.append(
+ (network_service_gs_path, 'network_service.mojo')
+ )
+ return artifacts
+
+
+def _download_from_gs(gs_path, output_path, depot_tools_path, verbose):
+ """Downloads the file at the given gs_path using depot_tools."""
+ # We're downloading from a public bucket which does not need authentication,
+ # but the user might have busted credential files somewhere such as ~/.boto
+ # that the gsutil script will try (and fail) to use. Setting these
+ # environment variables convinces gsutil not to attempt to use these.
+ env = os.environ.copy()
+ env['AWS_CREDENTIAL_FILE'] = ""
+ env['BOTO_CONFIG'] = ""
+
+ gsutil_exe = os.path.join(depot_tools_path, 'third_party', 'gsutil', 'gsutil')
+ if verbose:
+ print('Fetching ' + gs_path)
+
+ try:
+ subprocess.check_output(
+ [gsutil_exe,
+ '--bypass_prodaccess',
+ 'cp',
+ gs_path,
+ output_path],
+ stderr=subprocess.STDOUT,
+ env=env)
+ except subprocess.CalledProcessError as e:
+ print e.output
+ sys.exit(1)
+
+
+def _extract_file(archive_path, file_name, output_dir):
+ with zipfile.ZipFile(archive_path) as z:
+ zi = z.getinfo(file_name)
+ mode = zi.external_attr >> 16
+ z.extract(zi, output_dir)
+ os.chmod(os.path.join(output_dir, file_name), mode)
+
+
+def download_shell(mojo_version, platform, root_output_dir, verbose):
+ """Downloads the shell (along with corresponding artifacts if needed) at the
+ given version.
+ """
+ depot_tools_path = paths.find_depot_tools()
+ artifacts = _get_artifacts_to_download(mojo_version, platform, verbose)
+ output_dir = os.path.join(root_output_dir, mojo_version, platform)
+
+ for (gs_path, file_name) in artifacts:
+ if os.path.isfile(os.path.join(output_dir, file_name)):
+ continue
+
+ with tempfile.NamedTemporaryFile() as temp_zip_file:
+ _download_from_gs(gs_path, temp_zip_file.name, depot_tools_path, verbose)
+ _extract_file(temp_zip_file.name, file_name, output_dir)
+
+ return os.path.join(output_dir, _SHELL_FILE_NAME[platform])
diff --git a/devtoolslib/paths.py b/devtoolslib/paths.py
index 870dd0a..a1b1b41 100644
--- a/devtoolslib/paths.py
+++ b/devtoolslib/paths.py
@@ -12,6 +12,8 @@
import os.path
import sys
+DEVTOOLS_ROOT = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
+
def find_ancestor_with(relpath, start_path=None):
"""Returns the lowest ancestor of this file that contains |relpath|."""
diff --git a/devtoolslib/shell_arguments.py b/devtoolslib/shell_arguments.py
index 616615b..fd012f2 100644
--- a/devtoolslib/shell_arguments.py
+++ b/devtoolslib/shell_arguments.py
@@ -11,6 +11,8 @@
import os.path
import urlparse
+from devtoolslib import download
+from devtoolslib import paths
from devtoolslib.android_shell import AndroidShell
from devtoolslib.linux_shell import LinuxShell
from devtoolslib.shell_config import ShellConfigurationException
@@ -193,6 +195,20 @@
Throws:
ShellConfigurationException if shell abstraction could not be configured.
"""
+ platform = 'android-arm' if shell_config.android else 'linux-x64'
+
+ shell_path = shell_config.shell_path
+ if not shell_path and shell_config.mojo_version:
+ download_dir = os.path.join(paths.DEVTOOLS_ROOT, '_prebuilt')
+ shell_path = download.download_shell(shell_config.mojo_version, platform,
+ download_dir, shell_config.verbose)
+ if shell_config.verbose:
+ if shell_path:
+ print('Using shell binary at ' + shell_path)
+ else:
+ print('No shell path given, only running on Android with pre-installed '
+ 'shell will be possible.')
+
if shell_config.android:
shell = AndroidShell(shell_config.adb_path, shell_config.target_device,
logcat_tags=shell_config.logcat_tags,
@@ -201,13 +217,14 @@
device_status, error = shell.check_device()
if not device_status:
raise ShellConfigurationException('Device check failed: ' + error)
- if shell_config.shell_path:
- shell.install_apk(shell_config.shell_path)
+ if shell_path:
+ shell.install_apk(shell_path)
else:
- if not shell_config.shell_path:
+ if not shell_path:
raise ShellConfigurationException('Can not run without a shell binary. '
- 'Please pass --shell-path.')
- shell = LinuxShell(shell_config.shell_path)
+ 'Please pass --mojo-version or '
+ '--shell-path.')
+ shell = LinuxShell(shell_path)
if shell_config.use_osmesa:
shell_args.append('--args-for=mojo:native_viewport_service --use-osmesa')
@@ -215,7 +232,9 @@
shell_config.map_origin_list,
shell_config.reuse_servers)
+ # Configure origin for mojo: urls.
if shell_config.origin:
+ # If origin was set on the command line, this takes precedence.
if _is_web_url(shell_config.origin):
shell_args.append('--origin=' + shell_config.origin)
else:
@@ -223,6 +242,13 @@
shell_args.extend(configure_local_origin(shell, shell_config.origin,
local_origin_port,
shell_config.reuse_servers))
+ elif shell_config.mojo_version:
+ # Otherwise we infer the origin from the mojo_version.
+ web_origin = "https://storage.googleapis.com/mojo/services/%s/%s" % (
+ platform, shell_config.mojo_version)
+ if shell_config.verbose:
+ print('Inferring origin from `MOJO_VERSION` as: ' + web_origin)
+ shell_args.append('--origin=' + web_origin)
if shell_config.content_handlers:
for (mime_type,
diff --git a/devtoolslib/shell_config.py b/devtoolslib/shell_config.py
index e1c04fe..298a54f 100644
--- a/devtoolslib/shell_config.py
+++ b/devtoolslib/shell_config.py
@@ -24,6 +24,7 @@
def __init__(self):
self.android = None
+ self.mojo_version = None
self.shell_path = None
self.origin = None
self.map_url_list = []
@@ -59,6 +60,9 @@
# Arguments configuring the shell run.
parser.add_argument('--android', help='Run on Android',
action='store_true')
+ parser.add_argument('--mojo-version', help='Version of the Mojo to use. '
+ 'This will set the origin and download the shell binary '
+ 'at the given version')
parser.add_argument('--shell-path', help='Path of the Mojo shell binary.')
parser.add_argument('--origin', help='Origin for mojo: URLs. This can be a '
'web url or a local directory path. If MOJO_VERSION file '
@@ -144,33 +148,33 @@
Returns:
An instance of ShellConfig.
"""
- # Infer paths based on the Chromium configuration options
- # (--debug/--release, etc.), if running within a Chromium-like checkout.
- inferred_params = paths.infer_params(script_args.android, script_args.debug,
- script_args.target_cpu)
- inferred_origin = None
- if inferred_params['mojo_version']:
- inferred_origin = "https://storage.googleapis.com/mojo/services"
- if script_args.android:
- inferred_origin += "/android-arm"
- else:
- inferred_origin += "/linux-x64"
- # Get the versions that were built against Modular's version of the SDK.
- inferred_origin += "/%s" % inferred_params['mojo_version']
- if script_args.verbose:
- print('Inferring origin from `MOJO_VERSION` as: ' +
- inferred_origin)
shell_config = ShellConfig()
shell_config.android = script_args.android
- shell_config.shell_path = (script_args.shell_path or
- inferred_params['shell_path'])
- shell_config.origin = script_args.origin or inferred_origin
+ shell_config.mojo_version = script_args.mojo_version
+ shell_config.shell_path = script_args.shell_path
+ shell_config.origin = script_args.origin
shell_config.map_url_list = script_args.map_url
shell_config.map_origin_list = script_args.map_origin
shell_config.reuse_servers = script_args.reuse_servers
shell_config.verbose = script_args.verbose
+ # Infer paths based on the Chromium configuration options
+ # (--debug/--release, etc.), if running within a Chromium-like checkout.
+ inferred_params = paths.infer_params(script_args.android, script_args.debug,
+ script_args.target_cpu)
+ # Infer |mojo_version| if the client sets it in a MOJO_VERSION file.
+ shell_config.mojo_version = (shell_config.mojo_version or
+ inferred_params['mojo_version'])
+ # Use the shell binary and mojo: apps from the build directory, unless
+ # |mojo_version| is set.
+ if not shell_config.mojo_version:
+ shell_config.shell_path = (shell_config.shell_path or
+ inferred_params['shell_path'])
+ if shell_config.android:
+ shell_config.origin = (shell_config.origin or
+ inferred_params['build_dir_path'])
+
# Android-only.
shell_config.adb_path = (script_args.adb_path or inferred_params['adb_path']
or 'adb')
@@ -180,10 +184,6 @@
# Desktop-only.
shell_config.use_osmesa = script_args.use_osmesa
- if (shell_config.android and not shell_config.origin and
- inferred_params['build_dir_path']):
- shell_config.origin = inferred_params['build_dir_path']
-
# Read the mojoconfig file.
config_file = script_args.config_file
if not script_args.no_config_file:
diff --git a/docs/mojo_run.md b/docs/mojo_run.md
index aa60937..9279cd6 100644
--- a/docs/mojo_run.md
+++ b/docs/mojo_run.md
@@ -4,18 +4,34 @@
Android device.
```sh
-mojo_run APP_URL # Run on the host.
+mojo_run APP_URL # Run on Linux host.
mojo_run APP_URL --android # Run on Android device.
mojo_run "APP_URL APP_ARGUMENTS" # Run an app with startup arguments
```
-Unless running within a Mojo checkout, we need to indicate the path to the shell
-binary:
+## mojo version
+
+`mojo_run` will download mojo shell and configure it to use `mojo:` apps built
+at the corresponding version, if you pass the git commit sha of the
+https://github.com/domokit/mojo repository as `--mojo-version`:
```sh
-mojo_run --shell-path path/to/shell/binary APP_URL
+mojo_run APP_URL --mojo-version SOME_HASH
```
+If your project uses a pinned version of mojo, you can put the pinned hash in
+a `MOJO_VERSION` file in any ancestor directory of `mojo_run`. This will make
+`mojo_run` infer the parameter automatically.
+
+If you don't want to use prebuilt binaries at the given version, you can
+configure the shell binary and the origin to use manually:
+
+```sh
+mojo_run APP_URL --shell-path path/to/shell/binary --origin ORIGIN_URL
+```
+
+## Running applications in a view
+
Some applications implement ViewProvider and are run embedded in a view. To run
these, you can pass the app url using the `--embed` flag:
@@ -65,12 +81,3 @@
mkdir ~/another_home
HOME=~/another_home mojo_run APP_URL --reuse-servers
```
-
-## Setting default mojo origin
-
-When run outside of the `domokit/mojo` repository, `mojo_run` needs `--origin`
-parameter to indicate where binaries of the core mojo services come from. If a
-`MOJO_VERSION` file is present among ancestors of `mojo_run` and `--origin`
-parameter is not set, origin will point to Google Storage location storing
-binaries of core mojo services built at the git revision indicated in
-`MOJO_VERSION`.
