devtools: new workflow for multiple simultaneous `mojo_run` runs.

This patch provides a new workflow for multiple simultaneous runs of
`mojo_run`. In the new workflow first instance of `mojo_run` is started
normally, while the subsequent ones get the `--reuse-servers` flag.

`--reuse-servers` skips spawning any dev servers, instead it assumes
that they are already running and only sets up port forwarding if
needed. This preserves caching and should work over adb_remote_setup.

The previous workaround switches: `--free-ports` and `--free-host-ports`
are removed.

Fixes https://github.com/domokit/devtools/issues/55 .

R=qsr@chromium.org

Review URL: https://codereview.chromium.org/1436503002 .

Cr-Mirrored-From: https://github.com/domokit/mojo
Cr-Mirrored-Commit: b80202520e08230ac7c114569ffc0840581b4e0c

devtoolslib/android_shell.py

diff --git a/devtoolslib/android_shell.py b/devtoolslib/android_shell.py
index 400195c..7f69c08 100644
--- a/devtoolslib/android_shell.py
+++ b/devtoolslib/android_shell.py
@@ -16,7 +16,7 @@
 import time
 import uuid
 
-from devtoolslib.http_server import start_http_server
+from devtoolslib import http_server
 from devtoolslib.shell import Shell
 from devtoolslib.utils import overrides
 
@@ -421,10 +421,13 @@
     logcat_watch_thread.start()
 
   @overrides(Shell)
-  def serve_local_directories(self, mappings, port=0, free_host_port=False):
+  def serve_local_directories(self, mappings, port=0, reuse_servers=False):
     assert mappings
-    host_port = 0 if free_host_port else port
-    server_address = start_http_server(mappings, host_port=host_port)
+    if reuse_servers:
+      assert port, 'Cannot reuse the server when |port| is 0.'
+      server_address = ('127.0.0.1', port)
+    else:
+      server_address = http_server.start_http_server(mappings, port)
 
     return 'http://127.0.0.1:%d/' % self._forward_device_port_to_host(
         port, server_address[1])

devtoolslib/linux_shell.py

diff --git a/devtoolslib/linux_shell.py b/devtoolslib/linux_shell.py
index 48a2859..8a0ec7f 100644
--- a/devtoolslib/linux_shell.py
+++ b/devtoolslib/linux_shell.py
@@ -25,8 +25,14 @@
     self.command_prefix = command_prefix if command_prefix else []
 
   @overrides(Shell)
-  def serve_local_directories(self, mappings, port=0, free_host_port=False):
-    return 'http://%s:%d/' % http_server.start_http_server(mappings, port)
+  def serve_local_directories(self, mappings, port=0, reuse_servers=False):
+    if reuse_servers:
+      assert port, 'Cannot reuse the server when |port| is 0.'
+      server_address = ('127.0.0.1', port)
+    else:
+      server_address = http_server.start_http_server(mappings, port)
+
+    return 'http://%s:%d/' % server_address
 
   @overrides(Shell)
   def forward_host_port_to_shell(self, host_port):

devtoolslib/shell.py

diff --git a/devtoolslib/shell.py b/devtoolslib/shell.py
index 7e1d395..bc47856 100644
--- a/devtoolslib/shell.py
+++ b/devtoolslib/shell.py
@@ -6,7 +6,7 @@
 class Shell(object):
   """Represents an abstract Mojo shell."""
 
-  def serve_local_directories(self, mappings, port=0, free_host_port=False):
+  def serve_local_directories(self, mappings, port=0, reuse_servers=False):
     """Serves the content of the local (host) directories, making it available
     to the shell under the url returned by the function.
 
@@ -22,9 +22,9 @@
       port: port at which the server will be available to the shell. On Android
           this can be different from the port on which the server runs on the
           host.
-      free_host_port: spawn the server a system allocated port. This is ignored
-          on Linux, where |port| indicates the port on which the server will be
-          spawned.
+      reuse_servers: don't actually spawn the server. Instead assume that the
+          server is already running on |port|, and only set up forwarding if
+          needed.
 
     Returns:
       The url that the shell can use to access the server.

devtoolslib/shell_arguments.py

diff --git a/devtoolslib/shell_arguments.py b/devtoolslib/shell_arguments.py
index 65e6d3c..024ee51 100644
--- a/devtoolslib/shell_arguments.py
+++ b/devtoolslib/shell_arguments.py
@@ -26,7 +26,7 @@
   return True if urlparse.urlparse(dest).scheme else False
 
 
-def _host_local_url_destination(shell, dest_file, port, free_host_port):
+def _host_local_url_destination(shell, dest_file, port, reuse_servers):
   """Starts a local server to host |dest_file|.
 
   Returns:
@@ -37,21 +37,21 @@
     raise ValueError('local path passed as --map-url destination '
                      'does not exist')
   mappings = [('', [directory])]
-  server_url = shell.serve_local_directories(mappings, port, free_host_port)
+  server_url = shell.serve_local_directories(mappings, port, reuse_servers)
   return server_url + os.path.relpath(dest_file, directory)
 
 
-def _host_local_origin_destination(shell, dest_dir, port, free_host_port):
+def _host_local_origin_destination(shell, dest_dir, port, reuse_servers):
   """Starts a local server to host |dest_dir|.
 
   Returns:
     Url of the hosted directory.
   """
   mappings = [('', [dest_dir])]
-  return shell.serve_local_directories(mappings, port, free_host_port)
+  return shell.serve_local_directories(mappings, port, reuse_servers)
 
 
-def _rewrite(mapping, host_destination_functon, shell, port, free_host_port):
+def _rewrite(mapping, host_destination_functon, shell, port, reuse_servers):
   """Takes a mapping given as <src>=<dest> and rewrites the <dest> part to be
   hosted locally using the given function if <dest> is not a web url.
   """
@@ -64,12 +64,12 @@
     return mapping
 
   src = parts[0]
-  dest = host_destination_functon(shell, parts[1], port, free_host_port)
+  dest = host_destination_functon(shell, parts[1], port, reuse_servers)
   return src + '=' + dest
 
 
 def _apply_mappings(shell, original_arguments, map_urls, map_origins,
-                    free_ports, free_host_ports):
+                    reuse_servers):
   """Applies mappings for specified urls and origins. For each local path
   specified as destination a local server will be spawned and the mapping will
   be rewritten accordingly.
@@ -81,39 +81,34 @@
         <url>=<url-or-local-path>.
     map_origins: List of origin mappings, each in the form of
         <origin>=<url-or-local-path>.
-    free_ports: Iff True, run local development servers on system-allocated
-        ports. This defeats any performance benefits from caching.
-    free_host_ports: Only applicable on Android. Iff True, local development
-        servers are run on system-allocated ports, but are still forwarded from
-        fixed ports on the device.
+    reuse_servers: Assume that the development servers are already running and
+        do not spawn any.
 
   Returns:
     The updated argument list.
   """
-  next_port = 0 if free_ports else _MAPPINGS_BASE_PORT
+  next_port = _MAPPINGS_BASE_PORT
   args = original_arguments
   if map_urls:
     # Sort the mappings to preserve caching regardless of argument order.
     for map_url in sorted(map_urls):
       mapping = _rewrite(map_url, _host_local_url_destination, shell, next_port,
-                         free_host_ports)
-      if not free_ports:
-        next_port += 1
+                         reuse_servers)
+      next_port += 1
       # All url mappings need to be coalesced into one shell argument.
       args = append_to_argument(args, '--url-mappings=', mapping)
 
   if map_origins:
     for map_origin in sorted(map_origins):
       mapping = _rewrite(map_origin, _host_local_origin_destination, shell,
-                         next_port, free_host_ports)
-      if not free_ports:
-        next_port += 1
+                         next_port, reuse_servers)
+      next_port += 1
       # Origin mappings are specified as separate, repeated shell arguments.
       args.append('--map-origin=' + mapping)
   return args
 
 
-def configure_local_origin(shell, local_dir, port, free_host_port):
+def configure_local_origin(shell, local_dir, port, reuse_servers):
   """Sets up a local http server to serve files in |local_dir| along with
   device port forwarding if needed.
 
@@ -121,7 +116,7 @@
     The list of arguments to be appended to the shell argument list.
   """
   mappings = [('', [local_dir])]
-  origin_url = shell.serve_local_directories(mappings, port, free_host_port)
+  origin_url = shell.serve_local_directories(mappings, port, reuse_servers)
   return ["--origin=" + origin_url]
 
 
@@ -155,7 +150,7 @@
   return arguments
 
 
-def _configure_dev_server(shell, shell_args, dev_server_config, free_host_port,
+def _configure_dev_server(shell, shell_args, dev_server_config, reuse_servers,
                           verbose):
   """Sets up a dev server on the host according to |dev_server_config|.
 
@@ -169,9 +164,8 @@
     The updated argument list.
   """
   port = dev_server_config.port if dev_server_config.port else 0
-  server_url = shell.serve_local_directories(dev_server_config.mappings,
-                                             port=port,
-                                             free_host_port=free_host_port)
+  server_url = shell.serve_local_directories(
+      dev_server_config.mappings, port, reuse_servers)
   shell_args.append('--map-origin=%s=%s' % (dev_server_config.host, server_url))
 
   if verbose:
@@ -223,17 +217,16 @@
 
   shell_args = _apply_mappings(shell, shell_args, shell_config.map_url_list,
                                shell_config.map_origin_list,
-                               shell_config.free_ports,
-                               shell_config.free_host_ports)
+                               shell_config.reuse_servers)
 
   if shell_config.origin:
     if _is_web_url(shell_config.origin):
       shell_args.append('--origin=' + shell_config.origin)
     else:
-      local_origin_port = 0 if shell_config.free_ports else _LOCAL_ORIGIN_PORT
+      local_origin_port = _LOCAL_ORIGIN_PORT
       shell_args.extend(configure_local_origin(shell, shell_config.origin,
                                                local_origin_port,
-                                               shell_config.free_host_ports))
+                                               shell_config.reuse_servers))
 
   if shell_config.content_handlers:
     for (mime_type,
@@ -244,7 +237,7 @@
 
   for dev_server_config in shell_config.dev_servers:
     shell_args = _configure_dev_server(shell, shell_args, dev_server_config,
-                                       shell_config.free_host_ports,
+                                       shell_config.reuse_servers,
                                        shell_config.verbose)
 
   return shell, shell_args

devtoolslib/shell_config.py

diff --git a/devtoolslib/shell_config.py b/devtoolslib/shell_config.py
index 79a14c2..fde356f 100644
--- a/devtoolslib/shell_config.py
+++ b/devtoolslib/shell_config.py
@@ -29,7 +29,7 @@
     self.map_url_list = []
     self.map_origin_list = []
     self.dev_servers = []
-    self.free_ports = False
+    self.reuse_servers = False
     self.content_handlers = dict()
     self.verbose = None
 
@@ -38,7 +38,6 @@
     self.target_device = None
     self.logcat_tags = None
     self.require_root = False
-    self.free_host_ports = False
 
     # Desktop-only.
     self.use_osmesa = None
@@ -70,10 +69,10 @@
   parser.add_argument('--map-origin', action='append',
                       help='Define a mapping for a url origin in the format '
                       '<origin>=<url-or-local-file-path>')
-  parser.add_argument('--free-ports', action='store_true',
-                      help='Use system-allocated ports when spawning local '
-                      'servers. This defeats caching and thus hurts '
-                      'performance.')
+  parser.add_argument('--reuse-servers', action='store_true',
+                      help='Do not spawn any development servers. Assume that '
+                      'dev servers are already running and only forward them '
+                      'to a device if needed.')
   parser.add_argument('-v', '--verbose', action="store_true",
                       help="Increase output verbosity")
 
@@ -83,11 +82,6 @@
   android_group.add_argument('--target-device', help='Device to run on.')
   android_group.add_argument('--logcat-tags', help='Comma-separated list of '
                              'additional logcat tags to display.')
-  android_group.add_argument('--free-host-ports', action='store_true',
-                             help='Use system-allocated ports on the host when '
-                             'spawning local servers. This still forwards to '
-                             'fixed ports on the device, so that caching '
-                             'works.')
 
   desktop_group = parser.add_argument_group('Desktop-only',
       'These arguments apply only when running on desktop.')
@@ -160,7 +154,7 @@
   shell_config.origin = script_args.origin
   shell_config.map_url_list = script_args.map_url
   shell_config.map_origin_list = script_args.map_origin
-  shell_config.free_ports = script_args.free_ports
+  shell_config.reuse_servers = script_args.reuse_servers
   shell_config.verbose = script_args.verbose
 
   # Android-only.
@@ -168,7 +162,6 @@
                            'adb')
   shell_config.target_device = script_args.target_device
   shell_config.logcat_tags = script_args.logcat_tags
-  shell_config.free_host_ports = script_args.free_host_ports
 
   # Desktop-only.
   shell_config.use_osmesa = script_args.use_osmesa

docs/mojo_run.md

diff --git a/docs/mojo_run.md b/docs/mojo_run.md
index 0675c9c..11758b6 100644
--- a/docs/mojo_run.md
+++ b/docs/mojo_run.md
@@ -27,22 +27,45 @@
 By default, `mojo_run` uses mojo:kiosk_wm as the window manager. You can pass a
 different window manager url using the `--window-manager` flag to override this.
 
-## Running on multiple Android devices
+## Running multiple instances simultaneously
 
-Two or more instances of `mojo_run` can simultaneously run on separate Android
-devices. For that, run in individual shells:
+`mojo_run` sets up development servers on fixed ports to facilitate caching
+between runs and allow the script to work remotely using `adb_remote_setup`.
+This would normally prevent two or more instances of `mojo_run` from running
+simulatenously as the development servers cannot be spawned twice on the same
+ports.
+
+In order to run the same set of binaries simultaneously one can use the
+`--reuse-servers` switch for second and further instances. This will make the
+second and further instances assume that development servers are already
+spawned.
+
+On **Android** one needs to indicate the id of the device to be targeted in each
+run. For example, we could run the following in one shell:
 
 ```sh
-mojo_run APP_URL --android --target-device DEVICE_ID --free-host-ports
+mojo_run APP_URL --android --target-device DEVICE_ID
 ```
 
+and the following in another:
+
 ```sh
-mojo_run APP_URL --android --target-device ANOTHER_DEVICE_ID --free-host-ports
+mojo_run APP_URL --android --target-device ANOTHER_DEVICE_ID --reuse-servers
 ```
 
-`--free-host-ports` makes `mojo_run` spawn the development servers on
-system-allocated ports on the server (so that multiple instances can run in
-parallel) while still forwarding them to fixed ports on the device (so that
-caching still works). This breaks the remote workflow over `adb_remote_setup`.
+Device id can be obtained from `adb devices`.
 
-DEVICE_ID can be obtained from `adb devices`.
+On **Linux** one needs to use a different $HOME directory for each run, to avoid
+collision of the cache storage. For example, we could run the following in one
+shell:
+
+```sh
+mojo_run APP_URL
+```
+
+and the following in another:
+
+```sh
+mkdir ~/another_home
+HOME=~/another_home mojo_run APP_URL --reuse-servers
+```