Add mojo/tools/mopy with paths and version utilities
This consolidates some common mojo python script utilities into
mojo/tools/mopy and updates scripts under mojo/tools to use them.
Currently this has some path manipulation, version querying and python
test running utilities but I expect this to grow over time. A skypy
could also resuse a lot of of this functionality as well.
R=eseidel@chromium.org
Review URL: https://codereview.chromium.org/707893005
diff --git a/mojo/PRESUBMIT.py b/mojo/PRESUBMIT.py
index 663c3de..557883e 100644
--- a/mojo/PRESUBMIT.py
+++ b/mojo/PRESUBMIT.py
@@ -28,6 +28,8 @@
# For the roll tools scripts:
mojo_roll_tools_path = os.path.join(
input_api.PresubmitLocalPath(), "tools", "roll")
+ # For all mojo/tools scripts
+ mopy_path = os.path.join(input_api.PresubmitLocalPath(), "tools")
# TODO(vtl): Don't lint these files until the (many) problems are fixed
# (possibly by deleting/rewriting some files).
temporary_black_list = input_api.DEFAULT_BLACK_LIST + \
@@ -43,6 +45,7 @@
mojo_python_bindings_path,
mojo_python_bindings_tests_path,
mojo_roll_tools_path,
+ mopy_path,
]
results += input_api.canned_checks.RunPylint(
input_api, output_api, extra_paths_list=pylint_extra_paths,
diff --git a/mojo/tools/check_mojom_golden_files.py b/mojo/tools/check_mojom_golden_files.py
index 9af7a86..ae80ceb 100755
--- a/mojo/tools/check_mojom_golden_files.py
+++ b/mojo/tools/check_mojom_golden_files.py
@@ -9,11 +9,11 @@
from filecmp import dircmp
from shutil import rmtree
from tempfile import mkdtemp
+from mopy.paths import Paths
-_script_dir = os.path.dirname(os.path.abspath(__file__))
-_mojo_dir = os.path.join(_script_dir, os.pardir)
-_chromium_src_dir = os.path.join(_mojo_dir, os.pardir)
-sys.path.insert(0, os.path.join(_mojo_dir, "public", "tools", "bindings",
+paths = Paths()
+
+sys.path.insert(0, os.path.join(paths.mojo_dir, "public", "tools", "bindings",
"pylib"))
from mojom_tests.support.find_files import FindFiles
from mojom_tests.support.run_bindings_generator import RunBindingsGenerator
@@ -68,14 +68,14 @@
if args.verbose:
print "Generating files to %s ..." % out_dir
- mojom_files = FindFiles(_mojo_dir, "*.mojom")
+ mojom_files = FindFiles(paths.mojo_dir, "*.mojom")
for mojom_file in mojom_files:
if args.verbose:
- print " Processing %s ..." % os.path.relpath(mojom_file, _mojo_dir)
+ print " Processing %s ..." % os.path.relpath(mojom_file, paths.mojo_dir)
# TODO(vtl): This may wrong, since the path can be overridden in the .gyp
# file.
- RunBindingsGenerator(out_dir, _mojo_dir, mojom_file,
- ["-I", os.path.abspath(_chromium_src_dir)])
+ RunBindingsGenerator(out_dir, paths.mojo_dir, mojom_file,
+ ["-I", paths.src_root])
if args.generate_golden_files:
return 0
diff --git a/mojo/tools/mopy/__init__.py b/mojo/tools/mopy/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mojo/tools/mopy/__init__.py
diff --git a/mojo/tools/mopy/mojo_python_tests_runner.py b/mojo/tools/mopy/mojo_python_tests_runner.py
new file mode 100644
index 0000000..ef9a6b2
--- /dev/null
+++ b/mojo/tools/mopy/mojo_python_tests_runner.py
@@ -0,0 +1,52 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import os
+import sys
+import unittest
+
+import mopy.paths
+
+
+class MojoPythonTestRunner(object):
+ """Helper class to run python tests on the bots."""
+
+ def __init__(self, test_dir):
+ self._test_dir = test_dir
+
+ def run(self):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-v', '--verbose', action='count', default=0)
+ parser.add_argument('tests', nargs='*')
+
+ self.add_custom_commandline_options(parser)
+ args = parser.parse_args()
+ self.apply_customization(args)
+
+ loader = unittest.loader.TestLoader()
+ print "Running Python unit tests under %s..." % self._test_dir
+
+ src_root = mopy.paths.Paths().src_root
+ pylib_dir = os.path.abspath(os.path.join(src_root, self._test_dir))
+ if args.tests:
+ if pylib_dir not in sys.path:
+ sys.path.append(pylib_dir)
+ suite = unittest.TestSuite()
+ for test_name in args.tests:
+ suite.addTests(loader.loadTestsFromName(test_name))
+ else:
+ suite = loader.discover(pylib_dir, pattern='*_unittest.py')
+
+ runner = unittest.runner.TextTestRunner(verbosity=(args.verbose + 1))
+ result = runner.run(suite)
+ return 0 if result.wasSuccessful() else 1
+
+ def add_custom_commandline_options(self, parser):
+ """Allow to add custom option to the runner script."""
+ pass
+
+ def apply_customization(self, args):
+ """Allow to apply any customization to the runner."""
+ pass
diff --git a/mojo/tools/mopy/paths.py b/mojo/tools/mopy/paths.py
new file mode 100644
index 0000000..2ad3726
--- /dev/null
+++ b/mojo/tools/mopy/paths.py
@@ -0,0 +1,19 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+
+class Paths(object):
+ """Provides commonly used paths"""
+
+ def __init__(self, build_directory=None):
+ """Specify a build_directory to generate paths to binary artifacts"""
+ self.src_root = os.path.abspath(os.path.join(__file__,
+ os.pardir, os.pardir, os.pardir, os.pardir))
+ self.mojo_dir = os.path.join(self.src_root, "mojo")
+ if build_directory:
+ self.mojo_shell_path = os.path.join(self.src_root, build_directory,
+ "mojo_shell")
+ else:
+ self.mojo_shell_path = None
diff --git a/mojo/tools/mopy/transitive_hash.py b/mojo/tools/mopy/transitive_hash.py
new file mode 100644
index 0000000..6864ae3
--- /dev/null
+++ b/mojo/tools/mopy/transitive_hash.py
@@ -0,0 +1,93 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import subprocess
+import sys
+
+# pylint: disable=E0611
+from hashlib import sha256
+# pylint: enable=E0611
+from os.path import basename, realpath
+
+_logging = logging.getLogger()
+
+# pylint: disable=C0301
+# Based on/taken from
+# http://code.activestate.com/recipes/578231-probably-the-fastest-memoization-decorator-in-the-/
+# (with cosmetic changes).
+# pylint: enable=C0301
+def _memoize(f):
+ """Memoization decorator for a function taking a single argument."""
+ class Memoize(dict):
+ def __missing__(self, key):
+ rv = self[key] = f(key)
+ return rv
+ return Memoize().__getitem__
+
+@_memoize
+def _file_hash(filename):
+ """Returns a string representing the hash of the given file."""
+ _logging.debug("Hashing %s ...", filename)
+ rv = subprocess.check_output(['sha256sum', '-b', filename]).split(None, 1)[0]
+ _logging.debug(" => %s", rv)
+ return rv
+
+@_memoize
+def _get_dependencies(filename):
+ """Returns a list of filenames for files that the given file depends on."""
+ _logging.debug("Getting dependencies for %s ...", filename)
+ lines = subprocess.check_output(['ldd', filename]).splitlines()
+ rv = []
+ for line in lines:
+ i = line.find('/')
+ if i < 0:
+ _logging.debug(" => no file found in line: %s", line)
+ continue
+ rv.append(line[i:].split(None, 1)[0])
+ _logging.debug(" => %s", rv)
+ return rv
+
+def transitive_hash(filename):
+ """Returns a string that represents the "transitive" hash of the given
+ file. The transitive hash is a hash of the file and all the shared libraries
+ on which it depends (done in an order-independent way)."""
+ hashes = set()
+ to_hash = [filename]
+ while to_hash:
+ current_filename = realpath(to_hash.pop())
+ current_hash = _file_hash(current_filename)
+ if current_hash in hashes:
+ _logging.debug("Already seen %s (%s) ...", current_filename, current_hash)
+ continue
+ _logging.debug("Haven't seen %s (%s) ...", current_filename, current_hash)
+ hashes.add(current_hash)
+ to_hash.extend(_get_dependencies(current_filename))
+ return sha256('|'.join(sorted(hashes))).hexdigest()
+
+def main(argv):
+ logging.basicConfig()
+ # Uncomment to debug:
+ # _logging.setLevel(logging.DEBUG)
+
+ if len(argv) < 2:
+ print """\
+Usage: %s [file] ...
+
+Prints the \"transitive\" hash of each (executable) file. The transitive
+hash is a hash of the file and all the shared libraries on which it
+depends (done in an order-independent way).""" % basename(argv[0])
+ return 0
+
+ rv = 0
+ for filename in argv[1:]:
+ try:
+ print transitive_hash(filename), filename
+ except subprocess.CalledProcessError:
+ print "ERROR", filename
+ rv = 1
+ return rv
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/mojo/tools/mopy/version.py b/mojo/tools/mopy/version.py
new file mode 100644
index 0000000..8f8e132
--- /dev/null
+++ b/mojo/tools/mopy/version.py
@@ -0,0 +1,14 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import subprocess
+
+import mopy.paths
+
+class Version(object):
+ """Computes the version (git committish) of the mojo repo"""
+
+ def __init__(self):
+ self.version = subprocess.check_output(["git", "rev-parse", "HEAD"],
+ cwd=mopy.paths.Paths().src_root).strip()
diff --git a/mojo/tools/run_mojo_python_bindings_tests.py b/mojo/tools/run_mojo_python_bindings_tests.py
index 916c68d..d64f49c 100755
--- a/mojo/tools/run_mojo_python_bindings_tests.py
+++ b/mojo/tools/run_mojo_python_bindings_tests.py
@@ -6,14 +6,10 @@
import os
import sys
-_script_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.join(_script_dir, "pylib"))
-
-from mojo_python_tests_runner import MojoPythonTestRunner
+from mopy.mojo_python_tests_runner import MojoPythonTestRunner
class PythonBindingsTestRunner(MojoPythonTestRunner):
-
def add_custom_commandline_options(self, parser):
parser.add_argument('--build-dir', action='store',
help='path to the build output directory')
diff --git a/mojo/tools/run_mojo_python_tests.py b/mojo/tools/run_mojo_python_tests.py
index d83ca54..127e3fc 100755
--- a/mojo/tools/run_mojo_python_tests.py
+++ b/mojo/tools/run_mojo_python_tests.py
@@ -6,10 +6,7 @@
import os
import sys
-_script_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.join(_script_dir, "pylib"))
-
-from mojo_python_tests_runner import MojoPythonTestRunner
+from mopy.mojo_python_tests_runner import MojoPythonTestRunner
def main():
diff --git a/mojo/tools/test_runner.py b/mojo/tools/test_runner.py
index 9174912..28eb3be 100755
--- a/mojo/tools/test_runner.py
+++ b/mojo/tools/test_runner.py
@@ -12,10 +12,7 @@
_logging = logging.getLogger()
-_script_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.join(_script_dir, "pylib"))
-
-from transitive_hash import transitive_hash
+from mopy.transitive_hash import transitive_hash
def main(argv):
logging.basicConfig()
diff --git a/mojo/tools/upload_shell_binary.py b/mojo/tools/upload_shell_binary.py
index f9c1ff1..91c0ed9 100755
--- a/mojo/tools/upload_shell_binary.py
+++ b/mojo/tools/upload_shell_binary.py
@@ -11,30 +11,30 @@
import time
import zipfile
-root_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
- "..", "..")
+from mopy.paths import Paths
+from mopy.version import Version
-sys.path.insert(0, os.path.join(root_path, "tools"))
+paths = Paths(os.path.join("out", "Release"))
+
+sys.path.insert(0, os.path.join(paths.src_root, "tools"))
# pylint: disable=F0401
import find_depot_tools
-binary_path = os.path.join(root_path, "out", "Release", "mojo_shell")
-
depot_tools_path = find_depot_tools.add_depot_tools_to_path()
gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
-def upload(dry_run):
- version = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root_path)
- version = version.strip()
- dest = "gs://mojo/shell/" + version + "/linux-x64.zip"
+def upload(dry_run, verbose):
+ dest = "gs://mojo/shell/" + Version().version + "/linux-x64.zip"
with tempfile.NamedTemporaryFile() as zip_file:
with zipfile.ZipFile(zip_file, 'w') as z:
- with open(binary_path) as shell_binary:
+ with open(paths.mojo_shell_path) as shell_binary:
zipinfo = zipfile.ZipInfo("mojo_shell")
zipinfo.external_attr = 0777 << 16L
zipinfo.compress_type = zipfile.ZIP_DEFLATED
- zipinfo.date_time = time.gmtime(os.path.getmtime(binary_path))
+ zipinfo.date_time = time.gmtime(os.path.getmtime(paths.mojo_shell_path))
+ if verbose:
+ print "zipping %s" % paths.mojo_shell_path
z.writestr(zipinfo, shell_binary.read())
if dry_run:
print str([gsutil_exe, "cp", zip_file.name, dest])
@@ -46,8 +46,10 @@
"google storage")
parser.add_argument("-n", "--dry_run", help="Dry run, do not actually "+
"upload", action="store_true")
+ parser.add_argument("-v", "--verbose", help="Verbose mode",
+ action="store_true")
args = parser.parse_args()
- upload(args.dry_run)
+ upload(args.dry_run, args.verbose)
return 0
if __name__ == "__main__":
