# HG changeset patch # User Andrew Halberstadt # Date 1566875288 0 # Node ID d5ef7a5e02a4d12e6349beb11a2cee666ea75b44 # Parent 99c3d631300b868db6d6de2c612868b32c8deb19 Bug 1473498 - [mach] Support running mach commands with python 3 r=glandium,mars Credit: mars for making the shell POSIX compliant This embeds a blacklist of every mach command that needs to run with Python 2 directly in the mach driver itself. Initially this is every mach command. We then use a bit of shell to determine whether the command being run needs Python 2 or 3. While this approach may seem a bit hacky, it has several benefits: 1. No need to add complex machinery in mach's registration code. 2. No need to spawn two separate Python interpreters in the event a different Python from the original interpreter is needed. 3. Perf impact is negligible. 4. New commands are Python 3 by default. It is also only a temporary hack. Once all commands are running with Python 3, we can revert back to the original mach driver. Differential Revision: https://phabricator.services.mozilla.com/D36103 diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py --- a/build/mach_bootstrap.py +++ b/build/mach_bootstrap.py @@ -150,21 +150,22 @@ def search_path(mozilla_dir, packages_tx for path in handle_package(package): yield path def bootstrap(topsrcdir, mozilla_dir=None): if mozilla_dir is None: mozilla_dir = topsrcdir - # Ensure we are running Python 2.7+. We put this check here so we generate a - # user-friendly error message rather than a cryptic stack trace on module - # import. - if sys.version_info[0] != 2 or sys.version_info[1] < 7: - print('Python 2.7 or above (but not Python 3) is required to run mach.') + # Ensure we are running Python 2.7 or 3.5+. We put this check here so we + # generate a user-friendly error message rather than a cryptic stack trace + # on module import. + major, minor = sys.version_info[:2] + if (major == 2 and minor < 7) or (major == 3 and minor < 5): + print('Python 2.7 or Python 3.5+ is required to run mach.') print('You are running Python', platform.python_version()) sys.exit(1) # Global build system and mach state is stored in a central directory. By # default, this is ~/.mozbuild. However, it can be defined via an # environment variable. We detect first run (by lack of this directory # existing) and notify the user that it will be created. The logic for # creation is much simpler for the "advanced" environment variable use diff --git a/mach b/mach --- a/mach +++ b/mach @@ -1,37 +1,212 @@ #!/bin/sh # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# The beginning of this script is both valid shell and valid python, +# The beginning of this script is both valid POSIX shell and valid Python, # such that the script starts with the shell and is reexecuted with -# the right python. -'''which' python2.7 > /dev/null && exec python2.7 "$0" "$@" || exec python "$0" "$@" +# the right Python. + +# Embeds a shell script inside a Python triple quote. This pattern is valid +# shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op. +''':' +py2commands=" + addtest + analyze + android + android-emulator + artifact + awsy-test + bootstrap + browsertime + build + build-backend + buildsymbols + busted + cargo + check-spidermonkey + clang-format + clobber + compare-locales + compileflags + configure + cppunittest + cramtest + crashtest + devtools-css-db + doc + doctor + dxr + empty-makefiles + environment + eslint + file-info + firefox-ui-functional + fluent-migration-test + geckodriver + geckodriver-test + geckoview-junit + google + gradle + gtest + ide + import-pr + install + install-android + install-desktop + jsapi-tests + jsshell-bench + jstestbrowser + jstests + lint + mach-commands + mach-completion + mach-debug-commands + marionette-test + mdn + mochitest + mozbuild-reference + mozharness + mozregression + package + package-multi-locale + pastebin + power + prettier-format + puppeteer-test + python + python-safety + python-test + raptor + raptor-test + reftest + release + release-history + remote + repackage + resource-usage + robocop + run + run-android + run-desktop + rusttests + search + searchfox + settings + show-log + static-analysis + talos-test + taskcluster-build-image + taskcluster-load-image + taskgraph + telemetry-tests-client + test + test-info + tps-build + try + uuid + valgrind-test + vcs-setup + vendor + visualmetrics + warnings-list + warnings-summary + watch + web-platform-tests + web-platform-tests-update + webidl-example + webidl-parser-test + webrtc-gtest + wpt + wpt-manifest-update + wpt-metadata-merge + wpt-metadata-summary + wpt-serve + wpt-unittest + wpt-update + xpcshell-test +" + +run_py() { + # Try to run a specific Python interpreter. Fall back to the system + # default Python if the specific interpreter couldn't be found. + py_executable="$1" + shift + if which "$py_executable" > /dev/null + then + exec "$py_executable" "$0" "$@" + elif [ "$py_executable" = "python2.7" ]; then + exec python "$0" "$@" + else + echo "This mach command requires $py_executable, which wasn't found on the system!" + exit 1 + fi +} + +first_arg=$1 +if [ -z "$first_arg" ]; then + run_py python3 +fi + +case "${first_arg}" in + "-"*) + # We have global arguments which are tricky to parse from this shell + # script. So invoke `mach` with a special --print-command argument to + # return the name of the command. This adds extra overhead when using + # global arguments, but global arguments are an edge case and this hack + # is only needed temporarily for the Python 3 migration. We use Python + # 2.7 because using Python 3 hits this error in build tasks: + # https://searchfox.org/mozilla-central/rev/c7e8bc4996f9/build/moz.configure/init.configure#319 + command=`run_py python2.7 --print-command "$@" | tail -n1` + ;; + *) + # In the common case, the first argument is the command. + command=${first_arg}; + ;; +esac + +# Check for the mach subcommand in the Python 2 commands list and run it +# with the correct interpreter. +case " $(echo $py2commands) " in + *\ $command\ *) + run_py python2.7 "$@" + ;; + *) + run_py python3 "$@" + ;; +esac + +# Run Python 3 for everything else. +run_py python3 "$@" ''' -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import os import sys def ancestors(path): while path: yield path (path, child) = os.path.split(path) if child == "": break def load_mach(dir_path, mach_path): - import imp - with open(mach_path, 'r') as fh: - imp.load_module('mach_bootstrap', fh, mach_path, - ('.py', 'r', imp.PY_SOURCE)) - import mach_bootstrap + if sys.version_info < (3, 5): + import imp + mach_bootstrap = imp.load_source('mach_bootstrap', mach_path) + else: + import importlib.util + spec = importlib.util.spec_from_file_location('mach_bootstrap', mach_path) + mach_bootstrap = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mach_bootstrap) + return mach_bootstrap.bootstrap(dir_path) def check_and_get_mach(dir_path): bootstrap_paths = ( 'build/mach_bootstrap.py', # test package bootstrap 'tools/mach_bootstrap.py', diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py --- a/python/mach/mach/main.py +++ b/python/mach/mach/main.py @@ -422,16 +422,27 @@ To see more help for a specific command, print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message)) return 1 except UnrecognizedArgumentError as e: print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, ' '.join(e.arguments))) return 1 + if not hasattr(args, 'mach_handler'): + raise MachError('ArgumentParser result missing mach handler info.') + + handler = getattr(args, 'mach_handler') + + # This is used by the `mach` driver to find the command name amidst + # global arguments. + if args.print_command: + print(handler.name) + sys.exit(0) + # Add JSON logging to a file if requested. if args.logfile: self.log_manager.add_json_handler(args.logfile) # Up the logging level if requested. log_level = logging.INFO if args.verbose: log_level = logging.DEBUG @@ -448,21 +459,16 @@ To see more help for a specific command, write_interval=args.log_interval, write_times=write_times) if args.settings_file: # Argument parsing has already happened, so settings that apply # to command line handling (e.g alias, defaults) will be ignored. self.load_settings(args.settings_file) - if not hasattr(args, 'mach_handler'): - raise MachError('ArgumentParser result missing mach handler info.') - - handler = getattr(args, 'mach_handler') - try: return Registrar._run_command_handler(handler, context=context, debug_command=args.debug_command, **vars(args.command_args)) except KeyboardInterrupt as ki: raise ki except FailedCommandError as e: print(e.message) @@ -595,16 +601,18 @@ To see more help for a specific command, global_group.add_argument('-h', '--help', dest='help', action='store_true', default=False, help='Show this help message.') global_group.add_argument('--debug-command', action='store_true', help='Start a Python debugger when command is dispatched.') global_group.add_argument('--settings', dest='settings_file', metavar='FILENAME', default=None, help='Path to settings file.') + global_group.add_argument('--print-command', action='store_true', + help=argparse.SUPPRESS) for args, kwargs in self.global_arguments: global_group.add_argument(*args, **kwargs) # We need to be last because CommandAction swallows all remaining # arguments and argparse parses arguments in the order they were added. parser.add_argument('command', action=CommandAction, registrar=Registrar, context=context) diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py --- a/python/mach/mach/registrar.py +++ b/python/mach/mach/registrar.py @@ -1,17 +1,19 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import print_function -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals + +import time + +import six from .base import MachError -import time INVALID_COMMAND_CONTEXT = r''' It looks like you tried to run a mach command from an invalid context. The %s command failed to meet the following conditions: %s Run |mach help| to show a list of all commands available to the current context. '''.lstrip() @@ -104,17 +106,17 @@ class MachRegistrar(object): import pdb result = pdb.runcall(fn, **kwargs) else: result = fn(**kwargs) end_time = time.time() result = result or 0 - assert isinstance(result, (int, long)) + assert isinstance(result, six.integer_types) if context and not debug_command: postrun = getattr(context, 'post_dispatch_handler', None) if postrun: postrun(context, handler, instance, result, start_time, end_time, self.command_depth, args=kwargs) self.command_depth -= 1