# HG changeset patch # User M?ris Fogels # Date 1566513770 0 # Thu Aug 22 22:42:50 2019 +0000 # Node ID 41bebeeebc277d0b23b6a55331ad97319be63d60 # Parent d01e41f0133614cb428caf54b2cb9d2d17512ef2 Bug 1210157 - Add Python 3 support to mozboot.base and mozboot.bootstrap r=firefox-build-system-reviewers,mshal Add support for both Python 3 and Python 2.7 to the mozboot.base and mozboot.bootstrap modules. Remove legacy Python 2.6 code or mark it for later removal. Depends on D39359 Differential Revision: https://phabricator.services.mozilla.com/D39360 diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py --- a/python/mozboot/mozboot/base.py +++ b/python/mozboot/mozboot/base.py @@ -4,21 +4,30 @@ from __future__ import absolute_import, print_function, unicode_literals import hashlib import os import re import subprocess import sys -import urllib2 from distutils.version import LooseVersion from mozboot import rust +# NOTE: This script is intended to be run with a vanilla Python install. We +# have to rely on the standard library instead of Python 2+3 helpers like +# the six module. +if sys.version_info < (3,): + from urllib2 import urlopen + input = raw_input +else: + from urllib.request import urlopen + + NO_MERCURIAL = ''' Could not find Mercurial (hg) in the current shell's path. Try starting a new shell and running the bootstrapper again. ''' MERCURIAL_UNABLE_UPGRADE = ''' You are currently running Mercurial %s. Running %s or newer is recommended for performance and stability reasons. @@ -396,40 +405,28 @@ class BaseBootstrapper(object): def apt_add_architecture(self, arch): command = ['dpkg', '--add-architecture'] command.extend(arch) self.run_as_root(command) def check_output(self, *args, **kwargs): """Run subprocess.check_output even if Python doesn't provide it.""" - fn = getattr(subprocess, 'check_output', BaseBootstrapper._check_output) - - return fn(*args, **kwargs) - - @staticmethod - def _check_output(*args, **kwargs): - """Python 2.6 compatible implementation of subprocess.check_output.""" - proc = subprocess.Popen(stdout=subprocess.PIPE, *args, **kwargs) - output, unused_err = proc.communicate() - retcode = proc.poll() - if retcode: - cmd = kwargs.get('args', args[0]) - e = subprocess.CalledProcessError(retcode, cmd) - e.output = output - raise e - - return output + # TODO Legacy Python 2.6 code, can be removed. + # We had a custom check_output() function for Python 2.6 backward + # compatibility. Since py2.6 support was dropped we can remove this + # method. + return subprocess.check_output(*args, **kwargs) def prompt_int(self, prompt, low, high, limit=5): ''' Prompts the user with prompt and requires an integer between low and high. ''' valid = False while not valid and limit > 0: try: - choice = int(raw_input(prompt)) + choice = int(input(prompt)) if not low <= choice <= high: print("ERROR! Please enter a valid option!") limit -= 1 else: valid = True except ValueError: print("ERROR! Please enter a valid option!") limit -= 1 @@ -438,17 +435,17 @@ class BaseBootstrapper(object): return choice else: raise Exception("Error! Reached max attempts of entering option.") def prompt_yesno(self, prompt): ''' Prompts the user with prompt and requires a yes/no answer.''' valid = False while not valid: - choice = raw_input(prompt + ' (Yn): ').strip().lower()[:1] + choice = input(prompt + ' (Yn): ').strip().lower()[:1] if choice == '': choice = 'y' if choice not in ('y', 'n'): print('ERROR! Please enter y or n!') else: valid = True return choice == 'y' @@ -483,17 +480,18 @@ class BaseBootstrapper(object): ''' if not name: name = os.path.basename(path) if name.endswith('.exe'): name = name[:-4] info = self.check_output([path, version_param], env=env, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) match = re.search(name + ' ([a-z0-9\.]+)', info) if not match: print('ERROR! Unable to identify %s version.' % name) return None return LooseVersion(match.group(1)) def _parse_version(self, path, name=None, env=None): @@ -696,17 +694,19 @@ class BaseBootstrapper(object): sys.exit(1) else: # No rustup. Download and run the installer. print('Will try to install Rust.') self.install_rust() def ensure_rust_targets(self, rustup): """Make sure appropriate cross target libraries are installed.""" - target_list = subprocess.check_output([rustup, 'target', 'list']) + target_list = subprocess.check_output( + [rustup, 'target', 'list'], universal_newlines=True + ) targets = [line.split()[0] for line in target_list.splitlines() if 'installed' in line or 'default' in line] print('Rust supports %s targets.' % ', '.join(targets)) # Support 32-bit Windows on 64-bit Windows. win32 = 'i686-pc-windows-msvc' win64 = 'x86_64-pc-windows-msvc' if rust.platform() == win64 and win32 not in targets: @@ -751,17 +751,17 @@ class BaseBootstrapper(object): if e.errno != errno.ENOENT: raise def http_download_and_save(self, url, dest, hexhash, digest='sha256'): """Download the given url and save it to dest. hexhash is a checksum that will be used to validate the downloaded file using the given digest algorithm. The value of digest can be any value accepted by hashlib.new. The default digest used is 'sha256'.""" - f = urllib2.urlopen(url) + f = urlopen(url) h = hashlib.new(digest) with open(dest, 'wb') as out: while True: data = f.read(4096) if data: out.write(data) h.update(data) else: @@ -791,17 +791,18 @@ class BaseBootstrapper(object): raise Exception('You need to have Java version 1.8 installed. ' 'Please visit http://www.java.com/en/download ' 'to get version 1.8.') try: output = subprocess.check_output([java, '-XshowSettings:properties', '-version'], - stderr=subprocess.STDOUT).rstrip() + stderr=subprocess.STDOUT, + universal_newlines=True).rstrip() # -version strings are pretty free-form, like: 'java version # "1.8.0_192"' or 'openjdk version "11.0.1" 2018-10-16', but the # -XshowSettings:properties gives the information (to stderr, sigh) # like 'java.specification.version = 8'. That flag is non-standard # but has been around since at least 2011. version = [line for line in output.splitlines() if 'java.specification.version' in line] diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py --- a/python/mozboot/mozboot/bootstrap.py +++ b/python/mozboot/mozboot/bootstrap.py @@ -3,22 +3,27 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals import platform import sys import os import subprocess -try: + +# NOTE: This script is intended to be run with a vanilla Python install. We +# have to rely on the standard library instead of Python 2+3 helpers like +# the six module. +if sys.version_info < (3,): from ConfigParser import ( Error as ConfigParserError, RawConfigParser, ) -except ImportError: + input = raw_input +else: from configparser import ( Error as ConfigParserError, RawConfigParser, ) # Don't forgot to add new mozboot modules to the bootstrap download # list in bin/bootstrap.py! from mozboot.base import MODERN_RUST_VERSION @@ -52,16 +57,17 @@ Your choice: ''' APPLICATIONS_LIST = [ ('Firefox for Desktop Artifact Mode', 'browser_artifact_mode'), ('Firefox for Desktop', 'browser'), ('Firefox for Android Artifact Mode', 'mobile_android_artifact_mode'), ('Firefox for Android', 'mobile_android'), ] +# TODO Legacy Python 2.6 code, can be removed. # This is a workaround for the fact that we must support python2.6 (which has # no OrderedDict) APPLICATIONS = dict( browser_artifact_mode=APPLICATIONS_LIST[0], browser=APPLICATIONS_LIST[1], mobile_android_artifact_mode=APPLICATIONS_LIST[2], mobile_android=APPLICATIONS_LIST[3], ) @@ -256,17 +262,17 @@ class Bootstrapper(object): def input_clone_dest(self, with_hg=True): repo_name = 'mozilla-unified' vcs = 'Mercurial' if not with_hg: vcs = 'Git' print(CLONE_VCS.format(repo_name, vcs)) while True: - dest = raw_input(CLONE_VCS_PROMPT.format(vcs)) + dest = input(CLONE_VCS_PROMPT.format(vcs)) dest = dest.strip() if not dest: return '' dest = os.path.expanduser(dest) if not os.path.exists(dest): return dest @@ -578,17 +584,18 @@ def current_firefox_checkout(check_outpu while path: hg_dir = os.path.join(path, '.hg') git_dir = os.path.join(path, '.git') if hg and os.path.exists(hg_dir): # Verify the hg repo is a Firefox repo by looking at rev 0. try: node = check_output([hg, 'log', '-r', '0', '--template', '{node}'], cwd=path, - env=env) + env=env, + universal_newlines=True) if node in HG_ROOT_REVISIONS: return ('hg', path) # Else the root revision is different. There could be nested # repos. So keep traversing the parents. except subprocess.CalledProcessError: pass # Just check for known-good files in the checkout, to prevent attempted