# HG changeset patch # User Dave Hunt # Date 1522163994 -3600 # Tue Mar 27 16:19:54 2018 +0100 # Node ID c235820f3d76e6cb2ac43c4771af1c859a3da017 # Parent 9345ea9afa12839400e03d8d2a4748247041647f Bug 1437593 - Vendor virtualenv-clone 0.3.0; r=ted MozReview-Commit-ID: JkPPc9xcGyU diff --git a/third_party/python/virtualenv-clone/LICENSE b/third_party/python/virtualenv-clone/LICENSE new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2011, Edward George, based on code contained within the +virtualenv project. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/third_party/python/virtualenv-clone/MANIFEST.in b/third_party/python/virtualenv-clone/MANIFEST.in new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/MANIFEST.in @@ -0,0 +1,2 @@ +include README +include LICENSE diff --git a/third_party/python/virtualenv-clone/PKG-INFO b/third_party/python/virtualenv-clone/PKG-INFO new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/PKG-INFO @@ -0,0 +1,21 @@ +Metadata-Version: 1.1 +Name: virtualenv-clone +Version: 0.3.0 +Summary: script to clone virtualenvs. +Home-page: http://github.com/edwardgeorge/virtualenv-clone +Author: Edward George +Author-email: edwardgeorge@gmail.com +License: MIT +Description-Content-Type: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Intended Audience :: Developers +Classifier: Development Status :: 3 - Alpha +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 diff --git a/third_party/python/virtualenv-clone/README b/third_party/python/virtualenv-clone/README new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/README @@ -0,0 +1,33 @@ +virtualenv cloning script. + +A script for cloning a non-relocatable virtualenv. + +Virtualenv provides a way to make virtualenv's relocatable which could then be +copied as we wanted. However making a virtualenv relocatable this way breaks +the no-site-packages isolation of the virtualenv as well as other aspects that +come with relative paths and '/usr/bin/env' shebangs that may be undesirable. + +Also, the .pth and .egg-link rewriting doesn't seem to work as intended. This +attempts to overcome these issues and provide a way to easily clone an +existing virtualenv. + +It performs the following: + +- copies sys.argv[1] dir to sys.argv[2] +- updates the hardcoded VIRTUAL_ENV variable in the activate script to the + new repo location. (--relocatable doesn't touch this) +- updates the shebangs of the various scripts in bin to the new python if + they pointed to the old python. (version numbering is retained.) + + it can also change '/usr/bin/env python' shebangs to be absolute too, + though this functionality is not exposed at present. + +- checks sys.path of the cloned virtualenv and if any of the paths are from + the old environment it finds any .pth or .egg-link files within sys.path + located in the new environment and makes sure any absolute paths to the + old environment are updated to the new environment. + +- finally it double checks sys.path again and will fail if there are still + paths from the old environment present. + +NOTE: This script requires Python >= 2.5 diff --git a/third_party/python/virtualenv-clone/clonevirtualenv.py b/third_party/python/virtualenv-clone/clonevirtualenv.py new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/clonevirtualenv.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python +from __future__ import with_statement + +import logging +import optparse +import os +import os.path +import re +import shutil +import subprocess +import sys +import itertools + +version_info = (0, 2, 6) +__version__ = '.'.join(map(str, version_info)) + + +logger = logging.getLogger() + + +if sys.version_info < (2, 6): + next = lambda gen: gen.next() + + +env_bin_dir = 'bin' +if sys.platform == 'win32': + env_bin_dir = 'Scripts' + + +class UserError(Exception): + pass + + +def _dirmatch(path, matchwith): + """Check if path is within matchwith's tree. + + >>> _dirmatch('/home/foo/bar', '/home/foo/bar') + True + >>> _dirmatch('/home/foo/bar/', '/home/foo/bar') + True + >>> _dirmatch('/home/foo/bar/etc', '/home/foo/bar') + True + >>> _dirmatch('/home/foo/bar2', '/home/foo/bar') + False + >>> _dirmatch('/home/foo/bar2/etc', '/home/foo/bar') + False + """ + matchlen = len(matchwith) + if (path.startswith(matchwith) + and path[matchlen:matchlen + 1] in [os.sep, '']): + return True + return False + + +def _virtualenv_sys(venv_path): + "obtain version and path info from a virtualenv." + executable = os.path.join(venv_path, env_bin_dir, 'python') + # Must use "executable" as the first argument rather than as the + # keyword argument "executable" to get correct value from sys.path + p = subprocess.Popen([executable, + '-c', 'import sys;' + 'print (sys.version[:3]);' + 'print ("\\n".join(sys.path));'], + env={}, + stdout=subprocess.PIPE) + stdout, err = p.communicate() + assert not p.returncode and stdout + lines = stdout.decode('utf-8').splitlines() + return lines[0], filter(bool, lines[1:]) + + +def clone_virtualenv(src_dir, dst_dir): + if not os.path.exists(src_dir): + raise UserError('src dir %r does not exist' % src_dir) + if os.path.exists(dst_dir): + raise UserError('dest dir %r exists' % dst_dir) + #sys_path = _virtualenv_syspath(src_dir) + logger.info('cloning virtualenv \'%s\' => \'%s\'...' % + (src_dir, dst_dir)) + shutil.copytree(src_dir, dst_dir, symlinks=True, + ignore=shutil.ignore_patterns('*.pyc')) + version, sys_path = _virtualenv_sys(dst_dir) + logger.info('fixing scripts in bin...') + fixup_scripts(src_dir, dst_dir, version) + + has_old = lambda s: any(i for i in s if _dirmatch(i, src_dir)) + + if has_old(sys_path): + # only need to fix stuff in sys.path if we have old + # paths in the sys.path of new python env. right? + logger.info('fixing paths in sys.path...') + fixup_syspath_items(sys_path, src_dir, dst_dir) + v_sys = _virtualenv_sys(dst_dir) + remaining = has_old(v_sys[1]) + assert not remaining, v_sys + fix_symlink_if_necessary(src_dir, dst_dir) + +def fix_symlink_if_necessary(src_dir, dst_dir): + #sometimes the source virtual environment has symlinks that point to itself + #one example is $OLD_VIRTUAL_ENV/local/lib points to $OLD_VIRTUAL_ENV/lib + #this function makes sure + #$NEW_VIRTUAL_ENV/local/lib will point to $NEW_VIRTUAL_ENV/lib + #usually this goes unnoticed unless one tries to upgrade a package though pip, so this bug is hard to find. + logger.info("scanning for internal symlinks that point to the original virtual env") + for dirpath, dirnames, filenames in os.walk(dst_dir): + for a_file in itertools.chain(filenames, dirnames): + full_file_path = os.path.join(dirpath, a_file) + if os.path.islink(full_file_path): + target = os.path.realpath(full_file_path) + if target.startswith(src_dir): + new_target = target.replace(src_dir, dst_dir) + logger.debug('fixing symlink in %s' % (full_file_path,)) + os.remove(full_file_path) + os.symlink(new_target, full_file_path) + + +def fixup_scripts(old_dir, new_dir, version, rewrite_env_python=False): + bin_dir = os.path.join(new_dir, env_bin_dir) + root, dirs, files = next(os.walk(bin_dir)) + pybinre = re.compile(r'pythonw?([0-9]+(\.[0-9]+(\.[0-9]+)?)?)?$') + for file_ in files: + filename = os.path.join(root, file_) + if file_ in ['python', 'python%s' % version, 'activate_this.py']: + continue + elif file_.startswith('python') and pybinre.match(file_): + # ignore other possible python binaries + continue + elif file_.endswith('.pyc'): + # ignore compiled files + continue + elif file_ == 'activate' or file_.startswith('activate.'): + fixup_activate(os.path.join(root, file_), old_dir, new_dir) + elif os.path.islink(filename): + fixup_link(filename, old_dir, new_dir) + elif os.path.isfile(filename): + fixup_script_(root, file_, old_dir, new_dir, version, + rewrite_env_python=rewrite_env_python) + + +def fixup_script_(root, file_, old_dir, new_dir, version, + rewrite_env_python=False): + old_shebang = '#!%s/bin/python' % os.path.normcase(os.path.abspath(old_dir)) + new_shebang = '#!%s/bin/python' % os.path.normcase(os.path.abspath(new_dir)) + env_shebang = '#!/usr/bin/env python' + + filename = os.path.join(root, file_) + with open(filename, 'rb') as f: + if f.read(2) != b'#!': + # no shebang + return + f.seek(0) + lines = f.readlines() + + if not lines: + # warn: empty script + return + + def rewrite_shebang(version=None): + logger.debug('fixing %s' % filename) + shebang = new_shebang + if version: + shebang = shebang + version + shebang = (shebang + '\n').encode('utf-8') + with open(filename, 'wb') as f: + f.write(shebang) + f.writelines(lines[1:]) + + try: + bang = lines[0].decode('utf-8').strip() + except UnicodeDecodeError: + # binary file + return + + # This takes care of the scheme in which shebang is of type + # '#!/venv/bin/python3' while the version of system python + # is of type 3.x e.g. 3.5. + short_version = bang[len(old_shebang):] + + if not bang.startswith('#!'): + return + elif bang == old_shebang: + rewrite_shebang() + elif (bang.startswith(old_shebang) + and bang[len(old_shebang):] == version): + rewrite_shebang(version) + elif (bang.startswith(old_shebang) + and short_version + and bang[len(old_shebang):] == short_version): + rewrite_shebang(short_version) + elif rewrite_env_python and bang.startswith(env_shebang): + if bang == env_shebang: + rewrite_shebang() + elif bang[len(env_shebang):] == version: + rewrite_shebang(version) + else: + # can't do anything + return + + +def fixup_activate(filename, old_dir, new_dir): + logger.debug('fixing %s' % filename) + with open(filename, 'rb') as f: + data = f.read().decode('utf-8') + + data = data.replace(old_dir, new_dir) + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + + +def fixup_link(filename, old_dir, new_dir, target=None): + logger.debug('fixing %s' % filename) + if target is None: + target = os.readlink(filename) + + origdir = os.path.dirname(os.path.abspath(filename)).replace( + new_dir, old_dir) + if not os.path.isabs(target): + target = os.path.abspath(os.path.join(origdir, target)) + rellink = True + else: + rellink = False + + if _dirmatch(target, old_dir): + if rellink: + # keep relative links, but don't keep original in case it + # traversed up out of, then back into the venv. + # so, recreate a relative link from absolute. + target = target[len(origdir):].lstrip(os.sep) + else: + target = target.replace(old_dir, new_dir, 1) + + # else: links outside the venv, replaced with absolute path to target. + _replace_symlink(filename, target) + + +def _replace_symlink(filename, newtarget): + tmpfn = "%s.new" % filename + os.symlink(newtarget, tmpfn) + os.rename(tmpfn, filename) + + +def fixup_syspath_items(syspath, old_dir, new_dir): + for path in syspath: + if not os.path.isdir(path): + continue + path = os.path.normcase(os.path.abspath(path)) + if _dirmatch(path, old_dir): + path = path.replace(old_dir, new_dir, 1) + if not os.path.exists(path): + continue + elif not _dirmatch(path, new_dir): + continue + root, dirs, files = next(os.walk(path)) + for file_ in files: + filename = os.path.join(root, file_) + if filename.endswith('.pth'): + fixup_pth_file(filename, old_dir, new_dir) + elif filename.endswith('.egg-link'): + fixup_egglink_file(filename, old_dir, new_dir) + + +def fixup_pth_file(filename, old_dir, new_dir): + logger.debug('fixup_pth_file %s' % filename) + + with open(filename, 'r') as f: + lines = f.readlines() + + has_change = False + + for num, line in enumerate(lines): + line = (line.decode('utf-8') if hasattr(line, 'decode') else line).strip() + + if not line or line.startswith('#') or line.startswith('import '): + continue + elif _dirmatch(line, old_dir): + lines[num] = line.replace(old_dir, new_dir, 1) + has_change = True + + if has_change: + with open(filename, 'w') as f: + payload = os.linesep.join([l.strip() for l in lines]) + os.linesep + f.write(payload) + + +def fixup_egglink_file(filename, old_dir, new_dir): + logger.debug('fixing %s' % filename) + with open(filename, 'rb') as f: + link = f.read().decode('utf-8').strip() + if _dirmatch(link, old_dir): + link = link.replace(old_dir, new_dir, 1) + with open(filename, 'wb') as f: + link = (link + '\n').encode('utf-8') + f.write(link) + + +def main(): + parser = optparse.OptionParser("usage: %prog [options]" + " /path/to/existing/venv /path/to/cloned/venv") + parser.add_option('-v', + action="count", + dest='verbose', + default=False, + help='verbosity') + options, args = parser.parse_args() + try: + old_dir, new_dir = args + except ValueError: + print("virtualenv-clone %s" % (__version__,)) + parser.error("not enough arguments given.") + old_dir = os.path.realpath(old_dir) + new_dir = os.path.realpath(new_dir) + loglevel = (logging.WARNING, logging.INFO, logging.DEBUG)[min(2, + options.verbose)] + logging.basicConfig(level=loglevel, format='%(message)s') + try: + clone_virtualenv(old_dir, new_dir) + except UserError: + e = sys.exc_info()[1] + parser.error(str(e)) + + +if __name__ == '__main__': + main() diff --git a/third_party/python/virtualenv-clone/setup.cfg b/third_party/python/virtualenv-clone/setup.cfg new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/setup.cfg @@ -0,0 +1,4 @@ +[egg_info] +tag_build = +tag_date = 0 + diff --git a/third_party/python/virtualenv-clone/setup.py b/third_party/python/virtualenv-clone/setup.py new file mode 100644 --- /dev/null +++ b/third_party/python/virtualenv-clone/setup.py @@ -0,0 +1,53 @@ +import sys +from setuptools.command.test import test as TestCommand +from setuptools import setup + + +if __name__ == '__main__' and sys.version_info < (2, 5): + raise SystemExit("Python >= 2.5 required for virtualenv-clone") + +test_requirements = [ + 'virtualenv', + 'tox', + 'pytest' +] + + +class ToxTest(TestCommand): + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import tox + tox.cmdline() + + +setup(name="virtualenv-clone", + version='0.3.0', + description='script to clone virtualenvs.', + author='Edward George', + author_email='edwardgeorge@gmail.com', + url='http://github.com/edwardgeorge/virtualenv-clone', + license="MIT", + py_modules=["clonevirtualenv"], + entry_points={ + 'console_scripts': [ + 'virtualenv-clone=clonevirtualenv:main', + ]}, + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Intended Audience :: Developers", + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + ], + tests_require=test_requirements, + cmdclass={'test': ToxTest} +)