From b3bae3748a297573251cfff66bdedaf268325804 Mon Sep 17 00:00:00 2001 From: "sergiyb@chromium.org" Date: Wed, 24 Jun 2015 05:23:07 +0000 Subject: [PATCH] Added virtualenv for depot_tools R=pgervais@chromium.org BUG=503067 TEST=tested on Mac, Windows and Linux bots Review URL: https://codereview.chromium.org/1200843003. git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295811 0039d316-1c4b-4281-b951-d872f2087c98 --- .gitignore | 3 + PRESUBMIT.py | 3 +- bootstrap/.gitignore | 3 + bootstrap/bootstrap.py | 226 +++++++++++++++++++++++++++++++++++++++++ bootstrap/deps.pyl | 15 +++ bootstrap/util.py | 87 ++++++++++++++++ update_depot_tools | 4 + update_depot_tools.bat | 5 + 8 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 bootstrap/.gitignore create mode 100755 bootstrap/bootstrap.py create mode 100644 bootstrap/deps.pyl create mode 100644 bootstrap/util.py diff --git a/.gitignore b/.gitignore index 4db19bfbbf..a242c60359 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ /tests/subversion_config/servers /tests/svn/ /tests/svnrepo/ + +# Ignore virtualenv created during bootstrapping. +/ENV diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 2877144f35..41c56d32f4 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -22,7 +22,8 @@ def CommonChecks(input_api, output_api, tests_to_black_list): r'^python[0-9]*_bin[\/\\].+', r'^site-packages-py[0-9]\.[0-9][\/\\].+', r'^svn_bin[\/\\].+', - r'^testing_support[\/\\]_rietveld[\/\\].+'] + r'^testing_support[\/\\]_rietveld[\/\\].+', + r'^bootstrap[\/\\].+'] if os.path.exists('.gitignore'): with open('.gitignore') as fh: lines = [l.strip() for l in fh.readlines()] diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 0000000000..fcbfa83f30 --- /dev/null +++ b/bootstrap/.gitignore @@ -0,0 +1,3 @@ +BUILD_ENV +wheelhouse +virtualenv diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py new file mode 100755 index 0000000000..df14e858de --- /dev/null +++ b/bootstrap/bootstrap.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import contextlib +import glob +import logging +import os +import shutil +import subprocess +import sys +import tempfile + +from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL +from util import read_deps, merge_deps, print_deps, platform_tag + +LOGGER = logging.getLogger(__name__) + +# /path/to/infra +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*' + + +class NoWheelException(Exception): + def __init__(self, name, version, build, source_sha): + super(NoWheelException, self).__init__( + 'No matching wheel found for (%s==%s (build %s_%s))' % + (name, version, build, source_sha)) + + +def check_pydistutils(): + if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')): + print >> sys.stderr, '\n'.join([ + '', + '', + '=========== ERROR ===========', + 'You have a ~/.pydistutils.cfg file, which interferes with the ', + 'infra virtualenv environment. Please move it to the side and bootstrap ', + 'again. Once infra has bootstrapped, you may move it back.', + '', + 'Upstream bug: https://github.com/pypa/virtualenv/issues/88/', + '' + ]) + sys.exit(1) + + +def ls(prefix): + from pip._vendor import requests # pylint: disable=E0611 + data = requests.get(STORAGE_URL, params=dict( + prefix=prefix, + fields='items(name,md5Hash)' + )).json() + entries = data.get('items', []) + for entry in entries: + entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex') + entry['local'] = False + # Also look in the local cache + entries.extend([ + {'name': fname, 'md5Hash': None, 'local': True} + for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH, + prefix.split('/')[-1] + '*'))]) + return entries + + +def sha_for(deps_entry): + if 'rev' in deps_entry: + return deps_entry['rev'] + else: + return deps_entry['gs'].split('.')[0] + + +def get_links(deps): + import pip.wheel # pylint: disable=E0611 + plat_tag = platform_tag() + + links = [] + + for name, dep in deps.iteritems(): + version, source_sha = dep['version'] , sha_for(dep) + prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'], + source_sha) + generic_link = None + binary_link = None + local_link = None + + for entry in ls(prefix): + fname = entry['name'].split('/')[-1] + md5hash = entry['md5Hash'] + wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname) + if not wheel_info: + LOGGER.warn('Skipping invalid wheel: %r', fname) + continue + + if pip.wheel.Wheel(fname).supported(): + if entry['local']: + link = LOCAL_OBJECT_URL.format(entry['name']) + local_link = link + continue + else: + link = OBJECT_URL.format(entry['name'], md5hash) + if fname.endswith('none-any.whl'): + if generic_link: + LOGGER.error( + 'Found more than one generic matching wheel for %r: %r', + prefix, dep) + continue + generic_link = link + elif plat_tag in fname: + if binary_link: + LOGGER.error( + 'Found more than one binary matching wheel for %r: %r', + prefix, dep) + continue + binary_link = link + + if not binary_link and not generic_link and not local_link: + raise NoWheelException(name, version, dep['build'], source_sha) + + links.append(local_link or binary_link or generic_link) + + return links + + +@contextlib.contextmanager +def html_index(links): + tf = tempfile.mktemp('.html') + try: + with open(tf, 'w') as f: + print >> f, '' + for link in links: + print >> f, 'wat' % link + print >> f, '' + yield tf + finally: + os.unlink(tf) + + +def install(deps): + bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin' + pip = os.path.join(sys.prefix, bin_dir, 'pip') + + links = get_links(deps) + with html_index(links) as ipath: + requirements = [] + # TODO(iannucci): Do this as a requirements.txt + for name, deps_entry in deps.iteritems(): + if not deps_entry.get('implicit'): + requirements.append('%s==%s' % (name, deps_entry['version'])) + subprocess.check_call( + [pip, 'install', '--no-index', '--download-cache', + os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements) + + +def activate_env(env, deps): + if hasattr(sys, 'real_prefix'): + LOGGER.error('Already activated environment!') + return + + print 'Activating environment: %r' % env + assert isinstance(deps, dict) + + manifest_path = os.path.join(env, 'manifest.pyl') + cur_deps = read_deps(manifest_path) + if cur_deps != deps: + print ' Removing old environment: %r' % cur_deps + shutil.rmtree(env, ignore_errors=True) + cur_deps = None + + if cur_deps is None: + check_pydistutils() + + print ' Building new environment' + # Add in bundled virtualenv lib + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv')) + import virtualenv # pylint: disable=F0401 + virtualenv.create_environment( + env, search_dirs=virtualenv.file_search_dirs()) + + print ' Activating environment' + # Ensure hermeticity during activation. + os.environ.pop('PYTHONPATH', None) + bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin' + activate_this = os.path.join(env, bin_dir, 'activate_this.py') + execfile(activate_this, dict(__file__=activate_this)) + + if cur_deps is None: + print ' Installing deps' + print_deps(deps, indent=2, with_implicit=False) + install(deps) + virtualenv.make_environment_relocatable(env) + with open(manifest_path, 'wb') as f: + f.write(repr(deps) + '\n') + + # Create bin\python.bat on Windows to unify path where Python is found. + if sys.platform.startswith('win'): + bin_path = os.path.join(env, 'bin') + if not os.path.isdir(bin_path): + os.makedirs(bin_path) + python_bat_path = os.path.join(bin_path, 'python.bat') + if not os.path.isfile(python_bat_path): + with open(python_bat_path, 'w') as python_bat_file: + python_bat_file.write(PYTHON_BAT_WIN) + + print 'Done creating environment' + + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument('--deps-file', '--deps_file', action='append', + help='Path to deps.pyl file (may be used multiple times)') + parser.add_argument('env_path', + help='Path to place environment (default: %(default)s)', + default='ENV') + opts = parser.parse_args(args) + + deps = merge_deps(opts.deps_file) + activate_env(opts.env_path, deps) + + +if __name__ == '__main__': + logging.basicConfig() + LOGGER.setLevel(logging.DEBUG) + sys.exit(main(sys.argv[1:])) diff --git a/bootstrap/deps.pyl b/bootstrap/deps.pyl new file mode 100644 index 0000000000..c6236d48ea --- /dev/null +++ b/bootstrap/deps.pyl @@ -0,0 +1,15 @@ +#vim: ft=python: +{ + 'wheel': { + 'version': '0.24.0', + 'build': '0', + 'gs': 'c02262299489646af253067e8136c060a93572e3.tar.gz', + }, + + 'protobuf': { + 'version': '2.6.0', + 'build': '0', + 'repo': 'external/github.com/google/protobuf', + 'rev': '629a556879cc84e0f52546f0484b65b72ce44fe8', + }, +} diff --git a/bootstrap/util.py b/bootstrap/util.py new file mode 100644 index 0000000000..d64b142bfa --- /dev/null +++ b/bootstrap/util.py @@ -0,0 +1,87 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import ast +import contextlib +import os +import platform +import shutil +import sys +import tempfile + + +ROOT = os.path.dirname(os.path.abspath(__file__)) +WHEELHOUSE = os.path.join(ROOT, 'wheelhouse') + +BUCKET = 'chrome-python-wheelhouse' +STORAGE_URL = 'https://www.googleapis.com/storage/v1/b/{}/o'.format(BUCKET) +OBJECT_URL = 'https://storage.googleapis.com/{}/{{}}#md5={{}}'.format(BUCKET) +LOCAL_OBJECT_URL = 'file://{}' + +LOCAL_STORAGE_PATH = os.path.join(ROOT, 'wheelhouse_cache') + +SOURCE_URL = 'gs://{}/sources/{{}}'.format(BUCKET) +WHEELS_URL = 'gs://{}/wheels/'.format(BUCKET) + + +class DepsConflictException(Exception): + def __init__(self, name): + super(DepsConflictException, self).__init__( + 'Package \'%s\' is defined twice in deps.pyl' % name) + + +def platform_tag(): + if sys.platform.startswith('linux'): + return '_{0}_{1}'.format(*platform.linux_distribution()) + return '' + + +def print_deps(deps, indent=1, with_implicit=True): + for dep, entry in deps.iteritems(): + if not with_implicit and entry.get('implicit'): + continue + print ' ' * indent + '%s: %r' % (dep, entry) + print + + +@contextlib.contextmanager +def tempdir(*args, **kwargs): + tdir = None + try: + tdir = tempfile.mkdtemp(*args, **kwargs) + yield tdir + finally: + if tdir: + shutil.rmtree(tdir, ignore_errors=True) + + +@contextlib.contextmanager +def tempname(*args, **kwargs): + tmp = None + try: + tmp = tempfile.mktemp(*args, **kwargs) + yield tmp + finally: + if tmp: + try: + os.unlink(tmp) + except OSError: + pass + + +def read_deps(path): + if os.path.exists(path): + with open(path, 'rb') as f: + return ast.literal_eval(f.read()) + + +def merge_deps(paths): + deps = {} + for path in paths: + d = read_deps(path) + for key in d: + if key in deps: + raise DepsConflictException(key) + deps.update(d) + return deps diff --git a/update_depot_tools b/update_depot_tools index 8528b8bab5..6ba2a9a156 100755 --- a/update_depot_tools +++ b/update_depot_tools @@ -153,3 +153,7 @@ then fi find "$base_dir" -iname "*.pyc" -exec rm {} \; + +# Initialize/update virtualenv. +cd $base_dir +python -u ./bootstrap/bootstrap.py --deps_file bootstrap/deps.pyl ENV diff --git a/update_depot_tools.bat b/update_depot_tools.bat index 33fa40b220..43bdbabb7d 100644 --- a/update_depot_tools.bat +++ b/update_depot_tools.bat @@ -27,6 +27,11 @@ if errorlevel 1 goto :EOF :: Now clear errorlevel so it can be set by other programs later. set errorlevel= +:: Initialize/update virtualenv. +cd "%DEPOT_TOOLS_DIR%" +python -u bootstrap\bootstrap.py --deps_file bootstrap\deps.pyl ENV +if errorlevel 1 goto :EOF + :: Shall skip automatic update? IF "%DEPOT_TOOLS_UPDATE%" == "0" GOTO :EOF