#!/usr/bin/env python # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse import contextlib import glob import logging import os import shutil import subprocess import sys import tempfile from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL from util import read_deps, merge_deps, print_deps, platform_tag LOGGER = logging.getLogger(__name__) # /path/to/infra ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*' class NoWheelException(Exception): def __init__(self, name, version, build, source_sha): super(NoWheelException, self).__init__( 'No matching wheel found for (%s==%s (build %s_%s))' % (name, version, build, source_sha)) def check_pydistutils(): if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')): print >> sys.stderr, '\n'.join([ '', '', '=========== ERROR ===========', 'You have a ~/.pydistutils.cfg file, which interferes with the ', 'infra virtualenv environment. Please move it to the side and bootstrap ', 'again. Once infra has bootstrapped, you may move it back.', '', 'Upstream bug: https://github.com/pypa/virtualenv/issues/88/', '' ]) sys.exit(1) def ls(prefix): from pip._vendor import requests # pylint: disable=E0611 data = requests.get(STORAGE_URL, params=dict( prefix=prefix, fields='items(name,md5Hash)' )).json() entries = data.get('items', []) for entry in entries: entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex') entry['local'] = False # Also look in the local cache entries.extend([ {'name': fname, 'md5Hash': None, 'local': True} for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH, prefix.split('/')[-1] + '*'))]) return entries def sha_for(deps_entry): if 'rev' in deps_entry: return deps_entry['rev'] else: return deps_entry['gs'].split('.')[0] def get_links(deps): import pip.wheel # pylint: disable=E0611 plat_tag = platform_tag() links = [] for name, dep in deps.iteritems(): version, source_sha = dep['version'] , sha_for(dep) prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'], source_sha) generic_link = None binary_link = None local_link = None for entry in ls(prefix): fname = entry['name'].split('/')[-1] md5hash = entry['md5Hash'] wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname) if not wheel_info: LOGGER.warn('Skipping invalid wheel: %r', fname) continue if pip.wheel.Wheel(fname).supported(): if entry['local']: link = LOCAL_OBJECT_URL.format(entry['name']) local_link = link continue else: link = OBJECT_URL.format(entry['name'], md5hash) if fname.endswith('none-any.whl'): if generic_link: LOGGER.error( 'Found more than one generic matching wheel for %r: %r', prefix, dep) continue generic_link = link elif plat_tag in fname: if binary_link: LOGGER.error( 'Found more than one binary matching wheel for %r: %r', prefix, dep) continue binary_link = link if not binary_link and not generic_link and not local_link: raise NoWheelException(name, version, dep['build'], source_sha) links.append(local_link or binary_link or generic_link) return links @contextlib.contextmanager def html_index(links): tf = tempfile.mktemp('.html') try: with open(tf, 'w') as f: print >> f, '' for link in links: print >> f, 'wat' % link print >> f, '' yield tf finally: os.unlink(tf) def install(deps): bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin' pip = os.path.join(sys.prefix, bin_dir, 'pip') links = get_links(deps) with html_index(links) as ipath: requirements = [] # TODO(iannucci): Do this as a requirements.txt for name, deps_entry in deps.iteritems(): if not deps_entry.get('implicit'): requirements.append('%s==%s' % (name, deps_entry['version'])) subprocess.check_call( [pip, 'install', '--no-index', '--download-cache', os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements) def activate_env(env, deps, quiet=False): if hasattr(sys, 'real_prefix'): LOGGER.error('Already activated environment!') return if not quiet: print 'Activating environment: %r' % env assert isinstance(deps, dict) manifest_path = os.path.join(env, 'manifest.pyl') cur_deps = read_deps(manifest_path) if cur_deps != deps: if not quiet: print ' Removing old environment: %r' % cur_deps shutil.rmtree(env, ignore_errors=True) cur_deps = None if cur_deps is None: check_pydistutils() if not quiet: print ' Building new environment' # Add in bundled virtualenv lib sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv')) import virtualenv # pylint: disable=F0401 virtualenv.create_environment( env, search_dirs=virtualenv.file_search_dirs()) if not quiet: print ' Activating environment' # Ensure hermeticity during activation. os.environ.pop('PYTHONPATH', None) bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin' activate_this = os.path.join(env, bin_dir, 'activate_this.py') execfile(activate_this, dict(__file__=activate_this)) if cur_deps is None: if not quiet: print ' Installing deps' print_deps(deps, indent=2, with_implicit=False) install(deps) virtualenv.make_environment_relocatable(env) with open(manifest_path, 'wb') as f: f.write(repr(deps) + '\n') # Create bin\python.bat on Windows to unify path where Python is found. if sys.platform.startswith('win'): bin_path = os.path.join(env, 'bin') if not os.path.isdir(bin_path): os.makedirs(bin_path) python_bat_path = os.path.join(bin_path, 'python.bat') if not os.path.isfile(python_bat_path): with open(python_bat_path, 'w') as python_bat_file: python_bat_file.write(PYTHON_BAT_WIN) if not quiet: print 'Done creating environment' def main(args): parser = argparse.ArgumentParser() parser.add_argument('--deps-file', '--deps_file', action='append', help='Path to deps.pyl file (may be used multiple times)') parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Supress all output') parser.add_argument('env_path', help='Path to place environment (default: %(default)s)', default='ENV') opts = parser.parse_args(args) deps = merge_deps(opts.deps_file) activate_env(opts.env_path, deps, opts.quiet) if __name__ == '__main__': logging.basicConfig() LOGGER.setLevel(logging.DEBUG) sys.exit(main(sys.argv[1:]))