aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adds Python package management to depot_tools.

Browse Source 
This CL introduces package_management.py, a module for managing third-party Python packages in a site directory within depot_tools. A future CL will integrate this with depot_tools, causing the packages to be downloaded and installed on first run after that CL lands. Once in place presubmit_canned_checks will be extended to make use of the locally installed version of pylint.

BUG=
TEST=


Review URL: http://codereview.chromium.org/8965033

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@120225 0039d316-1c4b-4281-b951-d872f2087c98
experimental/szager/collated-output
chrisha@chromium.org 14 years ago
parent 8f927567f9
commit c920ad2e24
3 changed files with 619 additions and 4 deletions
Show Stats Download Patch File Download Diff File

18
.gitignore vendored
Unescape Escape View File

@ -1,12 +1,24 @@
# Ignore any python bytecode.
*.pyc
# Ignore the batch files produced by the Windows bootstrapping.
/git.bat
/gitk.bat
/python
/python.bat
/python_bin
/ssh.bat
/ssh-keygen.bat
/svn
/svn.bat
/svn_bin
/svnversion.bat
/depot_tools_testing_lib/_rietveld
# Ignore locations where third-party tools are placed during bootstrapping.
/python_bin
/git_bin
/site-packages-py*
/svn_bin
# Ignore unittest related files.
/testing_support/_rietveld
/tests/subversion_config/README.txt
/tests/subversion_config/auth

3
PRESUBMIT.py
Unescape Escape View File

@ -1,4 +1,4 @@
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@ -28,6 +28,7 @@ def CommonChecks(input_api, output_api, tests_to_black_list):
r'^cpplint\.py$',
r'^cpplint_chromium\.py$',
r'^python_bin[\/\\].+',
r'^site-packages-py[0-9]\.[0-9][\/\\].+',
r'^svn_bin[\/\\].+',
r'^testing_support[\/\\]_rietveld[\/\\].+']
results.extend(input_api.canned_checks.RunPylint(

602
package_management.py
Unescape Escape View File

@ -0,0 +1,602 @@
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Package Management
This module is used to bring in external Python dependencies via eggs from the
PyPi repository.
The approach is to create a site directory in depot_tools root which will
contain those packages that are listed as dependencies in the module variable
PACKAGES. Since we can't guarantee that setuptools is available in all
distributions this module also contains the ability to bootstrap the site
directory by manually downloading and installing setuptools. Once setuptools is
available it uses that to install the other packages in the traditional
manner.
Use is simple:
import package_management
# Before any imports from the site directory, call this. This only needs
# to be called in one place near the beginning of the program.
package_management.SetupSiteDirectory()
# If 'SetupSiteDirectory' fails it will complain with an error message but
# continue happily. Expect ImportErrors when trying to import any third
# party modules from the site directory.
import some_third_party_module
... etc ...
"""
import cStringIO
import os
import re
import shutil
import site
import subprocess
import sys
import tempfile
import urllib2
# This is the version of setuptools that we will download if the local
# python distribution does not include one.
SETUPTOOLS = ('setuptools', '0.6c11')
# These are the packages that are to be installed in the site directory.
# easy_install makes it so that the most recently installed version of a
# package is the one that takes precedence, even if a newer version exists
# in the site directory. This allows us to blindly install these one on top
# of the other without worrying about whats already installed.
#
# NOTE: If we are often rolling these dependencies then users' site
# directories will grow monotonically. We could be purging any orphaned
# packages using the tools provided by pkg_resources.
PACKAGES = (('logilab-common', '0.57.1'),
('logilab-astng', '0.23.1'),
('pylint', '0.25.1'))
# The Python version suffix used in generating the site directory and in
# requesting packages from PyPi.
VERSION_SUFFIX = "%d.%d" % sys.version_info[0:2]
# This is the root directory of the depot_tools installation.
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
# This is the path of the site directory we will use. We make this
# python version specific so that this will continue to work even if the
# python version is rolled.
SITE_DIR = os.path.join(ROOT_DIR, 'site-packages-py%s' % VERSION_SUFFIX)
# This status file is created the last time PACKAGES were rolled successfully.
# It is used to determine if packages need to be rolled by comparing against
# the age of __file__.
LAST_ROLLED = os.path.join(SITE_DIR, 'last_rolled.txt')
class Error(Exception):
"""The base class for all module errors."""
pass
class InstallError(Error):
"""Thrown if an installation is unable to complete."""
pass
class Package(object):
"""A package represents a release of a project.
We use this as a lightweight version of pkg_resources, allowing us to
perform an end-run around setuptools for the purpose of bootstrapping. Its
functionality is very limited.
Attributes:
name: the name of the package.
version: the version of the package.
safe_name: the safe name of the package.
safe_version: the safe version string of the package.
file_name: the filename-safe name of the package.
file_version: the filename-safe version string of the package.
"""
def __init__(self, name, version):
"""Initialize this package.
Args:
name: the name of the package.
version: the version of the package.
"""
self.name = name
self.version = version
self.safe_name = Package._MakeSafeName(self.name)
self.safe_version = Package._MakeSafeVersion(self.version)
self.file_name = Package._MakeSafeForFilename(self.safe_name)
self.file_version = Package._MakeSafeForFilename(self.safe_version)
@staticmethod
def _MakeSafeName(name):
"""Makes a safe package name, as per pkg_resources."""
return re.sub('[^A-Za-z0-9]+', '-', name)
@staticmethod
def _MakeSafeVersion(version):
"""Makes a safe package version string, as per pkg_resources."""
version = re.sub('\s+', '.', version)
return re.sub('[^A-Za-z0-9\.]+', '-', version)
@staticmethod
def _MakeSafeForFilename(safe_name_or_version):
"""Makes a safe name or safe version safe to use in a file name.
|safe_name_or_version| must be a safe name or version string as returned
by GetSafeName or GetSafeVersion.
"""
return re.sub('-', '_', safe_name_or_version)
def GetAsRequirementString(self):
"""Builds an easy_install requirements string representing this package."""
return '%s==%s' % (self.name, self.version)
def GetFilename(self, extension=None):
"""Builds a filename for this package using the setuptools convention.
If |extensions| is provided it will be appended to the generated filename,
otherwise only the basename is returned.
The following url discusses the filename format:
http://svn.python.org/projects/sandbox/trunk/setuptools/doc/formats.txt
"""
filename = '%s-%s-py%s' % (self.file_name, self.file_version,
VERSION_SUFFIX)
if extension:
if not extension.startswith('.'):
filename += '.'
filename += extension
return filename
def GetPyPiUrl(self, extension):
"""Returns the URL where this package is hosted on PyPi."""
return 'http://pypi.python.org/packages/%s/%c/%s/%s' % (VERSION_SUFFIX,
self.file_name[0], self.file_name, self.GetFilename(extension))
def DownloadEgg(self, dest_dir, overwrite=False):
"""Downloads the EGG for this URL.
Args:
dest_dir: The directory where the EGG should be written. If the EGG
has already been downloaded and cached the returned path may not
be in this directory.
overwite: If True the destination path will be overwritten even if
it already exists. Defaults to False.
Returns:
The path to the written EGG.
Raises:
Error: if dest_dir doesn't exist, the EGG is unable to be written,
the URL doesn't exist, or the server returned an error, or the
transmission was interrupted.
"""
if not os.path.exists(dest_dir):
raise Error('Path does not exist: %s' % dest_dir)
if not os.path.isdir(dest_dir):
raise Error('Path is not a directory: %s' % dest_dir)
filename = os.path.abspath(os.path.join(dest_dir, self.GetFilename('egg')))
if os.path.exists(filename):
if os.path.isdir(filename):
raise Error('Path is a directory: %s' % filename)
if not overwrite:
return filename
url = self.GetPyPiUrl('egg')
try:
url_stream = urllib2.urlopen(url)
local_file = open(filename, 'wb')
local_file.write(url_stream.read())
local_file.close()
return filename
except (IOError, urllib2.HTTPError, urllib2.URLError):
# Reraise with a new error type, keeping the original message and
# traceback.
raise Error, sys.exc_info()[1], sys.exc_info()[2]
def AddToPythonPath(path):
"""Adds the provided path to the head of PYTHONPATH and sys.path."""
path = os.path.abspath(path)
if path not in sys.path:
sys.path.insert(0, path)
paths = os.environ.get('PYTHONPATH', '').split(os.pathsep)
if path not in paths:
paths.insert(0, path)
os.environ['PYTHONPATH'] = os.pathsep.join(paths)
def AddSiteDirectory(path):
"""Adds the provided path to the runtime as a site directory.
Any modules that are in the site directory will be available for importing
after this returns. If modules are added or deleted this must be called
again for the changes to be reflected in the runtime.
This calls both AddToPythonPath and site.addsitedir. Both are needed to
convince easy_install to treat |path| as a site directory.
"""
AddToPythonPath(path)
site.addsitedir(path) # pylint: disable=E1101
def EnsureSiteDirectory(path):
"""Creates and/or adds the provided path to the runtime as a site directory.
This works like AddSiteDirectory but it will create the directory if it
does not yet exist.
Raise:
Error: if the site directory is unable to be created, or if it exists and
is not a directory.
"""
if os.path.exists(path):
if not os.path.isdir(path):
raise Error('Path is not a directory: %s' % path)
else:
try:
os.mkdir(path)
except IOError:
raise Error('Unable to create directory: %s' % path)
AddSiteDirectory(path)
def ModuleIsFromPackage(module, package_path):
"""Determines if a module has been imported from a given package.
Args:
module: the module to test.
package_path: the path to the package to test.
Returns:
True if |module| has been imported from |package_path|, False otherwise.
"""
try:
m = os.path.abspath(module.__file__)
p = os.path.abspath(package_path)
if len(m) <= len(p):
return False
if m[0:len(p)] != p:
return False
return m[len(p)] == os.sep
except AttributeError:
return False
def _CaptureStdStreams(function, *args, **kwargs):
"""Captures stdout and stderr while running the provided function.
This only works if |function| only accesses sys.stdout and sys.stderr. If
we need more than this we'll have to use subprocess.Popen.
Args:
function: the function to be called.
args: the arguments to pass to |function|.
kwargs: the keyword arguments to pass to |function|.
"""
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = cStringIO.StringIO()
sys.stderr = cStringIO.StringIO()
try:
return function(*args, **kwargs)
finally:
sys.stdout = orig_stdout
sys.stderr = orig_stderr
def InstallPackage(url_or_req, site_dir):
"""Installs a package to a site directory.
|site_dir| must exist and already be an active site directory. setuptools
must in the path. Uses easy_install which may involve a download from
pypi.python.org, so this also requires network access.
Args:
url_or_req: the package to install, expressed as an URL (may be local),
or a requirement string.
site_dir: the site directory in which to install it.
Raises:
InstallError: if installation fails for any reason.
"""
args = ['--quiet', '--install-dir', site_dir, '--exclude-scripts',
'--always-unzip', '--no-deps', url_or_req]
# The easy_install script only calls SystemExit if something goes wrong.
# Otherwise, it falls through returning None.
try:
import setuptools.command.easy_install
_CaptureStdStreams(setuptools.command.easy_install.main, args)
except (ImportError, SystemExit):
# Re-raise the error, preserving the stack trace and message.
raise InstallError, sys.exc_info()[1], sys.exc_info()[2]
def _RunInSubprocess(pycode):
"""Launches a python subprocess with the provided code.
The subprocess will be launched with the same stdout and stderr. The
subprocess will use the same instance of python as is currently running,
passing |pycode| as arguments to this script. |pycode| will be interpreted
as python code in the context of this module.
Returns:
True if the subprocess returned 0, False if it returned an error.
"""
return not subprocess.call([sys.executable, __file__, pycode])
def _LoadSetupToolsFromEggAndInstall(egg_path):
"""Loads setuptools from the provided egg |egg_path|, and installs it to
SITE_DIR.
This is intended to be run from a subprocess as it pollutes the running
instance of Python by importing a module and then forcibly deleting its
source.
Returns:
True on success, False on failure.
"""
AddToPythonPath(egg_path)
try:
# Import setuptools and ensure it comes from the EGG.
import setuptools
if not ModuleIsFromPackage(setuptools, egg_path):
raise ImportError()
except ImportError:
print ' Unable to import downloaded package!'
return False
try:
print ' Using setuptools to install itself ...'
InstallPackage(egg_path, SITE_DIR)
except InstallError:
print ' Unable to install setuptools!'
return False
return True
def BootstrapSetupTools():
"""Bootstraps the runtime with setuptools.
Will try to import setuptools directly. If not found it will attempt to
download it and load it from there. If the download is successful it will
then use setuptools to install itself in the site directory.
This is meant to be run from a child process as it modifies the running
instance of Python by importing modules and then physically deleting them
from disk.
Returns:
Returns True if 'import setuptools' will succeed, False otherwise.
"""
AddSiteDirectory(SITE_DIR)
# Check if setuptools is already available. If so, we're done.
try:
import setuptools # pylint: disable=W0612
return True
except ImportError:
pass
print 'Bootstrapping setuptools ...'
EnsureSiteDirectory(SITE_DIR)
# Download the egg to a temp directory.
dest_dir = tempfile.mkdtemp('depot_tools')
path = None
try:
package = Package(*SETUPTOOLS)
print ' Downloading %s ...' % package.GetFilename()
path = package.DownloadEgg(dest_dir)
except Error:
print ' Download failed!'
shutil.rmtree(dest_dir)
return False
try:
# Load the downloaded egg, and install it to the site directory. Do this
# in a subprocess so as not to pollute this runtime.
pycode = '_LoadSetupToolsFromEggAndInstall(%s)' % repr(path)
if not _RunInSubprocess(pycode):
raise Error()
# Reload our site directory, which should now contain setuptools.
AddSiteDirectory(SITE_DIR)
# Try to import setuptools
import setuptools
except ImportError:
print ' Unable to import setuptools!'
return False
except Error:
# This happens if RunInSubProcess fails, and the appropriate error has
# already been written to stdout.
return False
finally:
# Delete the temp directory.
shutil.rmtree(dest_dir)
return True
def _GetModTime(path):
"""Gets the last modification time associated with |path| in seconds since
epoch, returning 0 if |path| does not exist.
"""
try:
return os.stat(path).st_mtime
except: # pylint: disable=W0702
# This error is different depending on the OS, hence no specified type.
return 0
def _SiteDirectoryIsUpToDate():
return _GetModTime(LAST_ROLLED) > _GetModTime(__file__)
def UpdateSiteDirectory():
"""Installs the packages from PACKAGES if they are not already installed.
At this point we must have setuptools in the site directory.
This is intended to be run in a subprocess *prior* to the site directory
having been added to the parent process as it may cause packages to be
added and/or removed.
Returns:
True on success, False otherwise.
"""
if _SiteDirectoryIsUpToDate():
return True
try:
AddSiteDirectory(SITE_DIR)
import pkg_resources
# Determine if any packages actually need installing.
missing_packages = []
for package in [SETUPTOOLS] + list(PACKAGES):
pkg = Package(*package)
req = pkg.GetAsRequirementString()
# It may be that this package is already available in the site
# directory. If so, we can skip past it without trying to install it.
pkg_req = pkg_resources.Requirement.parse(req)
try:
dist = pkg_resources.working_set.find(pkg_req)
if dist:
continue
except pkg_resources.VersionConflict:
# This happens if another version of the package is already
# installed in another site directory (ie: the system site directory).
pass
missing_packages.append(pkg)
# Install the missing packages.
if missing_packages:
print 'Updating python packages ...'
for pkg in missing_packages:
print ' Installing %s ...' % pkg.GetFilename()
InstallPackage(pkg.GetAsRequirementString(), SITE_DIR)
# Touch the status file so we know that we're up to date next time.
open(LAST_ROLLED, 'wb')
except InstallError, e:
print ' Installation failed: %s' % str(e)
return False
return True
def SetupSiteDirectory():
"""Sets up the site directory, bootstrapping setuptools if necessary.
If this finishes successfully then SITE_DIR will exist and will contain
the appropriate version of setuptools and all of the packages listed in
PACKAGES.
This is the main workhorse of this module. Calling this will do everything
necessary to ensure that you have the desired packages installed in the
site directory, and the site directory enabled in this process.
Returns:
True on success, False on failure.
"""
if _SiteDirectoryIsUpToDate():
AddSiteDirectory(SITE_DIR)
return True
if not _RunInSubprocess('BootstrapSetupTools()'):
return False
if not _RunInSubprocess('UpdateSiteDirectory()'):
return False
# Process the site directory so that the packages within it are available
# for import.
AddSiteDirectory(SITE_DIR)
return True
def CanImportFromSiteDirectory(package_name):
"""Determines if the given package can be imported from the site directory.
Args:
package_name: the name of the package to import.
Returns:
True if 'import package_name' will succeed and return a module from the
site directory, False otherwise.
"""
try:
return ModuleIsFromPackage(__import__(package_name), SITE_DIR)
except ImportError:
return False
def Test():
"""Runs SetupSiteDirectory and then tries to load pylint, ensuring that it
comes from the site directory just created. This is an end-to-end unittest
and allows for simple testing from the command-line by running
./package_management.py 'Test()'
"""
print 'Testing package_management.'
if not SetupSiteDirectory():
print 'SetupSiteDirectory failed.'
return False
if not CanImportFromSiteDirectory('pylint'):
print 'CanImportFromSiteDirectory failed.'
return False
print 'Success!'
return True
def Main():
"""The main entry for the package management script.
If no arguments are provided simply runs SetupSiteDirectory. If arguments
have been passed we execute the first argument as python code in the
context of this module. This mechanism is used during the bootstrap
process so that the main instance of Python does not have its runtime
polluted by various intermediate packages and imports.
Returns:
0 on success, 1 otherwise.
"""
if len(sys.argv) == 2:
result = False
exec('result = %s' % sys.argv[1])
# Translate the success state to a return code.
return not result
else:
return not SetupSiteDirectory()
if __name__ == '__main__':
sys.exit(Main())
Write Preview
Loading…
Cancel
Save