From 5f3eee3babcdb5043f5b3d129091bb6a98e727cd Mon Sep 17 00:00:00 2001 From: "maruel@chromium.org" Date: Thu, 17 Sep 2009 00:34:30 +0000 Subject: [PATCH] gclient: remove wildcard import from git_scm Part of a larger refactoring to abstract SCM-specific bits. presubmit_support, revert, gcl: modify to import gclient_scm and gclient_utils Part of a larger refactoring to abstract SCM-specific bits. revert, gcl: modify to import gclient_scm and gclient_utils Part of a larger refactoring to abstract SCM-specific bits. gclient: pull out SCM bits Pulled out SCMWrapper into gcliet_scm.py as part of a larger refactoring to abstract SCM-specific bits. Plan is to evenutally add git support. Pulling out SCMWrapper also required pulling out utility functions into a gclient_utility.py. Patch contributed by msb@ TEST=none BUG=none git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@26423 0039d316-1c4b-4281-b951-d872f2087c98 --- gcl.py | 26 +- gclient.py | 791 +----------------------------------- gclient_scm.py | 577 ++++++++++++++++++++++++++ gclient_utils.py | 239 +++++++++++ presubmit_support.py | 12 +- revert.py | 14 +- tests/gcl_unittest.py | 11 +- tests/gclient_test.py | 156 +++---- tests/presubmit_unittest.py | 31 +- tests/revert_unittest.py | 7 +- 10 files changed, 961 insertions(+), 903 deletions(-) create mode 100644 gclient_scm.py create mode 100644 gclient_utils.py diff --git a/gcl.py b/gcl.py index 6eadf3c72..38e27c583 100755 --- a/gcl.py +++ b/gcl.py @@ -20,7 +20,8 @@ import urllib2 import xml.dom.minidom # gcl now depends on gclient. -import gclient +import gclient_scm +import gclient_utils __version__ = '1.1.1' @@ -50,7 +51,7 @@ FILES_CACHE = {} def IsSVNMoved(filename): """Determine if a file has been added through svn mv""" - info = gclient.CaptureSVNInfo(filename) + info = gclient_scm.CaptureSVNInfo(filename) return (info.get('Copied From URL') and info.get('Copied From Rev') and info.get('Schedule') == 'add') @@ -82,7 +83,7 @@ def UnknownFiles(extra_args): Any args in |extra_args| are passed to the tool to support giving alternate code locations. """ - return [item[1] for item in gclient.CaptureSVNStatus(extra_args) + return [item[1] for item in gclient_scm.CaptureSVNStatus(extra_args) if item[0][0] == '?'] @@ -93,15 +94,15 @@ def GetRepositoryRoot(): """ global REPOSITORY_ROOT if not REPOSITORY_ROOT: - infos = gclient.CaptureSVNInfo(os.getcwd(), print_error=False) + infos = gclient_scm.CaptureSVNInfo(os.getcwd(), print_error=False) cur_dir_repo_root = infos.get("Repository Root") if not cur_dir_repo_root: - raise gclient.Error("gcl run outside of repository") + raise gclient_utils.Error("gcl run outside of repository") REPOSITORY_ROOT = os.getcwd() while True: parent = os.path.dirname(REPOSITORY_ROOT) - if (gclient.CaptureSVNInfo(parent, print_error=False).get( + if (gclient_scm.CaptureSVNInfo(parent, print_error=False).get( "Repository Root") != cur_dir_repo_root): break REPOSITORY_ROOT = parent @@ -140,11 +141,11 @@ def GetCachedFile(filename, max_age=60*60*24*3, use_root=False): # First we check if we have a cached version. try: cached_file = os.path.join(GetCacheDir(), filename) - except gclient.Error: + except gclient_utils.Error: return None if (not os.path.exists(cached_file) or os.stat(cached_file).st_mtime > max_age): - dir_info = gclient.CaptureSVNInfo(".") + dir_info = gclient_scm.CaptureSVNInfo(".") repo_root = dir_info["Repository Root"] if use_root: url_path = repo_root @@ -461,7 +462,7 @@ class ChangeInfo(object): if update_status: for file in files: filename = os.path.join(local_root, file[1]) - status_result = gclient.CaptureSVNStatus(filename) + status_result = gclient_scm.CaptureSVNStatus(filename) if not status_result or not status_result[0][0]: # File has been reverted. save = True @@ -545,7 +546,7 @@ def GetModifiedFiles(): files_in_cl[filename] = change_info.name # Get all the modified files. - status_result = gclient.CaptureSVNStatus(None) + status_result = gclient_scm.CaptureSVNStatus(None) for line in status_result: status = line[0] filename = line[1] @@ -724,7 +725,8 @@ def GenerateDiff(files, root=None): for file in files: # Use svn info output instead of os.path.isdir because the latter fails # when the file is deleted. - if gclient.CaptureSVNInfo(file).get("Node Kind") in ("dir", "directory"): + if gclient_scm.CaptureSVNInfo(file).get("Node Kind") in ("dir", + "directory"): continue # If the user specified a custom diff command in their svn config file, # then it'll be used when we do svn diff, which we don't want to happen @@ -1149,7 +1151,7 @@ def main(argv=None): shutil.move(file_path, GetChangesDir()) if not os.path.exists(GetCacheDir()): os.mkdir(GetCacheDir()) - except gclient.Error: + except gclient_utils.Error: # Will throw an exception if not run in a svn checkout. pass diff --git a/gclient.py b/gclient.py index 743557c41..e56a56ac9 100755 --- a/gclient.py +++ b/gclient.py @@ -73,16 +73,14 @@ import optparse import os import re import stat -import subprocess import sys import time import urlparse -import xml.dom.minidom import urllib - -SVN_COMMAND = "svn" - +import gclient_scm +import gclient_utils +from gclient_utils import Error, FileRead, FileWrite # default help text DEFAULT_USAGE_TEXT = ( @@ -288,774 +286,6 @@ solutions = [ """) -## Generic utils - -def ParseXML(output): - try: - return xml.dom.minidom.parseString(output) - except xml.parsers.expat.ExpatError: - return None - - -def GetNamedNodeText(node, node_name): - child_nodes = node.getElementsByTagName(node_name) - if not child_nodes: - return None - assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1 - return child_nodes[0].firstChild.nodeValue - - -def GetNodeNamedAttributeText(node, node_name, attribute_name): - child_nodes = node.getElementsByTagName(node_name) - if not child_nodes: - return None - assert len(child_nodes) == 1 - return child_nodes[0].getAttribute(attribute_name) - - -class Error(Exception): - """gclient exception class.""" - pass - -class PrintableObject(object): - def __str__(self): - output = '' - for i in dir(self): - if i.startswith('__'): - continue - output += '%s = %s\n' % (i, str(getattr(self, i, ''))) - return output - - -def FileRead(filename): - content = None - f = open(filename, "rU") - try: - content = f.read() - finally: - f.close() - return content - - -def FileWrite(filename, content): - f = open(filename, "w") - try: - f.write(content) - finally: - f.close() - - -def RemoveDirectory(*path): - """Recursively removes a directory, even if it's marked read-only. - - Remove the directory located at *path, if it exists. - - shutil.rmtree() doesn't work on Windows if any of the files or directories - are read-only, which svn repositories and some .svn files are. We need to - be able to force the files to be writable (i.e., deletable) as we traverse - the tree. - - Even with all this, Windows still sometimes fails to delete a file, citing - a permission error (maybe something to do with antivirus scans or disk - indexing). The best suggestion any of the user forums had was to wait a - bit and try again, so we do that too. It's hand-waving, but sometimes it - works. :/ - - On POSIX systems, things are a little bit simpler. The modes of the files - to be deleted doesn't matter, only the modes of the directories containing - them are significant. As the directory tree is traversed, each directory - has its mode set appropriately before descending into it. This should - result in the entire tree being removed, with the possible exception of - *path itself, because nothing attempts to change the mode of its parent. - Doing so would be hazardous, as it's not a directory slated for removal. - In the ordinary case, this is not a problem: for our purposes, the user - will never lack write permission on *path's parent. - """ - file_path = os.path.join(*path) - if not os.path.exists(file_path): - return - - if os.path.islink(file_path) or not os.path.isdir(file_path): - raise Error("RemoveDirectory asked to remove non-directory %s" % file_path) - - has_win32api = False - if sys.platform == 'win32': - has_win32api = True - # Some people don't have the APIs installed. In that case we'll do without. - try: - win32api = __import__('win32api') - win32con = __import__('win32con') - except ImportError: - has_win32api = False - else: - # On POSIX systems, we need the x-bit set on the directory to access it, - # the r-bit to see its contents, and the w-bit to remove files from it. - # The actual modes of the files within the directory is irrelevant. - os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - for fn in os.listdir(file_path): - fullpath = os.path.join(file_path, fn) - - # If fullpath is a symbolic link that points to a directory, isdir will - # be True, but we don't want to descend into that as a directory, we just - # want to remove the link. Check islink and treat links as ordinary files - # would be treated regardless of what they reference. - if os.path.islink(fullpath) or not os.path.isdir(fullpath): - if sys.platform == 'win32': - os.chmod(fullpath, stat.S_IWRITE) - if has_win32api: - win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL) - try: - os.remove(fullpath) - except OSError, e: - if e.errno != errno.EACCES or sys.platform != 'win32': - raise - print 'Failed to delete %s: trying again' % fullpath - time.sleep(0.1) - os.remove(fullpath) - else: - RemoveDirectory(fullpath) - - if sys.platform == 'win32': - os.chmod(file_path, stat.S_IWRITE) - if has_win32api: - win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL) - try: - os.rmdir(file_path) - except OSError, e: - if e.errno != errno.EACCES or sys.platform != 'win32': - raise - print 'Failed to remove %s: trying again' % file_path - time.sleep(0.1) - os.rmdir(file_path) - - -def SubprocessCall(command, in_directory, fail_status=None): - """Runs command, a list, in directory in_directory. - - This function wraps SubprocessCallAndFilter, but does not perform the - filtering functions. See that function for a more complete usage - description. - """ - # Call subprocess and capture nothing: - SubprocessCallAndFilter(command, in_directory, True, True, fail_status) - - -def SubprocessCallAndFilter(command, - in_directory, - print_messages, - print_stdout, - fail_status=None, filter=None): - """Runs command, a list, in directory in_directory. - - If print_messages is true, a message indicating what is being done - is printed to stdout. If print_stdout is true, the command's stdout - is also forwarded to stdout. - - If a filter function is specified, it is expected to take a single - string argument, and it will be called with each line of the - subprocess's output. Each line has had the trailing newline character - trimmed. - - If the command fails, as indicated by a nonzero exit status, gclient will - exit with an exit status of fail_status. If fail_status is None (the - default), gclient will raise an Error exception. - """ - - if print_messages: - print("\n________ running \'%s\' in \'%s\'" - % (' '.join(command), in_directory)) - - # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the - # executable, but shell=True makes subprocess on Linux fail when it's called - # with a list because it only tries to execute the first item in the list. - kid = subprocess.Popen(command, bufsize=0, cwd=in_directory, - shell=(sys.platform == 'win32'), stdout=subprocess.PIPE) - - # Also, we need to forward stdout to prevent weird re-ordering of output. - # This has to be done on a per byte basis to make sure it is not buffered: - # normally buffering is done for each line, but if svn requests input, no - # end-of-line character is output after the prompt and it would not show up. - in_byte = kid.stdout.read(1) - in_line = "" - while in_byte: - if in_byte != "\r": - if print_stdout: - sys.stdout.write(in_byte) - if in_byte != "\n": - in_line += in_byte - if in_byte == "\n" and filter: - filter(in_line) - in_line = "" - in_byte = kid.stdout.read(1) - rv = kid.wait() - - if rv: - msg = "failed to run command: %s" % " ".join(command) - - if fail_status != None: - print >>sys.stderr, msg - sys.exit(fail_status) - - raise Error(msg) - - -def IsUsingGit(root, paths): - """Returns True if we're using git to manage any of our checkouts. - |entries| is a list of paths to check.""" - for path in paths: - if os.path.exists(os.path.join(root, path, '.git')): - return True - return False - -# ----------------------------------------------------------------------------- -# SVN utils: - - -def RunSVN(args, in_directory): - """Runs svn, sending output to stdout. - - Args: - args: A sequence of command line parameters to be passed to svn. - in_directory: The directory where svn is to be run. - - Raises: - Error: An error occurred while running the svn command. - """ - c = [SVN_COMMAND] - c.extend(args) - - SubprocessCall(c, in_directory) - - -def CaptureSVN(args, in_directory=None, print_error=True): - """Runs svn, capturing output sent to stdout as a string. - - Args: - args: A sequence of command line parameters to be passed to svn. - in_directory: The directory where svn is to be run. - - Returns: - The output sent to stdout as a string. - """ - c = [SVN_COMMAND] - c.extend(args) - - # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for - # the svn.exe executable, but shell=True makes subprocess on Linux fail - # when it's called with a list because it only tries to execute the - # first string ("svn"). - stderr = None - if not print_error: - stderr = subprocess.PIPE - return subprocess.Popen(c, - cwd=in_directory, - shell=(sys.platform == 'win32'), - stdout=subprocess.PIPE, - stderr=stderr).communicate()[0] - - -def RunSVNAndGetFileList(args, in_directory, file_list): - """Runs svn checkout, update, or status, output to stdout. - - The first item in args must be either "checkout", "update", or "status". - - svn's stdout is parsed to collect a list of files checked out or updated. - These files are appended to file_list. svn's stdout is also printed to - sys.stdout as in RunSVN. - - Args: - args: A sequence of command line parameters to be passed to svn. - in_directory: The directory where svn is to be run. - - Raises: - Error: An error occurred while running the svn command. - """ - command = [SVN_COMMAND] - command.extend(args) - - # svn update and svn checkout use the same pattern: the first three columns - # are for file status, property status, and lock status. This is followed - # by two spaces, and then the path to the file. - update_pattern = '^... (.*)$' - - # The first three columns of svn status are the same as for svn update and - # svn checkout. The next three columns indicate addition-with-history, - # switch, and remote lock status. This is followed by one space, and then - # the path to the file. - status_pattern = '^...... (.*)$' - - # args[0] must be a supported command. This will blow up if it's something - # else, which is good. Note that the patterns are only effective when - # these commands are used in their ordinary forms, the patterns are invalid - # for "svn status --show-updates", for example. - pattern = { - 'checkout': update_pattern, - 'status': status_pattern, - 'update': update_pattern, - }[args[0]] - - compiled_pattern = re.compile(pattern) - - def CaptureMatchingLines(line): - match = compiled_pattern.search(line) - if match: - file_list.append(match.group(1)) - - RunSVNAndFilterOutput(args, - in_directory, - True, - True, - CaptureMatchingLines) - -def RunSVNAndFilterOutput(args, - in_directory, - print_messages, - print_stdout, - filter): - """Runs svn checkout, update, status, or diff, optionally outputting - to stdout. - - The first item in args must be either "checkout", "update", - "status", or "diff". - - svn's stdout is passed line-by-line to the given filter function. If - print_stdout is true, it is also printed to sys.stdout as in RunSVN. - - Args: - args: A sequence of command line parameters to be passed to svn. - in_directory: The directory where svn is to be run. - print_messages: Whether to print status messages to stdout about - which Subversion commands are being run. - print_stdout: Whether to forward Subversion's output to stdout. - filter: A function taking one argument (a string) which will be - passed each line (with the ending newline character removed) of - Subversion's output for filtering. - - Raises: - Error: An error occurred while running the svn command. - """ - command = [SVN_COMMAND] - command.extend(args) - - SubprocessCallAndFilter(command, - in_directory, - print_messages, - print_stdout, - filter=filter) - -def CaptureSVNInfo(relpath, in_directory=None, print_error=True): - """Returns a dictionary from the svn info output for the given file. - - Args: - relpath: The directory where the working copy resides relative to - the directory given by in_directory. - in_directory: The directory where svn is to be run. - """ - output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error) - dom = ParseXML(output) - result = {} - if dom: - def C(item, f): - if item is not None: return f(item) - # /info/entry/ - # url - # reposityory/(root|uuid) - # wc-info/(schedule|depth) - # commit/(author|date) - # str() the results because they may be returned as Unicode, which - # interferes with the higher layers matching up things in the deps - # dictionary. - # TODO(maruel): Fix at higher level instead (!) - result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str) - result['URL'] = C(GetNamedNodeText(dom, 'url'), str) - result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str) - result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'), - int) - result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'), - str) - result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str) - result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str) - result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str) - result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str) - return result - - -def CaptureSVNHeadRevision(url): - """Get the head revision of a SVN repository. - - Returns: - Int head revision - """ - info = CaptureSVN(["info", "--xml", url], os.getcwd()) - dom = xml.dom.minidom.parseString(info) - return int(dom.getElementsByTagName('entry')[0].getAttribute('revision')) - - -def CaptureSVNStatus(files): - """Returns the svn 1.5 svn status emulated output. - - @files can be a string (one file) or a list of files. - - Returns an array of (status, file) tuples.""" - command = ["status", "--xml"] - if not files: - pass - elif isinstance(files, basestring): - command.append(files) - else: - command.extend(files) - - status_letter = { - None: ' ', - '': ' ', - 'added': 'A', - 'conflicted': 'C', - 'deleted': 'D', - 'external': 'X', - 'ignored': 'I', - 'incomplete': '!', - 'merged': 'G', - 'missing': '!', - 'modified': 'M', - 'none': ' ', - 'normal': ' ', - 'obstructed': '~', - 'replaced': 'R', - 'unversioned': '?', - } - dom = ParseXML(CaptureSVN(command)) - results = [] - if dom: - # /status/target/entry/(wc-status|commit|author|date) - for target in dom.getElementsByTagName('target'): - base_path = target.getAttribute('path') - for entry in target.getElementsByTagName('entry'): - file = entry.getAttribute('path') - wc_status = entry.getElementsByTagName('wc-status') - assert len(wc_status) == 1 - # Emulate svn 1.5 status ouput... - statuses = [' ' for i in range(7)] - # Col 0 - xml_item_status = wc_status[0].getAttribute('item') - if xml_item_status in status_letter: - statuses[0] = status_letter[xml_item_status] - else: - raise Exception('Unknown item status "%s"; please implement me!' % - xml_item_status) - # Col 1 - xml_props_status = wc_status[0].getAttribute('props') - if xml_props_status == 'modified': - statuses[1] = 'M' - elif xml_props_status == 'conflicted': - statuses[1] = 'C' - elif (not xml_props_status or xml_props_status == 'none' or - xml_props_status == 'normal'): - pass - else: - raise Exception('Unknown props status "%s"; please implement me!' % - xml_props_status) - # Col 2 - if wc_status[0].getAttribute('wc-locked') == 'true': - statuses[2] = 'L' - # Col 3 - if wc_status[0].getAttribute('copied') == 'true': - statuses[3] = '+' - item = (''.join(statuses), file) - results.append(item) - return results - - -### SCM abstraction layer - - -class SCMWrapper(object): - """Add necessary glue between all the supported SCM. - - This is the abstraction layer to bind to different SCM. Since currently only - subversion is supported, a lot of subersionism remains. This can be sorted out - once another SCM is supported.""" - def __init__(self, url=None, root_dir=None, relpath=None, - scm_name='svn'): - # TODO(maruel): Deduce the SCM from the url. - self.scm_name = scm_name - self.url = url - self._root_dir = root_dir - if self._root_dir: - self._root_dir = self._root_dir.replace('/', os.sep) - self.relpath = relpath - if self.relpath: - self.relpath = self.relpath.replace('/', os.sep) - - def FullUrlForRelativeUrl(self, url): - # Find the forth '/' and strip from there. A bit hackish. - return '/'.join(self.url.split('/')[:4]) + url - - def RunCommand(self, command, options, args, file_list=None): - # file_list will have all files that are modified appended to it. - - if file_list == None: - file_list = [] - - commands = { - 'cleanup': self.cleanup, - 'export': self.export, - 'update': self.update, - 'revert': self.revert, - 'status': self.status, - 'diff': self.diff, - 'pack': self.pack, - 'runhooks': self.status, - } - - if not command in commands: - raise Error('Unknown command %s' % command) - - return commands[command](options, args, file_list) - - def cleanup(self, options, args, file_list): - """Cleanup working copy.""" - command = ['cleanup'] - command.extend(args) - RunSVN(command, os.path.join(self._root_dir, self.relpath)) - - def diff(self, options, args, file_list): - # NOTE: This function does not currently modify file_list. - command = ['diff'] - command.extend(args) - RunSVN(command, os.path.join(self._root_dir, self.relpath)) - - def export(self, options, args, file_list): - assert len(args) == 1 - export_path = os.path.abspath(os.path.join(args[0], self.relpath)) - try: - os.makedirs(export_path) - except OSError: - pass - assert os.path.exists(export_path) - command = ['export', '--force', '.'] - command.append(export_path) - RunSVN(command, os.path.join(self._root_dir, self.relpath)) - - def update(self, options, args, file_list): - """Runs SCM to update or transparently checkout the working copy. - - All updated files will be appended to file_list. - - Raises: - Error: if can't get URL for relative path. - """ - # Only update if git is not controlling the directory. - checkout_path = os.path.join(self._root_dir, self.relpath) - git_path = os.path.join(self._root_dir, self.relpath, '.git') - if os.path.exists(git_path): - print("________ found .git directory; skipping %s" % self.relpath) - return - - if args: - raise Error("Unsupported argument(s): %s" % ",".join(args)) - - url = self.url - components = url.split("@") - revision = None - forced_revision = False - if options.revision: - # Override the revision number. - url = '%s@%s' % (components[0], str(options.revision)) - revision = int(options.revision) - forced_revision = True - elif len(components) == 2: - revision = int(components[1]) - forced_revision = True - - rev_str = "" - if revision: - rev_str = ' at %d' % revision - - if not os.path.exists(checkout_path): - # We need to checkout. - command = ['checkout', url, checkout_path] - if revision: - command.extend(['--revision', str(revision)]) - RunSVNAndGetFileList(command, self._root_dir, file_list) - return - - # Get the existing scm url and the revision number of the current checkout. - from_info = CaptureSVNInfo(os.path.join(checkout_path, '.'), '.') - if not from_info: - raise Error("Can't update/checkout %r if an unversioned directory is " - "present. Delete the directory and try again." % - checkout_path) - - if options.manually_grab_svn_rev: - # Retrieve the current HEAD version because svn is slow at null updates. - if not revision: - from_info_live = CaptureSVNInfo(from_info['URL'], '.') - revision = int(from_info_live['Revision']) - rev_str = ' at %d' % revision - - if from_info['URL'] != components[0]: - to_info = CaptureSVNInfo(url, '.') - can_switch = ((from_info['Repository Root'] != to_info['Repository Root']) - and (from_info['UUID'] == to_info['UUID'])) - if can_switch: - print("\n_____ relocating %s to a new checkout" % self.relpath) - # We have different roots, so check if we can switch --relocate. - # Subversion only permits this if the repository UUIDs match. - # Perform the switch --relocate, then rewrite the from_url - # to reflect where we "are now." (This is the same way that - # Subversion itself handles the metadata when switch --relocate - # is used.) This makes the checks below for whether we - # can update to a revision or have to switch to a different - # branch work as expected. - # TODO(maruel): TEST ME ! - command = ["switch", "--relocate", - from_info['Repository Root'], - to_info['Repository Root'], - self.relpath] - RunSVN(command, self._root_dir) - from_info['URL'] = from_info['URL'].replace( - from_info['Repository Root'], - to_info['Repository Root']) - else: - if CaptureSVNStatus(checkout_path): - raise Error("Can't switch the checkout to %s; UUID don't match and " - "there is local changes in %s. Delete the directory and " - "try again." % (url, checkout_path)) - # Ok delete it. - print("\n_____ switching %s to a new checkout" % self.relpath) - RemoveDirectory(checkout_path) - # We need to checkout. - command = ['checkout', url, checkout_path] - if revision: - command.extend(['--revision', str(revision)]) - RunSVNAndGetFileList(command, self._root_dir, file_list) - return - - - # If the provided url has a revision number that matches the revision - # number of the existing directory, then we don't need to bother updating. - if not options.force and from_info['Revision'] == revision: - if options.verbose or not forced_revision: - print("\n_____ %s%s" % (self.relpath, rev_str)) - return - - command = ["update", checkout_path] - if revision: - command.extend(['--revision', str(revision)]) - RunSVNAndGetFileList(command, self._root_dir, file_list) - - def revert(self, options, args, file_list): - """Reverts local modifications. Subversion specific. - - All reverted files will be appended to file_list, even if Subversion - doesn't know about them. - """ - path = os.path.join(self._root_dir, self.relpath) - if not os.path.isdir(path): - # svn revert won't work if the directory doesn't exist. It needs to - # checkout instead. - print("\n_____ %s is missing, synching instead" % self.relpath) - # Don't reuse the args. - return self.update(options, [], file_list) - - files = CaptureSVNStatus(path) - # Batch the command. - files_to_revert = [] - for file in files: - file_path = os.path.join(path, file[1]) - print(file_path) - # Unversioned file or unexpected unversioned file. - if file[0][0] in ('?', '~'): - # Remove extraneous file. Also remove unexpected unversioned - # directories. svn won't touch them but we want to delete these. - file_list.append(file_path) - try: - os.remove(file_path) - except EnvironmentError: - RemoveDirectory(file_path) - - if file[0][0] != '?': - # For any other status, svn revert will work. - file_list.append(file_path) - files_to_revert.append(file[1]) - - # Revert them all at once. - if files_to_revert: - accumulated_paths = [] - accumulated_length = 0 - command = ['revert'] - for p in files_to_revert: - # Some shell have issues with command lines too long. - if accumulated_length and accumulated_length + len(p) > 3072: - RunSVN(command + accumulated_paths, - os.path.join(self._root_dir, self.relpath)) - accumulated_paths = [] - accumulated_length = 0 - else: - accumulated_paths.append(p) - accumulated_length += len(p) - if accumulated_paths: - RunSVN(command + accumulated_paths, - os.path.join(self._root_dir, self.relpath)) - - def status(self, options, args, file_list): - """Display status information.""" - path = os.path.join(self._root_dir, self.relpath) - command = ['status'] - command.extend(args) - if not os.path.isdir(path): - # svn status won't work if the directory doesn't exist. - print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory " - "does not exist." - % (' '.join(command), path)) - # There's no file list to retrieve. - else: - RunSVNAndGetFileList(command, path, file_list) - - def pack(self, options, args, file_list): - """Generates a patch file which can be applied to the root of the - repository.""" - path = os.path.join(self._root_dir, self.relpath) - command = ['diff'] - command.extend(args) - # Simple class which tracks which file is being diffed and - # replaces instances of its file name in the original and - # working copy lines of the svn diff output. - class DiffFilterer(object): - index_string = "Index: " - original_prefix = "--- " - working_prefix = "+++ " - - def __init__(self, relpath): - # Note that we always use '/' as the path separator to be - # consistent with svn's cygwin-style output on Windows - self._relpath = relpath.replace("\\", "/") - self._current_file = "" - self._replacement_file = "" - - def SetCurrentFile(self, file): - self._current_file = file - # Note that we always use '/' as the path separator to be - # consistent with svn's cygwin-style output on Windows - self._replacement_file = self._relpath + '/' + file - - def ReplaceAndPrint(self, line): - print(line.replace(self._current_file, self._replacement_file)) - - def Filter(self, line): - if (line.startswith(self.index_string)): - self.SetCurrentFile(line[len(self.index_string):]) - self.ReplaceAndPrint(line) - else: - if (line.startswith(self.original_prefix) or - line.startswith(self.working_prefix)): - self.ReplaceAndPrint(line) - else: - print line - - filterer = DiffFilterer(self.relpath) - RunSVNAndFilterOutput(command, path, False, False, filterer.Filter) - ## GClient implementation. @@ -1325,7 +555,8 @@ class GClient(object): raise Error( "relative DEPS entry \"%s\" must begin with a slash" % d) # Create a scm just to query the full url. - scm = SCMWrapper(solution["url"], self._root_dir, None) + scm = gclient_scm.SCMWrapper(solution["url"], self._root_dir, + None) url = scm.FullUrlForRelativeUrl(url) if d in deps and deps[d] != url: raise Error( @@ -1354,7 +585,7 @@ class GClient(object): # Use a discrete exit status code of 2 to indicate that a hook action # failed. Users of this script may wish to treat hook action failures # differently from VC failures. - SubprocessCall(command, self._root_dir, fail_status=2) + gclient_utils.SubprocessCall(command, self._root_dir, fail_status=2) def _RunHooks(self, command, file_list, is_using_git): """Evaluates all hooks, running actions as needed. @@ -1441,7 +672,7 @@ class GClient(object): entries[name] = url if run_scm: self._options.revision = revision_overrides.get(name) - scm = SCMWrapper(url, self._root_dir, name) + scm = gclient_scm.SCMWrapper(url, self._root_dir, name) scm.RunCommand(command, self._options, args, file_list) file_list = [os.path.join(name, file.strip()) for file in file_list] self._options.revision = None @@ -1467,7 +698,7 @@ class GClient(object): entries[d] = url if run_scm: self._options.revision = revision_overrides.get(d) - scm = SCMWrapper(url, self._root_dir, d) + scm = gclient_scm.SCMWrapper(url, self._root_dir, d) scm.RunCommand(command, self._options, args, file_list) self._options.revision = None @@ -1484,7 +715,7 @@ class GClient(object): entries[d] = url if run_scm: self._options.revision = revision_overrides.get(d) - scm = SCMWrapper(url, self._root_dir, d) + scm = gclient_scm.SCMWrapper(url, self._root_dir, d) scm.RunCommand(command, self._options, args, file_list) self._options.revision = None @@ -1503,7 +734,7 @@ class GClient(object): while file_list[i].startswith('\\') or file_list[i].startswith('/'): file_list[i] = file_list[i][1:] - is_using_git = IsUsingGit(self._root_dir, entries.keys()) + is_using_git = gclient_utils.IsUsingGit(self._root_dir, entries.keys()) self._RunHooks(command, file_list, is_using_git) if command == 'update': @@ -1529,7 +760,7 @@ class GClient(object): # Delete the entry print("\n________ deleting \'%s\' " + "in \'%s\'") % (entry_fixed, self._root_dir) - RemoveDirectory(e_dir) + gclient_utils.RemoveDirectory(e_dir) # record the current list of entries for next time self._SaveEntries(entries) diff --git a/gclient_scm.py b/gclient_scm.py new file mode 100644 index 000000000..2096496b3 --- /dev/null +++ b/gclient_scm.py @@ -0,0 +1,577 @@ +# Copyright 2009 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import re +import subprocess +import sys +import xml.dom.minidom + +import gclient_utils + +SVN_COMMAND = "svn" + + +### SCM abstraction layer + + +class SCMWrapper(object): + """Add necessary glue between all the supported SCM. + + This is the abstraction layer to bind to different SCM. Since currently only + subversion is supported, a lot of subersionism remains. This can be sorted out + once another SCM is supported.""" + def __init__(self, url=None, root_dir=None, relpath=None, + scm_name='svn'): + # TODO(maruel): Deduce the SCM from the url. + self.scm_name = scm_name + self.url = url + self._root_dir = root_dir + if self._root_dir: + self._root_dir = self._root_dir.replace('/', os.sep) + self.relpath = relpath + if self.relpath: + self.relpath = self.relpath.replace('/', os.sep) + + def FullUrlForRelativeUrl(self, url): + # Find the forth '/' and strip from there. A bit hackish. + return '/'.join(self.url.split('/')[:4]) + url + + def RunCommand(self, command, options, args, file_list=None): + # file_list will have all files that are modified appended to it. + + file_list = file_list or [] + + commands = { + 'cleanup': self.cleanup, + 'export': self.export, + 'update': self.update, + 'revert': self.revert, + 'status': self.status, + 'diff': self.diff, + 'pack': self.pack, + 'runhooks': self.status, + } + + if not command in commands: + raise gclient_utils.Error('Unknown command %s' % command) + + return commands[command](options, args, file_list) + + def cleanup(self, options, args, file_list): + """Cleanup working copy.""" + command = ['cleanup'] + command.extend(args) + RunSVN(command, os.path.join(self._root_dir, self.relpath)) + + def diff(self, options, args, file_list): + # NOTE: This function does not currently modify file_list. + command = ['diff'] + command.extend(args) + RunSVN(command, os.path.join(self._root_dir, self.relpath)) + + def export(self, options, args, file_list): + assert len(args) == 1 + export_path = os.path.abspath(os.path.join(args[0], self.relpath)) + try: + os.makedirs(export_path) + except OSError: + pass + assert os.path.exists(export_path) + command = ['export', '--force', '.'] + command.append(export_path) + RunSVN(command, os.path.join(self._root_dir, self.relpath)) + + def update(self, options, args, file_list): + """Runs SCM to update or transparently checkout the working copy. + + All updated files will be appended to file_list. + + Raises: + Error: if can't get URL for relative path. + """ + # Only update if git is not controlling the directory. + checkout_path = os.path.join(self._root_dir, self.relpath) + git_path = os.path.join(self._root_dir, self.relpath, '.git') + if os.path.exists(git_path): + print("________ found .git directory; skipping %s" % self.relpath) + return + + if args: + raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args)) + + url = self.url + components = url.split("@") + revision = None + forced_revision = False + if options.revision: + # Override the revision number. + url = '%s@%s' % (components[0], str(options.revision)) + revision = int(options.revision) + forced_revision = True + elif len(components) == 2: + revision = int(components[1]) + forced_revision = True + + rev_str = "" + if revision: + rev_str = ' at %d' % revision + + if not os.path.exists(checkout_path): + # We need to checkout. + command = ['checkout', url, checkout_path] + if revision: + command.extend(['--revision', str(revision)]) + RunSVNAndGetFileList(command, self._root_dir, file_list) + return + + # Get the existing scm url and the revision number of the current checkout. + from_info = CaptureSVNInfo(os.path.join(checkout_path, '.'), '.') + if not from_info: + raise gclient_utils.Error("Can't update/checkout %r if an unversioned " + "directory is present. Delete the directory " + "and try again." % + checkout_path) + + if options.manually_grab_svn_rev: + # Retrieve the current HEAD version because svn is slow at null updates. + if not revision: + from_info_live = CaptureSVNInfo(from_info['URL'], '.') + revision = int(from_info_live['Revision']) + rev_str = ' at %d' % revision + + if from_info['URL'] != components[0]: + to_info = CaptureSVNInfo(url, '.') + can_switch = ((from_info['Repository Root'] != to_info['Repository Root']) + and (from_info['UUID'] == to_info['UUID'])) + if can_switch: + print("\n_____ relocating %s to a new checkout" % self.relpath) + # We have different roots, so check if we can switch --relocate. + # Subversion only permits this if the repository UUIDs match. + # Perform the switch --relocate, then rewrite the from_url + # to reflect where we "are now." (This is the same way that + # Subversion itself handles the metadata when switch --relocate + # is used.) This makes the checks below for whether we + # can update to a revision or have to switch to a different + # branch work as expected. + # TODO(maruel): TEST ME ! + command = ["switch", "--relocate", + from_info['Repository Root'], + to_info['Repository Root'], + self.relpath] + RunSVN(command, self._root_dir) + from_info['URL'] = from_info['URL'].replace( + from_info['Repository Root'], + to_info['Repository Root']) + else: + if CaptureSVNStatus(checkout_path): + raise gclient_utils.Error("Can't switch the checkout to %s; UUID " + "don't match and there is local changes " + "in %s. Delete the directory and " + "try again." % (url, checkout_path)) + # Ok delete it. + print("\n_____ switching %s to a new checkout" % self.relpath) + RemoveDirectory(checkout_path) + # We need to checkout. + command = ['checkout', url, checkout_path] + if revision: + command.extend(['--revision', str(revision)]) + RunSVNAndGetFileList(command, self._root_dir, file_list) + return + + + # If the provided url has a revision number that matches the revision + # number of the existing directory, then we don't need to bother updating. + if not options.force and from_info['Revision'] == revision: + if options.verbose or not forced_revision: + print("\n_____ %s%s" % (self.relpath, rev_str)) + return + + command = ["update", checkout_path] + if revision: + command.extend(['--revision', str(revision)]) + RunSVNAndGetFileList(command, self._root_dir, file_list) + + def revert(self, options, args, file_list): + """Reverts local modifications. Subversion specific. + + All reverted files will be appended to file_list, even if Subversion + doesn't know about them. + """ + path = os.path.join(self._root_dir, self.relpath) + if not os.path.isdir(path): + # svn revert won't work if the directory doesn't exist. It needs to + # checkout instead. + print("\n_____ %s is missing, synching instead" % self.relpath) + # Don't reuse the args. + return self.update(options, [], file_list) + + files = CaptureSVNStatus(path) + # Batch the command. + files_to_revert = [] + for file in files: + file_path = os.path.join(path, file[1]) + print(file_path) + # Unversioned file or unexpected unversioned file. + if file[0][0] in ('?', '~'): + # Remove extraneous file. Also remove unexpected unversioned + # directories. svn won't touch them but we want to delete these. + file_list.append(file_path) + try: + os.remove(file_path) + except EnvironmentError: + RemoveDirectory(file_path) + + if file[0][0] != '?': + # For any other status, svn revert will work. + file_list.append(file_path) + files_to_revert.append(file[1]) + + # Revert them all at once. + if files_to_revert: + accumulated_paths = [] + accumulated_length = 0 + command = ['revert'] + for p in files_to_revert: + # Some shell have issues with command lines too long. + if accumulated_length and accumulated_length + len(p) > 3072: + RunSVN(command + accumulated_paths, + os.path.join(self._root_dir, self.relpath)) + accumulated_paths = [] + accumulated_length = 0 + else: + accumulated_paths.append(p) + accumulated_length += len(p) + if accumulated_paths: + RunSVN(command + accumulated_paths, + os.path.join(self._root_dir, self.relpath)) + + def status(self, options, args, file_list): + """Display status information.""" + path = os.path.join(self._root_dir, self.relpath) + command = ['status'] + command.extend(args) + if not os.path.isdir(path): + # svn status won't work if the directory doesn't exist. + print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory " + "does not exist." + % (' '.join(command), path)) + # There's no file list to retrieve. + else: + RunSVNAndGetFileList(command, path, file_list) + + def pack(self, options, args, file_list): + """Generates a patch file which can be applied to the root of the + repository.""" + path = os.path.join(self._root_dir, self.relpath) + command = ['diff'] + command.extend(args) + # Simple class which tracks which file is being diffed and + # replaces instances of its file name in the original and + # working copy lines of the svn diff output. + class DiffFilterer(object): + index_string = "Index: " + original_prefix = "--- " + working_prefix = "+++ " + + def __init__(self, relpath): + # Note that we always use '/' as the path separator to be + # consistent with svn's cygwin-style output on Windows + self._relpath = relpath.replace("\\", "/") + self._current_file = "" + self._replacement_file = "" + + def SetCurrentFile(self, file): + self._current_file = file + # Note that we always use '/' as the path separator to be + # consistent with svn's cygwin-style output on Windows + self._replacement_file = self._relpath + '/' + file + + def ReplaceAndPrint(self, line): + print(line.replace(self._current_file, self._replacement_file)) + + def Filter(self, line): + if (line.startswith(self.index_string)): + self.SetCurrentFile(line[len(self.index_string):]) + self.ReplaceAndPrint(line) + else: + if (line.startswith(self.original_prefix) or + line.startswith(self.working_prefix)): + self.ReplaceAndPrint(line) + else: + print line + + filterer = DiffFilterer(self.relpath) + RunSVNAndFilterOutput(command, path, False, False, filterer.Filter) + + +# ----------------------------------------------------------------------------- +# SVN utils: + + +def RunSVN(args, in_directory): + """Runs svn, sending output to stdout. + + Args: + args: A sequence of command line parameters to be passed to svn. + in_directory: The directory where svn is to be run. + + Raises: + Error: An error occurred while running the svn command. + """ + c = [SVN_COMMAND] + c.extend(args) + + gclient_utils.SubprocessCall(c, in_directory) + + +def CaptureSVN(args, in_directory=None, print_error=True): + """Runs svn, capturing output sent to stdout as a string. + + Args: + args: A sequence of command line parameters to be passed to svn. + in_directory: The directory where svn is to be run. + + Returns: + The output sent to stdout as a string. + """ + c = [SVN_COMMAND] + c.extend(args) + + # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for + # the svn.exe executable, but shell=True makes subprocess on Linux fail + # when it's called with a list because it only tries to execute the + # first string ("svn"). + stderr = None + if not print_error: + stderr = subprocess.PIPE + return subprocess.Popen(c, + cwd=in_directory, + shell=(sys.platform == 'win32'), + stdout=subprocess.PIPE, + stderr=stderr).communicate()[0] + + +def RunSVNAndGetFileList(args, in_directory, file_list): + """Runs svn checkout, update, or status, output to stdout. + + The first item in args must be either "checkout", "update", or "status". + + svn's stdout is parsed to collect a list of files checked out or updated. + These files are appended to file_list. svn's stdout is also printed to + sys.stdout as in RunSVN. + + Args: + args: A sequence of command line parameters to be passed to svn. + in_directory: The directory where svn is to be run. + + Raises: + Error: An error occurred while running the svn command. + """ + command = [SVN_COMMAND] + command.extend(args) + + # svn update and svn checkout use the same pattern: the first three columns + # are for file status, property status, and lock status. This is followed + # by two spaces, and then the path to the file. + update_pattern = '^... (.*)$' + + # The first three columns of svn status are the same as for svn update and + # svn checkout. The next three columns indicate addition-with-history, + # switch, and remote lock status. This is followed by one space, and then + # the path to the file. + status_pattern = '^...... (.*)$' + + # args[0] must be a supported command. This will blow up if it's something + # else, which is good. Note that the patterns are only effective when + # these commands are used in their ordinary forms, the patterns are invalid + # for "svn status --show-updates", for example. + pattern = { + 'checkout': update_pattern, + 'status': status_pattern, + 'update': update_pattern, + }[args[0]] + + compiled_pattern = re.compile(pattern) + + def CaptureMatchingLines(line): + match = compiled_pattern.search(line) + if match: + file_list.append(match.group(1)) + + RunSVNAndFilterOutput(args, + in_directory, + True, + True, + CaptureMatchingLines) + +def RunSVNAndFilterOutput(args, + in_directory, + print_messages, + print_stdout, + filter): + """Runs svn checkout, update, status, or diff, optionally outputting + to stdout. + + The first item in args must be either "checkout", "update", + "status", or "diff". + + svn's stdout is passed line-by-line to the given filter function. If + print_stdout is true, it is also printed to sys.stdout as in RunSVN. + + Args: + args: A sequence of command line parameters to be passed to svn. + in_directory: The directory where svn is to be run. + print_messages: Whether to print status messages to stdout about + which Subversion commands are being run. + print_stdout: Whether to forward Subversion's output to stdout. + filter: A function taking one argument (a string) which will be + passed each line (with the ending newline character removed) of + Subversion's output for filtering. + + Raises: + Error: An error occurred while running the svn command. + """ + command = [SVN_COMMAND] + command.extend(args) + + gclient_utils.SubprocessCallAndFilter(command, + in_directory, + print_messages, + print_stdout, + filter=filter) + +def CaptureSVNInfo(relpath, in_directory=None, print_error=True): + """Returns a dictionary from the svn info output for the given file. + + Args: + relpath: The directory where the working copy resides relative to + the directory given by in_directory. + in_directory: The directory where svn is to be run. + """ + output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error) + dom = gclient_utils.ParseXML(output) + result = {} + if dom: + GetNamedNodeText = gclient_utils.GetNamedNodeText + GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText + def C(item, f): + if item is not None: return f(item) + # /info/entry/ + # url + # reposityory/(root|uuid) + # wc-info/(schedule|depth) + # commit/(author|date) + # str() the results because they may be returned as Unicode, which + # interferes with the higher layers matching up things in the deps + # dictionary. + # TODO(maruel): Fix at higher level instead (!) + result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str) + result['URL'] = C(GetNamedNodeText(dom, 'url'), str) + result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str) + result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'), + int) + result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'), + str) + result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str) + result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str) + result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str) + result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str) + return result + + +def CaptureSVNHeadRevision(url): + """Get the head revision of a SVN repository. + + Returns: + Int head revision + """ + info = CaptureSVN(["info", "--xml", url], os.getcwd()) + dom = xml.dom.minidom.parseString(info) + return int(dom.getElementsByTagName('entry')[0].getAttribute('revision')) + + +def CaptureSVNStatus(files): + """Returns the svn 1.5 svn status emulated output. + + @files can be a string (one file) or a list of files. + + Returns an array of (status, file) tuples.""" + command = ["status", "--xml"] + if not files: + pass + elif isinstance(files, basestring): + command.append(files) + else: + command.extend(files) + + status_letter = { + None: ' ', + '': ' ', + 'added': 'A', + 'conflicted': 'C', + 'deleted': 'D', + 'external': 'X', + 'ignored': 'I', + 'incomplete': '!', + 'merged': 'G', + 'missing': '!', + 'modified': 'M', + 'none': ' ', + 'normal': ' ', + 'obstructed': '~', + 'replaced': 'R', + 'unversioned': '?', + } + dom = gclient_utils.ParseXML(CaptureSVN(command)) + results = [] + if dom: + # /status/target/entry/(wc-status|commit|author|date) + for target in dom.getElementsByTagName('target'): + base_path = target.getAttribute('path') + for entry in target.getElementsByTagName('entry'): + file = entry.getAttribute('path') + wc_status = entry.getElementsByTagName('wc-status') + assert len(wc_status) == 1 + # Emulate svn 1.5 status ouput... + statuses = [' ' for i in range(7)] + # Col 0 + xml_item_status = wc_status[0].getAttribute('item') + if xml_item_status in status_letter: + statuses[0] = status_letter[xml_item_status] + else: + raise Exception('Unknown item status "%s"; please implement me!' % + xml_item_status) + # Col 1 + xml_props_status = wc_status[0].getAttribute('props') + if xml_props_status == 'modified': + statuses[1] = 'M' + elif xml_props_status == 'conflicted': + statuses[1] = 'C' + elif (not xml_props_status or xml_props_status == 'none' or + xml_props_status == 'normal'): + pass + else: + raise Exception('Unknown props status "%s"; please implement me!' % + xml_props_status) + # Col 2 + if wc_status[0].getAttribute('wc-locked') == 'true': + statuses[2] = 'L' + # Col 3 + if wc_status[0].getAttribute('copied') == 'true': + statuses[3] = '+' + item = (''.join(statuses), file) + results.append(item) + return results diff --git a/gclient_utils.py b/gclient_utils.py new file mode 100644 index 000000000..ff8a24038 --- /dev/null +++ b/gclient_utils.py @@ -0,0 +1,239 @@ +# Copyright 2009 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +import xml.dom.minidom + +## Generic utils + + +def ParseXML(output): + try: + return xml.dom.minidom.parseString(output) + except xml.parsers.expat.ExpatError: + return None + + +def GetNamedNodeText(node, node_name): + child_nodes = node.getElementsByTagName(node_name) + if not child_nodes: + return None + assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1 + return child_nodes[0].firstChild.nodeValue + + +def GetNodeNamedAttributeText(node, node_name, attribute_name): + child_nodes = node.getElementsByTagName(node_name) + if not child_nodes: + return None + assert len(child_nodes) == 1 + return child_nodes[0].getAttribute(attribute_name) + + +class Error(Exception): + """gclient exception class.""" + pass + + +class PrintableObject(object): + def __str__(self): + output = '' + for i in dir(self): + if i.startswith('__'): + continue + output += '%s = %s\n' % (i, str(getattr(self, i, ''))) + return output + + +def FileRead(filename): + content = None + f = open(filename, "rU") + try: + content = f.read() + finally: + f.close() + return content + + +def FileWrite(filename, content): + f = open(filename, "w") + try: + f.write(content) + finally: + f.close() + + +def RemoveDirectory(*path): + """Recursively removes a directory, even if it's marked read-only. + + Remove the directory located at *path, if it exists. + + shutil.rmtree() doesn't work on Windows if any of the files or directories + are read-only, which svn repositories and some .svn files are. We need to + be able to force the files to be writable (i.e., deletable) as we traverse + the tree. + + Even with all this, Windows still sometimes fails to delete a file, citing + a permission error (maybe something to do with antivirus scans or disk + indexing). The best suggestion any of the user forums had was to wait a + bit and try again, so we do that too. It's hand-waving, but sometimes it + works. :/ + + On POSIX systems, things are a little bit simpler. The modes of the files + to be deleted doesn't matter, only the modes of the directories containing + them are significant. As the directory tree is traversed, each directory + has its mode set appropriately before descending into it. This should + result in the entire tree being removed, with the possible exception of + *path itself, because nothing attempts to change the mode of its parent. + Doing so would be hazardous, as it's not a directory slated for removal. + In the ordinary case, this is not a problem: for our purposes, the user + will never lack write permission on *path's parent. + """ + file_path = os.path.join(*path) + if not os.path.exists(file_path): + return + + if os.path.islink(file_path) or not os.path.isdir(file_path): + raise Error("RemoveDirectory asked to remove non-directory %s" % file_path) + + has_win32api = False + if sys.platform == 'win32': + has_win32api = True + # Some people don't have the APIs installed. In that case we'll do without. + try: + win32api = __import__('win32api') + win32con = __import__('win32con') + except ImportError: + has_win32api = False + else: + # On POSIX systems, we need the x-bit set on the directory to access it, + # the r-bit to see its contents, and the w-bit to remove files from it. + # The actual modes of the files within the directory is irrelevant. + os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + for fn in os.listdir(file_path): + fullpath = os.path.join(file_path, fn) + + # If fullpath is a symbolic link that points to a directory, isdir will + # be True, but we don't want to descend into that as a directory, we just + # want to remove the link. Check islink and treat links as ordinary files + # would be treated regardless of what they reference. + if os.path.islink(fullpath) or not os.path.isdir(fullpath): + if sys.platform == 'win32': + os.chmod(fullpath, stat.S_IWRITE) + if has_win32api: + win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL) + try: + os.remove(fullpath) + except OSError, e: + if e.errno != errno.EACCES or sys.platform != 'win32': + raise + print 'Failed to delete %s: trying again' % fullpath + time.sleep(0.1) + os.remove(fullpath) + else: + RemoveDirectory(fullpath) + + if sys.platform == 'win32': + os.chmod(file_path, stat.S_IWRITE) + if has_win32api: + win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL) + try: + os.rmdir(file_path) + except OSError, e: + if e.errno != errno.EACCES or sys.platform != 'win32': + raise + print 'Failed to remove %s: trying again' % file_path + time.sleep(0.1) + os.rmdir(file_path) + + +def SubprocessCall(command, in_directory, fail_status=None): + """Runs command, a list, in directory in_directory. + + This function wraps SubprocessCallAndFilter, but does not perform the + filtering functions. See that function for a more complete usage + description. + """ + # Call subprocess and capture nothing: + SubprocessCallAndFilter(command, in_directory, True, True, fail_status) + + +def SubprocessCallAndFilter(command, + in_directory, + print_messages, + print_stdout, + fail_status=None, filter=None): + """Runs command, a list, in directory in_directory. + + If print_messages is true, a message indicating what is being done + is printed to stdout. If print_stdout is true, the command's stdout + is also forwarded to stdout. + + If a filter function is specified, it is expected to take a single + string argument, and it will be called with each line of the + subprocess's output. Each line has had the trailing newline character + trimmed. + + If the command fails, as indicated by a nonzero exit status, gclient will + exit with an exit status of fail_status. If fail_status is None (the + default), gclient will raise an Error exception. + """ + + if print_messages: + print("\n________ running \'%s\' in \'%s\'" + % (' '.join(command), in_directory)) + + # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the + # executable, but shell=True makes subprocess on Linux fail when it's called + # with a list because it only tries to execute the first item in the list. + kid = subprocess.Popen(command, bufsize=0, cwd=in_directory, + shell=(sys.platform == 'win32'), stdout=subprocess.PIPE) + + # Also, we need to forward stdout to prevent weird re-ordering of output. + # This has to be done on a per byte basis to make sure it is not buffered: + # normally buffering is done for each line, but if svn requests input, no + # end-of-line character is output after the prompt and it would not show up. + in_byte = kid.stdout.read(1) + in_line = "" + while in_byte: + if in_byte != "\r": + if print_stdout: + sys.stdout.write(in_byte) + if in_byte != "\n": + in_line += in_byte + if in_byte == "\n" and filter: + filter(in_line) + in_line = "" + in_byte = kid.stdout.read(1) + rv = kid.wait() + + if rv: + msg = "failed to run command: %s" % " ".join(command) + + if fail_status != None: + print >>sys.stderr, msg + sys.exit(fail_status) + + raise Error(msg) + + +def IsUsingGit(root, paths): + """Returns True if we're using git to manage any of our checkouts. + |entries| is a list of paths to check.""" + for path in paths: + if os.path.exists(os.path.join(root, path, '.git')): + return True + return False diff --git a/presubmit_support.py b/presubmit_support.py index 78dae3093..c666e5274 100755 --- a/presubmit_support.py +++ b/presubmit_support.py @@ -38,7 +38,7 @@ import warnings # TODO(joi) Would be cleaner to factor out utils in gcl to separate module, but # for now it would only be a couple of functions so hardly worth it. import gcl -import gclient +import gclient_scm import presubmit_canned_checks @@ -238,7 +238,7 @@ class InputApi(object): Remember to check for the None case and show an appropriate error! """ - local_path = gclient.CaptureSVNInfo(depot_path).get('Path') + local_path = gclient_scm.CaptureSVNInfo(depot_path).get('Path') if local_path: return local_path @@ -251,7 +251,7 @@ class InputApi(object): Returns: The depot path (SVN URL) of the file if mapped, otherwise None. """ - depot_path = gclient.CaptureSVNInfo(local_path).get('URL') + depot_path = gclient_scm.CaptureSVNInfo(local_path).get('URL') if depot_path: return depot_path @@ -461,7 +461,7 @@ class SvnAffectedFile(AffectedFile): def ServerPath(self): if self._server_path is None: - self._server_path = gclient.CaptureSVNInfo( + self._server_path = gclient_scm.CaptureSVNInfo( self.AbsoluteLocalPath()).get('URL', '') return self._server_path @@ -473,7 +473,7 @@ class SvnAffectedFile(AffectedFile): # querying subversion, especially on Windows. self._is_directory = os.path.isdir(path) else: - self._is_directory = gclient.CaptureSVNInfo( + self._is_directory = gclient_scm.CaptureSVNInfo( path).get('Node Kind') in ('dir', 'directory') return self._is_directory @@ -947,7 +947,7 @@ def Main(argv): options.files = ParseFiles(args, options.recursive) else: # Grab modified files. - files = gclient.CaptureSVNStatus([options.root]) + files = gclient_scm.CaptureSVNStatus([options.root]) else: # Doesn't seem under source control. change_class = Change diff --git a/revert.py b/revert.py index 4056c4ee9..71016654f 100755 --- a/revert.py +++ b/revert.py @@ -13,6 +13,8 @@ import xml import gcl import gclient +import gclient_scm +import gclient_utils class ModifiedFile(exceptions.Exception): pass @@ -32,7 +34,7 @@ def UniqueFast(list): def GetRepoBase(): """Returns the repository base of the root local checkout.""" - info = gclient.CaptureSVNInfo('.') + info = gclient_scm.CaptureSVNInfo('.') root = info['Repository Root'] url = info['URL'] if not root or not url: @@ -46,8 +48,8 @@ def CaptureSVNLog(args): command = ['log', '--xml'] if args: command += args - output = gclient.CaptureSVN(command) - dom = gclient.ParseXML(output) + output = gclient_scm.CaptureSVN(command) + dom = gclient_utils.ParseXML(output) entries = [] if dom: # /log/logentry/ @@ -66,8 +68,8 @@ def CaptureSVNLog(args): paths.append(item) entry = { 'revision': int(node.getAttribute('revision')), - 'author': gclient.GetNamedNodeText(node, 'author'), - 'date': gclient.GetNamedNodeText(node, 'date'), + 'author': gclient_utils.GetNamedNodeText(node, 'author'), + 'date': gclient_utils.GetNamedNodeText(node, 'date'), 'paths': paths, } entries.append(entry) @@ -144,7 +146,7 @@ def Revert(revisions, force=False, commit=True, send_email=True, message=None, print "" # Make sure these files are unmodified with svn status. - status = gclient.CaptureSVNStatus(files) + status = gclient_scm.CaptureSVNStatus(files) if status: if force: # TODO(maruel): Use the tool to correctly revert '?' files. diff --git a/tests/gcl_unittest.py b/tests/gcl_unittest.py index 814b664f8..bf73d917c 100755 --- a/tests/gcl_unittest.py +++ b/tests/gcl_unittest.py @@ -19,7 +19,7 @@ class GclTestsBase(super_mox.SuperMoxTestBase): super_mox.SuperMoxTestBase.setUp(self) self.fake_root_dir = self.RootDir() self.mox.StubOutWithMock(gcl, 'RunShell') - self.mox.StubOutWithMock(gcl.gclient, 'CaptureSVNInfo') + self.mox.StubOutWithMock(gcl.gclient_scm, 'CaptureSVNInfo') self.mox.StubOutWithMock(gcl.os, 'getcwd') self.mox.StubOutWithMock(gcl.os, 'chdir') self.mox.StubOutWithMock(gcl.os, 'close') @@ -55,7 +55,7 @@ class GclUnittest(GclTestsBase): 'PresubmitCL', 'ReadFile', 'REPOSITORY_ROOT', 'RunShell', 'RunShellWithReturnCode', 'SendToRietveld', 'TryChange', 'UnknownFiles', 'UploadCL', 'Warn', 'WriteFile', - 'gclient', 'getpass', 'main', 'os', 'random', 're', + 'gclient_scm', 'gclient_utils', 'getpass', 'main', 'os', 'random', 're', 'shutil', 'string', 'subprocess', 'sys', 'tempfile', 'upload', 'urllib2', 'xml', ] @@ -80,7 +80,7 @@ class GclUnittest(GclTestsBase): result = { "Repository Root": "" } - gcl.gclient.CaptureSVNInfo("/bleh/prout", print_error=False).AndReturn( + gcl.gclient_scm.CaptureSVNInfo("/bleh/prout", print_error=False).AndReturn( result) self.mox.ReplayAll() self.assertRaises(Exception, gcl.GetRepositoryRoot) @@ -90,10 +90,11 @@ class GclUnittest(GclTestsBase): root_path = gcl.os.path.join('bleh', 'prout', 'pouet') gcl.os.getcwd().AndReturn(root_path) result1 = { "Repository Root": "Some root" } - gcl.gclient.CaptureSVNInfo(root_path, print_error=False).AndReturn(result1) + gcl.gclient_scm.CaptureSVNInfo(root_path, + print_error=False).AndReturn(result1) gcl.os.getcwd().AndReturn(root_path) results2 = { "Repository Root": "A different root" } - gcl.gclient.CaptureSVNInfo( + gcl.gclient_scm.CaptureSVNInfo( gcl.os.path.dirname(root_path), print_error=False).AndReturn(results2) self.mox.ReplayAll() diff --git a/tests/gclient_test.py b/tests/gclient_test.py index e5aa13815..eb3cd7798 100644 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -25,6 +25,8 @@ import StringIO import unittest import gclient +import gclient_scm +import gclient_utils import super_mox from super_mox import mox @@ -35,12 +37,12 @@ class BaseTestCase(super_mox.SuperMoxTestBase): self.mox.StubOutWithMock(gclient.os.path, 'exists') self.mox.StubOutWithMock(gclient.os.path, 'isdir') self.mox.StubOutWithMock(gclient.sys, 'stdout') - self.mox.StubOutWithMock(gclient, 'subprocess') + self.mox.StubOutWithMock(gclient_utils, 'subprocess') # These are not tested. self.mox.StubOutWithMock(gclient, 'FileRead') self.mox.StubOutWithMock(gclient, 'FileWrite') - self.mox.StubOutWithMock(gclient, 'SubprocessCall') - self.mox.StubOutWithMock(gclient, 'RemoveDirectory') + self.mox.StubOutWithMock(gclient_utils, 'SubprocessCall') + self.mox.StubOutWithMock(gclient_utils, 'RemoveDirectory') # Like unittest's assertRaises, but checks for Gclient.Error. def assertRaisesError(self, msg, fn, *args, **kwargs): @@ -59,20 +61,20 @@ class GClientBaseTestCase(BaseTestCase): def setUp(self): BaseTestCase.setUp(self) # Mock them to be sure nothing bad happens. - self.mox.StubOutWithMock(gclient, 'CaptureSVN') - self._CaptureSVNInfo = gclient.CaptureSVNInfo - self.mox.StubOutWithMock(gclient, 'CaptureSVNInfo') - self.mox.StubOutWithMock(gclient, 'CaptureSVNStatus') - self.mox.StubOutWithMock(gclient, 'RunSVN') - self.mox.StubOutWithMock(gclient, 'RunSVNAndGetFileList') + self.mox.StubOutWithMock(gclient_scm, 'CaptureSVN') + self._CaptureSVNInfo = gclient_scm.CaptureSVNInfo + self.mox.StubOutWithMock(gclient_scm, 'CaptureSVNInfo') + self.mox.StubOutWithMock(gclient_scm, 'CaptureSVNStatus') + self.mox.StubOutWithMock(gclient_scm, 'RunSVN') + self.mox.StubOutWithMock(gclient_scm, 'RunSVNAndGetFileList') self._gclient_gclient = gclient.GClient gclient.GClient = self.mox.CreateMockAnything() - self._scm_wrapper = gclient.SCMWrapper - gclient.SCMWrapper = self.mox.CreateMockAnything() + self._scm_wrapper = gclient_scm.SCMWrapper + gclient_scm.SCMWrapper = self.mox.CreateMockAnything() def tearDown(self): gclient.GClient = self._gclient_gclient - gclient.SCMWrapper = self._scm_wrapper + gclient_scm.SCMWrapper = self._scm_wrapper BaseTestCase.tearDown(self) @@ -352,9 +354,11 @@ class GClientClassTestCase(GclientTestCase): solution_name = 'solution name' solution_url = 'solution url' safesync_url = 'safesync url' - default_text = gclient.DEFAULT_CLIENT_FILE_TEXT % (solution_name, - solution_url, - safesync_url) + default_text = gclient.DEFAULT_CLIENT_FILE_TEXT % { + 'solution_name' : solution_name, + 'solution_url' : solution_url, + 'safesync_url' : safesync_url + } client.SetDefaultConfig(solution_name, solution_url, safesync_url) self.assertEqual(client.ConfigContent(), default_text) solutions = [{ @@ -403,7 +407,7 @@ class GClientClassTestCase(GclientTestCase): # An scm will be requested for the solution. scm_wrapper_sol = self.mox.CreateMockAnything() - gclient.SCMWrapper(self.url, self.root_dir, solution_name + gclient_scm.SCMWrapper(self.url, self.root_dir, solution_name ).AndReturn(scm_wrapper_sol) # Then an update will be performed. scm_wrapper_sol.RunCommand('update', options, self.args, []) @@ -460,7 +464,7 @@ class GClientClassTestCase(GclientTestCase): ).AndReturn(False) # An scm will be requested for the solution. - gclient.SCMWrapper(self.url, self.root_dir, solution_name + gclient_scm.SCMWrapper(self.url, self.root_dir, solution_name ).AndReturn(scm_wrapper_sol) # Then an update will be performed. scm_wrapper_sol.RunCommand('update', options, self.args, []) @@ -472,7 +476,7 @@ class GClientClassTestCase(GclientTestCase): # Next we expect an scm to be request for dep src/t but it should # use the url specified in deps and the relative path should now # be relative to the DEPS file. - gclient.SCMWrapper( + gclient_scm.SCMWrapper( 'svn://scm.t/trunk', self.root_dir, os.path.join(solution_name, "src", "t")).AndReturn(scm_wrapper_t) @@ -534,7 +538,7 @@ class GClientClassTestCase(GclientTestCase): ).AndReturn(False) # An scm will be requested for the solution. - gclient.SCMWrapper(self.url, self.root_dir, solution_name + gclient_scm.SCMWrapper(self.url, self.root_dir, solution_name ).AndReturn(scm_wrapper_sol) # Then an update will be performed. scm_wrapper_sol.RunCommand('update', options, self.args, []) @@ -544,13 +548,13 @@ class GClientClassTestCase(GclientTestCase): # Next we expect an scm to be request for dep src/n even though it does not # exist in the DEPS file. - gclient.SCMWrapper('svn://custom.n/trunk', + gclient_scm.SCMWrapper('svn://custom.n/trunk', self.root_dir, "src/n").AndReturn(scm_wrapper_n) # Next we expect an scm to be request for dep src/t but it should # use the url specified in custom_deps. - gclient.SCMWrapper('svn://custom.t/trunk', + gclient_scm.SCMWrapper('svn://custom.t/trunk', self.root_dir, "src/t").AndReturn(scm_wrapper_t) @@ -620,7 +624,7 @@ class GClientClassTestCase(GclientTestCase): ).AndReturn(False) # An scm will be requested for the first solution. - gclient.SCMWrapper(url_a, self.root_dir, name_a).AndReturn( + gclient_scm.SCMWrapper(url_a, self.root_dir, name_a).AndReturn( scm_wrapper_a) # Then an attempt will be made to read it's DEPS file. gclient.FileRead(os.path.join(self.root_dir, name_a, options.deps_file) @@ -629,7 +633,7 @@ class GClientClassTestCase(GclientTestCase): scm_wrapper_a.RunCommand('update', options, self.args, []) # An scm will be requested for the second solution. - gclient.SCMWrapper(url_b, self.root_dir, name_b).AndReturn( + gclient_scm.SCMWrapper(url_b, self.root_dir, name_b).AndReturn( scm_wrapper_b) # Then an attempt will be made to read its DEPS file. gclient.FileRead(os.path.join(self.root_dir, name_b, options.deps_file) @@ -638,7 +642,7 @@ class GClientClassTestCase(GclientTestCase): scm_wrapper_b.RunCommand('update', options, self.args, []) # Finally, an scm is requested for the shared dep. - gclient.SCMWrapper('http://svn.t/trunk', self.root_dir, 'src/t' + gclient_scm.SCMWrapper('http://svn.t/trunk', self.root_dir, 'src/t' ).AndReturn(scm_wrapper_dep) # And an update is run on it. scm_wrapper_dep.RunCommand('update', options, self.args, []) @@ -666,9 +670,9 @@ class GClientClassTestCase(GclientTestCase): ).AndReturn(False) gclient.os.path.exists(os.path.join(self.root_dir, options.entries_filename) ).AndReturn(False) - gclient.SCMWrapper(self.url, self.root_dir, name).AndReturn( - gclient.SCMWrapper) - gclient.SCMWrapper.RunCommand('update', options, self.args, []) + gclient_scm.SCMWrapper(self.url, self.root_dir, name).AndReturn( + gclient_scm.SCMWrapper) + gclient_scm.SCMWrapper.RunCommand('update', options, self.args, []) gclient.FileRead(os.path.join(self.root_dir, name, options.deps_file) ).AndReturn("Boo = 'a'") gclient.FileWrite(os.path.join(self.root_dir, options.entries_filename), @@ -760,36 +764,36 @@ deps_os = { gclient.os.path.exists(os.path.join(self.root_dir, options.entries_filename) ).AndReturn(False) - gclient.SCMWrapper(self.url, self.root_dir, 'src').AndReturn( + gclient_scm.SCMWrapper(self.url, self.root_dir, 'src').AndReturn( scm_wrapper_src) scm_wrapper_src.RunCommand('update', mox.Func(OptIsRev123), self.args, []) - gclient.SCMWrapper(self.url, self.root_dir, + gclient_scm.SCMWrapper(self.url, self.root_dir, None).AndReturn(scm_wrapper_src2) scm_wrapper_src2.FullUrlForRelativeUrl('/trunk/deps/third_party/cygwin@3248' ).AndReturn(cygwin_path) - gclient.SCMWrapper(self.url, self.root_dir, + gclient_scm.SCMWrapper(self.url, self.root_dir, None).AndReturn(scm_wrapper_src2) scm_wrapper_src2.FullUrlForRelativeUrl('/trunk/deps/third_party/WebKit' ).AndReturn(webkit_path) - gclient.SCMWrapper(webkit_path, self.root_dir, + gclient_scm.SCMWrapper(webkit_path, self.root_dir, 'foo/third_party/WebKit').AndReturn(scm_wrapper_webkit) scm_wrapper_webkit.RunCommand('update', mox.Func(OptIsRev42), self.args, []) - gclient.SCMWrapper( + gclient_scm.SCMWrapper( 'http://google-breakpad.googlecode.com/svn/trunk/src@285', self.root_dir, 'src/breakpad/bar').AndReturn(scm_wrapper_breakpad) scm_wrapper_breakpad.RunCommand('update', mox.Func(OptIsRevNone), self.args, []) - gclient.SCMWrapper(cygwin_path, self.root_dir, + gclient_scm.SCMWrapper(cygwin_path, self.root_dir, 'src/third_party/cygwin').AndReturn(scm_wrapper_cygwin) scm_wrapper_cygwin.RunCommand('update', mox.Func(OptIsRev333), self.args, []) - gclient.SCMWrapper('svn://random_server:123/trunk/python_24@5580', + gclient_scm.SCMWrapper('svn://random_server:123/trunk/python_24@5580', self.root_dir, 'src/third_party/python_24').AndReturn( scm_wrapper_python) @@ -865,18 +869,18 @@ deps = { ).AndReturn(False) gclient.os.path.exists(os.path.join(self.root_dir, options.entries_filename) ).AndReturn(False) - gclient.SCMWrapper(self.url, self.root_dir, name).AndReturn( - gclient.SCMWrapper) - gclient.SCMWrapper.RunCommand('update', options, self.args, []) + gclient_scm.SCMWrapper(self.url, self.root_dir, name).AndReturn( + gclient_scm.SCMWrapper) + gclient_scm.SCMWrapper.RunCommand('update', options, self.args, []) - gclient.SCMWrapper(self.url, self.root_dir, + gclient_scm.SCMWrapper(self.url, self.root_dir, None).AndReturn(scm_wrapper_src) scm_wrapper_src.FullUrlForRelativeUrl('/trunk/bar/WebKit' ).AndReturn(webkit_path) - gclient.SCMWrapper(webkit_path, self.root_dir, - 'foo/third_party/WebKit').AndReturn(gclient.SCMWrapper) - gclient.SCMWrapper.RunCommand('update', options, self.args, []) + gclient_scm.SCMWrapper(webkit_path, self.root_dir, + 'foo/third_party/WebKit').AndReturn(gclient_scm.SCMWrapper) + gclient_scm.SCMWrapper.RunCommand('update', options, self.args, []) self.mox.ReplayAll() client = self._gclient_gclient(self.root_dir, options) @@ -921,18 +925,18 @@ deps = { ).AndReturn(False) gclient.os.path.exists(os.path.join(self.root_dir, options.entries_filename) ).AndReturn(False) - gclient.SCMWrapper(self.url, self.root_dir, name).AndReturn( - gclient.SCMWrapper) - gclient.SCMWrapper.RunCommand('update', options, self.args, []) + gclient_scm.SCMWrapper(self.url, self.root_dir, name).AndReturn( + gclient_scm.SCMWrapper) + gclient_scm.SCMWrapper.RunCommand('update', options, self.args, []) - gclient.SCMWrapper(self.url, self.root_dir, + gclient_scm.SCMWrapper(self.url, self.root_dir, None).AndReturn(scm_wrapper_src) scm_wrapper_src.FullUrlForRelativeUrl('/trunk/bar_custom/WebKit' ).AndReturn(webkit_path) - gclient.SCMWrapper(webkit_path, self.root_dir, - 'foo/third_party/WebKit').AndReturn(gclient.SCMWrapper) - gclient.SCMWrapper.RunCommand('update', options, self.args, []) + gclient_scm.SCMWrapper(webkit_path, self.root_dir, + 'foo/third_party/WebKit').AndReturn(gclient_scm.SCMWrapper) + gclient_scm.SCMWrapper.RunCommand('update', options, self.args, []) self.mox.ReplayAll() client = self._gclient_gclient(self.root_dir, options) @@ -956,9 +960,9 @@ deps = { options = self.Options() gclient.FileRead(os.path.join(self.root_dir, name, options.deps_file) ).AndReturn(deps_content) - gclient.SCMWrapper(self.url, self.root_dir, name).AndReturn( - gclient.SCMWrapper) - gclient.SCMWrapper.RunCommand('update', options, self.args, []) + gclient_scm.SCMWrapper(self.url, self.root_dir, name).AndReturn( + gclient_scm.SCMWrapper) + gclient_scm.SCMWrapper.RunCommand('update', options, self.args, []) self.mox.ReplayAll() client = self._gclient_gclient(self.root_dir, options) @@ -1073,7 +1077,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): # Checkout. gclient.os.path.exists(base_path).AndReturn(False) files_list = self.mox.CreateMockAnything() - gclient.RunSVNAndGetFileList(['checkout', self.url, base_path], + gclient_scm.RunSVNAndGetFileList(['checkout', self.url, base_path], self.root_dir, files_list) self.mox.ReplayAll() @@ -1085,7 +1089,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): options = self.Options(verbose=True) base_path = os.path.join(self.root_dir, self.relpath) gclient.os.path.isdir(base_path).AndReturn(True) - gclient.CaptureSVNStatus(base_path).AndReturn([]) + gclient_scm.CaptureSVNStatus(base_path).AndReturn([]) self.mox.ReplayAll() scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, @@ -1101,11 +1105,11 @@ class SCMWrapperTestCase(GClientBaseTestCase): ('M ', 'a'), ('A ', 'b'), ] - gclient.CaptureSVNStatus(base_path).AndReturn(items) + gclient_scm.CaptureSVNStatus(base_path).AndReturn(items) print(os.path.join(base_path, 'a')) print(os.path.join(base_path, 'b')) - gclient.RunSVN(['revert', 'a', 'b'], base_path) + gclient_scm.RunSVN(['revert', 'a', 'b'], base_path) self.mox.ReplayAll() scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, @@ -1117,7 +1121,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): options = self.Options(verbose=True) base_path = os.path.join(self.root_dir, self.relpath) gclient.os.path.isdir(base_path).AndReturn(True) - gclient.RunSVNAndGetFileList(['status'] + self.args, base_path, + gclient_scm.RunSVNAndGetFileList(['status'] + self.args, base_path, []).AndReturn(None) self.mox.ReplayAll() @@ -1132,7 +1136,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): def testUpdateCheckout(self): options = self.Options(verbose=True) base_path = os.path.join(self.root_dir, self.relpath) - file_info = gclient.PrintableObject() + file_info = gclient_utils.PrintableObject() file_info.root = 'blah' file_info.url = self.url file_info.uuid = 'ABC' @@ -1141,7 +1145,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): # Checkout. gclient.os.path.exists(base_path).AndReturn(False) files_list = self.mox.CreateMockAnything() - gclient.RunSVNAndGetFileList(['checkout', self.url, base_path], + gclient_scm.RunSVNAndGetFileList(['checkout', self.url, base_path], self.root_dir, files_list) self.mox.ReplayAll() scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, @@ -1162,15 +1166,15 @@ class SCMWrapperTestCase(GClientBaseTestCase): gclient.os.path.exists(os.path.join(base_path, '.git')).AndReturn(False) # Checkout or update. gclient.os.path.exists(base_path).AndReturn(True) - gclient.CaptureSVNInfo(os.path.join(base_path, "."), '.' + gclient_scm.CaptureSVNInfo(os.path.join(base_path, "."), '.' ).AndReturn(file_info) # Cheat a bit here. - gclient.CaptureSVNInfo(file_info['URL'], '.').AndReturn(file_info) + gclient_scm.CaptureSVNInfo(file_info['URL'], '.').AndReturn(file_info) additional_args = [] if options.manually_grab_svn_rev: additional_args = ['--revision', str(file_info['Revision'])] files_list = [] - gclient.RunSVNAndGetFileList(['update', base_path] + additional_args, + gclient_scm.RunSVNAndGetFileList(['update', base_path] + additional_args, self.root_dir, files_list) self.mox.ReplayAll() @@ -1206,7 +1210,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): """ % self.url - gclient.CaptureSVN(['info', '--xml', self.url], + gclient_scm.CaptureSVN(['info', '--xml', self.url], '.', True).AndReturn(xml_text) expected = { 'URL': 'http://src.chromium.org/svn/trunk/src/chrome/app/d', @@ -1247,7 +1251,7 @@ class SCMWrapperTestCase(GClientBaseTestCase): """ % (self.url, self.root_dir) - gclient.CaptureSVN(['info', '--xml', self.url], + gclient_scm.CaptureSVN(['info', '--xml', self.url], '.', True).AndReturn(xml_text) self.mox.ReplayAll() file_info = self._CaptureSVNInfo(self.url, '.', True) @@ -1268,15 +1272,15 @@ class SCMWrapperTestCase(GClientBaseTestCase): class RunSVNTestCase(BaseTestCase): def testRunSVN(self): param2 = 'bleh' - gclient.SubprocessCall(['svn', 'foo', 'bar'], param2).AndReturn(None) + gclient_utils.SubprocessCall(['svn', 'foo', 'bar'], param2).AndReturn(None) self.mox.ReplayAll() - gclient.RunSVN(['foo', 'bar'], param2) + gclient_scm.RunSVN(['foo', 'bar'], param2) class SubprocessCallAndFilterTestCase(BaseTestCase): def setUp(self): BaseTestCase.setUp(self) - self.mox.StubOutWithMock(gclient, 'CaptureSVN') + self.mox.StubOutWithMock(gclient_scm, 'CaptureSVN') def testSubprocessCallAndFilter(self): command = ['boo', 'foo', 'bar'] @@ -1292,9 +1296,9 @@ class SubprocessCallAndFilterTestCase(BaseTestCase): print("\n________ running 'boo foo bar' in 'bleh'") for i in test_string: gclient.sys.stdout.write(i) - gclient.subprocess.Popen(command, bufsize=0, cwd=in_directory, - shell=(gclient.sys.platform == 'win32'), - stdout=gclient.subprocess.PIPE).AndReturn(kid) + gclient_utils.subprocess.Popen(command, bufsize=0, cwd=in_directory, + shell=(gclient.sys.platform == 'win32'), + stdout=gclient_utils.subprocess.PIPE).AndReturn(kid) self.mox.ReplayAll() compiled_pattern = re.compile(pattern) line_list = [] @@ -1304,9 +1308,9 @@ class SubprocessCallAndFilterTestCase(BaseTestCase): match = compiled_pattern.search(line) if match: capture_list.append(match.group(1)) - gclient.SubprocessCallAndFilter(command, in_directory, - True, True, - fail_status, FilterLines) + gclient_utils.SubprocessCallAndFilter(command, in_directory, + True, True, + fail_status, FilterLines) self.assertEquals(line_list, ['ahah', 'accb', 'allo', 'addb']) self.assertEquals(capture_list, ['cc', 'dd']) @@ -1353,8 +1357,8 @@ class SubprocessCallAndFilterTestCase(BaseTestCase): """ - gclient.CaptureSVN = CaptureSVNMock - info = gclient.CaptureSVNStatus('.') + gclient_scm.CaptureSVN = CaptureSVNMock + info = gclient_scm.CaptureSVNStatus('.') expected = [ ('? ', 'unversionned_file.txt'), ('M ', 'build\\internal\\essential.vsprops'), @@ -1376,8 +1380,8 @@ class SubprocessCallAndFilterTestCase(BaseTestCase): """ - gclient.CaptureSVN = CaptureSVNMock - info = gclient.CaptureSVNStatus(None) + gclient_scm.CaptureSVN = CaptureSVNMock + info = gclient_scm.CaptureSVNStatus(None) self.assertEquals(info, []) diff --git a/tests/presubmit_unittest.py b/tests/presubmit_unittest.py index e30c41104..568d343c3 100755 --- a/tests/presubmit_unittest.py +++ b/tests/presubmit_unittest.py @@ -59,7 +59,7 @@ def CheckChangeOnUpload(input_api, output_api): presubmit.os.path.abspath = MockAbsPath presubmit.os.path.commonprefix = os_path_commonprefix self.fake_root_dir = self.RootDir() - self.mox.StubOutWithMock(presubmit.gclient, 'CaptureSVNInfo') + self.mox.StubOutWithMock(presubmit.gclient_scm, 'CaptureSVNInfo') self.mox.StubOutWithMock(presubmit.gcl, 'GetSVNFileProperty') self.mox.StubOutWithMock(presubmit.gcl, 'ReadFile') @@ -75,7 +75,7 @@ class PresubmitUnittest(PresubmitTestsBase): 'OutputApi', 'ParseFiles', 'PresubmitExecuter', 'ScanSubDirs', 'SvnAffectedFile', 'SvnChange', 'cPickle', 'cStringIO', 'exceptions', - 'fnmatch', 'gcl', 'gclient', 'glob', 'logging', 'marshal', 'normpath', + 'fnmatch', 'gcl', 'gclient_scm', 'glob', 'logging', 'marshal', 'normpath', 'optparse', 'os', 'pickle', 'presubmit_canned_checks', 'random', 're', 'subprocess', 'sys', 'time', 'tempfile', 'traceback', 'types', 'unittest', 'urllib2', 'warnings', @@ -151,19 +151,19 @@ class PresubmitUnittest(PresubmitTestsBase): presubmit.os.path.exists(notfound).AndReturn(True) presubmit.os.path.isdir(notfound).AndReturn(False) presubmit.os.path.exists(flap).AndReturn(False) - presubmit.gclient.CaptureSVNInfo(flap + presubmit.gclient_scm.CaptureSVNInfo(flap ).AndReturn({'Node Kind': 'file'}) presubmit.gcl.GetSVNFileProperty(blat, 'svn:mime-type').AndReturn(None) presubmit.gcl.GetSVNFileProperty( binary, 'svn:mime-type').AndReturn('application/octet-stream') presubmit.gcl.GetSVNFileProperty( notfound, 'svn:mime-type').AndReturn('') - presubmit.gclient.CaptureSVNInfo(blat).AndReturn( + presubmit.gclient_scm.CaptureSVNInfo(blat).AndReturn( {'URL': 'svn:/foo/foo/blat.cc'}) - presubmit.gclient.CaptureSVNInfo(binary).AndReturn( + presubmit.gclient_scm.CaptureSVNInfo(binary).AndReturn( {'URL': 'svn:/foo/binary.dll'}) - presubmit.gclient.CaptureSVNInfo(notfound).AndReturn({}) - presubmit.gclient.CaptureSVNInfo(flap).AndReturn( + presubmit.gclient_scm.CaptureSVNInfo(notfound).AndReturn({}) + presubmit.gclient_scm.CaptureSVNInfo(flap).AndReturn( {'URL': 'svn:/foo/boo/flap.h'}) presubmit.gcl.ReadFile(blat).AndReturn('boo!\nahh?') presubmit.gcl.ReadFile(notfound).AndReturn('look!\nthere?') @@ -520,9 +520,9 @@ class InputApiUnittest(PresubmitTestsBase): self.compareMembers(presubmit.InputApi(None, './.', False), members) def testDepotToLocalPath(self): - presubmit.gclient.CaptureSVNInfo('svn://foo/smurf').AndReturn( + presubmit.gclient_scm.CaptureSVNInfo('svn://foo/smurf').AndReturn( {'Path': 'prout'}) - presubmit.gclient.CaptureSVNInfo('svn:/foo/notfound/burp').AndReturn({}) + presubmit.gclient_scm.CaptureSVNInfo('svn:/foo/notfound/burp').AndReturn({}) self.mox.ReplayAll() path = presubmit.InputApi(None, './p', False).DepotToLocalPath( @@ -533,8 +533,9 @@ class InputApiUnittest(PresubmitTestsBase): self.failUnless(path == None) def testLocalToDepotPath(self): - presubmit.gclient.CaptureSVNInfo('smurf').AndReturn({'URL': 'svn://foo'}) - presubmit.gclient.CaptureSVNInfo('notfound-food').AndReturn({}) + presubmit.gclient_scm.CaptureSVNInfo('smurf').AndReturn({'URL': + 'svn://foo'}) + presubmit.gclient_scm.CaptureSVNInfo('notfound-food').AndReturn({}) self.mox.ReplayAll() path = presubmit.InputApi(None, './p', False).LocalToDepotPath('smurf') @@ -585,8 +586,8 @@ class InputApiUnittest(PresubmitTestsBase): presubmit.os.path.exists(notfound).AndReturn(False) presubmit.os.path.exists(flap).AndReturn(True) presubmit.os.path.isdir(flap).AndReturn(False) - presubmit.gclient.CaptureSVNInfo(beingdeleted).AndReturn({}) - presubmit.gclient.CaptureSVNInfo(notfound).AndReturn({}) + presubmit.gclient_scm.CaptureSVNInfo(beingdeleted).AndReturn({}) + presubmit.gclient_scm.CaptureSVNInfo(notfound).AndReturn({}) presubmit.gcl.GetSVNFileProperty(blat, 'svn:mime-type').AndReturn(None) presubmit.gcl.GetSVNFileProperty(readme, 'svn:mime-type').AndReturn(None) presubmit.gcl.GetSVNFileProperty(binary, 'svn:mime-type').AndReturn( @@ -910,7 +911,7 @@ class AffectedFileUnittest(PresubmitTestsBase): presubmit.os.path.exists(path).AndReturn(True) presubmit.os.path.isdir(path).AndReturn(False) presubmit.gcl.ReadFile(path).AndReturn('whatever\ncookie') - presubmit.gclient.CaptureSVNInfo(path).AndReturn( + presubmit.gclient_scm.CaptureSVNInfo(path).AndReturn( {'URL': 'svn:/foo/foo/blat.cc'}) self.mox.ReplayAll() af = presubmit.SvnAffectedFile('foo/blat.cc', 'M') @@ -934,7 +935,7 @@ class AffectedFileUnittest(PresubmitTestsBase): def testIsDirectoryNotExists(self): presubmit.os.path.exists('foo.cc').AndReturn(False) - presubmit.gclient.CaptureSVNInfo('foo.cc').AndReturn({}) + presubmit.gclient_scm.CaptureSVNInfo('foo.cc').AndReturn({}) self.mox.ReplayAll() affected_file = presubmit.SvnAffectedFile('foo.cc', 'A') # Verify cache coherency. diff --git a/tests/revert_unittest.py b/tests/revert_unittest.py index eee176d2e..477eb2ada 100644 --- a/tests/revert_unittest.py +++ b/tests/revert_unittest.py @@ -20,6 +20,7 @@ class RevertTestsBase(super_mox.SuperMoxTestBase): super_mox.SuperMoxTestBase.setUp(self) self.mox.StubOutWithMock(revert, 'gcl') self.mox.StubOutWithMock(revert, 'gclient') + self.mox.StubOutWithMock(revert, 'gclient_scm') self.mox.StubOutWithMock(revert, 'os') self.mox.StubOutWithMock(revert.os, 'path') self.mox.StubOutWithMock(revert.sys, 'stdout') @@ -35,7 +36,8 @@ class RevertUnittest(RevertTestsBase): members = [ 'CaptureSVNLog', 'GetRepoBase', 'Main', 'ModifiedFile', 'NoBlameList', 'NoModifiedFile', 'OutsideOfCheckout', 'Revert', 'UniqueFast', - 'exceptions', 'gcl', 'gclient', 'optparse', 'os', 'sys', 'xml' + 'exceptions', 'gcl', 'gclient', 'gclient_scm', 'gclient_utils', + 'optparse', 'os', 'sys', 'xml' ] # If this test fails, you should add the relevant test. self.compareMembers(revert, members) @@ -45,7 +47,6 @@ class RevertMainUnittest(RevertTestsBase): def setUp(self): RevertTestsBase.setUp(self) self.mox.StubOutWithMock(revert, 'gcl') - self.mox.StubOutWithMock(revert, 'gclient') self.mox.StubOutWithMock(revert, 'os') self.mox.StubOutWithMock(revert.os, 'path') self.mox.StubOutWithMock(revert, 'sys') @@ -78,7 +79,7 @@ class RevertRevertUnittest(RevertTestsBase): }] revert.CaptureSVNLog(['-r', '42', '-v']).AndReturn(entries) revert.GetRepoBase().AndReturn('proto://fqdn/repo/') - revert.gclient.CaptureSVNStatus(['random_file']).AndReturn([]) + revert.gclient_scm.CaptureSVNStatus(['random_file']).AndReturn([]) revert.gcl.RunShell(['svn', 'up', 'random_file']) revert.os.path.isdir('random_file').AndReturn(False) status = """--- Reverse-merging r42 into '.':