From 92bec4f56f991d35ddcf9c4d8a0bdd8ce1b10735 Mon Sep 17 00:00:00 2001 From: agable Date: Wed, 24 Aug 2016 09:27:27 -0700 Subject: [PATCH] Delete gcl, drover, and trychange These tools are relatively standalone, and only ever worked for SVN. Removing these is a good start to removing other SVN support code. R=maruel@chromium.org BUG=475321 Review-Url: https://codereview.chromium.org/2269413002 --- codereview.settings | 2 +- drover | 25 - drover.bat | 9 - drover.py | 648 --------------- gcl | 8 - gcl.bat | 6 - gcl.py | 1523 ----------------------------------- gclient_utils.py | 12 +- git_cl.py | 2 +- presubmit_support.py | 30 +- tests/trychange_unittest.py | 161 ---- trychange.py | 1265 ----------------------------- 12 files changed, 7 insertions(+), 3684 deletions(-) delete mode 100755 drover delete mode 100644 drover.bat delete mode 100755 drover.py delete mode 100755 gcl delete mode 100755 gcl.bat delete mode 100755 gcl.py delete mode 100755 tests/trychange_unittest.py delete mode 100755 trychange.py diff --git a/codereview.settings b/codereview.settings index f84203bd8..9f2ff618d 100644 --- a/codereview.settings +++ b/codereview.settings @@ -1,4 +1,4 @@ -# This file is used by gcl to get repository specific information. +# This file is used by git cl to get repository specific information. CODE_REVIEW_SERVER: codereview.chromium.org CC_LIST: chromium-reviews@chromium.org VIEW_VC: https://chromium.googlesource.com/chromium/tools/depot_tools/+/ diff --git a/drover b/drover deleted file mode 100755 index 12dc53e4e..000000000 --- a/drover +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2009 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This script will try to sync the bootstrap directories and then defer control. - -base_dir=$(dirname "$0") - -# Use the batch file as an entry point if on cygwin. -if [ "${OSTYPE}" = "cygwin" -a "${TERM}" != "xterm" ]; then - ${base_dir}/drover.bat "$@" - exit -fi - - -# We're on POSIX (not cygwin). We can now safely look for svn checkout. -if [ "X$DEPOT_TOOLS_UPDATE" != "X0" -a -e "${base_dir}/.svn" ] -then - # Update the bootstrap directory to stay up-to-date with the latest - # depot_tools. - svn -q up "${base_dir}" -fi - -PYTHONDONTWRITEBYTECODE=1 exec python "${base_dir}/drover.py" "$@" diff --git a/drover.bat b/drover.bat deleted file mode 100644 index 7aa62468e..000000000 --- a/drover.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off -:: Copyright (c) 2009 The Chromium Authors. All rights reserved. -:: Use of this source code is governed by a BSD-style license that can be -:: found in the LICENSE file. - -setlocal -set PATH=%~dp0svn;%PATH% -set PYTHONDONTWRITEBYTECODE=1 -call python "%~dp0drover.py" %* diff --git a/drover.py b/drover.py deleted file mode 100755 index a73f1d347..000000000 --- a/drover.py +++ /dev/null @@ -1,648 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import datetime -import optparse -import os -import re -import sys -import urlparse - - -import gclient_utils -import subprocess2 - -USAGE = """ -WARNING: Please use this tool in an empty directory -(or at least one that you don't mind clobbering.) - -REQUIRES: SVN 1.5+ -NOTE: NO NEED TO CHECKOUT ANYTHING IN ADVANCE OF USING THIS TOOL. -Valid parameters: - -[Merge from trunk to branch] ---merge --branch -Example: %(app)s --merge 12345 --branch 187 - -[Merge from trunk to local copy] ---merge --local -Example: %(app)s --merge 12345 --local - -[Merge from branch to branch] ---merge --sbranch --branch -Example: %(app)s --merge 12345 --sbranch 248 --branch 249 - -[Revert from trunk] ---revert -Example: %(app)s --revert 12345 - -[Revert from branch] ---revert --branch -Example: %(app)s --revert 12345 --branch 187 -""" - -export_map_ = None -files_info_ = None -delete_map_ = None -file_pattern_ = r"[ ]+([MADUC])[ ]+/((?:trunk|branches/.*?)/src(.*)/(.*))" -depot_tools_dir_ = os.path.dirname(os.path.abspath(__file__)) - - -def runGcl(subcommand): - gcl_path = os.path.join(depot_tools_dir_, "gcl") - if not os.path.exists(gcl_path): - print "WARNING: gcl not found beside drover.py. Using system gcl instead..." - gcl_path = 'gcl' - - command = "%s %s" % (gcl_path, subcommand) - return os.system(command) - -def gclUpload(revision, author): - command = ("upload " + str(revision) + - " --send_mail --no_presubmit --reviewers=" + author) - return runGcl(command) - -def getSVNInfo(url, revision): - info = {} - svn_info = subprocess2.capture( - ['svn', 'info', '--non-interactive', '%s@%s' % (url, revision)], - stderr=subprocess2.VOID).splitlines() - for line in svn_info: - match = re.search(r"(.*?):(.*)", line) - if match: - info[match.group(1).strip()] = match.group(2).strip() - return info - -def isSVNDirty(): - svn_status = subprocess2.check_output(['svn', 'status']).splitlines() - for line in svn_status: - match = re.search(r"^[^X?]", line) - if match: - return True - - return False - -def getAuthor(url, revision): - info = getSVNInfo(url, revision) - if (info.has_key("Last Changed Author")): - return info["Last Changed Author"] - return None - -def isSVNFile(url, revision): - info = getSVNInfo(url, revision) - if (info.has_key("Node Kind")): - if (info["Node Kind"] == "file"): - return True - return False - -def isSVNDirectory(url, revision): - info = getSVNInfo(url, revision) - if (info.has_key("Node Kind")): - if (info["Node Kind"] == "directory"): - return True - return False - -def inCheckoutRoot(path): - info = getSVNInfo(path, "HEAD") - if (not info.has_key("Repository Root")): - return False - repo_root = info["Repository Root"] - info = getSVNInfo(os.path.dirname(os.path.abspath(path)), "HEAD") - if (info.get("Repository Root", None) != repo_root): - return True - return False - -def getRevisionLog(url, revision): - """Takes an svn url and gets the associated revision.""" - svn_log = subprocess2.check_output( - ['svn', 'log', url, '-r', str(revision)], - universal_newlines=True).splitlines(True) - # Don't include the header lines and the trailing "---..." line. - return ''.join(svn_log[3:-1]) - -def getSVNVersionInfo(): - """Extract version information from SVN""" - svn_info = subprocess2.check_output(['svn', '--version']).splitlines() - info = {} - for line in svn_info: - match = re.search(r"svn, version ((\d+)\.(\d+)\.(\d+))", line) - if match: - info['version'] = match.group(1) - info['major'] = int(match.group(2)) - info['minor'] = int(match.group(3)) - info['patch'] = int(match.group(4)) - return info - - return None - -def isMinimumSVNVersion(major, minor, patch=0): - """Test for minimum SVN version""" - return _isMinimumSVNVersion(getSVNVersionInfo(), major, minor, patch) - -def _isMinimumSVNVersion(version, major, minor, patch=0): - """Test for minimum SVN version, internal method""" - if not version: - return False - - if (version['major'] > major): - return True - elif (version['major'] < major): - return False - - if (version['minor'] > minor): - return True - elif (version['minor'] < minor): - return False - - if (version['patch'] >= patch): - return True - else: - return False - -def checkoutRevision(url, revision, branch_url, revert=False, pop=True): - files_info = getFileInfo(url, revision) - paths = getBestMergePaths2(files_info, revision) - export_map = getBestExportPathsMap2(files_info, revision) - - command = 'svn checkout -N ' + branch_url - print command - os.system(command) - - match = re.search(r"^[a-z]+://.*/(.*)", branch_url) - - if match: - os.chdir(match.group(1)) - - # This line is extremely important due to the way svn behaves in the - # set-depths action. If parents aren't handled before children, the child - # directories get clobbered and the merge step fails. - paths.sort() - - # Checkout the directories that already exist - for path in paths: - if (export_map.has_key(path) and not revert): - print "Exclude new directory " + path - continue - subpaths = path.split('/') - #In the normal case, where no url override is specified and it's just - # chromium source, it's necessary to remove the 'trunk' from the filepath, - # since in the checkout we include 'trunk' or 'branch/\d+'. - # - # However, when a url is specified we want to preserve that because it's - # a part of the filepath and necessary for path operations on svn (because - # frankly, we are checking out the correct top level, and not hacking it). - if pop: - subpaths.pop(0) - base = '' - for subpath in subpaths: - base += '/' + subpath - # This logic ensures that you don't empty out any directories - if not os.path.exists("." + base): - command = ('svn update --depth empty ' + "." + base) - print command - os.system(command) - - if (revert): - files = getAllFilesInRevision(files_info) - else: - files = getExistingFilesInRevision(files_info) - - for f in files: - # Prevent the tool from clobbering the src directory - if (f == ""): - continue - command = ('svn up ".' + f + '"') - print command - os.system(command) - -def mergeRevision(url, revision): - paths = getBestMergePaths(url, revision) - export_map = getBestExportPathsMap(url, revision) - - for path in paths: - if export_map.has_key(path): - continue - command = ('svn merge -N -r ' + str(revision-1) + ":" + str(revision) + " ") - command += " --ignore-ancestry " - command += " -x --ignore-eol-style " - command += url + path + "@" + str(revision) + " ." + path - - print command - os.system(command) - -def exportRevision(url, revision): - paths = getBestExportPathsMap(url, revision).keys() - paths.sort() - - for path in paths: - command = ('svn export -N ' + url + path + "@" + str(revision) + " ." + - path) - print command - os.system(command) - - command = 'svn add .' + path - print command - os.system(command) - -def deleteRevision(url, revision): - paths = getBestDeletePathsMap(url, revision).keys() - paths.sort() - paths.reverse() - - for path in paths: - command = "svn delete ." + path - print command - os.system(command) - -def revertExportRevision(url, revision): - paths = getBestExportPathsMap(url, revision).keys() - paths.sort() - paths.reverse() - - for path in paths: - command = "svn delete ." + path - print command - os.system(command) - -def revertRevision(url, revision): - command = ('svn merge --ignore-ancestry -c -%d %s .' % (revision, url)) - print command - os.system(command) - -def getFileInfo(url, revision): - global files_info_ - - if (files_info_ != None): - return files_info_ - - svn_log = subprocess2.check_output( - ['svn', 'log', url, '-r', str(revision), '-v']).splitlines() - - info = [] - for line in svn_log: - # A workaround to dump the (from .*) stuff, regex not so friendly in the 2nd - # pass... - match = re.search(r"(.*) \(from.*\)", line) - if match: - line = match.group(1) - match = re.search(file_pattern_, line) - if match: - info.append([match.group(1).strip(), match.group(2).strip(), - match.group(3).strip(),match.group(4).strip()]) - - files_info_ = info - return info - -def getBestMergePaths(url, revision): - """Takes an svn url and gets the associated revision.""" - return getBestMergePaths2(getFileInfo(url, revision), revision) - -def getBestMergePaths2(files_info, revision): - """Takes an svn url and gets the associated revision.""" - return list(set([f[2] for f in files_info])) - -def getBestExportPathsMap(url, revision): - return getBestExportPathsMap2(getFileInfo(url, revision), revision) - -def getBestExportPathsMap2(files_info, revision): - """Takes an svn url and gets the associated revision.""" - global export_map_ - - if export_map_: - return export_map_ - - result = {} - for file_info in files_info: - if (file_info[0] == "A"): - if(isSVNDirectory("svn://svn.chromium.org/chrome/" + file_info[1], - revision)): - result[file_info[2] + "/" + file_info[3]] = "" - - export_map_ = result - return result - -def getBestDeletePathsMap(url, revision): - return getBestDeletePathsMap2(getFileInfo(url, revision), revision) - -def getBestDeletePathsMap2(files_info, revision): - """Takes an svn url and gets the associated revision.""" - global delete_map_ - - if delete_map_: - return delete_map_ - - result = {} - for file_info in files_info: - if (file_info[0] == "D"): - if(isSVNDirectory("svn://svn.chromium.org/chrome/" + file_info[1], - revision)): - result[file_info[2] + "/" + file_info[3]] = "" - - delete_map_ = result - return result - - -def getExistingFilesInRevision(files_info): - """Checks for existing files in the revision. - - Anything that's A will require special treatment (either a merge or an - export + add) - """ - return ['%s/%s' % (f[2], f[3]) for f in files_info if f[0] != 'A'] - - -def getAllFilesInRevision(files_info): - """Checks for existing files in the revision. - - Anything that's A will require special treatment (either a merge or an - export + add) - """ - return ['%s/%s' % (f[2], f[3]) for f in files_info] - - -def getSVNAuthInfo(folder=None): - """Fetches SVN authorization information in the subversion auth folder and - returns it as a dictionary of dictionaries.""" - if not folder: - if sys.platform == 'win32': - folder = '%%APPDATA%\\Subversion\\auth' - else: - folder = '~/.subversion/auth' - folder = os.path.expandvars(os.path.expanduser(folder)) - svn_simple_folder = os.path.join(folder, 'svn.simple') - results = {} - try: - for auth_file in os.listdir(svn_simple_folder): - # Read the SVN auth file, convert it into a dictionary, and store it. - results[auth_file] = dict(re.findall(r'K [0-9]+\n(.*)\nV [0-9]+\n(.*)\n', - open(os.path.join(svn_simple_folder, auth_file)).read())) - except Exception as _: - pass - return results - - -def getCurrentSVNUsers(url): - """Tries to fetch the current SVN in the current checkout by scanning the - SVN authorization folder for a match with the current SVN URL.""" - netloc = urlparse.urlparse(url)[1] - auth_infos = getSVNAuthInfo() - results = [] - for _, auth_info in auth_infos.iteritems(): - if ('svn:realmstring' in auth_info - and netloc in auth_info['svn:realmstring']): - username = auth_info['username'] - results.append(username) - if 'google.com' in username: - results.append(username.replace('google.com', 'chromium.org')) - return results - - -def prompt(question): - while True: - print question + " [y|n]:", - answer = sys.stdin.readline() - if answer.lower().startswith('n'): - return False - elif answer.lower().startswith('y'): - return True - - -def text_prompt(question, default): - print question + " [" + default + "]:" - answer = sys.stdin.readline() - if answer.strip() == "": - return default - return answer - - -def drover(options, args): - revision = options.revert or options.merge - - # Initialize some variables used below. They can be overwritten by - # the drover.properties file. - BASE_URL = "svn://svn.chromium.org/chrome" - REVERT_ALT_URLS = ['svn://svn.chromium.org/blink', - 'svn://svn.chromium.org/chrome-internal', - 'svn://svn.chromium.org/native_client'] - TRUNK_URL = BASE_URL + "/trunk/src" - BRANCH_URL = BASE_URL + "/branches/$branch/src" - SKIP_CHECK_WORKING = True - PROMPT_FOR_AUTHOR = False - NO_ALT_URLS = options.no_alt_urls - - DEFAULT_WORKING = "drover_" + str(revision) - if options.branch: - DEFAULT_WORKING += ("_" + options.branch) - - if not isMinimumSVNVersion(1, 5): - print "You need to use at least SVN version 1.5.x" - return 1 - - # Override the default properties if there is a drover.properties file. - global file_pattern_ - if os.path.exists("drover.properties"): - print 'Using options from %s' % os.path.join( - os.getcwd(), 'drover.properties') - FILE_PATTERN = file_pattern_ - f = open("drover.properties") - exec(f) - f.close() - if FILE_PATTERN: - file_pattern_ = FILE_PATTERN - NO_ALT_URLS = True - - if options.revert and options.branch: - print 'Note: --branch is usually not needed for reverts.' - url = BRANCH_URL.replace("$branch", options.branch) - elif options.merge and options.sbranch: - url = BRANCH_URL.replace("$branch", options.sbranch) - elif options.revert: - url = options.url or BASE_URL - file_pattern_ = r"[ ]+([MADUC])[ ]+((/.*)/(.*))" - else: - url = TRUNK_URL - - working = options.workdir or DEFAULT_WORKING - - if options.local: - working = os.getcwd() - if not inCheckoutRoot(working): - print "'%s' appears not to be the root of a working copy" % working - return 1 - if (isSVNDirty() and not - prompt("Working copy contains uncommitted files. Continue?")): - return 1 - - if options.revert and not NO_ALT_URLS and not options.url: - for cur_url in [url] + REVERT_ALT_URLS: - try: - commit_date_str = getSVNInfo( - cur_url, options.revert).get('Last Changed Date', 'x').split()[0] - commit_date = datetime.datetime.strptime(commit_date_str, '%Y-%m-%d') - if (datetime.datetime.now() - commit_date).days < 180: - if cur_url != url: - print 'Guessing svn repo: %s.' % cur_url, - print 'Use --no-alt-urls to disable heuristic.' - url = cur_url - break - except ValueError: - pass - command = 'svn log ' + url + " -r "+str(revision) + " -v" - os.system(command) - - if not (options.revertbot or prompt("Is this the correct revision?")): - return 0 - - if (os.path.exists(working)) and not options.local: - if not (options.revertbot or SKIP_CHECK_WORKING or - prompt("Working directory: '%s' already exists, clobber?" % working)): - return 0 - gclient_utils.rmtree(working) - - if not options.local: - os.makedirs(working) - os.chdir(working) - - if options.merge: - action = "Merge" - if not options.local: - branch_url = BRANCH_URL.replace("$branch", options.branch) - # Checkout everything but stuff that got added into a new dir - checkoutRevision(url, revision, branch_url) - # Merge everything that changed - mergeRevision(url, revision) - # "Export" files that were added from the source and add them to branch - exportRevision(url, revision) - # Delete directories that were deleted (file deletes are handled in the - # merge). - deleteRevision(url, revision) - elif options.revert: - action = "Revert" - pop_em = not options.url - checkoutRevision(url, revision, url, True, pop_em) - revertRevision(url, revision) - revertExportRevision(url, revision) - - # Check the base url so we actually find the author who made the change - if options.auditor: - author = options.auditor - else: - author = getAuthor(url, revision) - if not author: - author = getAuthor(TRUNK_URL, revision) - - # Check that the author of the CL is different than the user making - # the revert. If they're the same, then we'll want to prompt the user - # for a different reviewer to TBR. - current_users = getCurrentSVNUsers(BASE_URL) - is_self_revert = options.revert and author in current_users - - filename = str(revision)+".txt" - out = open(filename,"w") - drover_title = '%s %s' % (action, revision) - revision_log = getRevisionLog(url, revision).splitlines() - if revision_log: - commit_title = revision_log[0] - # Limit title to 68 chars so git log --oneline is <80 chars. - max_commit_title = 68 - (len(drover_title) + 3) - if len(commit_title) > max_commit_title: - commit_title = commit_title[:max_commit_title-3] + '...' - drover_title += ' "%s"' % commit_title - out.write(drover_title + '\n\n') - for line in revision_log: - out.write('> %s\n' % line) - if author: - out.write("\nTBR=" + author) - out.close() - - change_cmd = 'change ' + str(revision) + " " + filename - if options.revertbot: - if sys.platform == 'win32': - os.environ['SVN_EDITOR'] = 'cmd.exe /c exit' - else: - os.environ['SVN_EDITOR'] = 'true' - runGcl(change_cmd) - os.unlink(filename) - - if options.local: - return 0 - - print author - print revision - print ("gcl upload " + str(revision) + - " --send_mail --no_presubmit --reviewers=" + author) - - if options.revertbot or prompt("Would you like to upload?"): - if PROMPT_FOR_AUTHOR or is_self_revert: - author = text_prompt("Enter new author or press enter to accept default", - author) - if options.revertbot and options.revertbot_reviewers: - author += "," - author += options.revertbot_reviewers - gclUpload(revision, author) - else: - print "Deleting the changelist." - print "gcl delete " + str(revision) - runGcl("delete " + str(revision)) - return 0 - - # We commit if the reverbot is set to commit automatically, or if this is - # not the revertbot and the user agrees. - if options.revertbot_commit or (not options.revertbot and - prompt("Would you like to commit?")): - print "gcl commit " + str(revision) + " --no_presubmit --force" - return runGcl("commit " + str(revision) + " --no_presubmit --force") - else: - return 0 - - -def main(): - option_parser = optparse.OptionParser(usage=USAGE % {"app": sys.argv[0]}) - option_parser.add_option('-m', '--merge', type="int", - help='Revision to merge from trunk to branch') - option_parser.add_option('-b', '--branch', - help='Branch to revert or merge from') - option_parser.add_option('-l', '--local', action='store_true', - help='Local working copy to merge to') - option_parser.add_option('-s', '--sbranch', - help='Source branch for merge') - option_parser.add_option('-r', '--revert', type="int", - help='Revision to revert') - option_parser.add_option('-w', '--workdir', - help='subdir to use for the revert') - option_parser.add_option('-u', '--url', - help='svn url to use for the revert') - option_parser.add_option('-a', '--auditor', - help='overrides the author for reviewer') - option_parser.add_option('--revertbot', action='store_true', - default=False) - option_parser.add_option('--no-alt-urls', action='store_true', - help='Disable heuristics used to determine svn url') - option_parser.add_option('--revertbot-commit', action='store_true', - default=False) - option_parser.add_option('--revertbot-reviewers') - options, args = option_parser.parse_args() - - if not options.merge and not options.revert: - option_parser.error("You need at least --merge or --revert") - return 1 - - if options.merge and not (options.branch or options.local): - option_parser.error("--merge requires --branch or --local") - return 1 - - if options.local and (options.revert or options.branch): - option_parser.error("--local cannot be used with --revert or --branch") - return 1 - - return drover(options, args) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) diff --git a/gcl b/gcl deleted file mode 100755 index e48956de5..000000000 --- a/gcl +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2009 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -base_dir=$(dirname "$0") - -PYTHONDONTWRITEBYTECODE=1 exec python "$base_dir/gcl.py" "$@" diff --git a/gcl.bat b/gcl.bat deleted file mode 100755 index 786ceea06..000000000 --- a/gcl.bat +++ /dev/null @@ -1,6 +0,0 @@ -@echo off -setlocal -:: This is required with cygwin only. -PATH=%~dp0;%PATH% -set PYTHONDONTWRITEBYTECODE=1 -call python "%~dp0gcl.py" %* diff --git a/gcl.py b/gcl.py deleted file mode 100755 index 1480187c0..000000000 --- a/gcl.py +++ /dev/null @@ -1,1523 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""\ -Wrapper script around Rietveld's upload.py that simplifies working with groups -of files. -""" - -import json -import optparse -import os -import random -import re -import ssl -import string -import sys -import tempfile -import time -import urllib2 - - -import auth -import fix_encoding -import gclient_utils -import git_cl -import presubmit_support -import rietveld -from scm import SVN -import subprocess2 -from third_party import upload - -__version__ = '1.2.1' - - -CODEREVIEW_SETTINGS = { - # To make gcl send reviews to a server, check in a file named - # "codereview.settings" (see |CODEREVIEW_SETTINGS_FILE| below) to your - # project's base directory and add the following line to codereview.settings: - # CODE_REVIEW_SERVER: codereview.yourserver.org -} - -# globals that store the root of the current repository and the directory where -# we store information about changelists. -REPOSITORY_ROOT = "" - -# Printed when people upload patches using svn. -SWITCH_TO_GIT = """You're using svn to work on depot_tools. -Consider switching to git today, so that you're ready when svn stops working -and you need a functional checkout for a future fire.""" - -# Filename where we store repository specific information for gcl. -CODEREVIEW_SETTINGS_FILE = "codereview.settings" -CODEREVIEW_SETTINGS_FILE_NOT_FOUND = ( - 'No %s file found. Please add one.' % CODEREVIEW_SETTINGS_FILE) - -# Warning message when the change appears to be missing tests. -MISSING_TEST_MSG = "Change contains new or modified methods, but no new tests!" - -# Global cache of files cached in GetCacheDir(). -FILES_CACHE = {} - -# Valid extensions for files we want to lint. -DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)" -DEFAULT_LINT_IGNORE_REGEX = r"$^" - -def CheckHomeForFile(filename): - """Checks the users home dir for the existence of the given file. Returns - the path to the file if it's there, or None if it is not. - """ - full_path = os.path.expanduser(os.path.join('~', filename)) - if os.path.exists(full_path): - return full_path - return None - - -def UnknownFiles(): - """Runs svn status and returns unknown files.""" - return [ - item[1] for item in SVN.CaptureStatus([], GetRepositoryRoot()) - if item[0][0] == '?' - ] - - -def GetRepositoryRoot(): - """Returns the top level directory of the current repository. - - The directory is returned as an absolute path. - """ - global REPOSITORY_ROOT - if not REPOSITORY_ROOT: - REPOSITORY_ROOT = SVN.GetCheckoutRoot(os.getcwd()) - if not REPOSITORY_ROOT: - raise gclient_utils.Error("gcl run outside of repository") - return REPOSITORY_ROOT - - -def GetInfoDir(): - """Returns the directory where gcl info files are stored.""" - return os.path.join(GetRepositoryRoot(), '.svn', 'gcl_info') - - -def GetChangesDir(): - """Returns the directory where gcl change files are stored.""" - return os.path.join(GetInfoDir(), 'changes') - - -def GetCacheDir(): - """Returns the directory where gcl change files are stored.""" - return os.path.join(GetInfoDir(), 'cache') - - -def GetCachedFile(filename, max_age=60*60*24*3, use_root=False): - """Retrieves a file from the repository and caches it in GetCacheDir() for - max_age seconds. - - use_root: If False, look up the arborescence for the first match, otherwise go - directory to the root repository. - - Note: The cache will be inconsistent if the same file is retrieved with both - use_root=True and use_root=False. Don't be stupid. - """ - if filename not in FILES_CACHE: - # Don't try to look up twice. - FILES_CACHE[filename] = None - # First we check if we have a cached version. - try: - cached_file = os.path.join(GetCacheDir(), filename) - except (gclient_utils.Error, subprocess2.CalledProcessError): - return None - if (not os.path.exists(cached_file) or - (time.time() - os.stat(cached_file).st_mtime) > max_age): - dir_info = SVN.CaptureLocalInfo([], '.') - repo_root = dir_info['Repository Root'] - if use_root: - url_path = repo_root - else: - url_path = dir_info['URL'] - while True: - # Look in the repository at the current level for the file. - for _ in range(5): - content = None - try: - # Take advantage of the fact that svn won't output to stderr in case - # of success but will do in case of failure so don't mind putting - # stderr into content_array. - content_array = [] - svn_path = url_path + '/' + filename - args = ['svn', 'cat', svn_path] - if sys.platform != 'darwin': - # MacOSX 10.5.2 has a bug with svn 1.4.4 that will trigger the - # 'Can\'t get username or password' and can be fixed easily. - # The fix doesn't work if the user upgraded to svn 1.6.x. Bleh. - # I don't have time to fix their broken stuff. - args.append('--non-interactive') - gclient_utils.CheckCallAndFilter( - args, cwd='.', filter_fn=content_array.append) - # Exit the loop if the file was found. Override content. - content = '\n'.join(content_array) - break - except (gclient_utils.Error, subprocess2.CalledProcessError): - if content_array[0].startswith( - 'svn: Can\'t get username or password'): - ErrorExit('Your svn credentials expired. Please run svn update ' - 'to fix the cached credentials') - if content_array[0].startswith('svn: Can\'t get password'): - ErrorExit('If are using a Mac and svn --version shows 1.4.x, ' - 'please hack gcl.py to remove --non-interactive usage, it\'s' - 'a bug on your installed copy') - if (content_array[0].startswith('svn: File not found:') or - content_array[0].endswith('path not found')): - break - # Otherwise, fall through to trying again. - if content: - break - if url_path == repo_root: - # Reached the root. Abandoning search. - break - # Go up one level to try again. - url_path = os.path.dirname(url_path) - if content is not None or filename != CODEREVIEW_SETTINGS_FILE: - # Write a cached version even if there isn't a file, so we don't try to - # fetch it each time. codereview.settings must always be present so do - # not cache negative. - gclient_utils.FileWrite(cached_file, content or '') - else: - content = gclient_utils.FileRead(cached_file, 'r') - # Keep the content cached in memory. - FILES_CACHE[filename] = content - return FILES_CACHE[filename] - - -def GetCodeReviewSetting(key): - """Returns a value for the given key for this repository.""" - # Use '__just_initialized' as a flag to determine if the settings were - # already initialized. - if '__just_initialized' not in CODEREVIEW_SETTINGS: - settings_file = GetCachedFile(CODEREVIEW_SETTINGS_FILE) - if settings_file: - CODEREVIEW_SETTINGS.update( - gclient_utils.ParseCodereviewSettingsContent(settings_file)) - CODEREVIEW_SETTINGS.setdefault('__just_initialized', None) - return CODEREVIEW_SETTINGS.get(key, "") - - -def Warn(msg): - print >> sys.stderr, msg - - -def ErrorExit(msg): - print >> sys.stderr, msg - sys.exit(1) - - -def RunShellWithReturnCode(command, print_output=False): - """Executes a command and returns the output and the return code.""" - p = subprocess2.Popen( - command, - cwd=GetRepositoryRoot(), - stdout=subprocess2.PIPE, - stderr=subprocess2.STDOUT, - universal_newlines=True) - if print_output: - output_array = [] - while True: - line = p.stdout.readline() - if not line: - break - if print_output: - print line.strip('\n') - output_array.append(line) - output = "".join(output_array) - else: - output = p.stdout.read() - p.wait() - p.stdout.close() - return output, p.returncode - - -def RunShell(command, print_output=False): - """Executes a command and returns the output.""" - return RunShellWithReturnCode(command, print_output)[0] - - -def FilterFlag(args, flag): - """Returns True if the flag is present in args list. - - The flag is removed from args if present. - """ - if flag in args: - args.remove(flag) - return True - return False - - -class ChangeInfo(object): - """Holds information about a changelist. - - name: change name. - issue: the Rietveld issue number or 0 if it hasn't been uploaded yet. - patchset: the Rietveld latest patchset number or 0. - description: the description. - files: a list of 2 tuple containing (status, filename) of changed files, - with paths being relative to the top repository directory. - local_root: Local root directory - rietveld: rietveld server for this change - """ - # Kept for unit test support. This is for the old format, it's deprecated. - SEPARATOR = "\n-----\n" - - def __init__(self, name, issue, patchset, description, files, local_root, - rietveld_url, needs_upload): - # Defer the description processing to git_cl.ChangeDescription. - self._desc = git_cl.ChangeDescription(description) - self.name = name - self.issue = int(issue) - self.patchset = int(patchset) - self._files = files or [] - self.patch = None - self._local_root = local_root - self.needs_upload = needs_upload - self.rietveld = gclient_utils.UpgradeToHttps( - rietveld_url or GetCodeReviewSetting('CODE_REVIEW_SERVER')) - self._rpc_server = None - - @property - def description(self): - return self._desc.description - - def force_description(self, new_description): - self._desc = git_cl.ChangeDescription(new_description) - self.needs_upload = True - - def append_footer(self, line): - self._desc.append_footer(line) - - def get_reviewers(self): - return self._desc.get_reviewers() - - def update_reviewers(self, reviewers): - self._desc.update_reviewers(reviewers) - - def NeedsUpload(self): - return self.needs_upload - - def GetFileNames(self): - """Returns the list of file names included in this change.""" - return [f[1] for f in self._files] - - def GetFiles(self): - """Returns the list of files included in this change with their status.""" - return self._files - - def GetLocalRoot(self): - """Returns the local repository checkout root directory.""" - return self._local_root - - def Exists(self): - """Returns True if this change already exists (i.e., is not new).""" - return (self.issue or self.description or self._files) - - def _NonDeletedFileList(self): - """Returns a list of files in this change, not including deleted files.""" - return [f[1] for f in self.GetFiles() - if not f[0].startswith("D")] - - def _AddedFileList(self): - """Returns a list of files added in this change.""" - return [f[1] for f in self.GetFiles() if f[0].startswith("A")] - - def Save(self): - """Writes the changelist information to disk.""" - data = json.dumps({ - 'issue': self.issue, - 'patchset': self.patchset, - 'needs_upload': self.NeedsUpload(), - 'files': self.GetFiles(), - 'description': self.description, - 'rietveld': self.rietveld, - }, sort_keys=True, indent=2) - gclient_utils.FileWrite(GetChangelistInfoFile(self.name), data) - - def Delete(self): - """Removes the changelist information from disk.""" - os.remove(GetChangelistInfoFile(self.name)) - - def RpcServer(self): - if not self._rpc_server: - if not self.rietveld: - ErrorExit(CODEREVIEW_SETTINGS_FILE_NOT_FOUND) - # TODO(vadimsh): glc.py should be deleted soon. Do not bother much about - # authentication options and always use defaults. - self._rpc_server = rietveld.CachingRietveld( - self.rietveld, auth.make_auth_config()) - return self._rpc_server - - def CloseIssue(self): - """Closes the Rietveld issue for this changelist.""" - # Newer versions of Rietveld require us to pass an XSRF token to POST, so - # we fetch it from the server. - xsrf_token = self.SendToRietveld( - '/xsrf_token', - extra_headers={'X-Requesting-XSRF-Token': '1'}) - - # You cannot close an issue with a GET. - # We pass an empty string for the data so it is a POST rather than a GET. - data = [("description", self.description), - ("xsrf_token", xsrf_token)] - ctype, body = upload.EncodeMultipartFormData(data, []) - self.SendToRietveld('/%d/close' % self.issue, payload=body, - content_type=ctype) - - def UpdateRietveldDescription(self): - """Sets the description for an issue on Rietveld.""" - data = [("description", self.description),] - ctype, body = upload.EncodeMultipartFormData(data, []) - self.SendToRietveld('/%d/description' % self.issue, payload=body, - content_type=ctype) - self.needs_upload = False - - def GetIssueDescription(self): - """Returns the issue description from Rietveld.""" - return self.SendToRietveld('/%d/description' % self.issue).replace('\r\n', - '\n') - - def UpdateDescriptionFromIssue(self): - """Updates self.description with the issue description from Rietveld.""" - self._desc = git_cl.ChangeDescription(self.GetIssueDescription()) - - def GetApprovingReviewers(self): - """Returns the issue reviewers list from Rietveld.""" - return git_cl.get_approving_reviewers( - self.RpcServer().get_issue_properties(self.issue, True)) - - def AddComment(self, comment): - """Adds a comment for an issue on Rietveld. - As a side effect, this will email everyone associated with the issue.""" - return self.RpcServer().add_comment(self.issue, comment) - - def PrimeLint(self): - """Do background work on Rietveld to lint the file so that the results are - ready when the issue is viewed.""" - if self.issue and self.patchset: - try: - self.SendToRietveld('/lint/issue%s_%s' % (self.issue, self.patchset), - timeout=60) - except ssl.SSLError as e: - # It takes more than 60 seconds to lint some CLs. Silently ignore - # the expected timeout. - if e.message != 'The read operation timed out': - raise - - def SendToRietveld(self, request_path, timeout=None, **kwargs): - """Send a POST/GET to Rietveld. Returns the response body.""" - try: - return self.RpcServer().Send(request_path, timeout=timeout, **kwargs) - except urllib2.URLError: - if timeout is None: - ErrorExit('Error accessing url %s' % request_path) - else: - return None - - def MissingTests(self): - """Returns True if the change looks like it needs unit tests but has none. - - A change needs unit tests if it contains any new source files or methods. - """ - SOURCE_SUFFIXES = [".cc", ".cpp", ".c", ".m", ".mm"] - # Ignore third_party entirely. - files = [f for f in self._NonDeletedFileList() - if f.find("third_party") == -1] - added_files = [f for f in self._AddedFileList() - if f.find("third_party") == -1] - - # If the change is entirely in third_party, we're done. - if len(files) == 0: - return False - - # Any new or modified test files? - # A test file's name ends with "test.*" or "tests.*". - test_files = [test for test in files - if os.path.splitext(test)[0].rstrip("s").endswith("test")] - if len(test_files) > 0: - return False - - # Any new source files? - source_files = [item for item in added_files - if os.path.splitext(item)[1] in SOURCE_SUFFIXES] - if len(source_files) > 0: - return True - - # Do the long test, checking the files for new methods. - return self._HasNewMethod() - - def _HasNewMethod(self): - """Returns True if the changeset contains any new functions, or if a - function signature has been changed. - - A function is identified by starting flush left, containing a "(" before - the next flush-left line, and either ending with "{" before the next - flush-left line or being followed by an unindented "{". - - Currently this returns True for new methods, new static functions, and - methods or functions whose signatures have been changed. - - Inline methods added to header files won't be detected by this. That's - acceptable for purposes of determining if a unit test is needed, since - inline methods should be trivial. - """ - # To check for methods added to source or header files, we need the diffs. - # We'll generate them all, since there aren't likely to be many files - # apart from source and headers; besides, we'll want them all if we're - # uploading anyway. - if self.patch is None: - self.patch = GenerateDiff(self.GetFileNames()) - - definition = "" - for line in self.patch.splitlines(): - if not line.startswith("+"): - continue - line = line.strip("+").rstrip(" \t") - # Skip empty lines, comments, and preprocessor directives. - # TODO(pamg): Handle multiline comments if it turns out to be a problem. - if line == "" or line.startswith("/") or line.startswith("#"): - continue - - # A possible definition ending with "{" is complete, so check it. - if definition.endswith("{"): - if definition.find("(") != -1: - return True - definition = "" - - # A { or an indented line, when we're in a definition, continues it. - if (definition != "" and - (line == "{" or line.startswith(" ") or line.startswith("\t"))): - definition += line - - # A flush-left line starts a new possible function definition. - elif not line.startswith(" ") and not line.startswith("\t"): - definition = line - - return False - - @staticmethod - def Load(changename, local_root, fail_on_not_found, update_status): - """Gets information about a changelist. - - Args: - fail_on_not_found: if True, this function will quit the program if the - changelist doesn't exist. - update_status: if True, the svn status will be updated for all the files - and unchanged files will be removed. - - Returns: a ChangeInfo object. - """ - info_file = GetChangelistInfoFile(changename) - if not os.path.exists(info_file): - if fail_on_not_found: - ErrorExit("Changelist " + changename + " not found.") - return ChangeInfo(changename, 0, 0, '', None, local_root, None, False) - content = gclient_utils.FileRead(info_file) - save = False - try: - values = ChangeInfo._LoadNewFormat(content) - except ValueError: - try: - values = ChangeInfo._LoadOldFormat(content) - save = True - except ValueError: - ErrorExit( - ('Changelist file %s is corrupt.\n' - 'Either run "gcl delete %s" or manually edit the file') % ( - info_file, changename)) - files = values['files'] - if update_status: - for item in files[:]: - status_result = SVN.CaptureStatus(item[1], local_root) - if not status_result or not status_result[0][0]: - # File has been reverted. - save = True - files.remove(item) - continue - status = status_result[0][0] - if status != item[0]: - save = True - files[files.index(item)] = (status, item[1]) - change_info = ChangeInfo( - changename, - values['issue'], - values['patchset'], - values['description'], - files, - local_root, - values.get('rietveld'), - values['needs_upload']) - if save: - change_info.Save() - return change_info - - @staticmethod - def _LoadOldFormat(content): - # The info files have the following format: - # issue_id, patchset\n (, patchset is optional) - # SEPARATOR\n - # filepath1\n - # filepath2\n - # . - # . - # filepathn\n - # SEPARATOR\n - # description - split_data = content.split(ChangeInfo.SEPARATOR, 2) - if len(split_data) != 3: - raise ValueError('Bad change format') - values = { - 'issue': 0, - 'patchset': 0, - 'needs_upload': False, - 'files': [], - } - items = split_data[0].split(', ') - if items[0]: - values['issue'] = int(items[0]) - if len(items) > 1: - values['patchset'] = int(items[1]) - if len(items) > 2: - values['needs_upload'] = (items[2] == "dirty") - for line in split_data[1].splitlines(): - status = line[:7] - filename = line[7:] - values['files'].append((status, filename)) - values['description'] = split_data[2] - return values - - @staticmethod - def _LoadNewFormat(content): - return json.loads(content) - - def __str__(self): - out = ['%s:' % self.__class__.__name__] - for k in dir(self): - if k.startswith('__'): - continue - v = getattr(self, k) - if v is self or callable(getattr(self, k)): - continue - out.append(' %s: %r' % (k, v)) - return '\n'.join(out) - - -def GetChangelistInfoFile(changename): - """Returns the file that stores information about a changelist.""" - if not changename or re.search(r'[^\w-]', changename): - ErrorExit("Invalid changelist name: " + changename) - return os.path.join(GetChangesDir(), changename) - - -def LoadChangelistInfoForMultiple(changenames, local_root, fail_on_not_found, - update_status): - """Loads many changes and merge their files list into one pseudo change. - - This is mainly useful to concatenate many changes into one for a 'gcl try'. - """ - changes = changenames.split(',') - aggregate_change_info = ChangeInfo( - changenames, 0, 0, '', None, local_root, None, False) - for change in changes: - aggregate_change_info._files += ChangeInfo.Load( - change, local_root, fail_on_not_found, update_status).GetFiles() - return aggregate_change_info - - -def GetCLs(): - """Returns a list of all the changelists in this repository.""" - cls = os.listdir(GetChangesDir()) - if CODEREVIEW_SETTINGS_FILE in cls: - cls.remove(CODEREVIEW_SETTINGS_FILE) - return cls - - -def GenerateChangeName(): - """Generate a random changelist name.""" - random.seed() - current_cl_names = GetCLs() - while True: - cl_name = (random.choice(string.ascii_lowercase) + - random.choice(string.digits) + - random.choice(string.ascii_lowercase) + - random.choice(string.digits)) - if cl_name not in current_cl_names: - return cl_name - - -def GetModifiedFiles(): - """Returns a set that maps from changelist name to (status,filename) tuples. - - Files not in a changelist have an empty changelist name. Filenames are in - relation to the top level directory of the current repository. Note that - only the current directory and subdirectories are scanned, in order to - improve performance while still being flexible. - """ - files = {} - - # Since the files are normalized to the root folder of the repositary, figure - # out what we need to add to the paths. - dir_prefix = os.getcwd()[len(GetRepositoryRoot()):].strip(os.sep) - - # Get a list of all files in changelists. - files_in_cl = {} - for cl in GetCLs(): - change_info = ChangeInfo.Load(cl, GetRepositoryRoot(), - fail_on_not_found=True, update_status=False) - for status, filename in change_info.GetFiles(): - files_in_cl[filename] = change_info.name - - # Get all the modified files down the current directory. - for line in SVN.CaptureStatus(None, os.getcwd()): - status = line[0] - filename = line[1] - if status[0] == "?": - continue - if dir_prefix: - filename = os.path.join(dir_prefix, filename) - change_list_name = "" - if filename in files_in_cl: - change_list_name = files_in_cl[filename] - files.setdefault(change_list_name, []).append((status, filename)) - - return files - - -def GetFilesNotInCL(): - """Returns a list of tuples (status,filename) that aren't in any changelists. - - See docstring of GetModifiedFiles for information about path of files and - which directories are scanned. - """ - modified_files = GetModifiedFiles() - if "" not in modified_files: - return [] - return modified_files[""] - - -def ListFiles(show_unknown_files): - files = GetModifiedFiles() - cl_keys = files.keys() - cl_keys.sort() - for cl_name in cl_keys: - if not cl_name: - continue - note = "" - change_info = ChangeInfo.Load(cl_name, GetRepositoryRoot(), - fail_on_not_found=True, update_status=False) - if len(change_info.GetFiles()) != len(files[cl_name]): - note = " (Note: this changelist contains files outside this directory)" - print "\n--- Changelist " + cl_name + note + ":" - for filename in files[cl_name]: - print "".join(filename) - if show_unknown_files: - unknown_files = UnknownFiles() - if (files.get('') or (show_unknown_files and len(unknown_files))): - print "\n--- Not in any changelist:" - for item in files.get('', []): - print "".join(item) - if show_unknown_files: - for filename in unknown_files: - print "? %s" % filename - return 0 - - -def GenerateDiff(files): - return SVN.GenerateDiff( - files, GetRepositoryRoot(), full_move=False, revision=None) - - -def GetTreeStatus(): - tree_status_url = GetCodeReviewSetting('STATUS') - return git_cl.GetTreeStatus(tree_status_url) if tree_status_url else "unset" - - -def OptionallyDoPresubmitChecks(change_info, committing, args): - if FilterFlag(args, "--no_presubmit") or FilterFlag(args, "--force"): - return presubmit_support.PresubmitOutput() - return DoPresubmitChecks(change_info, committing, True) - - -def defer_attributes(a, b): - """Copy attributes from an object (like a function) to another.""" - for x in dir(a): - if not getattr(b, x, None): - setattr(b, x, getattr(a, x)) - - -def need_change(function): - """Converts args -> change_info.""" - # pylint: disable=W0612,W0621 - def hook(args): - if not len(args) == 1: - ErrorExit("You need to pass a change list name") - change_info = ChangeInfo.Load(args[0], GetRepositoryRoot(), True, True) - return function(change_info) - defer_attributes(function, hook) - hook.need_change = True - hook.no_args = True - return hook - - -def need_change_and_args(function): - """Converts args -> change_info.""" - # pylint: disable=W0612,W0621 - def hook(args): - if not args: - ErrorExit("You need to pass a change list name") - change_info = ChangeInfo.Load(args.pop(0), GetRepositoryRoot(), True, True) - return function(change_info, args) - defer_attributes(function, hook) - hook.need_change = True - return hook - - -def no_args(function): - """Make sure no args are passed.""" - # pylint: disable=W0612,W0621 - def hook(args): - if args: - ErrorExit("Doesn't support arguments") - return function() - defer_attributes(function, hook) - hook.no_args = True - return hook - - -def attrs(**kwargs): - """Decorate a function with new attributes.""" - def decorate(function): - for k in kwargs: - setattr(function, k, kwargs[k]) - return function - return decorate - - -@no_args -def CMDopened(): - """Lists modified files in the current directory down.""" - return ListFiles(False) - - -@no_args -def CMDstatus(): - """Lists modified and unknown files in the current directory down.""" - return ListFiles(True) - - -@need_change_and_args -@attrs(usage='[--no_presubmit] [--no_watchlists]') -def CMDupload(change_info, args): - """Uploads the changelist to the server for review. - - This does not submit a try job; use gcl try to submit a try job. - """ - if '-s' in args or '--server' in args: - ErrorExit('Don\'t use the -s flag, fix codereview.settings instead') - if not change_info.GetFiles(): - print "Nothing to upload, changelist is empty." - return 0 - - output = OptionallyDoPresubmitChecks(change_info, False, args) - if not output.should_continue(): - return 1 - no_watchlists = (FilterFlag(args, "--no_watchlists") or - FilterFlag(args, "--no-watchlists")) - - # Map --send-mail to --send_mail - if FilterFlag(args, "--send-mail"): - args.append("--send_mail") - - # Replace -m with -t and --message with --title, but make sure to - # preserve anything after the -m/--message. - found_deprecated_arg = [False] - def replace_message(a): - if a.startswith('-m'): - found_deprecated_arg[0] = True - return '-t' + a[2:] - elif a.startswith('--message'): - found_deprecated_arg[0] = True - return '--title' + a[9:] - return a - args = map(replace_message, args) - if found_deprecated_arg[0]: - print >> sys.stderr, ( - '\nWARNING: Use -t or --title to set the title of the patchset.\n' - 'In the near future, -m or --message will send a message instead.\n' - 'See http://goo.gl/JGg0Z for details.\n') - - upload_arg = ["upload.py", "-y"] - upload_arg.append("--server=%s" % change_info.rietveld.encode('utf-8')) - - reviewers = change_info.get_reviewers() or output.reviewers - if (reviewers and - not any(arg.startswith('-r') or arg.startswith('--reviewer') for - arg in args)): - upload_arg.append('--reviewers=%s' % ','.join(reviewers)) - - upload_arg.extend(args) - - desc_file = None - try: - if change_info.issue: - # Uploading a new patchset. - upload_arg.append("--issue=%d" % change_info.issue) - - project = GetCodeReviewSetting("PROJECT") - if project: - print SWITCH_TO_GIT - upload_arg.append("--project=%s" % project) - - if not any(i.startswith('--title') or i.startswith('-t') for i in args): - upload_arg.append('--title= ') - else: - # First time we upload. - handle, desc_file = tempfile.mkstemp(text=True) - os.write(handle, change_info.description) - os.close(handle) - - # Watchlist processing -- CC people interested in this changeset - # http://dev.chromium.org/developers/contributing-code/watchlists - if not no_watchlists: - import watchlists - watchlist = watchlists.Watchlists(change_info.GetLocalRoot()) - watchers = watchlist.GetWatchersForPaths(change_info.GetFileNames()) - - # We check this before applying the "PRIVATE" parameter of codereview - # settings assuming that the author of the settings file has put - # addresses which we can send private CLs to, and so we should ignore - # CC_LIST only when --private is specified explicitly on the command - # line. - if "--private" in upload_arg: - Warn("WARNING: CC_LIST and WATCHLISTS are ignored when --private is " - "specified. You need to review and add them manually if " - "necessary.") - cc_list = "" - no_watchlists = True - else: - cc_list = GetCodeReviewSetting("CC_LIST") - if not no_watchlists and watchers: - # Filter out all empty elements and join by ',' - cc_list = ','.join(filter(None, [cc_list] + watchers)) - if cc_list: - upload_arg.append("--cc=" + cc_list) - upload_arg.append("--file=%s" % desc_file) - - if GetCodeReviewSetting("PRIVATE") == "True": - upload_arg.append("--private") - - project = GetCodeReviewSetting("PROJECT") - if project: - print SWITCH_TO_GIT - upload_arg.append("--project=%s" % project) - - # If we have a lot of files with long paths, then we won't be able to fit - # the command to "svn diff". Instead, we generate the diff manually for - # each file and concatenate them before passing it to upload.py. - if change_info.patch is None: - change_info.patch = GenerateDiff(change_info.GetFileNames()) - - # Change the current working directory before calling upload.py so that it - # shows the correct base. - previous_cwd = os.getcwd() - os.chdir(change_info.GetLocalRoot()) - try: - try: - issue, patchset = upload.RealMain(upload_arg, change_info.patch) - except KeyboardInterrupt: - sys.exit(1) - if issue and patchset: - change_info.issue = int(issue) - change_info.patchset = int(patchset) - change_info.Save() - change_info.PrimeLint() - finally: - os.chdir(previous_cwd) - finally: - if desc_file: - os.remove(desc_file) - print "*** Upload does not submit a try; use gcl try to submit a try. ***" - return 0 - - -@need_change_and_args -@attrs(usage='[--upload]') -def CMDpresubmit(change_info, args): - """Runs presubmit checks on the change. - - The actual presubmit code is implemented in presubmit_support.py and looks - for PRESUBMIT.py files.""" - if not change_info.GetFiles(): - print('Nothing to presubmit check, changelist is empty.') - return 0 - parser = optparse.OptionParser() - parser.add_option('--upload', action='store_true') - options, args = parser.parse_args(args) - if args: - parser.error('Unrecognized args: %s' % args) - if options.upload: - print('*** Presubmit checks for UPLOAD would report: ***') - return not DoPresubmitChecks(change_info, False, False) - else: - print('*** Presubmit checks for COMMIT would report: ***') - return not DoPresubmitChecks(change_info, True, False) - - -def TryChange(change_info, args, swallow_exception): - """Create a diff file of change_info and send it to the try server.""" - try: - import trychange - except ImportError: - if swallow_exception: - return 1 - ErrorExit("You need to install trychange.py to use the try server.") - - trychange_args = [] - if change_info: - trychange_args.extend(['--name', change_info.name]) - if change_info.issue: - trychange_args.extend(["--issue", str(change_info.issue)]) - if change_info.patchset: - trychange_args.extend(["--patchset", str(change_info.patchset)]) - change = presubmit_support.SvnChange(change_info.name, - change_info.description, - change_info.GetLocalRoot(), - change_info.GetFiles(), - change_info.issue, - change_info.patchset, - None) - else: - change = None - - trychange_args.extend(args) - return trychange.TryChange( - trychange_args, - change=change, - swallow_exception=swallow_exception, - prog='gcl try', - extra_epilog='\n' - 'When called from gcl, use the format gcl try .\n') - - -@need_change_and_args -@attrs(usage='[--no_presubmit]') -def CMDcommit(change_info, args): - """Commits the changelist to the repository.""" - if not change_info.GetFiles(): - print "Nothing to commit, changelist is empty." - return 1 - - # OptionallyDoPresubmitChecks has a side-effect which eats these flags. - bypassed = '--no_presubmit' in args or '--force' in args - output = OptionallyDoPresubmitChecks(change_info, True, args) - if not output.should_continue(): - return 1 - - # We face a problem with svn here: Let's say change 'bleh' modifies - # svn:ignore on dir1\. but another unrelated change 'pouet' modifies - # dir1\foo.cc. When the user `gcl commit bleh`, foo.cc is *also committed*. - # The only fix is to use --non-recursive but that has its issues too: - # Let's say if dir1 is deleted, --non-recursive must *not* be used otherwise - # you'll get "svn: Cannot non-recursively commit a directory deletion of a - # directory with child nodes". Yay... - commit_cmd = ["svn", "commit"] - if change_info.issue: - # Get the latest description from Rietveld. - change_info.UpdateDescriptionFromIssue() - - change_info.update_reviewers(change_info.GetApprovingReviewers()) - - commit_desc = git_cl.ChangeDescription(change_info.description) - if change_info.issue: - server = change_info.rietveld - if not server.startswith("http://") and not server.startswith("https://"): - server = "http://" + server - commit_desc.append_footer('Review URL: %s/%d' % (server, change_info.issue)) - - handle, commit_filename = tempfile.mkstemp(text=True) - os.write(handle, commit_desc.description) - os.close(handle) - try: - handle, targets_filename = tempfile.mkstemp(text=True) - os.write(handle, "\n".join(change_info.GetFileNames())) - os.close(handle) - try: - commit_cmd += ['--file=' + commit_filename] - commit_cmd += ['--targets=' + targets_filename] - # Change the current working directory before calling commit. - output = '' - try: - output = RunShell(commit_cmd, True) - except subprocess2.CalledProcessError, e: - ErrorExit('Commit failed.\n%s' % e) - finally: - os.remove(commit_filename) - finally: - os.remove(targets_filename) - if output.find("Committed revision") != -1: - change_info.Delete() - - if change_info.issue: - revision = re.compile(".*?\nCommitted revision (\d+)", - re.DOTALL).match(output).group(1) - viewvc_url = GetCodeReviewSetting('VIEW_VC') - if viewvc_url and revision: - change_info.append_footer('Committed: ' + viewvc_url + revision) - elif revision: - change_info.append_footer('Committed: ' + revision) - change_info.CloseIssue() - props = change_info.RpcServer().get_issue_properties( - change_info.issue, False) - patch_num = len(props['patchsets']) - comment = "Committed patchset #%d (id:%d) manually as r%s" % ( - patch_num, props['patchsets'][-1], revision) - if bypassed: - comment += ' (tree was closed).' if GetTreeStatus() == 'closed' else '.' - else: - comment += ' (presubmit successful).' - change_info.AddComment(comment) - return 0 - - -def CMDchange(args): - """Creates or edits a changelist. - - Only scans the current directory and subdirectories. - """ - # Verify the user is running the change command from a read-write checkout. - svn_info = SVN.CaptureLocalInfo([], '.') - if not svn_info: - ErrorExit("Current checkout is unversioned. Please retry with a versioned " - "directory.") - - if len(args) == 0: - # Generate a random changelist name. - changename = GenerateChangeName() - elif args[0] == '--force': - changename = GenerateChangeName() - else: - changename = args[0] - change_info = ChangeInfo.Load(changename, GetRepositoryRoot(), False, True) - - if len(args) == 2: - if not os.path.isfile(args[1]): - ErrorExit('The change "%s" doesn\'t exist.' % args[1]) - f = open(args[1], 'rU') - override_description = f.read() - f.close() - else: - override_description = None - - if change_info.issue and not change_info.NeedsUpload(): - try: - description = change_info.GetIssueDescription() - except urllib2.HTTPError, err: - if err.code == 404: - # The user deleted the issue in Rietveld, so forget the old issue id. - description = change_info.description - change_info.issue = 0 - change_info.Save() - else: - ErrorExit("Error getting the description from Rietveld: " + err) - else: - if override_description: - description = override_description - else: - description = change_info.description - - other_files = GetFilesNotInCL() - - # Edited files (as opposed to files with only changed properties) will have - # a letter for the first character in the status string. - file_re = re.compile(r"^[a-z].+\Z", re.IGNORECASE) - affected_files = [x for x in other_files if file_re.match(x[0])] - unaffected_files = [x for x in other_files if not file_re.match(x[0])] - - description = description.rstrip() + '\n' - - separator1 = ("\n---All lines above this line become the description.\n" - "---Repository Root: " + change_info.GetLocalRoot() + "\n" - "---Paths in this changelist (" + change_info.name + "):\n") - separator2 = "\n\n---Paths modified but not in any changelist:\n\n" - - text = (description + separator1 + '\n' + - '\n'.join([f[0] + f[1] for f in change_info.GetFiles()])) - - if change_info.Exists(): - text += (separator2 + - '\n'.join([f[0] + f[1] for f in affected_files]) + '\n') - else: - text += ('\n'.join([f[0] + f[1] for f in affected_files]) + '\n' + - separator2) - text += '\n'.join([f[0] + f[1] for f in unaffected_files]) + '\n' - - result = gclient_utils.RunEditor(text, False) - if not result: - ErrorExit('Running editor failed') - - split_result = result.split(separator1, 1) - if len(split_result) != 2: - ErrorExit("Don't modify the text starting with ---!\n\n%r" % result) - - # Update the CL description if it has changed. - new_description = split_result[0] - cl_files_text = split_result[1] - if new_description != description or override_description: - change_info.force_description(new_description) - - new_cl_files = [] - for line in cl_files_text.splitlines(): - if not len(line): - continue - if line.startswith("---"): - break - status = line[:7] - filename = line[7:] - new_cl_files.append((status, filename)) - - if (not len(change_info.GetFiles()) and not change_info.issue and - not len(new_description) and not new_cl_files): - ErrorExit("Empty changelist not saved") - - change_info._files = new_cl_files - change_info.Save() - if svn_info.get('URL', '').startswith('http:'): - Warn("WARNING: Creating CL in a read-only checkout. You will need to " - "commit using a commit queue!") - - print change_info.name + " changelist saved." - if change_info.MissingTests(): - Warn("WARNING: " + MISSING_TEST_MSG) - - # Update the Rietveld issue. - if change_info.issue and change_info.NeedsUpload(): - change_info.UpdateRietveldDescription() - change_info.Save() - return 0 - - -@need_change_and_args -def CMDlint(change_info, args): - """Runs cpplint.py on all the files in the change list. - - Checks all the files in the changelist for possible style violations. - """ - # Access to a protected member _XX of a client class - # pylint: disable=W0212 - try: - import cpplint - import cpplint_chromium - except ImportError: - ErrorExit("You need to install cpplint.py to lint C++ files.") - # Change the current working directory before calling lint so that it - # shows the correct base. - previous_cwd = os.getcwd() - os.chdir(change_info.GetLocalRoot()) - try: - # Process cpplints arguments if any. - filenames = cpplint.ParseArguments(args + change_info.GetFileNames()) - - white_list = GetCodeReviewSetting("LINT_REGEX") - if not white_list: - white_list = DEFAULT_LINT_REGEX - white_regex = re.compile(white_list) - black_list = GetCodeReviewSetting("LINT_IGNORE_REGEX") - if not black_list: - black_list = DEFAULT_LINT_IGNORE_REGEX - black_regex = re.compile(black_list) - extra_check_functions = [cpplint_chromium.CheckPointerDeclarationWhitespace] - for filename in filenames: - if white_regex.match(filename): - if black_regex.match(filename): - print "Ignoring file %s" % filename - else: - cpplint.ProcessFile(filename, cpplint._cpplint_state.verbose_level, - extra_check_functions) - else: - print "Skipping file %s" % filename - finally: - os.chdir(previous_cwd) - - print "Total errors found: %d\n" % cpplint._cpplint_state.error_count - return 1 - - -def DoPresubmitChecks(change_info, committing, may_prompt): - """Imports presubmit, then calls presubmit.DoPresubmitChecks.""" - root_presubmit = GetCachedFile('PRESUBMIT.py', use_root=True) - change = presubmit_support.SvnChange(change_info.name, - change_info.description, - change_info.GetLocalRoot(), - change_info.GetFiles(), - change_info.issue, - change_info.patchset, - None) - output = presubmit_support.DoPresubmitChecks( - change=change, - committing=committing, - verbose=False, - output_stream=sys.stdout, - input_stream=sys.stdin, - default_presubmit=root_presubmit, - may_prompt=may_prompt, - rietveld_obj=change_info.RpcServer()) - if not output.should_continue() and may_prompt: - # TODO(dpranke): move into DoPresubmitChecks(), unify cmd line args. - print "\nPresubmit errors, can't continue (use --no_presubmit to bypass)" - - return output - - -@no_args -def CMDchanges(): - """Lists all the changelists and their files.""" - for cl in GetCLs(): - change_info = ChangeInfo.Load(cl, GetRepositoryRoot(), True, True) - print "\n--- Changelist " + change_info.name + ":" - for filename in change_info.GetFiles(): - print "".join(filename) - return 0 - - -@no_args -def CMDdeleteempties(): - """Delete all changelists that have no files.""" - print "\n--- Deleting:" - for cl in GetCLs(): - change_info = ChangeInfo.Load(cl, GetRepositoryRoot(), True, True) - if not len(change_info.GetFiles()): - print change_info.name - change_info.Delete() - return 0 - - -@no_args -def CMDnothave(): - """Lists files unknown to Subversion.""" - for filename in UnknownFiles(): - print "? " + "".join(filename) - return 0 - - -@attrs(usage='') -def CMDdiff(args): - """Diffs all files in the changelist or all files that aren't in a CL.""" - files = None - if args: - change_info = ChangeInfo.Load(args.pop(0), GetRepositoryRoot(), True, True) - files = change_info.GetFileNames() - else: - files = [f[1] for f in GetFilesNotInCL()] - - root = GetRepositoryRoot() - cmd = ['svn', 'diff'] - cmd.extend([os.path.join(root, x) for x in files]) - cmd.extend(args) - return RunShellWithReturnCode(cmd, print_output=True)[1] - - -@no_args -def CMDsettings(): - """Prints code review settings for this checkout.""" - # Force load settings - GetCodeReviewSetting("UNKNOWN") - del CODEREVIEW_SETTINGS['__just_initialized'] - print '\n'.join(("%s: %s" % (str(k), str(v)) - for (k,v) in CODEREVIEW_SETTINGS.iteritems())) - return 0 - - -@need_change -def CMDdescription(change_info): - """Prints the description of the specified change to stdout.""" - print change_info.description - return 0 - - -def CMDdelete(args): - """Deletes a changelist.""" - if not len(args) == 1: - ErrorExit('You need to pass a change list name') - filepath = GetChangelistInfoFile(args[0]) - if not os.path.isfile(filepath): - ErrorExit('You need to pass a valid change list name') - os.remove(filepath) - return 0 - - -def CMDtry(args): - """Sends the change to the tryserver to do a test run on your code. - - To send multiple changes as one path, use a comma-separated list of - changenames. Use 'gcl help try' for more information!""" - # When the change contains no file, send the "changename" positional - # argument to trychange.py. - # When the command is 'try' and --patchset is used, the patch to try - # is on the Rietveld server. - if not args: - ErrorExit("You need to pass a change list name") - if args[0].find(',') != -1: - change_info = LoadChangelistInfoForMultiple(args[0], GetRepositoryRoot(), - True, True) - else: - change_info = ChangeInfo.Load(args[0], GetRepositoryRoot(), - True, True) - - props = change_info.RpcServer().get_issue_properties( - change_info.issue, False) - if props.get('private'): - ErrorExit('Cannot use trybots on a private issue') - - if change_info.GetFiles(): - args = args[1:] - else: - change_info = None - return TryChange(change_info, args, swallow_exception=False) - - -@attrs(usage=' ') -def CMDrename(args): - """Renames an existing change.""" - if len(args) != 2: - ErrorExit("Usage: gcl rename .") - src, dst = args - src_file = GetChangelistInfoFile(src) - if not os.path.isfile(src_file): - ErrorExit("Change '%s' does not exist." % src) - dst_file = GetChangelistInfoFile(dst) - if os.path.isfile(dst_file): - ErrorExit("Change '%s' already exists; pick a new name." % dst) - os.rename(src_file, dst_file) - print "Change '%s' renamed '%s'." % (src, dst) - return 0 - - -def CMDpassthru(args): - """Everything else that is passed into gcl we redirect to svn. - - It assumes a change list name is passed and is converted with the files names. - """ - if not args or len(args) < 2: - ErrorExit("You need to pass a change list name for this svn fall-through " - "command") - cl_name = args[1] - args = ["svn", args[0]] - if len(args) > 1: - root = GetRepositoryRoot() - change_info = ChangeInfo.Load(cl_name, root, True, True) - args.extend([os.path.join(root, x) for x in change_info.GetFileNames()]) - return RunShellWithReturnCode(args, print_output=True)[1] - - -def Command(name): - return getattr(sys.modules[__name__], 'CMD' + name, None) - - -def GenUsage(command): - """Modify an OptParse object with the function's documentation.""" - obj = Command(command) - display = command - more = getattr(obj, 'usage', '') - if command == 'help': - display = '' - need_change_val = '' - if getattr(obj, 'need_change', None): - need_change_val = ' ' - options = ' [options]' - if getattr(obj, 'no_args', None): - options = '' - res = 'Usage: gcl %s%s%s %s\n\n' % (display, need_change_val, options, more) - res += re.sub('\n ', '\n', obj.__doc__) - return res - - -def CMDhelp(args): - """Prints this help or help for the given command.""" - if args and 'CMD' + args[0] in dir(sys.modules[__name__]): - print GenUsage(args[0]) - - # These commands defer to external tools so give this info too. - if args[0] == 'try': - TryChange(None, ['--help'], swallow_exception=False) - if args[0] == 'upload': - upload.RealMain(['upload.py', '--help']) - return 0 - - print GenUsage('help') - print sys.modules[__name__].__doc__ - print 'version ' + __version__ + '\n' - - print('Commands are:\n' + '\n'.join([ - ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n')[0].strip()) - for fn in dir(sys.modules[__name__]) if fn.startswith('CMD')])) - return 0 - - -def main(argv): - if sys.hexversion < 0x02060000: - print >> sys.stderr, ( - '\nYour python version %s is unsupported, please upgrade.\n' % - sys.version.split(' ', 1)[0]) - return 2 - - sys.stderr.write('Warning: gcl is going away soon. Get off subversion!\n') - sys.stderr.write('See http://crbug.com/475321 for more details.\n') - - if not argv: - argv = ['help'] - command = Command(argv[0]) - # Help can be run from anywhere. - if command == CMDhelp: - return command(argv[1:]) - - try: - GetRepositoryRoot() - except (gclient_utils.Error, subprocess2.CalledProcessError): - print >> sys.stderr, 'To use gcl, you need to be in a subversion checkout.' - return 1 - - # Create the directories where we store information about changelists if it - # doesn't exist. - try: - if not os.path.exists(GetInfoDir()): - os.mkdir(GetInfoDir()) - if not os.path.exists(GetChangesDir()): - os.mkdir(GetChangesDir()) - if not os.path.exists(GetCacheDir()): - os.mkdir(GetCacheDir()) - - if command: - return command(argv[1:]) - # Unknown command, try to pass that to svn - return CMDpassthru(argv) - except (gclient_utils.Error, subprocess2.CalledProcessError), e: - print >> sys.stderr, 'Got an exception' - print >> sys.stderr, str(e) - return 1 - except upload.ClientLoginError, e: - print >> sys.stderr, 'Got an exception logging in to Rietveld' - print >> sys.stderr, str(e) - return 1 - except urllib2.HTTPError, e: - if e.code != 500: - raise - print >> sys.stderr, ( - 'AppEngine is misbehaving and returned HTTP %d, again. Keep faith ' - 'and retry or visit go/isgaeup.\n%s') % (e.code, str(e)) - return 1 - - -if __name__ == "__main__": - fix_encoding.fix_encoding() - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) diff --git a/gclient_utils.py b/gclient_utils.py index a988b8242..602322b2c 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -1056,7 +1056,7 @@ class ExecutionQueue(object): work_queue.ready_cond.release() -def GetEditor(git, git_editor=None): +def GetEditor(git_editor=None): """Returns the most plausible editor to use. In order of preference: @@ -1068,14 +1068,8 @@ def GetEditor(git, git_editor=None): In the case of git-cl, this matches git's behaviour, except that it does not include dumb terminal detection. - - In the case of gcl, this matches svn's behaviour, except that it does not - accept a command-line flag or check the editor-cmd configuration variable. """ - if git: - editor = os.environ.get('GIT_EDITOR') or git_editor - else: - editor = os.environ.get('SVN_EDITOR') + editor = os.environ.get('GIT_EDITOR') or git_editor if not editor: editor = os.environ.get('VISUAL') if not editor: @@ -1105,7 +1099,7 @@ def RunEditor(content, git, git_editor=None): fileobj.close() try: - editor = GetEditor(git, git_editor=git_editor) + editor = GetEditor(git_editor=git_editor) if not editor: return None cmd = '%s %s' % (editor, filename) diff --git a/git_cl.py b/git_cl.py index 5831b121e..7dc9bace6 100755 --- a/git_cl.py +++ b/git_cl.py @@ -1025,7 +1025,7 @@ class Changelist(object): def GetCCList(self): """Return the users cc'd on this CL. - Return is a string suitable for passing to gcl with the --cc flag. + Return is a string suitable for passing to git cl with the --cc flag. """ if self.cc is None: base_cc = settings.GetDefaultCCList() diff --git a/presubmit_support.py b/presubmit_support.py index 84a1f9bcf..5851178f1 100755 --- a/presubmit_support.py +++ b/presubmit_support.py @@ -1009,32 +1009,6 @@ class SvnChange(Change): scm = 'svn' _changelists = None - def _GetChangeLists(self): - """Get all change lists.""" - if self._changelists == None: - previous_cwd = os.getcwd() - os.chdir(self.RepositoryRoot()) - # Need to import here to avoid circular dependency. - import gcl - self._changelists = gcl.GetModifiedFiles() - os.chdir(previous_cwd) - return self._changelists - - def GetAllModifiedFiles(self): - """Get all modified files.""" - changelists = self._GetChangeLists() - all_modified_files = [] - for cl in changelists.values(): - all_modified_files.extend( - [os.path.join(self.RepositoryRoot(), f[1]) for f in cl]) - return all_modified_files - - def GetModifiedFiles(self): - """Get modified files in the current CL.""" - changelists = self._GetChangeLists() - return [os.path.join(self.RepositoryRoot(), f[1]) - for f in changelists[self.Name()]] - def AllFiles(self, root=None): """List all files under source control in the repo.""" root = root or self.RepositoryRoot() @@ -1413,7 +1387,7 @@ class PresubmitExecuter(object): """ Args: change: The Change object. - committing: True if 'gcl commit' is running, False if 'gcl upload' is. + committing: True if 'git cl land' is running, False if 'git cl upload' is. rietveld_obj: rietveld.Rietveld client object. gerrit_obj: provides basic Gerrit codereview functionality. dry_run: if true, some Checks will be skipped. @@ -1500,7 +1474,7 @@ def DoPresubmitChecks(change, Args: change: The Change object. - committing: True if 'gcl commit' is running, False if 'gcl upload' is. + committing: True if 'git cl land' is running, False if 'git cl upload' is. verbose: Prints debug info. output_stream: A stream to write output from presubmit tests to. input_stream: A stream to read input from the user. diff --git a/tests/trychange_unittest.py b/tests/trychange_unittest.py deleted file mode 100755 index c4917c0f5..000000000 --- a/tests/trychange_unittest.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Unit tests for trychange.py.""" - -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from testing_support.super_mox import SuperMoxTestBase - -import subprocess2 -import trychange - - -class TryChangeTestsBase(SuperMoxTestBase): - """Setups and tear downs the mocks but doesn't test anything as-is.""" - def setUp(self): - SuperMoxTestBase.setUp(self) - self.mox.StubOutWithMock(subprocess2, 'communicate') - self.mox.StubOutWithMock(trychange, 'RunGit') - self.mox.StubOutWithMock(trychange.scm.GIT, 'Capture') - self.mox.StubOutWithMock(trychange.scm.GIT, 'GenerateDiff') - self.mox.StubOutWithMock(trychange.scm.GIT, 'GetCheckoutRoot') - self.mox.StubOutWithMock(trychange.scm.GIT, 'GetEmail') - self.mox.StubOutWithMock(trychange.scm.GIT, 'GetPatchName') - self.mox.StubOutWithMock(trychange.scm.GIT, 'GetUpstreamBranch') - self.mox.StubOutWithMock(trychange.scm.SVN, 'GenerateDiff') - self.mox.StubOutWithMock(trychange.scm.SVN, 'GetCheckoutRoot') - self.mox.StubOutWithMock(trychange.scm.SVN, 'GetEmail') - self.fake_root = self.Dir() - self.expected_files = ['foo.txt', 'bar.txt'] - self.options = trychange.optparse.Values() - self.options.files = self.expected_files - self.options.diff = None - self.options.name = None - self.options.email = None - self.options.exclude = [] - - -class TryChangeUnittest(TryChangeTestsBase): - """General trychange.py tests.""" - def testMembersChanged(self): - members = [ - 'DieWithError', 'EPILOG', 'Escape', 'GIT', 'GIT_PATCH_DIR_BASENAME', - 'GetMungedDiff', 'GuessVCS', 'GIT_BRANCH_FILE', - 'HELP_STRING', 'Error', 'InvalidScript', 'NoTryServerAccess', - 'OptionParser', 'PrintSuccess', - 'RunCommand', 'RunGit', 'SCM', 'SVN', 'TryChange', 'USAGE', 'contextlib', - 'datetime', 'errno', 'fix_encoding', 'gcl', 'gclient_utils', - 'gerrit_util', 'gen_parser', - 'getpass', 'itertools', 'json', 'logging', 'optparse', 'os', 'posixpath', - 're', 'scm', 'shutil', 'subprocess2', 'sys', 'tempfile', 'urllib', - 'urllib2', 'urlparse'] - # If this test fails, you should add the relevant test. - self.compareMembers(trychange, members) - - -class TryChangeSimpleTest(unittest.TestCase): - # Doesn't require supermox to run. - def test_flags(self): - cmd = [ - '--bot', 'bot1,bot2', - '--testfilter', 'test1', - '--testfilter', 'test2', - '--user', 'joe', - '--email', 'joe@example.com', - ] - options, args = trychange.gen_parser(None).parse_args(cmd) - self.assertEquals([], args) - # pylint: disable=W0212 - bot_spec = trychange._ParseBotList(options.bot, options.testfilter) - if options.testfilter: - bot_spec = trychange._ApplyTestFilter(options.testfilter, bot_spec) - values = trychange._ParseSendChangeOptions(bot_spec, options) - self.assertEquals( - [ - ('user', 'joe'), - ('name', None), - ('email', 'joe@example.com'), - ('bot', 'bot1:test1,test2'), - ('bot', 'bot2:test1,test2'), - ], - values) - - def test_flags_bad_combination(self): - cmd = [ - '--bot', 'bot1:test1', - '--testfilter', 'test2', - ] - options, args = trychange.gen_parser(None).parse_args(cmd) - self.assertEquals([], args) - try: - # pylint: disable=W0212 - trychange._ParseBotList(options.bot, options.testfilter) - self.fail() - except ValueError: - pass - - -class SVNUnittest(TryChangeTestsBase): - """trychange.SVN tests.""" - def testMembersChanged(self): - members = [ - 'AutomagicalSettings', 'CaptureStatus', 'GetCodeReviewSetting', - 'ReadRootFile', 'GenerateDiff', 'GetFileNames', 'files', 'file_tuples', - ] - # If this test fails, you should add the relevant test. - self.compareMembers(trychange.SVN, members) - - def testBasic(self): - # pylint: disable=E1103 - trychange.os.path.abspath(self.fake_root).AndReturn(self.fake_root) - trychange.scm.SVN.GetCheckoutRoot(self.fake_root).AndReturn(self.fake_root) - trychange.scm.SVN.GenerateDiff(['foo.txt', 'bar.txt'], - self.fake_root, - full_move=True, - revision=None).AndReturn('A diff') - trychange.scm.SVN.GetEmail(self.fake_root).AndReturn('georges@example.com') - self.mox.ReplayAll() - svn = trychange.SVN(self.options, self.fake_root, self.options.files) - self.assertEqual(svn.GetFileNames(), self.expected_files) - self.assertEqual(svn.checkout_root, self.fake_root) - self.assertEqual(svn.GenerateDiff(), 'A diff') - - -class GITUnittest(TryChangeTestsBase): - """trychange.GIT tests.""" - def testMembersChanged(self): - members = [ - 'AutomagicalSettings', 'CaptureStatus', 'GetCodeReviewSetting', - 'ReadRootFile', 'GenerateDiff', 'GetFileNames', 'files', 'file_tuples', - ] - # If this test fails, you should add the relevant test. - self.compareMembers(trychange.GIT, members) - - def testBasic(self): - # pylint: disable=E1103 - trychange.os.path.abspath(self.fake_root).AndReturn(self.fake_root) - trychange.scm.GIT.GetCheckoutRoot(self.fake_root).AndReturn(self.fake_root) - trychange.scm.GIT.GetUpstreamBranch(self.fake_root).AndReturn('somewhere') - trychange.RunGit(['diff-index', 'HEAD']) - trychange.scm.GIT.GenerateDiff(self.fake_root, - full_move=True, - files=['foo.txt', 'bar.txt'], - branch='somewhere').AndReturn('A diff') - trychange.scm.GIT.GetPatchName(self.fake_root).AndReturn('bleh-1233') - trychange.scm.GIT.GetEmail(self.fake_root).AndReturn('georges@example.com') - self.mox.ReplayAll() - git = trychange.GIT(self.options, self.fake_root, self.options.files) - self.assertEqual(git.GetFileNames(), self.expected_files) - self.assertEqual(git.checkout_root, self.fake_root) - self.assertEqual(git.GenerateDiff(), 'A diff') - - -if __name__ == '__main__': - unittest.main() diff --git a/trychange.py b/trychange.py deleted file mode 100755 index e752ae9ed..000000000 --- a/trychange.py +++ /dev/null @@ -1,1265 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Client-side script to send a try job to the try server. It communicates to -the try server by either writting to a svn/git repository or by directly -connecting to the server by HTTP. -""" - -import contextlib -import datetime -import errno -import getpass -import itertools -import json -import logging -import optparse -import os -import posixpath -import re -import shutil -import sys -import tempfile -import urllib -import urllib2 -import urlparse - -import fix_encoding -import gcl -import gclient_utils -import gerrit_util -import scm -import subprocess2 - - -__version__ = '1.2' - - -# Constants -HELP_STRING = "Sorry, Tryserver is not available." -USAGE = r"""%prog [options] - -Client-side script to send a try job to the try server. It communicates to -the try server by either writting to a svn repository or by directly connecting -to the server by HTTP.""" - -EPILOG = """ -Examples: - Send a patch directly from rietveld: - %(prog)s -R codereview.chromium.org/1337 - --email recipient@example.com --root src - - Try a change against a particular revision: - %(prog)s -r 123 - - Try a change including changes to a sub repository: - %(prog)s -s third_party/WebKit - - A git patch off a web site (git inserts a/ and b/) and fix the base dir: - %(prog)s --url http://url/to/patch.diff --patchlevel 1 --root src - - Use svn to store the try job, specify an alternate email address and use a - premade diff file on the local drive: - %(prog)s --email user@example.com - --svn_repo svn://svn.chromium.org/chrome-try/try --diff foo.diff - - Running only on a 'mac' slave with revision 123 and clobber first; specify - manually the 3 source files to use for the try job: - %(prog)s --bot mac --revision 123 --clobber -f src/a.cc -f src/a.h - -f include/b.h -""" - -GIT_PATCH_DIR_BASENAME = os.path.join('git-try', 'patches-git') -GIT_BRANCH_FILE = 'ref' -_GIT_PUSH_ATTEMPTS = 3 - -def DieWithError(message): - print >> sys.stderr, message - sys.exit(1) - - -def RunCommand(args, error_ok=False, error_message=None, **kwargs): - try: - return subprocess2.check_output(args, shell=False, **kwargs) - except subprocess2.CalledProcessError, e: - if not error_ok: - DieWithError( - 'Command "%s" failed.\n%s' % ( - ' '.join(args), error_message or e.stdout or '')) - return e.stdout - - -def RunGit(args, **kwargs): - """Returns stdout.""" - return RunCommand(['git'] + args, **kwargs) - -class Error(Exception): - """An error during a try job submission. - - For this error, trychange.py does not display stack trace, only message - """ - -class InvalidScript(Error): - def __str__(self): - return self.args[0] + '\n' + HELP_STRING - - -class NoTryServerAccess(Error): - def __str__(self): - return self.args[0] + '\n' + HELP_STRING - -def Escape(name): - """Escapes characters that could interfere with the file system or try job - parsing. - """ - return re.sub(r'[^\w#-]', '_', name) - - -class SCM(object): - """Simplistic base class to implement one function: ProcessOptions.""" - def __init__(self, options, path, file_list): - items = path.split('@') - assert len(items) <= 2 - self.checkout_root = os.path.abspath(items[0]) - items.append(None) - self.diff_against = items[1] - self.options = options - # Lazy-load file list from the SCM unless files were specified in options. - self._files = None - self._file_tuples = None - if file_list: - self._files = file_list - self._file_tuples = [('M', f) for f in self.files] - self.options.files = None - self.codereview_settings = None - self.codereview_settings_file = 'codereview.settings' - self.toplevel_root = None - - def GetFileNames(self): - """Return the list of files in the diff.""" - return self.files - - def GetCodeReviewSetting(self, key): - """Returns a value for the given key for this repository. - - Uses gcl-style settings from the repository. - """ - if gcl: - gcl_setting = gcl.GetCodeReviewSetting(key) - if gcl_setting != '': - return gcl_setting - if self.codereview_settings is None: - self.codereview_settings = {} - settings_file = self.ReadRootFile(self.codereview_settings_file) - if settings_file: - for line in settings_file.splitlines(): - if not line or line.lstrip().startswith('#'): - continue - k, v = line.split(":", 1) - self.codereview_settings[k.strip()] = v.strip() - return self.codereview_settings.get(key, '') - - def _GclStyleSettings(self): - """Set default settings based on the gcl-style settings from the repository. - - The settings in the self.options object will only be set if no previous - value exists (i.e. command line flags to the try command will override the - settings in codereview.settings). - """ - settings = { - 'port': self.GetCodeReviewSetting('TRYSERVER_HTTP_PORT'), - 'host': self.GetCodeReviewSetting('TRYSERVER_HTTP_HOST'), - 'svn_repo': self.GetCodeReviewSetting('TRYSERVER_SVN_URL'), - 'gerrit_url': self.GetCodeReviewSetting('TRYSERVER_GERRIT_URL'), - 'git_repo': self.GetCodeReviewSetting('TRYSERVER_GIT_URL'), - 'project': self.GetCodeReviewSetting('TRYSERVER_PROJECT'), - # Primarily for revision=auto - 'revision': self.GetCodeReviewSetting('TRYSERVER_REVISION'), - 'root': self.GetCodeReviewSetting('TRYSERVER_ROOT'), - 'patchlevel': self.GetCodeReviewSetting('TRYSERVER_PATCHLEVEL'), - } - logging.info('\n'.join(['%s: %s' % (k, v) - for (k, v) in settings.iteritems() if v])) - for (k, v) in settings.iteritems(): - # Avoid overwriting options already set using command line flags. - if v and getattr(self.options, k) is None: - setattr(self.options, k, v) - - def AutomagicalSettings(self): - """Determines settings based on supported code review and checkout tools. - """ - # Try to find gclient or repo root first. - if not self.options.no_search: - self.toplevel_root = gclient_utils.FindGclientRoot(self.checkout_root) - if self.toplevel_root: - logging.info('Found .gclient at %s' % self.toplevel_root) - else: - self.toplevel_root = gclient_utils.FindFileUpwards( - os.path.join('..', '.repo'), self.checkout_root) - if self.toplevel_root: - logging.info('Found .repo dir at %s' - % os.path.dirname(self.toplevel_root)) - - # Parse TRYSERVER_* settings from codereview.settings before falling back - # on setting self.options.root manually further down. Otherwise - # TRYSERVER_ROOT would never be used in codereview.settings. - self._GclStyleSettings() - - if self.toplevel_root and not self.options.root: - assert os.path.abspath(self.toplevel_root) == self.toplevel_root - self.options.root = gclient_utils.PathDifference(self.toplevel_root, - self.checkout_root) - else: - self._GclStyleSettings() - - def ReadRootFile(self, filename): - cur = self.checkout_root - root = self.toplevel_root or self.checkout_root - - assert cur.startswith(root), (root, cur) - while cur.startswith(root): - filepath = os.path.join(cur, filename) - if os.path.isfile(filepath): - logging.info('Found %s at %s' % (filename, cur)) - return gclient_utils.FileRead(filepath) - cur = os.path.dirname(cur) - logging.warning('Didn\'t find %s' % filename) - return None - - def _SetFileTuples(self, file_tuples): - excluded = ['!', '?', 'X', ' ', '~'] - def Excluded(f): - if f[0][0] in excluded: - return True - for r in self.options.exclude: - if re.search(r, f[1]): - logging.info('Ignoring "%s"' % f[1]) - return True - return False - - self._file_tuples = [f for f in file_tuples if not Excluded(f)] - self._files = [f[1] for f in self._file_tuples] - - def CaptureStatus(self): - """Returns the 'svn status' emulated output as an array of (status, file) - tuples.""" - raise NotImplementedError( - "abstract method -- subclass %s must override" % self.__class__) - - @property - def files(self): - if self._files is None: - self._SetFileTuples(self.CaptureStatus()) - return self._files - - @property - def file_tuples(self): - if self._file_tuples is None: - self._SetFileTuples(self.CaptureStatus()) - return self._file_tuples - - -class SVN(SCM): - """Gathers the options and diff for a subversion checkout.""" - def __init__(self, *args, **kwargs): - SCM.__init__(self, *args, **kwargs) - self.checkout_root = scm.SVN.GetCheckoutRoot(self.checkout_root) - if not self.options.email: - # Assumes the svn credential is an email address. - self.options.email = scm.SVN.GetEmail(self.checkout_root) - logging.info("SVN(%s)" % self.checkout_root) - - def ReadRootFile(self, filename): - data = SCM.ReadRootFile(self, filename) - if data: - return data - - # Try to search on the subversion repository for the file. - if not gcl: - return None - data = gcl.GetCachedFile(filename) - logging.debug('%s:\n%s' % (filename, data)) - return data - - def CaptureStatus(self): - return scm.SVN.CaptureStatus(None, self.checkout_root) - - def GenerateDiff(self): - """Returns a string containing the diff for the given file list. - - The files in the list should either be absolute paths or relative to the - given root. - """ - return scm.SVN.GenerateDiff(self.files, self.checkout_root, full_move=True, - revision=self.diff_against) - - -class GIT(SCM): - """Gathers the options and diff for a git checkout.""" - def __init__(self, *args, **kwargs): - SCM.__init__(self, *args, **kwargs) - self.checkout_root = scm.GIT.GetCheckoutRoot(self.checkout_root) - if not self.options.name: - self.options.name = scm.GIT.GetPatchName(self.checkout_root) - if not self.options.email: - self.options.email = scm.GIT.GetEmail(self.checkout_root) - if not self.diff_against: - self.diff_against = scm.GIT.GetUpstreamBranch(self.checkout_root) - if not self.diff_against: - raise NoTryServerAccess( - "Unable to determine default branch to diff against. " - "Verify this branch is set up to track another" - "(via the --track argument to \"git checkout -b ...\"") - logging.info("GIT(%s)" % self.checkout_root) - - def CaptureStatus(self): - return scm.GIT.CaptureStatus( - [], - self.checkout_root.replace(os.sep, '/'), - self.diff_against) - - def GenerateDiff(self): - if RunGit(['diff-index', 'HEAD']): - print 'Cannot try with a dirty tree. You must commit locally first.' - return None - return scm.GIT.GenerateDiff( - self.checkout_root, - files=self.files, - full_move=True, - branch=self.diff_against) - - -def _ParseBotList(botlist, testfilter): - """Parses bot configurations from a list of strings.""" - bots = [] - if testfilter: - for bot in itertools.chain.from_iterable(botspec.split(',') - for botspec in botlist): - tests = set() - if ':' in bot: - if bot.endswith(':compile'): - tests |= set(['compile']) - else: - raise ValueError( - 'Can\'t use both --testfilter and --bot builder:test formats ' - 'at the same time') - - bots.append((bot, tests)) - else: - for botspec in botlist: - botname = botspec.split(':')[0] - tests = set() - if ':' in botspec: - tests |= set(filter(None, botspec.split(':')[1].split(','))) - bots.append((botname, tests)) - return bots - - -def _ApplyTestFilter(testfilter, bot_spec): - """Applies testfilter from CLI. - - Specifying a testfilter strips off any builder-specified tests (except for - compile). - """ - if testfilter: - return [(botname, set(testfilter) | (tests & set(['compile']))) - for botname, tests in bot_spec] - else: - return bot_spec - - -def _GenTSBotSpec(checkouts, change, changed_files, options): - bot_spec = [] - # Get try slaves from PRESUBMIT.py files if not specified. - # Even if the diff comes from options.url, use the local checkout for bot - # selection. - try: - import presubmit_support - root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py') - if not change: - if not changed_files: - changed_files = checkouts[0].file_tuples - change = presubmit_support.Change(options.name, - '', - checkouts[0].checkout_root, - changed_files, - options.issue, - options.patchset, - options.email) - masters = presubmit_support.DoGetTryMasters( - change, - checkouts[0].GetFileNames(), - checkouts[0].checkout_root, - root_presubmit, - options.project, - options.verbose, - sys.stdout) - - # Compatibility for old checkouts and bots that were on tryserver.chromium. - try_bots = masters.get('tryserver.chromium', []) - - # Compatibility for checkouts that are not using tryserver.chromium - # but are stuck with git-try or gcl-try. - if not try_bots and len(masters) == 1: - try_bots = masters.values()[0] - - if try_bots: - old_style = filter(lambda x: isinstance(x, basestring), try_bots) - new_style = filter(lambda x: isinstance(x, tuple), try_bots) - - # _ParseBotList's testfilter is set to None otherwise it will complain. - bot_spec = _ApplyTestFilter(options.testfilter, - _ParseBotList(old_style, None)) - - bot_spec.extend(_ApplyTestFilter(options.testfilter, new_style)) - - except ImportError: - pass - - return bot_spec - - -def _ParseSendChangeOptions(bot_spec, options): - """Parse common options passed to _SendChangeHTTP, _SendChangeSVN and - _SendChangeGit. - """ - values = [ - ('user', options.user), - ('name', options.name), - ] - # A list of options to copy. - optional_values = ( - 'email', - 'revision', - 'root', - 'patchlevel', - 'issue', - 'patchset', - 'target', - 'project', - ) - for option_name in optional_values: - value = getattr(options, option_name) - if value: - values.append((option_name, value)) - - # Not putting clobber to optional_names - # because it used to have lower-case 'true'. - if options.clobber: - values.append(('clobber', 'true')) - - for bot, tests in bot_spec: - values.append(('bot', ('%s:%s' % (bot, ','.join(tests))))) - - return values - - -def _SendChangeHTTP(bot_spec, options): - """Send a change to the try server using the HTTP protocol.""" - if not options.host: - raise NoTryServerAccess('Please use the --host option to specify the try ' - 'server host to connect to.') - if not options.port: - raise NoTryServerAccess('Please use the --port option to specify the try ' - 'server port to connect to.') - - values = _ParseSendChangeOptions(bot_spec, options) - values.append(('patch', options.diff)) - - url = 'http://%s:%s/send_try_patch' % (options.host, options.port) - - logging.info('Sending by HTTP') - logging.info(''.join("%s=%s\n" % (k, v) for k, v in values)) - logging.info(url) - logging.info(options.diff) - if options.dry_run: - return - - try: - logging.info('Opening connection...') - connection = urllib2.urlopen(url, urllib.urlencode(values)) - logging.info('Done') - except IOError, e: - logging.info(str(e)) - if bot_spec and len(e.args) > 2 and e.args[2] == 'got a bad status line': - raise NoTryServerAccess('%s is unaccessible. Bad --bot argument?' % url) - else: - raise NoTryServerAccess('%s is unaccessible. Reason: %s' % (url, - str(e.args))) - if not connection: - raise NoTryServerAccess('%s is unaccessible.' % url) - logging.info('Reading response...') - response = connection.read() - logging.info('Done') - if response != 'OK': - raise NoTryServerAccess('%s is unaccessible. Got:\n%s' % (url, response)) - - PrintSuccess(bot_spec, options) - -@contextlib.contextmanager -def _TempFilename(name, contents=None): - """Create a temporary directory, append the specified name and yield. - - In contrast to NamedTemporaryFile, does not keep the file open. - Deletes the file on __exit__. - """ - temp_dir = tempfile.mkdtemp(prefix=name) - try: - path = os.path.join(temp_dir, name) - if contents is not None: - with open(path, 'wb') as f: - f.write(contents) - yield path - finally: - shutil.rmtree(temp_dir, True) - - -@contextlib.contextmanager -def _PrepareDescriptionAndPatchFiles(description, options): - """Creates temporary files with description and patch. - - __enter__ called on the return value returns a tuple of patch_filename and - description_filename. - - Args: - description: contents of description file. - options: patchset options object. Must have attributes: user, - name (of patch) and diff (contents of patch). - """ - current_time = str(datetime.datetime.now()).replace(':', '.') - patch_basename = '%s.%s.%s.diff' % (Escape(options.user), - Escape(options.name), current_time) - with _TempFilename('description', description) as description_filename: - with _TempFilename(patch_basename, options.diff) as patch_filename: - yield patch_filename, description_filename - - -def _SendChangeSVN(bot_spec, options): - """Send a change to the try server by committing a diff file on a subversion - server.""" - if not options.svn_repo: - raise NoTryServerAccess('Please use the --svn_repo option to specify the' - ' try server svn repository to connect to.') - - values = _ParseSendChangeOptions(bot_spec, options) - description = ''.join("%s=%s\n" % (k, v) for k, v in values) - logging.info('Sending by SVN') - logging.info(description) - logging.info(options.svn_repo) - logging.info(options.diff) - if options.dry_run: - return - - with _PrepareDescriptionAndPatchFiles(description, options) as ( - patch_filename, description_filename): - if sys.platform == "cygwin": - # Small chromium-specific issue here: - # git-try uses /usr/bin/python on cygwin but svn.bat will be used - # instead of /usr/bin/svn by default. That causes bad things(tm) since - # Windows' svn.exe has no clue about cygwin paths. Hence force to use - # the cygwin version in this particular context. - exe = "/usr/bin/svn" - else: - exe = "svn" - patch_dir = os.path.dirname(patch_filename) - command = [exe, 'import', '-q', patch_dir, options.svn_repo, '--file', - description_filename] - if scm.SVN.AssertVersion("1.5")[0]: - command.append('--no-ignore') - - try: - subprocess2.check_call(command) - except subprocess2.CalledProcessError, e: - raise NoTryServerAccess(str(e)) - - PrintSuccess(bot_spec, options) - -def _GetPatchGitRepo(git_url): - """Gets a path to a Git repo with patches. - - Stores patches in .git/git-try/patches-git directory, a git repo. If it - doesn't exist yet or its origin URL is different, cleans up and clones it. - If it existed before, then pulls changes. - - Does not support SVN repo. - - Returns a path to the directory with patches. - """ - git_dir = scm.GIT.GetGitDir(os.getcwd()) - patch_dir = os.path.join(git_dir, GIT_PATCH_DIR_BASENAME) - - logging.info('Looking for git repo for patches') - # Is there already a repo with the expected url or should we clone? - clone = True - if os.path.exists(patch_dir) and scm.GIT.IsInsideWorkTree(patch_dir): - existing_url = scm.GIT.Capture( - ['config', '--local', 'remote.origin.url'], - cwd=patch_dir) - clone = existing_url != git_url - - if clone: - if os.path.exists(patch_dir): - logging.info('Cleaning up') - shutil.rmtree(patch_dir, True) - logging.info('Cloning patch repo') - scm.GIT.Capture(['clone', git_url, GIT_PATCH_DIR_BASENAME], cwd=git_dir) - email = scm.GIT.GetEmail(cwd=os.getcwd()) - scm.GIT.Capture(['config', '--local', 'user.email', email], cwd=patch_dir) - else: - if scm.GIT.IsWorkTreeDirty(patch_dir): - logging.info('Work dir is dirty: hard reset!') - scm.GIT.Capture(['reset', '--hard'], cwd=patch_dir) - logging.info('Updating patch repo') - scm.GIT.Capture(['pull', 'origin', 'master'], cwd=patch_dir) - - return os.path.abspath(patch_dir) - - -def _SendChangeGit(bot_spec, options): - """Sends a change to the try server by committing a diff file to a GIT repo. - - Creates a temp orphan branch, commits patch.diff, creates a ref pointing to - that commit, deletes the temp branch, checks master out, adds 'ref' file - containing the name of the new ref, pushes master and the ref to the origin. - - TODO: instead of creating a temp branch, use git-commit-tree. - """ - - if not options.git_repo: - raise NoTryServerAccess('Please use the --git_repo option to specify the ' - 'try server git repository to connect to.') - - values = _ParseSendChangeOptions(bot_spec, options) - comment_subject = '%s.%s' % (options.user, options.name) - comment_body = ''.join("%s=%s\n" % (k, v) for k, v in values) - description = '%s\n\n%s' % (comment_subject, comment_body) - logging.info('Sending by GIT') - logging.info(description) - logging.info(options.git_repo) - logging.info(options.diff) - if options.dry_run: - return - - patch_dir = _GetPatchGitRepo(options.git_repo) - def patch_git(*args): - return scm.GIT.Capture(list(args), cwd=patch_dir) - def add_and_commit(filename, comment_filename): - patch_git('add', filename) - patch_git('commit', '-F', comment_filename) - - assert scm.GIT.IsInsideWorkTree(patch_dir) - assert not scm.GIT.IsWorkTreeDirty(patch_dir) - - with _PrepareDescriptionAndPatchFiles(description, options) as ( - patch_filename, description_filename): - logging.info('Committing patch') - - temp_branch = 'tmp_patch' - target_ref = 'refs/patches/%s/%s' % ( - Escape(options.user), - os.path.basename(patch_filename).replace(' ','_')) - target_filename = os.path.join(patch_dir, 'patch.diff') - branch_file = os.path.join(patch_dir, GIT_BRANCH_FILE) - - patch_git('checkout', 'master') - try: - # Try deleting an existing temp branch, if any. - try: - patch_git('branch', '-D', temp_branch) - logging.debug('Deleted an existing temp branch.') - except subprocess2.CalledProcessError: - pass - # Create a new branch and put the patch there. - patch_git('checkout', '--orphan', temp_branch) - patch_git('reset') - patch_git('clean', '-f') - shutil.copyfile(patch_filename, target_filename) - add_and_commit(target_filename, description_filename) - assert not scm.GIT.IsWorkTreeDirty(patch_dir) - - # Create a ref and point it to the commit referenced by temp_branch. - patch_git('update-ref', target_ref, temp_branch) - - # Delete the temp ref. - patch_git('checkout', 'master') - patch_git('branch', '-D', temp_branch) - - # Update the branch file in the master. - def update_branch(): - with open(branch_file, 'w') as f: - f.write(target_ref) - add_and_commit(branch_file, description_filename) - - update_branch() - - # Push master and target_ref to origin. - logging.info('Pushing patch') - for attempt in xrange(_GIT_PUSH_ATTEMPTS): - try: - patch_git('push', 'origin', 'master', target_ref) - except subprocess2.CalledProcessError as e: - is_last = attempt == _GIT_PUSH_ATTEMPTS - 1 - if is_last: - raise NoTryServerAccess(str(e)) - # Fetch, reset, update branch file again. - patch_git('fetch', 'origin') - patch_git('reset', '--hard', 'origin/master') - update_branch() - except subprocess2.CalledProcessError, e: - # Restore state. - patch_git('checkout', 'master') - patch_git('reset', '--hard', 'origin/master') - raise - - PrintSuccess(bot_spec, options) - -def _SendChangeGerrit(bot_spec, options): - """Posts a try job to a Gerrit change. - - Reads Change-Id from the HEAD commit, resolves the current revision, checks - that local revision matches the uploaded one, posts a try job in form of a - message, sets Tryjob-Request label to 1. - - Gerrit message format: starts with !tryjob, optionally followed by a try job - definition in JSON format: - buildNames: list of strings specifying build names. - build_properties: a dict of build properties. - """ - - logging.info('Sending by Gerrit') - if not options.gerrit_url: - raise NoTryServerAccess('Please use --gerrit_url option to specify the ' - 'Gerrit instance url to connect to') - gerrit_host = urlparse.urlparse(options.gerrit_url).hostname - logging.debug('Gerrit host: %s' % gerrit_host) - - def GetChangeId(commmitish): - """Finds Change-ID of the HEAD commit.""" - CHANGE_ID_RGX = '^Change-Id: (I[a-f0-9]{10,})' - comment = scm.GIT.Capture(['log', '-1', commmitish, '--format=%b'], - cwd=os.getcwd()) - change_id_match = re.search(CHANGE_ID_RGX, comment, re.I | re.M) - if not change_id_match: - raise Error('Change-Id was not found in the HEAD commit. Make sure you ' - 'have a Git hook installed that generates and inserts a ' - 'Change-Id into a commit message automatically.') - change_id = change_id_match.group(1) - return change_id - - def FormatMessage(): - # Build job definition. - job_def = {} - build_properties = {} - if options.testfilter: - build_properties['testfilter'] = options.testfilter - builderNames = [builder for builder, _ in bot_spec] - if builderNames: - job_def['builderNames'] = builderNames - if build_properties: - job_def['build_properties'] = build_properties - - # Format message. - msg = '!tryjob' - if job_def: - msg = '%s %s' % (msg, json.dumps(job_def, sort_keys=True)) - return msg - - def PostTryjob(message): - logging.info('Posting gerrit message: %s' % message) - if not options.dry_run: - # Post a message and set TryJob=1 label. - try: - gerrit_util.SetReview(gerrit_host, change_id, msg=message, - labels={'Tryjob-Request': 1}) - except gerrit_util.GerritError, e: - if e.http_status == 400: - raise Error(e.message) - else: - raise - - head_sha = scm.GIT.Capture(['log', '-1', '--format=%H'], cwd=os.getcwd()) - - change_id = GetChangeId(head_sha) - - try: - # Check that the uploaded revision matches the local one. - changes = gerrit_util.GetChangeCurrentRevision(gerrit_host, change_id) - except gerrit_util.GerritAuthenticationError, e: - raise NoTryServerAccess(e.message) - - assert len(changes) <= 1, 'Multiple changes with id %s' % change_id - if not changes: - raise Error('A change %s was not found on the server. Was it uploaded?' % - change_id) - logging.debug('Found Gerrit change: %s' % changes[0]) - if changes[0]['current_revision'] != head_sha: - raise Error('Please upload your latest local changes to Gerrit.') - - # Post a try job. - message = FormatMessage() - PostTryjob(message) - change_url = urlparse.urljoin(options.gerrit_url, - '/#/c/%s' % changes[0]['_number']) - print('A try job was posted on change %s' % change_url) - -def PrintSuccess(bot_spec, options): - if not options.dry_run: - text = 'Patch \'%s\' sent to try server' % options.name - if bot_spec: - text += ': %s' % ', '.join( - '%s:%s' % (b[0], ','.join(b[1])) for b in bot_spec) - print(text) - - -def GuessVCS(options, path, file_list): - """Helper to guess the version control system. - - NOTE: Very similar to upload.GuessVCS. Doesn't look for hg since we don't - support it yet. - - This examines the path directory, guesses which SCM we're using, and - returns an instance of the appropriate class. Exit with an error if we can't - figure it out. - - Returns: - A SCM instance. Exits if the SCM can't be guessed. - """ - __pychecker__ = 'no-returnvalues' - real_path = path.split('@')[0] - logging.info("GuessVCS(%s)" % path) - # Subversion has a .svn in all working directories. - if os.path.isdir(os.path.join(real_path, '.svn')): - return SVN(options, path, file_list) - - # Git has a command to test if you're in a git tree. - # Try running it, but don't die if we don't have git installed. - try: - subprocess2.check_output( - ['git', 'rev-parse', '--is-inside-work-tree'], cwd=real_path, - stderr=subprocess2.VOID) - return GIT(options, path, file_list) - except OSError, e: - if e.errno != errno.ENOENT: - raise - except subprocess2.CalledProcessError, e: - if e.returncode != errno.ENOENT and e.returncode != 128: - # ENOENT == 2 = they don't have git installed. - # 128 = git error code when not in a repo. - logging.warning('Unexpected error code: %s' % e.returncode) - raise - raise NoTryServerAccess( - ( 'Could not guess version control system for %s.\n' - 'Are you in a working copy directory?') % path) - - -def GetMungedDiff(path_diff, diff): - # Munge paths to match svn. - changed_files = [] - for i in range(len(diff)): - if diff[i].startswith('--- ') or diff[i].startswith('+++ '): - new_file = posixpath.join(path_diff, diff[i][4:]).replace('\\', '/') - if diff[i].startswith('--- '): - file_path = new_file.split('\t')[0].strip() - if file_path.startswith('a/'): - file_path = file_path[2:] - changed_files.append(('M', file_path)) - diff[i] = diff[i][0:4] + new_file - return (diff, changed_files) - - -class OptionParser(optparse.OptionParser): - def format_epilog(self, _): - """Removes epilog formatting.""" - return self.epilog or '' - - -def gen_parser(prog): - # Parse argv - parser = OptionParser(usage=USAGE, version=__version__, prog=prog) - parser.add_option("-v", "--verbose", action="count", default=0, - help="Prints debugging infos") - group = optparse.OptionGroup(parser, "Result and status") - group.add_option("-u", "--user", default=getpass.getuser(), - help="Owner user name [default: %default]") - group.add_option("-e", "--email", - default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS', - os.environ.get('EMAIL_ADDRESS')), - help="Email address where to send the results. Use either " - "the TRYBOT_RESULTS_EMAIL_ADDRESS environment " - "variable or EMAIL_ADDRESS to set the email address " - "the try bots report results to [default: %default]") - group.add_option("-n", "--name", - help="Descriptive name of the try job") - group.add_option("--issue", type='int', - help="Update rietveld issue try job status") - group.add_option("--patchset", type='int', - help="Update rietveld issue try job status. This is " - "optional if --issue is used, In that case, the " - "latest patchset will be used.") - group.add_option("--dry_run", action='store_true', - help="Don't send the try job. This implies --verbose, so " - "it will print the diff.") - parser.add_option_group(group) - - group = optparse.OptionGroup(parser, "Try job options") - group.add_option( - "-b", "--bot", action="append", - help=("IMPORTANT: specify ONE builder per --bot flag. Use it multiple " - "times to specify multiple builders. ex: " - "'-bwin_rel:ui_tests,webkit_unit_tests -bwin_layout'. See " - "the try server waterfall for the builders name and the tests " - "available. Can also be used to specify gtest_filter, e.g. " - "-bwin_rel:base_unittests:ValuesTest.*Value")) - group.add_option("-B", "--print_bots", action="store_true", - help="Print bots we would use (e.g. from PRESUBMIT.py)" - " and exit. Do not send patch. Like --dry_run" - " but less verbose.") - group.add_option("-r", "--revision", - help="Revision to use for the try job. If 'auto' is " - "specified, it is resolved to the revision a patch is " - "generated against (Git only). Default: the " - "revision will be determined by the try server; see " - "its waterfall for more info") - group.add_option("-c", "--clobber", action="store_true", - help="Force a clobber before building; e.g. don't do an " - "incremental build") - # TODO(maruel): help="Select a specific configuration, usually 'debug' or " - # "'release'" - group.add_option("--target", help=optparse.SUPPRESS_HELP) - - group.add_option("--project", - help="Override which project to use. Projects are defined " - "server-side to define what default bot set to use") - - group.add_option( - "-t", "--testfilter", action="append", default=[], - help=("Apply a testfilter to all the selected builders. Unless the " - "builders configurations are similar, use multiple " - "--bot : arguments.")) - - parser.add_option_group(group) - - group = optparse.OptionGroup(parser, "Patch to run") - group.add_option("-f", "--file", default=[], dest="files", - metavar="FILE", action="append", - help="Use many times to list the files to include in the " - "try, relative to the repository root") - group.add_option("--diff", - help="File containing the diff to try") - group.add_option("--url", - help="Url where to grab a patch, e.g. " - "http://example.com/x.diff") - group.add_option("-R", "--rietveld_url", default="codereview.chromium.org", - metavar="URL", - help="Has 2 usages, both refer to the rietveld instance: " - "Specify which code review patch to use as the try job " - "or rietveld instance to update the try job results " - "Default:%default") - group.add_option("--root", - help="Root to use for the patch; base subdirectory for " - "patch created in a subdirectory") - group.add_option("-p", "--patchlevel", type='int', metavar="LEVEL", - help="Used as -pN parameter to patch") - group.add_option("-s", "--sub_rep", action="append", default=[], - help="Subcheckout to use in addition. This is mainly " - "useful for gclient-style checkouts. In git, checkout " - "the branch with changes first. Use @rev or " - "@branch to specify the " - "revision/branch to diff against. If no @branch is " - "given the diff will be against the upstream branch. " - "If @branch then the diff is branch..HEAD. " - "All edits must be checked in.") - group.add_option("--no_search", action="store_true", - help=("Disable automatic search for gclient or repo " - "checkout root.")) - group.add_option("-E", "--exclude", action="append", - default=['ChangeLog'], metavar='REGEXP', - help="Regexp patterns to exclude files. Default: %default") - group.add_option("--upstream_branch", action="store", - help="Specify the upstream branch to diff against in the " - "main checkout") - parser.add_option_group(group) - - group = optparse.OptionGroup(parser, "Access the try server by HTTP") - group.add_option("--use_http", - action="store_const", - const=_SendChangeHTTP, - dest="send_patch", - help="Use HTTP to talk to the try server [default]") - group.add_option("-H", "--host", - help="Host address") - group.add_option("-P", "--port", type="int", - help="HTTP port") - parser.add_option_group(group) - - group = optparse.OptionGroup(parser, "Access the try server with SVN") - group.add_option("--use_svn", - action="store_const", - const=_SendChangeSVN, - dest="send_patch", - help="Use SVN to talk to the try server") - group.add_option("-S", "--svn_repo", - metavar="SVN_URL", - help="SVN url to use to write the changes in; --use_svn is " - "implied when using --svn_repo") - parser.add_option_group(group) - - group = optparse.OptionGroup(parser, "Access the try server with Git") - group.add_option("--use_git", - action="store_const", - const=_SendChangeGit, - dest="send_patch", - help="Use GIT to talk to the try server") - group.add_option("-G", "--git_repo", - metavar="GIT_URL", - help="GIT url to use to write the changes in; --use_git is " - "implied when using --git_repo") - parser.add_option_group(group) - - group = optparse.OptionGroup(parser, "Access the try server with Gerrit") - group.add_option("--use_gerrit", - action="store_const", - const=_SendChangeGerrit, - dest="send_patch", - help="Use Gerrit to talk to the try server") - group.add_option("--gerrit_url", - metavar="GERRIT_URL", - help="Gerrit url to post a try job to; --use_gerrit is " - "implied when using --gerrit_url") - parser.add_option_group(group) - - return parser - - -def TryChange(argv, - change, - swallow_exception, - prog=None, - extra_epilog=None): - """ - Args: - argv: Arguments and options. - change: Change instance corresponding to the CL. - swallow_exception: Whether we raise or swallow exceptions. - """ - parser = gen_parser(prog) - epilog = EPILOG % { 'prog': prog } - if extra_epilog: - epilog += extra_epilog - parser.epilog = epilog - - options, args = parser.parse_args(argv) - - # If they've asked for help, give it to them - if len(args) == 1 and args[0] == 'help': - parser.print_help() - return 0 - - # If they've said something confusing, don't spawn a try job until you - # understand what they want. - if args: - parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) - - if options.dry_run: - options.verbose += 1 - - LOG_FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s' - if not swallow_exception: - if options.verbose == 0: - logging.basicConfig(level=logging.WARNING, format=LOG_FORMAT) - elif options.verbose == 1: - logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) - elif options.verbose > 1: - logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - - logging.debug(argv) - - if (options.patchlevel is not None and - (options.patchlevel < 0 or options.patchlevel > 10)): - parser.error( - 'Have you tried --port instead? You probably confused -p and -P.') - - # Strip off any @ in the user, otherwise svn gets confused. - options.user = options.user.split('@', 1)[0] - - if options.rietveld_url: - # Try to extract the review number if possible and fix the protocol. - if not '://' in options.rietveld_url: - options.rietveld_url = 'http://' + options.rietveld_url - match = re.match(r'^(.*)/(\d+)/?$', options.rietveld_url) - if match: - if options.issue or options.patchset: - parser.error('Cannot use both --issue and use a review number url') - options.issue = int(match.group(2)) - options.rietveld_url = match.group(1) - - try: - changed_files = None - # Always include os.getcwd() in the checkout settings. - path = os.getcwd() - - file_list = [] - if options.files: - file_list = options.files - elif change: - file_list = [f.LocalPath() for f in change.AffectedFiles()] - - if options.upstream_branch: - path += '@' + options.upstream_branch - # Clear file list so that the correct list will be retrieved from the - # upstream branch. - file_list = [] - - current_vcs = GuessVCS(options, path, file_list) - current_vcs.AutomagicalSettings() - options = current_vcs.options - vcs_is_git = type(current_vcs) is GIT - - # So far, git_repo doesn't work with SVN - if options.git_repo and not vcs_is_git: - parser.error('--git_repo option is supported only for GIT repositories') - - # If revision==auto, resolve it - if options.revision and options.revision.lower() == 'auto': - if not vcs_is_git: - parser.error('--revision=auto is supported only for GIT repositories') - options.revision = scm.GIT.Capture( - ['rev-parse', current_vcs.diff_against], - cwd=path) - - checkouts = [current_vcs] - for item in options.sub_rep: - # Pass file_list=None because we don't know the sub repo's file list. - checkout = GuessVCS(options, - os.path.join(current_vcs.checkout_root, item), - None) - if checkout.checkout_root in [c.checkout_root for c in checkouts]: - parser.error('Specified the root %s two times.' % - checkout.checkout_root) - checkouts.append(checkout) - - can_http = options.port and options.host - can_svn = options.svn_repo - can_git = options.git_repo - can_gerrit = options.gerrit_url - can_something = can_http or can_svn or can_git or can_gerrit - # If there was no transport selected yet, now we must have enough data to - # select one. - if not options.send_patch and not can_something: - parser.error('Please specify an access method.') - - # Convert options.diff into the content of the diff. - if options.url: - if options.files: - parser.error('You cannot specify files and --url at the same time.') - options.diff = urllib2.urlopen(options.url).read() - elif options.diff: - if options.files: - parser.error('You cannot specify files and --diff at the same time.') - options.diff = gclient_utils.FileRead(options.diff, 'rb') - elif options.issue and options.patchset is None: - # Retrieve the patch from rietveld when the diff is not specified. - # When patchset is specified, it's because it's done by gcl/git-try. - api_url = '%s/api/%d' % (options.rietveld_url, options.issue) - logging.debug(api_url) - contents = json.loads(urllib2.urlopen(api_url).read()) - options.patchset = contents['patchsets'][-1] - diff_url = ('%s/download/issue%d_%d.diff' % - (options.rietveld_url, options.issue, options.patchset)) - diff = GetMungedDiff('', urllib2.urlopen(diff_url).readlines()) - options.diff = ''.join(diff[0]) - changed_files = diff[1] - else: - # Use this as the base. - root = checkouts[0].checkout_root - diffs = [] - for checkout in checkouts: - raw_diff = checkout.GenerateDiff() - if not raw_diff: - continue - diff = raw_diff.splitlines(True) - path_diff = gclient_utils.PathDifference(root, checkout.checkout_root) - # Munge it. - diffs.extend(GetMungedDiff(path_diff, diff)[0]) - if not diffs: - logging.error('Empty or non-existant diff, exiting.') - return 1 - options.diff = ''.join(diffs) - - if not options.name: - if options.issue: - options.name = 'Issue %s' % options.issue - else: - options.name = 'Unnamed' - print('Note: use --name NAME to change the try job name.') - - if not options.email: - parser.error('Using an anonymous checkout. Please use --email or set ' - 'the TRYBOT_RESULTS_EMAIL_ADDRESS environment variable.') - print('Results will be emailed to: ' + options.email) - - if options.bot: - bot_spec = _ApplyTestFilter( - options.testfilter, _ParseBotList(options.bot, options.testfilter)) - else: - bot_spec = _GenTSBotSpec(checkouts, change, changed_files, options) - - if options.testfilter: - bot_spec = _ApplyTestFilter(options.testfilter, bot_spec) - - if any('triggered' in b[0] for b in bot_spec): - print >> sys.stderr, ( - 'ERROR You are trying to send a job to a triggered bot. This type of' - ' bot requires an\ninitial job from a parent (usually a builder). ' - 'Instead send your job to the parent.\nBot list: %s' % bot_spec) - return 1 - - if options.print_bots: - print 'Bots which would be used:' - for bot in bot_spec: - if bot[1]: - print ' %s:%s' % (bot[0], ','.join(bot[1])) - else: - print ' %s' % (bot[0]) - return 0 - - # Determine sending protocol - if options.send_patch: - # If forced. - senders = [options.send_patch] - else: - # Try sending patch using avaialble protocols - all_senders = [ - (_SendChangeHTTP, can_http), - (_SendChangeSVN, can_svn), - (_SendChangeGerrit, can_gerrit), - (_SendChangeGit, can_git), - ] - senders = [sender for sender, can in all_senders if can] - - # Send the patch. - for sender in senders: - try: - sender(bot_spec, options) - return 0 - except NoTryServerAccess: - is_last = sender == senders[-1] - if is_last: - raise - assert False, "Unreachable code" - except Error, e: - if swallow_exception: - return 1 - print >> sys.stderr, e - return 1 - except (gclient_utils.Error, subprocess2.CalledProcessError), e: - print >> sys.stderr, e - return 1 - return 0 - - -if __name__ == "__main__": - fix_encoding.fix_encoding() - sys.exit(TryChange(None, None, False))