You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
511 lines
18 KiB
Python
511 lines
18 KiB
Python
#!/usr/bin/python
|
|
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
"""Client-side script to send a try job to the try server. It communicates to
|
|
the try server by either writting to a svn repository or by directly connecting
|
|
to the server by HTTP.
|
|
"""
|
|
|
|
|
|
import datetime
|
|
import getpass
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
import urllib
|
|
|
|
import gcl
|
|
import gclient
|
|
import upload
|
|
|
|
__version__ = '1.1'
|
|
|
|
|
|
# Constants
|
|
HELP_STRING = "Sorry, Tryserver is not available."
|
|
SCRIPT_PATH = os.path.join('tools', 'tryserver', 'tryserver.py')
|
|
USAGE = r"""%prog [options]
|
|
|
|
Client-side script to send a try job to the try server. It communicates to
|
|
the try server by either writting to a svn repository or by directly connecting
|
|
to the server by HTTP.
|
|
|
|
|
|
Examples:
|
|
A git patch off a web site (git inserts a/ and b/) and fix the base dir:
|
|
%prog --url http://url/to/patch.diff --patchlevel 1 --root src
|
|
|
|
Use svn to store the try job, specify an alternate email address and use a
|
|
premade diff file on the local drive:
|
|
%prog --email user@example.com
|
|
--svn_repo svn://svn.chromium.org/chrome-try/try --diff foo.diff
|
|
|
|
Running only on a 'mac' slave with revision src@123 and clobber first; specify
|
|
manually the 3 source files to use for the try job:
|
|
%prog --bot mac --revision src@123 --clobber -f src/a.cc -f src/a.h
|
|
-f include/b.h
|
|
|
|
"""
|
|
|
|
class InvalidScript(Exception):
|
|
def __str__(self):
|
|
return self.args[0] + '\n' + HELP_STRING
|
|
|
|
|
|
class NoTryServerAccess(Exception):
|
|
def __str__(self):
|
|
return self.args[0] + '\n' + HELP_STRING
|
|
|
|
|
|
def PathDifference(root, subpath):
|
|
"""Returns the difference subpath minus root."""
|
|
if subpath.find(root) != 0:
|
|
return None
|
|
# If the root does not have a trailing \ or /, we add it so the returned path
|
|
# starts immediately after the seperator regardless of whether it is provided.
|
|
if not root.endswith(os.sep):
|
|
root += os.sep
|
|
return subpath[len(root):]
|
|
|
|
|
|
def GetSourceRoot():
|
|
"""Returns the absolute directory one level up from the repository root."""
|
|
return os.path.abspath(os.path.join(gcl.GetRepositoryRoot(), '..'))
|
|
|
|
|
|
def ExecuteTryServerScript():
|
|
"""Locates the tryserver script, executes it and returns its dictionary.
|
|
|
|
The try server script contains the repository-specific try server commands."""
|
|
script_locals = {}
|
|
try:
|
|
# gcl.GetRepositoryRoot() may throw an exception.
|
|
script_path = os.path.join(gcl.GetRepositoryRoot(), SCRIPT_PATH)
|
|
except Exception:
|
|
return script_locals
|
|
if os.path.exists(script_path):
|
|
try:
|
|
exec(gcl.ReadFile(script_path), script_locals)
|
|
except Exception, e:
|
|
# TODO(maruel): Need to specialize the exception trapper.
|
|
traceback.print_exc()
|
|
raise InvalidScript('%s is invalid.' % script_path)
|
|
return script_locals
|
|
|
|
|
|
def EscapeDot(name):
|
|
return name.replace('.', '-')
|
|
|
|
|
|
def RunCommand(command):
|
|
output, retcode = gcl.RunShellWithReturnCode(command)
|
|
if retcode:
|
|
raise NoTryServerAccess(' '.join(command) + '\nOuput:\n' + output)
|
|
return output
|
|
|
|
|
|
class SCM(object):
|
|
"""Simplistic base class to implement one function: ProcessOptions."""
|
|
def __init__(self, options):
|
|
self.options = options
|
|
|
|
def ProcessOptions(self):
|
|
raise Unimplemented
|
|
|
|
|
|
class SVN(SCM):
|
|
"""Gathers the options and diff for a subversion checkout."""
|
|
def GenerateDiff(self, files, root):
|
|
"""Returns a string containing the diff for the given file list.
|
|
|
|
The files in the list should either be absolute paths or relative to the
|
|
given root. If no root directory is provided, the repository root will be
|
|
used.
|
|
"""
|
|
previous_cwd = os.getcwd()
|
|
if root is None:
|
|
os.chdir(gcl.GetRepositoryRoot())
|
|
else:
|
|
os.chdir(root)
|
|
|
|
diff = []
|
|
for file in files:
|
|
# Use svn info output instead of os.path.isdir because the latter fails
|
|
# when the file is deleted.
|
|
if gclient.CaptureSVNInfo(file).get("Node Kind") in ("dir", "directory"):
|
|
continue
|
|
# If the user specified a custom diff command in their svn config file,
|
|
# then it'll be used when we do svn diff, which we don't want to happen
|
|
# since we want the unified diff. Using --diff-cmd=diff doesn't always
|
|
# work, since they can have another diff executable in their path that
|
|
# gives different line endings. So we use a bogus temp directory as the
|
|
# config directory, which gets around these problems.
|
|
if sys.platform.startswith("win"):
|
|
parent_dir = tempfile.gettempdir()
|
|
else:
|
|
parent_dir = sys.path[0] # tempdir is not secure.
|
|
bogus_dir = os.path.join(parent_dir, "temp_svn_config")
|
|
if not os.path.exists(bogus_dir):
|
|
os.mkdir(bogus_dir)
|
|
# Grabs the diff data.
|
|
data = gcl.RunShell(["svn", "diff", "--config-dir", bogus_dir, file])
|
|
|
|
# We know the diff will be incorrectly formatted. Fix it.
|
|
if gcl.IsSVNMoved(file):
|
|
# The file is "new" in the patch sense. Generate a homebrew diff.
|
|
# We can't use ReadFile() since it's not using binary mode.
|
|
file_handle = open(file, 'rb')
|
|
file_content = file_handle.read()
|
|
file_handle.close()
|
|
# Prepend '+ ' to every lines.
|
|
file_content = ['+ ' + i for i in file_content.splitlines(True)]
|
|
nb_lines = len(file_content)
|
|
# We need to use / since patch on unix will fail otherwise.
|
|
file = file.replace('\\', '/')
|
|
data = "Index: %s\n" % file
|
|
data += ("============================================================="
|
|
"======\n")
|
|
# Note: Should we use /dev/null instead?
|
|
data += "--- %s\n" % file
|
|
data += "+++ %s\n" % file
|
|
data += "@@ -0,0 +1,%d @@\n" % nb_lines
|
|
data += ''.join(file_content)
|
|
diff.append(data)
|
|
os.chdir(previous_cwd)
|
|
return "".join(diff)
|
|
|
|
def ProcessOptions(self):
|
|
if not self.options.diff:
|
|
# Generate the diff with svn and write it to the submit queue path. The
|
|
# files are relative to the repository root, but we need patches relative
|
|
# to one level up from there (i.e., 'src'), so adjust both the file
|
|
# paths and the root of the diff.
|
|
source_root = GetSourceRoot()
|
|
prefix = PathDifference(source_root, gcl.GetRepositoryRoot())
|
|
adjusted_paths = [os.path.join(prefix, x) for x in self.options.files]
|
|
self.options.diff = self.GenerateDiff(adjusted_paths, root=source_root)
|
|
|
|
|
|
class GIT(SCM):
|
|
"""Gathers the options and diff for a git checkout."""
|
|
def GenerateDiff(self):
|
|
"""Get the diff we'll send to the try server. We ignore the files list."""
|
|
branch = upload.RunShell(['git', 'cl', 'upstream']).strip()
|
|
diff = upload.RunShell(['git', 'diff-tree', '-p', '--no-prefix',
|
|
branch, 'HEAD']).splitlines(True)
|
|
for i in range(len(diff)):
|
|
# In the case of added files, replace /dev/null with the path to the
|
|
# file being added.
|
|
if diff[i].startswith('--- /dev/null'):
|
|
diff[i] = '--- %s' % diff[i+1][4:]
|
|
return ''.join(diff)
|
|
|
|
def GetEmail(self):
|
|
# TODO: check for errors here?
|
|
return upload.RunShell(['git', 'config', 'user.email']).strip()
|
|
|
|
def GetPatchName(self):
|
|
"""Construct a name for this patch."""
|
|
# TODO: perhaps include the hash of the current commit, to distinguish
|
|
# patches?
|
|
branch = upload.RunShell(['git', 'symbolic-ref', 'HEAD']).strip()
|
|
if not branch.startswith('refs/heads/'):
|
|
raise "Couldn't figure out branch name"
|
|
branch = branch[len('refs/heads/'):]
|
|
return branch
|
|
|
|
def ProcessOptions(self):
|
|
if not self.options.diff:
|
|
self.options.diff = self.GenerateDiff()
|
|
if not self.options.name:
|
|
self.options.name = self.GetPatchName()
|
|
if not self.options.email:
|
|
self.options.email = self.GetEmail()
|
|
|
|
|
|
def _ParseSendChangeOptions(options):
|
|
"""Parse common options passed to _SendChangeHTTP and _SendChangeSVN."""
|
|
values = {}
|
|
if options.email:
|
|
values['email'] = options.email
|
|
values['user'] = options.user
|
|
values['name'] = options.name
|
|
if options.bot:
|
|
values['bot'] = ','.join(options.bot)
|
|
if options.revision:
|
|
values['revision'] = options.revision
|
|
if options.clobber:
|
|
values['clobber'] = 'true'
|
|
if options.tests:
|
|
values['tests'] = ','.join(options.tests)
|
|
if options.root:
|
|
values['root'] = options.root
|
|
if options.patchlevel:
|
|
values['patchlevel'] = options.patchlevel
|
|
if options.issue:
|
|
values['issue'] = options.issue
|
|
if options.patchset:
|
|
values['patchset'] = options.patchset
|
|
return values
|
|
|
|
|
|
def _SendChangeHTTP(options):
|
|
"""Send a change to the try server using the HTTP protocol."""
|
|
script_locals = ExecuteTryServerScript()
|
|
|
|
if not options.host:
|
|
options.host = script_locals.get('try_server_http_host', None)
|
|
if not options.host:
|
|
raise NoTryServerAccess('Please use the --host option to specify the try '
|
|
'server host to connect to.')
|
|
if not options.port:
|
|
options.port = script_locals.get('try_server_http_port', None)
|
|
if not options.port:
|
|
raise NoTryServerAccess('Please use the --port option to specify the try '
|
|
'server port to connect to.')
|
|
|
|
values = _ParseSendChangeOptions(options)
|
|
values['patch'] = options.diff
|
|
|
|
url = 'http://%s:%s/send_try_patch' % (options.host, options.port)
|
|
proxies = None
|
|
if options.proxy:
|
|
if options.proxy.lower() == 'none':
|
|
# Effectively disable HTTP_PROXY or Internet settings proxy setup.
|
|
proxies = {}
|
|
else:
|
|
proxies = {'http': options.proxy, 'https': options.proxy}
|
|
try:
|
|
connection = urllib.urlopen(url, urllib.urlencode(values), proxies=proxies)
|
|
except IOError, e:
|
|
# TODO(thestig) this probably isn't quite right.
|
|
if values.get('bot') and e[2] == 'got a bad status line':
|
|
raise NoTryServerAccess('%s is unaccessible. Bad --bot argument?' % url)
|
|
else:
|
|
raise NoTryServerAccess('%s is unaccessible.' % url)
|
|
if not connection:
|
|
raise NoTryServerAccess('%s is unaccessible.' % url)
|
|
if connection.read() != 'OK':
|
|
raise NoTryServerAccess('%s is unaccessible.' % url)
|
|
return options.name
|
|
|
|
|
|
def _SendChangeSVN(options):
|
|
"""Send a change to the try server by committing a diff file on a subversion
|
|
server."""
|
|
script_locals = ExecuteTryServerScript()
|
|
if not options.svn_repo:
|
|
options.svn_repo = script_locals.get('try_server_svn', None)
|
|
if not options.svn_repo:
|
|
raise NoTryServerAccess('Please use the --svn_repo option to specify the'
|
|
' try server svn repository to connect to.')
|
|
|
|
values = _ParseSendChangeOptions(options)
|
|
description = ''
|
|
for (k,v) in values.iteritems():
|
|
description += "%s=%s\n" % (k,v)
|
|
|
|
# Do an empty checkout.
|
|
temp_dir = tempfile.mkdtemp()
|
|
temp_file = tempfile.NamedTemporaryFile()
|
|
temp_file_name = temp_file.name
|
|
try:
|
|
RunCommand(['svn', 'checkout', '--depth', 'empty', '--non-interactive',
|
|
options.svn_repo, temp_dir])
|
|
# TODO(maruel): Use a subdirectory per user?
|
|
current_time = str(datetime.datetime.now()).replace(':', '.')
|
|
file_name = (EscapeDot(options.user) + '.' + EscapeDot(options.name) +
|
|
'.%s.diff' % current_time)
|
|
full_path = os.path.join(temp_dir, file_name)
|
|
full_url = options.svn_repo + '/' + file_name
|
|
file_found = False
|
|
try:
|
|
RunCommand(['svn', 'ls', '--non-interactive', full_url])
|
|
file_found = True
|
|
except NoTryServerAccess:
|
|
pass
|
|
if file_found:
|
|
# The file already exists in the repo. Note that commiting a file is a
|
|
# no-op if the file's content (the diff) is not modified. This is why the
|
|
# file name contains the date and time.
|
|
RunCommand(['svn', 'update', '--non-interactive', full_path])
|
|
file = open(full_path, 'wb')
|
|
file.write(options.diff)
|
|
file.close()
|
|
else:
|
|
# Add the file to the repo
|
|
file = open(full_path, 'wb')
|
|
file.write(options.diff)
|
|
file.close()
|
|
RunCommand(["svn", "add", '--non-interactive', full_path])
|
|
temp_file.write(description)
|
|
temp_file.flush()
|
|
RunCommand(["svn", "commit", '--non-interactive', full_path, '--file',
|
|
temp_file_name])
|
|
finally:
|
|
temp_file.close()
|
|
shutil.rmtree(temp_dir, True)
|
|
return options.name
|
|
|
|
|
|
def GuessVCS(options):
|
|
"""Helper to guess the version control system.
|
|
|
|
NOTE: Very similar to upload.GuessVCS. Doesn't look for hg since we don't
|
|
support it yet.
|
|
|
|
This examines the current directory, guesses which SCM we're using, and
|
|
returns an instance of the appropriate class. Exit with an error if we can't
|
|
figure it out.
|
|
|
|
Returns:
|
|
A SCM instance. Exits if the SCM can't be guessed.
|
|
"""
|
|
# Subversion has a .svn in all working directories.
|
|
if os.path.isdir('.svn'):
|
|
logging.info("Guessed VCS = Subversion")
|
|
return SVN(options)
|
|
|
|
# Git has a command to test if you're in a git tree.
|
|
# Try running it, but don't die if we don't have git installed.
|
|
try:
|
|
out, returncode = gcl.RunShellWithReturnCode(["git", "rev-parse",
|
|
"--is-inside-work-tree"])
|
|
if returncode == 0:
|
|
logging.info("Guessed VCS = Git")
|
|
return GIT(options)
|
|
except OSError, (errno, message):
|
|
if errno != 2: # ENOENT -- they don't have git installed.
|
|
raise
|
|
|
|
raise NoTryServerAccess("Could not guess version control system. "
|
|
"Are you in a working copy directory?")
|
|
|
|
|
|
def TryChange(argv,
|
|
file_list,
|
|
swallow_exception,
|
|
prog=None):
|
|
# Parse argv
|
|
parser = optparse.OptionParser(usage=USAGE,
|
|
version=__version__,
|
|
prog=prog)
|
|
|
|
group = optparse.OptionGroup(parser, "Result and status")
|
|
group.add_option("-u", "--user", default=getpass.getuser(),
|
|
help="Owner user name [default: %default]")
|
|
group.add_option("-e", "--email", default=os.environ.get('EMAIL_ADDRESS'),
|
|
help="Email address where to send the results. Use the "
|
|
"EMAIL_ADDRESS environment variable to set the default "
|
|
"email address [default: %default]")
|
|
group.add_option("-n", "--name", default='Unnamed',
|
|
help="Descriptive name of the try job")
|
|
group.add_option("--issue", type='int',
|
|
help="Update rietveld issue try job status")
|
|
group.add_option("--patchset", type='int',
|
|
help="Update rietveld issue try job status")
|
|
parser.add_option_group(group)
|
|
|
|
group = optparse.OptionGroup(parser, "Try job options")
|
|
group.add_option("-b", "--bot", action="append",
|
|
help="Only use specifics build slaves, ex: '--bot win' to "
|
|
"run the try job only on the 'win' slave; see the try "
|
|
"server watefall for the slave's name")
|
|
group.add_option("-r", "--revision",
|
|
help="Revision to use for the try job; default: the "
|
|
"revision will be determined by the try server; see "
|
|
"its waterfall for more info")
|
|
group.add_option("-c", "--clobber", action="store_true",
|
|
help="Force a clobber before building; e.g. don't do an "
|
|
"incremental build")
|
|
# Override the list of tests to run, use multiple times to list many tests
|
|
# (or comma separated)
|
|
group.add_option("-t", "--tests", action="append",
|
|
help=optparse.SUPPRESS_HELP)
|
|
parser.add_option_group(group)
|
|
|
|
group = optparse.OptionGroup(parser, "Patch to run")
|
|
group.add_option("-f", "--file", default=file_list, dest="files",
|
|
metavar="FILE", action="append",
|
|
help="Use many times to list the files to include in the "
|
|
"try, relative to the repository root")
|
|
group.add_option("--diff",
|
|
help="File containing the diff to try")
|
|
group.add_option("--url",
|
|
help="Url where to grab a patch")
|
|
group.add_option("--root",
|
|
help="Root to use for the patch; base subdirectory for "
|
|
"patch created in a subdirectory")
|
|
group.add_option("--patchlevel", type='int', metavar="LEVEL",
|
|
help="Used as -pN parameter to patch")
|
|
parser.add_option_group(group)
|
|
|
|
group = optparse.OptionGroup(parser, "Access the try server by HTTP")
|
|
group.add_option("--use_http", action="store_const", const=_SendChangeHTTP,
|
|
dest="send_patch", default=_SendChangeHTTP,
|
|
help="Use HTTP to talk to the try server [default]")
|
|
group.add_option("--host",
|
|
help="Host address")
|
|
group.add_option("--port",
|
|
help="HTTP port")
|
|
group.add_option("--proxy",
|
|
help="HTTP proxy")
|
|
parser.add_option_group(group)
|
|
|
|
group = optparse.OptionGroup(parser, "Access the try server with SVN")
|
|
group.add_option("--use_svn", action="store_const", const=_SendChangeSVN,
|
|
dest="send_patch",
|
|
help="Use SVN to talk to the try server")
|
|
group.add_option("--svn_repo", metavar="SVN_URL",
|
|
help="SVN url to use to write the changes in; --use_svn is "
|
|
"implied when using --svn_repo")
|
|
parser.add_option_group(group)
|
|
|
|
options, args = parser.parse_args(argv)
|
|
# Switch the default accordingly.
|
|
if options.svn_repo:
|
|
options.send_patch = _SendChangeSVN
|
|
|
|
if len(args) == 1 and args[0] == 'help':
|
|
parser.print_help()
|
|
if (not options.files and (not options.issue and options.patchset) and
|
|
not options.diff and not options.url):
|
|
# TODO(maruel): It should just try the modified files showing up in a
|
|
# svn status.
|
|
print "Nothing to try, changelist is empty."
|
|
return
|
|
|
|
try:
|
|
# Convert options.diff into the content of the diff.
|
|
if options.url:
|
|
options.diff = urllib.urlopen(options.url).read()
|
|
elif options.diff:
|
|
options.diff = gcl.ReadFile(options.diff)
|
|
# Process the VCS in any case at least to retrieve the email address.
|
|
try:
|
|
options.scm = GuessVCS(options)
|
|
options.scm.ProcessOptions()
|
|
except NoTryServerAccess, e:
|
|
# If we got the diff, we don't care.
|
|
if not options.diff:
|
|
raise
|
|
|
|
# Send the patch.
|
|
patch_name = options.send_patch(options)
|
|
print 'Patch \'%s\' sent to try server.' % patch_name
|
|
if patch_name == 'Unnamed':
|
|
print "Note: use --name NAME to change the try's name."
|
|
except (InvalidScript, NoTryServerAccess), e:
|
|
if swallow_exception:
|
|
return
|
|
print e
|
|
|
|
|
|
if __name__ == "__main__":
|
|
TryChange(None, None, False)
|