#!/usr/bin/env python # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Applies an issue from Rietveld. """ import json import logging import optparse import os import subprocess import sys import urllib2 import annotated_gclient import auth import checkout import fix_encoding import gclient_utils import rietveld import scm BASE_DIR = os.path.dirname(os.path.abspath(__file__)) RETURN_CODE_OK = 0 RETURN_CODE_OTHER_FAILURE = 1 # any other failure, likely patch apply one. RETURN_CODE_ARGPARSE_FAILURE = 2 # default in python. RETURN_CODE_INFRA_FAILURE = 3 # considered as infra failure. class Unbuffered(object): """Disable buffering on a file object.""" def __init__(self, stream): self.stream = stream def write(self, data): self.stream.write(data) self.stream.flush() def __getattr__(self, attr): return getattr(self.stream, attr) def _get_arg_parser(): parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option( '-v', '--verbose', action='count', default=0, help='Prints debugging infos') parser.add_option( '-e', '--email', help='Email address to access rietveld. If not specified, anonymous ' 'access will be used.') parser.add_option( '-E', '--email-file', help='File containing the email address to access rietveld. ' 'If not specified, anonymous access will be used.') parser.add_option( '-k', '--private-key-file', help='Path to file containing a private key in p12 format for OAuth2 ' 'authentication with "notasecret" password (as generated by Google ' 'Cloud Console).') parser.add_option( '-i', '--issue', type='int', help='Rietveld issue number') parser.add_option( '-p', '--patchset', type='int', help='Rietveld issue\'s patchset number') parser.add_option( '-r', '--root_dir', default=os.getcwd(), help='Root directory to apply the patch') parser.add_option( '-s', '--server', default='http://codereview.chromium.org', help='Rietveld server') parser.add_option('--no-auth', action='store_true', help='Do not attempt authenticated requests.') parser.add_option('--revision-mapping', default='{}', help='When running gclient, annotate the got_revisions ' 'using the revision-mapping.') parser.add_option('-f', '--force', action='store_true', help='Really run apply_issue, even if .update.flag ' 'is detected.') parser.add_option('-b', '--base_ref', help='DEPRECATED do not use.') parser.add_option('--whitelist', action='append', default=[], help='Patch only specified file(s).') parser.add_option('--blacklist', action='append', default=[], help='Don\'t patch specified file(s).') parser.add_option('-d', '--ignore_deps', action='store_true', help='Don\'t run gclient sync on DEPS changes.') parser.add_option('--extra_patchlevel', type='int', help='Number of directories the patch level number should ' 'be incremented (useful for patches from repos with ' 'different directory hierarchies).') auth.add_auth_options(parser) return parser def main(): # TODO(pgervais,tandrii): split this func, it's still too long. sys.stdout = Unbuffered(sys.stdout) parser = _get_arg_parser() options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if options.whitelist and options.blacklist: parser.error('Cannot specify both --whitelist and --blacklist') if options.email and options.email_file: parser.error('-e and -E options are incompatible') if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag')) and not options.force): print 'update.flag file found: bot_update has run and checkout is already ' print 'in a consistent state. No actions will be performed in this step.' return 0 logging.basicConfig( format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s', level=[logging.WARNING, logging.INFO, logging.DEBUG][ min(2, options.verbose)]) if args: parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) if not options.issue: parser.error('Require --issue') options.server = options.server.rstrip('/') if not options.server: parser.error('Require a valid server') options.revision_mapping = json.loads(options.revision_mapping) # read email if needed if options.email_file: if not os.path.exists(options.email_file): parser.error('file does not exist: %s' % options.email_file) with open(options.email_file, 'rb') as f: options.email = f.read().strip() print('Connecting to %s' % options.server) # Always try un-authenticated first, except for OAuth2 if options.private_key_file: # OAuth2 authentication rietveld_obj = rietveld.JwtOAuth2Rietveld(options.server, options.email, options.private_key_file) try: properties = rietveld_obj.get_issue_properties(options.issue, False) except urllib2.URLError: logging.exception('failed to fetch issue properties') sys.exit(RETURN_CODE_INFRA_FAILURE) else: # Passing None as auth_config disables authentication. rietveld_obj = rietveld.Rietveld(options.server, None) properties = None # Bad except clauses order (HTTPError is an ancestor class of # ClientLoginError) # pylint: disable=bad-except-order try: properties = rietveld_obj.get_issue_properties(options.issue, False) except urllib2.HTTPError as e: if e.getcode() != 302: raise if options.no_auth: exit('FAIL: Login detected -- is issue private?') # TODO(maruel): A few 'Invalid username or password.' are printed first, # we should get rid of those. except urllib2.URLError: logging.exception('failed to fetch issue properties') return RETURN_CODE_INFRA_FAILURE except rietveld.upload.ClientLoginError as e: # Fine, we'll do proper authentication. pass if properties is None: rietveld_obj = rietveld.Rietveld(options.server, auth_config, options.email) try: properties = rietveld_obj.get_issue_properties(options.issue, False) except rietveld.upload.ClientLoginError as e: print('Accessing the issue requires proper credentials.') return RETURN_CODE_OTHER_FAILURE except urllib2.URLError: logging.exception('failed to fetch issue properties') return RETURN_CODE_INFRA_FAILURE if not options.patchset: options.patchset = properties['patchsets'][-1] print('No patchset specified. Using patchset %d' % options.patchset) issues_patchsets_to_apply = [(options.issue, options.patchset)] try: depends_on_info = rietveld_obj.get_depends_on_patchset( options.issue, options.patchset) except urllib2.URLError: logging.exception('failed to fetch depends_on_patchset') return RETURN_CODE_INFRA_FAILURE while depends_on_info: depends_on_issue = int(depends_on_info['issue']) depends_on_patchset = int(depends_on_info['patchset']) try: depends_on_info = rietveld_obj.get_depends_on_patchset(depends_on_issue, depends_on_patchset) issues_patchsets_to_apply.insert(0, (depends_on_issue, depends_on_patchset)) except urllib2.HTTPError: print ('The patchset that was marked as a dependency no longer ' 'exists: %s/%d/#ps%d' % ( options.server, depends_on_issue, depends_on_patchset)) print 'Therefore it is likely that this patch will not apply cleanly.' print depends_on_info = None except urllib2.URLError: logging.exception('failed to fetch dependency issue') return RETURN_CODE_INFRA_FAILURE num_issues_patchsets_to_apply = len(issues_patchsets_to_apply) if num_issues_patchsets_to_apply > 1: print print 'apply_issue.py found %d dependent CLs.' % ( num_issues_patchsets_to_apply - 1) print 'They will be applied in the following order:' num = 1 for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: print ' #%d %s/%d/#ps%d' % ( num, options.server, issue_to_apply, patchset_to_apply) num += 1 print for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply, patchset_to_apply) print('Downloading patch from %s' % issue_url) try: patchset = rietveld_obj.get_patch(issue_to_apply, patchset_to_apply) except urllib2.HTTPError: print( 'Failed to fetch the patch for issue %d, patchset %d.\n' 'Try visiting %s/%d') % ( issue_to_apply, patchset_to_apply, options.server, issue_to_apply) # If we got this far, then this is likely missing patchset. # Thus, it's not infra failure. return RETURN_CODE_OTHER_FAILURE except urllib2.URLError: logging.exception( 'Failed to fetch the patch for issue %d, patchset %d', issue_to_apply, patchset_to_apply) return RETURN_CODE_INFRA_FAILURE if options.whitelist: patchset.patches = [patch for patch in patchset.patches if patch.filename in options.whitelist] if options.blacklist: patchset.patches = [patch for patch in patchset.patches if patch.filename not in options.blacklist] for patch in patchset.patches: print(patch) if options.extra_patchlevel: patch.patchlevel += options.extra_patchlevel full_dir = os.path.abspath(options.root_dir) scm_type = scm.determine_scm(full_dir) if scm_type == 'git': scm_obj = checkout.GitCheckout(full_dir, None, None, None, None) else: parser.error('Couldn\'t determine the scm') print('\nApplying the patch from %s' % issue_url) try: scm_obj.apply_patch(patchset, verbose=True) except checkout.PatchApplicationFailed as e: print(str(e)) print('CWD=%s' % os.getcwd()) print('Checkout path=%s' % scm_obj.project_path) return RETURN_CODE_OTHER_FAILURE if ('DEPS' in map(os.path.basename, patchset.filenames) and not options.ignore_deps): gclient_root = gclient_utils.FindGclientRoot(full_dir) if gclient_root and scm_type: print( 'A DEPS file was updated inside a gclient checkout, running gclient ' 'sync.') gclient_path = os.path.join(BASE_DIR, 'gclient') if sys.platform == 'win32': gclient_path += '.bat' with annotated_gclient.temp_filename(suffix='gclient') as f: cmd = [ gclient_path, 'sync', '--nohooks', '--delete_unversioned_trees', ] if options.revision_mapping: cmd.extend(['--output-json', f]) retcode = subprocess.call(cmd, cwd=gclient_root) if retcode == 0 and options.revision_mapping: revisions = annotated_gclient.parse_got_revision( f, options.revision_mapping) annotated_gclient.emit_buildprops(revisions) return retcode return RETURN_CODE_OK if __name__ == "__main__": fix_encoding.fix_encoding() try: sys.exit(main()) except KeyboardInterrupt: sys.stderr.write('interrupted\n') sys.exit(RETURN_CODE_OTHER_FAILURE)