From d90ba3f1363b699a2f9e5cd01846636387d5941c Mon Sep 17 00:00:00 2001 From: "nasser@codeaurora.org" Date: Tue, 23 Feb 2010 14:42:57 +0000 Subject: [PATCH] sync @branchname git support Also improve GIT.update error handling and verbosity levels TEST=unit tests BUG=http://crosbug.com/480 BUG=http://crosbug.com/1136 Review URL: http://codereview.chromium.org/559003 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@39717 0039d316-1c4b-4281-b951-d872f2087c98 --- gclient_scm.py | 321 +++++++++++++++++++++++++++++++++----- tests/gclient_scm_test.py | 40 ++++- 2 files changed, 317 insertions(+), 44 deletions(-) diff --git a/gclient_scm.py b/gclient_scm.py index 1f76910d2e..e8dd3cf336 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -167,41 +167,44 @@ class GitWrapper(SCMWrapper, scm.GIT): self._CheckMinVersion("1.6") + default_rev = "refs/heads/master" url, revision = gclient_utils.SplitUrlRevision(self.url) rev_str = "" if options.revision: # Override the revision number. revision = str(options.revision) - if revision: - rev_str = ' at %s' % revision + if not revision: + revision = default_rev + + rev_str = ' at %s' % revision + files = [] + printed_path = False + verbose = [] if options.verbose: print("\n_____ %s%s" % (self.relpath, rev_str)) + verbose = ['--verbose'] + printed_path = True + + if revision.startswith('refs/heads/'): + rev_type = "branch" + elif revision.startswith('origin/'): + # For compatability with old naming, translate 'origin' to 'refs/heads' + revision = revision.replace('origin/', 'refs/heads/') + rev_type = "branch" + else: + # hash is also a tag, only make a distinction at checkout + rev_type = "hash" - if not os.path.exists(self.checkout_path): - # Cloning - for i in range(3): - try: - self._Run(['clone', url, self.checkout_path], - cwd=self._root_dir, redirect_stdout=False) - break - except gclient_utils.Error, e: - # TODO(maruel): Hackish, should be fixed by moving _Run() to - # CheckCall(). - # Too bad we don't have access to the actual output. - # We should check for "transfer closed with NNN bytes remaining to - # read". In the meantime, just make sure .git exists. - if (e.args[0] == 'git command clone returned 128' and - os.path.exists(os.path.join(self.checkout_path, '.git'))): - print str(e) - print "Retrying..." - continue - raise e - if revision: - self._Run(['reset', '--hard', revision], redirect_stdout=False) + if not os.path.exists(self.checkout_path): + self._Clone(rev_type, revision, url, options.verbose) files = self._Run(['ls-files']).split() file_list.extend([os.path.join(self.checkout_path, f) for f in files]) + if not verbose: + # Make the output a little prettier. It's nice to have some whitespace + # between projects when cloning. + print "" return if not os.path.exists(os.path.join(self.checkout_path, '.git')): @@ -212,9 +215,6 @@ class GitWrapper(SCMWrapper, scm.GIT): '\tAnd run gclient sync again\n' % (self.relpath, rev_str, self.relpath)) - new_base = 'origin' - if revision: - new_base = revision cur_branch = self._GetCurrentBranch() # Check if we are in a rebase conflict @@ -226,18 +226,140 @@ class GitWrapper(SCMWrapper, scm.GIT): '\tSee man git-rebase for details.\n' % (self.relpath, rev_str)) - # TODO(maruel): Do we need to do an automatic retry here? Probably overkill - merge_base = self._Run(['merge-base', 'HEAD', new_base]) - self._Run(['remote', 'update'], redirect_stdout=False) - files = self._Run(['diff', new_base, '--name-only']).split() - file_list.extend([os.path.join(self.checkout_path, f) for f in files]) + # Cases: + # 1) current branch based on a hash (could be git-svn) + # - try to rebase onto the new upstream (hash or branch) + # 2) current branch based on a remote branch with local committed changes, + # but the DEPS file switched to point to a hash + # - rebase those changes on top of the hash + # 3) current branch based on a remote with or without changes, no switch + # - see if we can FF, if not, prompt the user for rebase, merge, or stop + # 4) current branch based on a remote, switches to a new remote + # - exit + + # GetUpstream returns something like 'refs/remotes/origin/master' for a + # tracking branch + # or 'master' if not a tracking branch (it's based on a specific rev/hash) + # or it returns None if it couldn't find an upstream + upstream_branch = self.GetUpstream(self.checkout_path) + if not upstream_branch or not upstream_branch.startswith('refs/remotes'): + current_type = "hash" + logging.debug("Current branch is based off a specific rev and is not " + "tracking an upstream.") + elif upstream_branch.startswith('refs/remotes'): + current_type = "branch" + else: + raise gclient_utils.Error('Invalid Upstream') + + # Update the remotes first so we have all the refs + remote_output, remote_err = self.Capture(['remote'] + verbose + ['update'], + self.checkout_path, + print_error=False) + if verbose: + print remote_output.strip() + # git remote update prints to stderr when used with --verbose + print remote_err.strip() + + # This is a big hammer, debatable if it should even be here... if options.force or options.reset: - self._Run(['reset', '--hard', merge_base], redirect_stdout=False) - try: - self._Run(['rebase', '-v', '--onto', new_base, merge_base, cur_branch], - redirect_stdout=False) - except gclient_utils.Error: - pass + self._Run(['reset', '--hard', 'HEAD'], redirect_stdout=False) + + if current_type is 'hash': + # case 1 + if self.IsGitSvn(self.checkout_path) and upstream_branch is not None: + # Our git-svn branch (upstream_branch) is our upstream + self._AttemptRebase(upstream_branch, files, verbose=options.verbose, + newbase=revision, printed_path=printed_path) + printed_path = True + else: + # Can't find a merge-base since we don't know our upstream. That makes + # this command VERY likely to produce a rebase failure. For now we + # assume origin is our upstream since that's what the old behavior was. + self._AttemptRebase('origin', files=files, verbose=options.verbose, + printed_path=printed_path) + printed_path = True + elif rev_type is 'hash': + # case 2 + self._AttemptRebase(upstream_branch, files, verbose=options.verbose, + newbase=revision, printed_path=printed_path) + printed_path = True + elif revision.replace('heads', 'remotes/origin') != upstream_branch: + # case 4 + new_base = revision.replace('heads', 'remotes/origin') + if not printed_path: + print("\n_____ %s%s" % (self.relpath, rev_str)) + switch_error = ("Switching upstream branch from %s to %s\n" + % (upstream_branch, new_base) + + "Please merge or rebase manually:\n" + + "cd %s; git rebase %s\n" % (self.checkout_path, new_base) + + "OR git checkout -b %s" % new_base) + raise gclient_utils.Error(switch_error) + else: + # case 3 - the default case + files = self._Run(['diff', upstream_branch, '--name-only']).split() + if verbose: + print "Trying fast-forward merge to branch : %s" % upstream_branch + try: + merge_output, merge_err = self.Capture(['merge', '--ff-only', + upstream_branch], + self.checkout_path, + print_error=False) + except gclient_utils.CheckCallError, e: + if re.match('fatal: Not possible to fast-forward, aborting.', e.stderr): + if not printed_path: + print("\n_____ %s%s" % (self.relpath, rev_str)) + printed_path = True + while True: + try: + action = str(raw_input("Cannot fast-forward merge, attempt to " + "rebase? (y)es / (q)uit / (s)kip : ")) + except ValueError: + gclient_utils.Error('Invalid Character') + continue + if re.match(r'yes|y', action, re.I): + self._AttemptRebase(upstream_branch, files, + verbose=options.verbose, + printed_path=printed_path) + printed_path = True + break + elif re.match(r'quit|q', action, re.I): + raise gclient_utils.Error("Can't fast-forward, please merge or " + "rebase manually.\n" + "cd %s && git " % self.checkout_path + + "rebase %s" % upstream_branch) + elif re.match(r'skip|s', action, re.I): + print "Skipping %s" % self.relpath + return + else: + print "Input not recognized" + elif re.match("error: Your local changes to '.*' would be " + "overwritten by merge. Aborting.\nPlease, commit your " + "changes or stash them before you can merge.\n", + e.stderr): + if not printed_path: + print("\n_____ %s%s" % (self.relpath, rev_str)) + printed_path = True + raise gclient_utils.Error(e.stderr) + else: + # Some other problem happened with the merge + logging.error("Error during fast-forward merge in %s!" % self.relpath) + print e.stderr + raise + else: + # Fast-forward merge was successful + if not re.match('Already up-to-date.', merge_output) or verbose: + if not printed_path: + print("\n_____ %s%s" % (self.relpath, rev_str)) + printed_path = True + print merge_output.strip() + if merge_err: + print "Merge produced error output:\n%s" % merge_err.strip() + if not verbose: + # Make the output a little prettier. It's nice to have some + # whitespace between projects when syncing. + print "" + + file_list.extend([os.path.join(self.checkout_path, f) for f in files]) # If the rebase generated a conflict, abort and ask user to fix if self._GetCurrentBranch() is None: @@ -247,7 +369,8 @@ class GitWrapper(SCMWrapper, scm.GIT): 'See man git-rebase for details.\n' % (self.relpath, rev_str)) - print "Checked out revision %s." % self.revinfo(options, (), None) + if verbose: + print "Checked out revision %s" % self.revinfo(options, (), None) def revert(self, options, args, file_list): """Reverts local modifications. @@ -293,6 +416,128 @@ class GitWrapper(SCMWrapper, scm.GIT): base_url = self.url return base_url[:base_url.rfind('/')] + url + def _Clone(self, rev_type, revision, url, verbose=False): + """Clone a git repository from the given URL. + + Once we've cloned the repo, we checkout a working branch based off the + specified revision.""" + if not verbose: + # git clone doesn't seem to insert a newline properly before printing + # to stdout + print "" + + clone_cmd = ['clone'] + if verbose: + clone_cmd.append('--verbose') + clone_cmd.extend([url, self.checkout_path]) + + for i in range(3): + try: + self._Run(clone_cmd, cwd=self._root_dir, redirect_stdout=False) + break + except gclient_utils.Error, e: + # TODO(maruel): Hackish, should be fixed by moving _Run() to + # CheckCall(). + # Too bad we don't have access to the actual output. + # We should check for "transfer closed with NNN bytes remaining to + # read". In the meantime, just make sure .git exists. + if (e.args[0] == 'git command clone returned 128' and + os.path.exists(os.path.join(self.checkout_path, '.git'))): + print str(e) + print "Retrying..." + continue + raise e + + if rev_type is "branch": + short_rev = revision.replace('refs/heads/', '') + new_branch = revision.replace('heads', 'remotes/origin') + elif revision.startswith('refs/tags/'): + short_rev = revision.replace('refs/tags/', '') + new_branch = revision + else: + # revision is a specific sha1 hash + short_rev = revision + new_branch = revision + + cur_branch = self._GetCurrentBranch() + if cur_branch != short_rev: + self._Run(['checkout', '-b', short_rev, new_branch], + redirect_stdout=False) + + def _AttemptRebase(self, upstream, files, verbose=False, newbase=None, + branch=None, printed_path=False): + """Attempt to rebase onto either upstream or, if specified, newbase.""" + files.extend(self._Run(['diff', upstream, '--name-only']).split()) + revision = upstream + if newbase: + revision = newbase + if not printed_path: + print "\n_____ %s : Attempting rebase onto %s..." % (self.relpath, + revision) + printed_path = True + else: + print "Attempting rebase onto %s..." % revision + + # Build the rebase command here using the args + # git rebase [options] [--onto ] [] + rebase_cmd = ['rebase'] + if verbose: + rebase_cmd.append('--verbose') + if newbase: + rebase_cmd.extend(['--onto', newbase]) + rebase_cmd.append(upstream) + if branch: + rebase_cmd.append(branch) + + try: + rebase_output, rebase_err = self.Capture(rebase_cmd, self.checkout_path, + print_error=False) + except gclient_utils.CheckCallError, e: + if re.match(r'cannot rebase: you have unstaged changes', e.stderr) or \ + re.match(r'cannot rebase: your index contains uncommitted changes', + e.stderr): + while True: + rebase_action = str(raw_input("Cannot rebase because of unstaged " + "changes.\n'git reset --hard HEAD' ?\n" + "WARNING: destroys any uncommitted " + "work in your current branch!" + " (y)es / (q)uit / (s)how : ")) + if re.match(r'yes|y', rebase_action, re.I): + self._Run(['reset', '--hard', 'HEAD'], redirect_stdout=False) + # Should this be recursive? + rebase_output, rebase_err = self.Capture(rebase_cmd, + self.checkout_path) + break + elif re.match(r'quit|q', rebase_action, re.I): + raise gclient_utils.Error("Please merge or rebase manually\n" + "cd %s && git " % self.checkout_path + + "%s" % ' '.join(rebase_cmd)) + elif re.match(r'show|s', rebase_action, re.I): + print "\n%s" % e.stderr.strip() + continue + else: + gclient_utils.Error("Input not recognized") + continue + elif re.search(r'^CONFLICT', e.stdout, re.M): + raise gclient_utils.Error("Conflict while rebasing this branch.\n" + "Fix the conflict and run gclient again.\n" + "See 'man git-rebase' for details.\n") + else: + print e.stdout.strip() + print "Rebase produced error output:\n%s" % e.stderr.strip() + raise gclient_utils.Error("Unrecognized error, please merge or rebase " + "manually.\ncd %s && git " % + self.checkout_path + + "%s" % ' '.join(rebase_cmd)) + + print rebase_output.strip() + if rebase_err: + print "Rebase produced error output:\n%s" % rebase_err.strip() + if not verbose: + # Make the output a little prettier. It's nice to have some + # whitespace between projects when syncing. + print "" + def _CheckMinVersion(self, min_version): def only_int(val): if val.isdigit(): diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index f03b9ee2c5..634ad9810a 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -10,9 +10,10 @@ from os import rename from shutil import rmtree from subprocess import Popen, PIPE, STDOUT import tempfile +import __builtin__ # Fixes include path. -from super_mox import mox, SuperMoxBaseTestBase +from super_mox import mox, SuperMoxBaseTestBase, SuperMoxTestBase import gclient_scm from gclient_test import BaseTestCase as GCBaseTestCase @@ -353,6 +354,14 @@ from :3 cwd=path).communicate(input=git_import) Popen(['git', 'checkout'], stdout=PIPE, stderr=STDOUT, cwd=path).communicate() + Popen(['git', 'remote', 'add', '-f', 'origin', '.'], stdout=PIPE, + stderr=STDOUT, cwd=path).communicate() + Popen(['git', 'checkout', '-b', 'new', 'origin/master'], stdout=PIPE, + stderr=STDOUT, cwd=path).communicate() + Popen(['git', 'push', 'origin', 'origin/origin:origin/master'], stdout=PIPE, + stderr=STDOUT, cwd=path).communicate() + Popen(['git', 'config', '--unset', 'remote.origin.fetch'], stdout=PIPE, + stderr=STDOUT, cwd=path).communicate() return True def setUp(self): @@ -512,6 +521,20 @@ from :3 self.assertEquals(scm.revinfo(options, (), None), 'a7142dc9f0009350b96a11f372b6ea658592aa95') + def testUpdateUnstagedConflict(self): + if not self.enabled: + return + options = self.Options() + scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, + relpath=self.relpath) + file_path = gclient_scm.os.path.join(self.base_path, 'b') + f = open(file_path, 'w').writelines('conflict\n') + exception = ( + "error: Your local changes to 'b' would be overwritten by merge. " + "Aborting.\n" + "Please, commit your changes or stash them before you can merge.\n") + self.assertRaisesError(exception, scm.update, options, (), []) + def testUpdateConflict(self): if not self.enabled: return @@ -521,14 +544,19 @@ from :3 file_path = gclient_scm.os.path.join(self.base_path, 'b') f = open(file_path, 'w').writelines('conflict\n') scm._Run(['commit', '-am', 'test']) + self.mox.StubOutWithMock(__builtin__, 'raw_input') + __builtin__.raw_input.__call__(mox.StrContains('Cannot fast-forward merge, ' + 'attempt to rebase? (y)es / ' + '(q)uit / (s)kip : ') + ).AndReturn('y') + self.mox.ReplayAll() exception = \ - '\n____ .\n' \ - '\nConflict while rebasing this branch.\n' \ + 'Conflict while rebasing this branch.\n' \ 'Fix the conflict and run gclient again.\n' \ - 'See man git-rebase for details.\n' + "See 'man git-rebase' for details.\n" self.assertRaisesError(exception, scm.update, options, (), []) exception = \ - '\n____ .\n' \ + '\n____ . at refs/heads/master\n' \ '\tAlready in a conflict, i.e. (no branch).\n' \ '\tFix the conflict and run gclient again.\n' \ '\tOr to abort run:\n\t\tgit-rebase --abort\n' \ @@ -544,7 +572,7 @@ from :3 git_path = gclient_scm.os.path.join(self.base_path, '.git') rename(git_path, git_path + 'foo') exception = \ - '\n____ .\n' \ + '\n____ . at refs/heads/master\n' \ '\tPath is not a git repo. No .git dir.\n' \ '\tTo resolve:\n' \ '\t\trm -rf .\n' \