diff --git a/gclient.py b/gclient.py index d1b5a8733..215e19408 100755 --- a/gclient.py +++ b/gclient.py @@ -87,6 +87,7 @@ import platform import posixpath import pprint import re +import socket import sys import time import urllib @@ -668,6 +669,39 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): command, options, parsed_url, self.parent.name, revision_overrides) self._used_scm = gclient_scm.CreateSCM( parsed_url, self.root.root_dir, self.name) + + def enable_deletion_of_conflicting_checkouts(): + """Determines whether to enable new checkout deletion behavior. + + Initially, enables the experimental functionality on a small set of + bots. + """ + # TODO(borenet): Remove this hack as soon as we've verified that it + # doesn't cause the bots to break. + return (os.environ.get('CHROME_HEADLESS') and + socket.gethostname() in ('vm859-m1', 'build1-m1', 'vm630-m1')) + + # When updating, determine whether the destination directory contains a + # checkout of the desired repository. If not, avoid conflicts by + # deleting the directory before running the update. + if command == 'update' and enable_deletion_of_conflicting_checkouts(): + logging.warning('Experimental deletion of mismatching checkouts ' + 'enabled.') + actual_remote_url = self._used_scm.GetRemoteURL(options) + url, _ = gclient_utils.SplitUrlRevision(parsed_url) + url = url.rstrip('/') + dest_dir = os.path.join(self.root.root_dir, self.name) + if os.path.isdir(dest_dir) and actual_remote_url != url: + if options.force: + logging.warning('%s does not contain a checkout of %s. Removing ' + ' %s' % (dest_dir, url, dest_dir)) + gclient_utils.rmtree(dest_dir) + else: + raise gclient_utils.Error('%s does not contain a checkout of %s. ' + 'Please fix the solution manually or ' + 'run with --force to delete ' + 'automatically.' % (dest_dir, url)) + self._got_revision = self._used_scm.RunCommand(command, options, args, file_list) if file_list: diff --git a/gclient_scm.py b/gclient_scm.py index 8572f094a..d26487eab 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -222,6 +222,13 @@ class GitWrapper(SCMWrapper): def GetCheckoutRoot(self): return scm.GIT.GetCheckoutRoot(self.checkout_path) + def GetRemoteURL(self, options, cwd=None): + try: + return self._Capture(['config', 'remote.%s.url' % self.remote], + cwd=cwd or self.checkout_path).rstrip() + except (OSError, subprocess2.CalledProcessError): + return None + def GetRevisionDate(self, _revision): """Returns the given revision's date in ISO-8601 format (which contains the time zone).""" @@ -277,22 +284,6 @@ class GitWrapper(SCMWrapper): gclient_utils.CheckCallAndFilter(cmd4, **kwargs) - def _FetchAndReset(self, revision, file_list, options): - """Equivalent to git fetch; git reset.""" - quiet = [] - if not options.verbose: - quiet = ['--quiet'] - self._UpdateBranchHeads(options, fetch=False) - - fetch_cmd = [ - '-c', 'core.deltaBaseCacheLimit=2g', 'fetch', self.remote, '--prune'] - self._Run(fetch_cmd + quiet, options, retry=True) - self._Run(['reset', '--hard', revision] + quiet, options) - self.UpdateSubmoduleConfig() - if file_list is not None: - files = self._Capture(['ls-files']).splitlines() - file_list.extend([os.path.join(self.checkout_path, f) for f in files]) - def update(self, options, args, file_list): """Runs git to update or transparently checkout the working copy. @@ -379,35 +370,10 @@ class GitWrapper(SCMWrapper): '\tAnd run gclient sync again\n' % (self.relpath, rev_str, self.relpath)) - # See if the url has changed (the unittests use git://foo for the url, let - # that through). - current_url = self._Capture(['config', 'remote.%s.url' % self.remote]) - return_early = False - # TODO(maruel): Delete url != 'git://foo' since it's just to make the - # unit test pass. (and update the comment above) - # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set. - # This allows devs to use experimental repos which have a different url - # but whose branch(s) are the same as official repos. - if (current_url != url and - url != 'git://foo' and - subprocess2.capture( - ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote], - cwd=self.checkout_path).strip() != 'False'): - print('_____ switching %s to a new upstream' % self.relpath) - # Make sure it's clean - self._CheckClean(rev_str) - # Switch over to the new upstream - self._Run(['remote', 'set-url', self.remote, url], options) - self._FetchAndReset(revision, file_list, options) - return_early = True - # Need to do this in the normal path as well as in the post-remote-switch # path. self._PossiblySwitchCache(url, options) - if return_early: - return self._Capture(['rev-parse', '--verify', 'HEAD']) - cur_branch = self._GetCurrentBranch() # Cases: @@ -813,8 +779,7 @@ class GitWrapper(SCMWrapper): # to relax this restriction in the future to allow for smarter cache # repo update schemes (such as pulling the same repo, but from a # different host). - existing_url = self._Capture(['config', 'remote.%s.url' % self.remote], - cwd=folder) + existing_url = self.GetRemoteURL(options, cwd=folder) assert self._NormalizeGitURL(existing_url) == self._NormalizeGitURL(url) if use_reference: @@ -1083,6 +1048,13 @@ class SVNWrapper(SCMWrapper): def GetCheckoutRoot(self): return scm.SVN.GetCheckoutRoot(self.checkout_path) + def GetRemoteURL(self, options): + try: + local_info = scm.SVN.CaptureLocalInfo([os.curdir], self.checkout_path) + except (OSError, subprocess2.CalledProcessError): + return None + return local_info.get('URL') + def GetRevisionDate(self, revision): """Returns the given revision's date in ISO-8601 format (which contains the time zone).""" @@ -1122,25 +1094,12 @@ class SVNWrapper(SCMWrapper): Raises: Error: if can't get URL for relative path. """ - # Only update if git or hg is not controlling the directory. - git_path = os.path.join(self.checkout_path, '.git') - if os.path.exists(git_path): - print('________ found .git directory; skipping %s' % self.relpath) - return - - hg_path = os.path.join(self.checkout_path, '.hg') - if os.path.exists(hg_path): - print('________ found .hg directory; skipping %s' % self.relpath) - return - if args: raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args)) # revision is the revision to match. It is None if no revision is specified, # i.e. the 'deps ain't pinned'. url, revision = gclient_utils.SplitUrlRevision(self.url) - # Keep the original unpinned url for reference in case the repo is switched. - base_url = url managed = True if options.revision: # Override the revision number. @@ -1287,53 +1246,6 @@ class SVNWrapper(SCMWrapper): revision = str(from_info_live['Revision']) rev_str = ' at %s' % revision - if from_info['URL'] != base_url: - # The repository url changed, need to switch. - try: - to_info = scm.SVN.CaptureRemoteInfo(url) - except (gclient_utils.Error, subprocess2.CalledProcessError): - # The url is invalid or the server is not accessible, it's safer to bail - # out right now. - raise gclient_utils.Error('This url is unreachable: %s' % url) - can_switch = ((from_info['Repository Root'] != to_info['Repository Root']) - and (from_info['UUID'] == to_info['UUID'])) - if can_switch: - print('\n_____ relocating %s to a new checkout' % self.relpath) - # We have different roots, so check if we can switch --relocate. - # Subversion only permits this if the repository UUIDs match. - # Perform the switch --relocate, then rewrite the from_url - # to reflect where we "are now." (This is the same way that - # Subversion itself handles the metadata when switch --relocate - # is used.) This makes the checks below for whether we - # can update to a revision or have to switch to a different - # branch work as expected. - # TODO(maruel): TEST ME ! - command = ['switch', '--relocate', - from_info['Repository Root'], - to_info['Repository Root'], - self.relpath] - self._Run(command, options, cwd=self._root_dir) - from_info['URL'] = from_info['URL'].replace( - from_info['Repository Root'], - to_info['Repository Root']) - else: - if not options.force and not options.reset: - # Look for local modifications but ignore unversioned files. - for status in scm.SVN.CaptureStatus(None, self.checkout_path): - if status[0][0] != '?': - raise gclient_utils.Error( - ('Can\'t switch the checkout to %s; UUID don\'t match and ' - 'there is local changes in %s. Delete the directory and ' - 'try again.') % (url, self.checkout_path)) - # Ok delete it. - print('\n_____ switching %s to a new checkout' % self.relpath) - gclient_utils.rmtree(self.checkout_path) - # We need to checkout. - command = ['checkout', url, self.checkout_path] - command = self._AddAdditionalUpdateFlags(command, options, revision) - self._RunAndGetFileList(command, options, file_list, self._root_dir) - return self.Svnversion() - # If the provided url has a revision number that matches the revision # number of the existing directory, then we don't need to bother updating. if not options.force and str(from_info['Revision']) == revision: @@ -1400,12 +1312,6 @@ class SVNWrapper(SCMWrapper): return self.update(options, [], file_list) if not os.path.isdir(os.path.join(self.checkout_path, '.svn')): - if os.path.isdir(os.path.join(self.checkout_path, '.git')): - print('________ found .git directory; skipping %s' % self.relpath) - return - if os.path.isdir(os.path.join(self.checkout_path, '.hg')): - print('________ found .hg directory; skipping %s' % self.relpath) - return if not options.force: raise gclient_utils.Error('Invalid checkout path, aborting') print( diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index 5607f85dd..99007b09b 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -102,6 +102,7 @@ class SVNWrapperTestCase(BaseTestCase): 'BinaryExists', 'FullUrlForRelativeUrl', 'GetCheckoutRoot', + 'GetRemoteURL', 'GetRevisionDate', 'GetUsableRev', 'Svnversion', @@ -162,8 +163,6 @@ class SVNWrapperTestCase(BaseTestCase): def testRunCommandException(self): options = self.Options(verbose=False) - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) self.mox.ReplayAll() scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, @@ -183,8 +182,6 @@ class SVNWrapperTestCase(BaseTestCase): gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None ).AndReturn('1.5.1') # It'll to a checkout instead. - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) # Checkout. gclient_scm.os.path.exists(self.base_path).AndReturn(False) parent = gclient_scm.os.path.dirname(self.base_path) @@ -212,11 +209,7 @@ class SVNWrapperTestCase(BaseTestCase): options = self.Options(verbose=True, force=True) gclient_scm.os.path.isdir(self.base_path).AndReturn(True) gclient_scm.os.path.isdir(join(self.base_path, '.svn')).AndReturn(False) - gclient_scm.os.path.isdir(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.isdir(join(self.base_path, '.hg')).AndReturn(False) # Checkout. - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(False) parent = gclient_scm.os.path.dirname(self.base_path) gclient_scm.os.path.exists(parent).AndReturn(False) @@ -341,8 +334,6 @@ class SVNWrapperTestCase(BaseTestCase): file_info.url = self.url file_info.uuid = 'ABC' file_info.revision = 42 - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) # Checkout. gclient_scm.os.path.exists(self.base_path).AndReturn(False) parent = gclient_scm.os.path.dirname(self.base_path) @@ -374,8 +365,6 @@ class SVNWrapperTestCase(BaseTestCase): 'UUID': 'ABC', 'Revision': 42, } - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) # Checkout or update. @@ -420,8 +409,6 @@ class SVNWrapperTestCase(BaseTestCase): 'UUID': 'ABC', 'Revision': 42, } - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) # Checkout or update. @@ -456,8 +443,6 @@ class SVNWrapperTestCase(BaseTestCase): 'UUID': 'ABC', 'Revision': 42, } - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) # Checkout or update. @@ -522,8 +507,6 @@ class SVNWrapperTestCase(BaseTestCase): file_list=files_list) # Now we fall back on scm.update(). - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info) gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None @@ -592,8 +575,6 @@ class SVNWrapperTestCase(BaseTestCase): file_list=files_list) # Now we fall back on scm.update(). - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) gclient_scm.scm.SVN._CaptureInfo( [], join(self.base_path, ".")).AndReturn(file_info) @@ -628,8 +609,6 @@ class SVNWrapperTestCase(BaseTestCase): # Now we fall back on scm.update(). files_list = self.mox.CreateMockAnything() - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) gclient_scm.scm.SVN._CaptureInfo( [], join(self.base_path, '.')).AndReturn(file_info) @@ -645,32 +624,6 @@ class SVNWrapperTestCase(BaseTestCase): scm.updatesingle(options, ['DEPS'], files_list) self.checkstdout('\n_____ %s at 42\n' % self.relpath) - def testUpdateGit(self): - options = self.Options(verbose=True) - file_path = gclient_scm.os.path.join(self.root_dir, self.relpath, '.git') - gclient_scm.os.path.exists(file_path).AndReturn(True) - - self.mox.ReplayAll() - scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) - file_list = [] - scm.update(options, self.args, file_list) - self.checkstdout( - ('________ found .git directory; skipping %s\n' % self.relpath)) - - def testUpdateHg(self): - options = self.Options(verbose=True) - gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False) - gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(True) - - self.mox.ReplayAll() - scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) - file_list = [] - scm.update(options, self.args, file_list) - self.checkstdout( - ('________ found .hg directory; skipping %s\n' % self.relpath)) - def testGetUsableRevSVN(self): # pylint: disable=E1101 options = self.Options(verbose=True) @@ -692,6 +645,48 @@ class SVNWrapperTestCase(BaseTestCase): self.assertRaises(gclient_scm.gclient_utils.Error, svn_scm.GetUsableRev, 'fake', options) + def testGetRemoteURL(self): + self.mox.UnsetStubs() + options = self.Options(verbose=True) + self.mox.StubOutWithMock(gclient_scm.scm.SVN, 'Capture', True) + svn_scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, + relpath=self.relpath) + + if svn_scm.relpath: + cwd = os.path.join(svn_scm._root_dir, svn_scm.relpath) + else: + cwd = svn_scm._root_dir + + gclient_scm.scm.SVN.Capture(['info', '--xml', os.curdir], cwd).AndReturn( +""" + + +%s + +https://dummy.repo.com/svn +FAKE + + +normal +infinity + + +fakedev@chromium.org +2013-11-14T15:08:21.757885Z + + + +""" % svn_scm.url) + + self.mox.ReplayAll() + + self.assertEquals(svn_scm.GetRemoteURL(options), self.url) + + class BaseGitWrapperTestCase(GCBaseTestCase, StdoutCheck, TestCaseUtils, unittest.TestCase): """This class doesn't use pymox.""" @@ -817,6 +812,7 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): 'BinaryExists', 'FullUrlForRelativeUrl', 'GetCheckoutRoot', + 'GetRemoteURL', 'GetRevisionDate', 'GetUsableRev', 'RunCommand', @@ -1185,6 +1181,18 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): self.assertRaises(gclient_scm.gclient_utils.Error, git_svn_scm.GetUsableRev, too_big, options) + def testGetRemoteURL(self): + options = self.Options(verbose=True) + self.mox.StubOutWithMock(gclient_scm.GitWrapper, '_Capture', True) + git_scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, + relpath=self.relpath) + git_scm._Capture(['config', 'remote.origin.url'], cwd='/tmp/fake' + ).AndReturn('%s\n' % git_scm.url) + + self.mox.ReplayAll() + + self.assertEquals(git_scm.GetRemoteURL(options), self.url) + class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): def testUpdateUpdate(self): diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index c05bbf339..bda14cf20 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -13,6 +13,7 @@ This test assumes GClientSmokeBase.URL_BASE is valid. import logging import os import re +import socket import subprocess import sys import unittest @@ -776,7 +777,7 @@ class GClientSmokeSVN(GClientSmokeBase): # Cripple src/third_party/foo and make sure gclient still succeeds. gclient_utils.rmtree(join(third_party, 'foo', '.svn')) - self.assertEquals(0, self.gclient(cmd)[-1]) + self.assertEquals(0, self.gclient(cmd + ['--force'])[-1]) class GClientSmokeSVNTransitive(GClientSmokeBase): @@ -1103,7 +1104,7 @@ class GClientSmokeGIT(GClientSmokeBase): self.assertTree(tree) # Pre-DEPS hooks run when syncing with --nohooks. - self.gclient(['sync', '--deps', 'mac', '--nohooks', + self.gclient(['sync', '--deps', 'mac', '--nohooks', '--force', '--revision', 'src@' + self.githash('repo_5', 2)]) tree = self.mangle_git_tree(('repo_5@2', 'src'), ('repo_1@2', 'src/repo1'), @@ -1115,7 +1116,7 @@ class GClientSmokeGIT(GClientSmokeBase): os.remove(join(self.root_dir, 'src', 'git_pre_deps_hooked')) # Pre-DEPS hooks don't run with --noprehooks - self.gclient(['sync', '--deps', 'mac', '--noprehooks', + self.gclient(['sync', '--deps', 'mac', '--noprehooks', '--force', '--revision', 'src@' + self.githash('repo_5', 2)]) tree = self.mangle_git_tree(('repo_5@2', 'src'), ('repo_1@2', 'src/repo1'), @@ -1343,6 +1344,81 @@ class GClientSmokeBoth(GClientSmokeBase): self.assertEquals(sorted(entries), sorted(expected)) + # TODO(borenet): Enable this at the same time that the guard is removed in + # gclient. + if (os.environ.get('CHROME_HEADLESS') and + socket.gethostname() in ('vm859-m1', 'build1-m1', 'vm630-m1')): + def testDeleteConflictingCheckout(self): + if not self.enabled: + return + + # Create an initial svn checkout. + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.svn_base + 'trunk/src"},' + ']' + ]) + results = self.gclient(['sync', '--deps', 'mac']) + self.assertEqual(results[2], 0, 'Sync failed!') + + # Verify that we have the expected svn checkout. + results = self.gclient(['revinfo', '--deps', 'mac']) + actual = results[0].splitlines() + expected = [ + 'src: %strunk/src' % self.svn_base, + 'src/file/other: File("%strunk/other/DEPS")' % self.svn_base, + 'src/other: %strunk/other' % self.svn_base, + 'src/third_party/foo: %strunk/third_party/foo@1' % self.svn_base, + ] + self.assertEquals(actual, expected) + + # Change the desired checkout to git. + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.git_base + 'repo_1"},' + ']' + ]) + + # Verify that the sync succeeds with --force. + results = self.gclient(['sync', '--deps', 'mac', '--force']) + self.assertEqual(results[2], 0, 'Sync failed!') + + # Verify that we got the desired git checkout. + results = self.gclient(['revinfo', '--deps', 'mac']) + actual = results[0].splitlines() + expected = [ + 'src: %srepo_1' % self.git_base, + 'src/repo2: %srepo_2@%s' % (self.git_base, + self.githash('repo_2', 1)[:7]), + 'src/repo2/repo_renamed: %srepo_3' % self.git_base, + ] + self.assertEquals(actual, expected) + + # Change the desired checkout back to svn. + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.svn_base + 'trunk/src"},' + ']' + ]) + + # Verify that the sync succeeds. + results = self.gclient(['sync', '--deps', 'mac', '--force']) + self.assertEqual(results[2], 0, 'Sync failed!') + + # Verify that we have the expected svn checkout. + results = self.gclient(['revinfo', '--deps', 'mac']) + actual = results[0].splitlines() + expected = [ + 'src: %strunk/src' % self.svn_base, + 'src/file/other: File("%strunk/other/DEPS")' % self.svn_base, + 'src/other: %strunk/other' % self.svn_base, + 'src/third_party/foo: %strunk/third_party/foo@1' % self.svn_base, + ] + self.assertEquals(actual, expected) + class GClientSmokeFromCheckout(GClientSmokeBase): # WebKit abuses this. It has a .gclient and a DEPS from a checkout.