From b2256212810bfb07545c8538e777b24ee8ffa0ac Mon Sep 17 00:00:00 2001 From: "borenet@google.com" Date: Wed, 7 May 2014 20:57:28 +0000 Subject: [PATCH] gclient: Actually move or delete mismatched checkouts Followup to https://codereview.chromium.org/189913020/ ("gclient: print a warning if a dep would get deleted or moved in the future") BUG=skia:1638 Review URL: https://codereview.chromium.org/225403015 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@268944 0039d316-1c4b-4281-b951-d872f2087c98 --- gclient_scm.py | 39 ++++++--- tests/gclient_scm_test.py | 75 ++++++++++------ tests/gclient_smoketest.py | 174 ++++++++++++++++++++++++++++++++++++- 3 files changed, 249 insertions(+), 39 deletions(-) diff --git a/gclient_scm.py b/gclient_scm.py index b0ca582817..00b8145bb4 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -6,6 +6,7 @@ from __future__ import print_function +import errno import logging import os import posixpath @@ -20,6 +21,7 @@ import download_from_google_storage import gclient_utils import git_cache import scm +import shutil import subprocess2 @@ -196,21 +198,36 @@ class SCMWrapper(object): # valid git or svn checkout. return False - # TODO(borenet): Remove this once SCMWrapper._DeleteOrMove is enabled. - # pylint: disable=R0201 def _DeleteOrMove(self, force): """Delete the checkout directory or move it out of the way. Args: force: bool; if True, delete the directory. Otherwise, just move it. """ - gclient_utils.AddWarning('WARNING: Upcoming change in ' - 'https://codereview.chromium.org/225403015 would ' - 'cause %s to be deleted or moved to the side. ' - 'This is intended to ease changes to DEPS in the ' - 'future. If you are seeing this warning and ' - 'haven\'t changed the DEPS file, please contact ' - 'borenet@ immediately.' % self.checkout_path) + if force and os.environ.get('CHROME_HEADLESS') == '1': + self.Print('_____ Conflicting directory found in %s. Removing.' + % self.checkout_path) + gclient_utils.AddWarning('Conflicting directory %s deleted.' + % self.checkout_path) + gclient_utils.rmtree(self.checkout_path) + else: + bad_scm_dir = os.path.join(self._root_dir, '_bad_scm', + os.path.dirname(self.relpath)) + + try: + os.makedirs(bad_scm_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + dest_path = tempfile.mkdtemp( + prefix=os.path.basename(self.relpath), + dir=bad_scm_dir) + self.Print('_____ Conflicting directory found in %s. Moving to %s.' + % (self.checkout_path, dest_path)) + gclient_utils.AddWarning('Conflicting directory %s moved to %s.' + % (self.checkout_path, dest_path)) + shutil.move(self.checkout_path, dest_path) class GitWrapper(SCMWrapper): @@ -1092,8 +1109,8 @@ class SVNWrapper(SCMWrapper): 'svn-remote.svn.url'], cwd=self.checkout_path).rstrip() if remote_url.rstrip('/') == base_url.rstrip('/'): - print('\n_____ %s looks like a git-svn checkout. Skipping.' - % self.relpath) + self.Print('\n_____ %s looks like a git-svn checkout. Skipping.' + % self.relpath) return # TODO(borenet): Get the svn revision number? # Get the existing scm url and the revision number of the current checkout. diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index 7c0b38670a..a79bc25571 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -646,6 +646,18 @@ class SVNWrapperTestCase(BaseTestCase): 1, 'cmd', '/cwd', 'stdout', 'stderr') gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.').AndRaise(error) + bad_scm_path = os.path.join(self.root_dir, '_bad_scm', + os.path.dirname(self.relpath)) + gclient_scm.os.makedirs(bad_scm_path) + dest_path = os.path.join(bad_scm_path, + os.path.basename(self.relpath) + 'ABCD') + self.mox.StubOutWithMock(gclient_scm.tempfile, 'mkdtemp', True) + gclient_scm.tempfile.mkdtemp( + prefix=os.path.basename(self.relpath), + dir=os.path.join(self.root_dir, '_bad_scm', + os.path.dirname(self.relpath))).AndReturn(dest_path) + self.mox.StubOutWithMock(gclient_scm.shutil, 'move', True) + gclient_scm.shutil.move(self.base_path, dest_path) gclient_scm.os.path.exists(self.root_dir).AndReturn(True) gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None ).AndReturn('1.5.1') @@ -662,34 +674,45 @@ class SVNWrapperTestCase(BaseTestCase): scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, relpath=self.relpath) scm.update(options, None, []) + self.checkstdout('_____ Conflicting directory found in %s. Moving to %s.\n' + % (self.base_path, dest_path)) def testUpdateGitForce(self): options = self.Options(verbose=True, force=True) - file_path = gclient_scm.os.path.join(self.root_dir, self.relpath, '.hg') - gclient_scm.os.path.exists(file_path).AndReturn(False) - gclient_scm.os.path.exists(self.base_path).AndReturn(True) - self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True) - gclient_scm.scm.GIT.IsGitSvn(self.base_path).AndReturn(False) - error = gclient_scm.subprocess2.CalledProcessError( - 1, 'cmd', '/cwd', 'stdout', 'stderr') - gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.').AndRaise(error) - gclient_scm.os.path.exists(self.root_dir).AndReturn(True) - gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None - ).AndReturn('1.5.1') - gclient_scm.scm.SVN.RunAndGetFileList( - options.verbose, - ['checkout', self.url, self.base_path, '--force', '--ignore-externals'], - cwd=self.root_dir, - file_list=[]) - - gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.' - ).AndReturn({'Revision': 100}) - - self.mox.ReplayAll() - scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) - file_list = [] - scm.update(options, None, file_list) + old_environ = dict(gclient_scm.os.environ) + gclient_scm.os.environ['CHROME_HEADLESS'] = '1' + try: + file_path = gclient_scm.os.path.join(self.root_dir, self.relpath, '.hg') + gclient_scm.os.path.exists(file_path).AndReturn(False) + gclient_scm.os.path.exists(self.base_path).AndReturn(True) + self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True) + gclient_scm.scm.GIT.IsGitSvn(self.base_path).AndReturn(False) + error = gclient_scm.subprocess2.CalledProcessError( + 1, 'cmd', '/cwd', 'stdout', 'stderr') + gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.').AndRaise(error) + gclient_scm.gclient_utils.rmtree(self.base_path) + gclient_scm.os.path.exists(self.root_dir).AndReturn(True) + gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None + ).AndReturn('1.5.1') + gclient_scm.scm.SVN.RunAndGetFileList( + options.verbose, + ['checkout', self.url, self.base_path, '--force', + '--ignore-externals'], + cwd=self.root_dir, + file_list=[]) + + gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.' + ).AndReturn({'Revision': 100}) + + self.mox.ReplayAll() + scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, + relpath=self.relpath) + file_list = [] + scm.update(options, None, file_list) + self.checkstdout('_____ Conflicting directory found in %s. Removing.\n' + % self.base_path) + finally: + gclient_scm.os.environ = old_environ def testUpdateGitSvn(self): options = self.Options(verbose=True) @@ -1297,7 +1320,6 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): gclient_scm.os.path.isdir(self.base_path).AndReturn(True) gclient_scm.os.path.exists(os.path.join(self.base_path, '.git') ).AndReturn(False) - self.mox.StubOutWithMock(gclient_scm.GitWrapper, '_Clone', True) # pylint: disable=E1120 gclient_scm.GitWrapper._Clone('refs/remotes/origin/master', self.url, @@ -1326,7 +1348,6 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): gclient_scm.os.path.isdir(self.base_path).AndReturn(True) gclient_scm.os.path.exists(os.path.join(self.base_path, '.git') ).AndReturn(False) - self.mox.StubOutWithMock(gclient_scm.GitWrapper, '_Clone', True) # pylint: disable=E1120 gclient_scm.GitWrapper._Clone( diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index bf06e22b30..046e2fe10a 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -21,7 +21,11 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT_DIR) from testing_support.fake_repos import join, write -from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive +from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive, \ + FakeRepoSkiaDEPS + +import gclient_utils +import scm as gclient_scm import subprocess2 @@ -761,6 +765,21 @@ class GClientSmokeSVN(GClientSmokeBase): self.checkBlock(res[0], ['running', 'running', 'running']) + def testUnversionedRepository(self): + # Check that gclient automatically deletes crippled SVN repositories. + if not self.enabled: + return + self.gclient(['config', self.svn_base + 'trunk/src/']) + cmd = ['sync', '--jobs', '1', '--delete_unversioned_trees', '--reset'] + self.assertEquals(0, self.gclient(cmd)[-1]) + third_party = join(self.root_dir, 'src', 'third_party') + subprocess2.check_call(['svn', 'propset', '-q', 'svn:ignore', 'foo', '.'], + cwd=third_party) + + # Cripple src/third_party/foo and make sure gclient still succeeds. + gclient_utils.rmtree(join(third_party, 'foo', '.svn')) + self.assertEquals(0, self.gclient(cmd)[-1]) + class GClientSmokeSVNTransitive(GClientSmokeBase): FAKE_REPOS_CLASS = FakeRepoTransitive @@ -1323,6 +1342,159 @@ class GClientSmokeBoth(GClientSmokeBase): self.assertEquals(sorted(entries), sorted(expected)) +class SkiaDEPSTransitionSmokeTest(GClientSmokeBase): + """Simulate the behavior of bisect bots as they transition across the Skia + DEPS change.""" + + FAKE_REPOS_CLASS = FakeRepoSkiaDEPS + + def setUp(self): + super(SkiaDEPSTransitionSmokeTest, self).setUp() + self.enabled = self.FAKE_REPOS.set_up_git() and self.FAKE_REPOS.set_up_svn() + + def testSkiaDEPSChangeSVN(self): + if not self.enabled: + return + + # Create an initial checkout: + # - Single checkout at the root. + # - Multiple checkouts in a shared subdirectory. + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.svn_base + 'trunk/src/",' + '}]']) + + checkout_path = os.path.join(self.root_dir, 'src') + skia = os.path.join(checkout_path, 'third_party', 'skia') + skia_gyp = os.path.join(skia, 'gyp') + skia_include = os.path.join(skia, 'include') + skia_src = os.path.join(skia, 'src') + + gyp_svn_url = self.svn_base + 'skia/gyp' + include_svn_url = self.svn_base + 'skia/include' + src_svn_url = self.svn_base + 'skia/src' + skia_git_url = self.git_base + 'repo_1' + + # Initial sync. Verify that we get the expected checkout. + res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@2']) + self.assertEqual(res[2], 0, 'Initial sync failed.') + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_gyp)['URL'], + gyp_svn_url) + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_include)['URL'], + include_svn_url) + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_src)['URL'], + src_svn_url) + + # Verify that the sync succeeds. Verify that we have the expected merged + # checkout. + res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@3']) + self.assertEqual(res[2], 0, 'DEPS change sync failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia), skia_git_url) + + # Sync again. Verify that we still have the expected merged checkout. + res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@3']) + self.assertEqual(res[2], 0, 'Subsequent sync failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia), skia_git_url) + + # Sync back to the original DEPS. Verify that we get the original structure. + res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@2']) + self.assertEqual(res[2], 0, 'Reverse sync failed.') + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_gyp)['URL'], + gyp_svn_url) + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_include)['URL'], + include_svn_url) + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_src)['URL'], + src_svn_url) + + # Sync again. Verify that we still have the original structure. + res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@2']) + self.assertEqual(res[2], 0, 'Subsequent sync #2 failed.') + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_gyp)['URL'], + gyp_svn_url) + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_include)['URL'], + include_svn_url) + self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_src)['URL'], + src_svn_url) + + def testSkiaDEPSChangeGit(self): + if not self.enabled: + return + + # Create an initial checkout: + # - Single checkout at the root. + # - Multiple checkouts in a shared subdirectory. + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.git_base + 'repo_2",' + '}]']) + + checkout_path = os.path.join(self.root_dir, 'src') + skia = os.path.join(checkout_path, 'third_party', 'skia') + skia_gyp = os.path.join(skia, 'gyp') + skia_include = os.path.join(skia, 'include') + skia_src = os.path.join(skia, 'src') + + gyp_git_url = self.git_base + 'repo_3' + include_git_url = self.git_base + 'repo_4' + src_git_url = self.git_base + 'repo_5' + skia_git_url = self.FAKE_REPOS.git_base + 'repo_1' + + pre_hash = self.githash('repo_2', 1) + post_hash = self.githash('repo_2', 2) + + # Initial sync. Verify that we get the expected checkout. + res = self.gclient(['sync', '--deps', 'mac', '--revision', + 'src@%s' % pre_hash]) + self.assertEqual(res[2], 0, 'Initial sync failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_gyp), gyp_git_url) + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_include), include_git_url) + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_src), src_git_url) + + # Verify that the sync succeeds. Verify that we have the expected merged + # checkout. + res = self.gclient(['sync', '--deps', 'mac', '--revision', + 'src@%s' % post_hash]) + self.assertEqual(res[2], 0, 'DEPS change sync failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia), skia_git_url) + + # Sync again. Verify that we still have the expected merged checkout. + res = self.gclient(['sync', '--deps', 'mac', '--revision', + 'src@%s' % post_hash]) + self.assertEqual(res[2], 0, 'Subsequent sync failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia), skia_git_url) + + # Sync back to the original DEPS. Verify that we get the original structure. + res = self.gclient(['sync', '--deps', 'mac', '--revision', + 'src@%s' % pre_hash]) + self.assertEqual(res[2], 0, 'Reverse sync failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_gyp), gyp_git_url) + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_include), include_git_url) + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_src), src_git_url) + + # Sync again. Verify that we still have the original structure. + res = self.gclient(['sync', '--deps', 'mac', '--revision', + 'src@%s' % pre_hash]) + self.assertEqual(res[2], 0, 'Subsequent sync #2 failed.') + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_gyp), gyp_git_url) + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_include), include_git_url) + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + skia_src), src_git_url) + + class GClientSmokeFromCheckout(GClientSmokeBase): # WebKit abuses this. It has a .gclient and a DEPS from a checkout. def setUp(self):