gclient: Actually move or delete mismatched checkouts

Followup to https://codereview.chromium.org/189913020/ ("gclient: print a warning if a dep would get deleted or moved in the future")

BUG=skia:1638

Review URL: https://codereview.chromium.org/225403015

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@268944 0039d316-1c4b-4281-b951-d872f2087c98

gclient_scm.py

@@ -6,6 +6,7 @@
 
 from __future__ import print_function
 
+import errno
 import logging
 import os
 import posixpath
@@ -20,6 +21,7 @@ import download_from_google_storage
 import gclient_utils
 import git_cache
 import scm
+import shutil
 import subprocess2
 
 
@@ -196,21 +198,36 @@ class SCMWrapper(object):
       # valid git or svn checkout.
       return False
 
-  # TODO(borenet): Remove this once SCMWrapper._DeleteOrMove is enabled.
-  # pylint: disable=R0201
   def _DeleteOrMove(self, force):
     """Delete the checkout directory or move it out of the way.
 
     Args:
         force: bool; if True, delete the directory. Otherwise, just move it.
     """
-    gclient_utils.AddWarning('WARNING: Upcoming change in '
-                             'https://codereview.chromium.org/225403015 would '
-                             'cause %s to be deleted or moved to the side. '
-                             'This is intended to ease changes to DEPS in the '
-                             'future. If you are seeing this warning and '
-                             'haven\'t changed the DEPS file, please contact '
-                             'borenet@ immediately.' % self.checkout_path)
+    if force and os.environ.get('CHROME_HEADLESS') == '1':
+      self.Print('_____ Conflicting directory found in %s. Removing.'
+                 % self.checkout_path)
+      gclient_utils.AddWarning('Conflicting directory %s deleted.'
+                               % self.checkout_path)
+      gclient_utils.rmtree(self.checkout_path)
+    else:
+      bad_scm_dir = os.path.join(self._root_dir, '_bad_scm',
+                                 os.path.dirname(self.relpath))
+
+      try:
+        os.makedirs(bad_scm_dir)
+      except OSError as e:
+        if e.errno != errno.EEXIST:
+          raise
+
+      dest_path = tempfile.mkdtemp(
+          prefix=os.path.basename(self.relpath),
+          dir=bad_scm_dir)
+      self.Print('_____ Conflicting directory found in %s. Moving to %s.'
+                 % (self.checkout_path, dest_path))
+      gclient_utils.AddWarning('Conflicting directory %s moved to %s.'
+                               % (self.checkout_path, dest_path))
+      shutil.move(self.checkout_path, dest_path)
 
 
 class GitWrapper(SCMWrapper):
@@ -1092,8 +1109,8 @@ class SVNWrapper(SCMWrapper):
                                       'svn-remote.svn.url'],
                                      cwd=self.checkout_path).rstrip()
         if remote_url.rstrip('/') == base_url.rstrip('/'):
-          print('\n_____ %s looks like a git-svn checkout. Skipping.'
-                % self.relpath)
+          self.Print('\n_____ %s looks like a git-svn checkout. Skipping.'
+                     % self.relpath)
           return # TODO(borenet): Get the svn revision number?
 
     # Get the existing scm url and the revision number of the current checkout.

tests/gclient_scm_test.py

@@ -646,6 +646,18 @@ class SVNWrapperTestCase(BaseTestCase):
         1, 'cmd', '/cwd', 'stdout', 'stderr')
     gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.').AndRaise(error)
 
+    bad_scm_path = os.path.join(self.root_dir, '_bad_scm',
+                                os.path.dirname(self.relpath))
+    gclient_scm.os.makedirs(bad_scm_path)
+    dest_path = os.path.join(bad_scm_path,
+                             os.path.basename(self.relpath) + 'ABCD')
+    self.mox.StubOutWithMock(gclient_scm.tempfile, 'mkdtemp', True)
+    gclient_scm.tempfile.mkdtemp(
+        prefix=os.path.basename(self.relpath),
+        dir=os.path.join(self.root_dir, '_bad_scm',
+                         os.path.dirname(self.relpath))).AndReturn(dest_path)
+    self.mox.StubOutWithMock(gclient_scm.shutil, 'move', True)
+    gclient_scm.shutil.move(self.base_path, dest_path)
     gclient_scm.os.path.exists(self.root_dir).AndReturn(True)
     gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None
         ).AndReturn('1.5.1')
@@ -662,34 +674,45 @@ class SVNWrapperTestCase(BaseTestCase):
     scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
                             relpath=self.relpath)
     scm.update(options, None, [])
+    self.checkstdout('_____ Conflicting directory found in %s. Moving to %s.\n'
+                     % (self.base_path, dest_path))
 
   def testUpdateGitForce(self):
     options = self.Options(verbose=True, force=True)
-    file_path = gclient_scm.os.path.join(self.root_dir, self.relpath, '.hg')
-    gclient_scm.os.path.exists(file_path).AndReturn(False)
-    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
-    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True)
-    gclient_scm.scm.GIT.IsGitSvn(self.base_path).AndReturn(False)
-    error = gclient_scm.subprocess2.CalledProcessError(
-        1, 'cmd', '/cwd', 'stdout', 'stderr')
-    gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.').AndRaise(error)
-    gclient_scm.os.path.exists(self.root_dir).AndReturn(True)
-    gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None
-        ).AndReturn('1.5.1')
-    gclient_scm.scm.SVN.RunAndGetFileList(
-        options.verbose,
-        ['checkout', self.url, self.base_path, '--force', '--ignore-externals'],
-        cwd=self.root_dir,
-        file_list=[])
-
-    gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.'
-        ).AndReturn({'Revision': 100})
-
-    self.mox.ReplayAll()
-    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
-                            relpath=self.relpath)
-    file_list = []
-    scm.update(options, None, file_list)
+    old_environ = dict(gclient_scm.os.environ)
+    gclient_scm.os.environ['CHROME_HEADLESS'] = '1'
+    try:
+      file_path = gclient_scm.os.path.join(self.root_dir, self.relpath, '.hg')
+      gclient_scm.os.path.exists(file_path).AndReturn(False)
+      gclient_scm.os.path.exists(self.base_path).AndReturn(True)
+      self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True)
+      gclient_scm.scm.GIT.IsGitSvn(self.base_path).AndReturn(False)
+      error = gclient_scm.subprocess2.CalledProcessError(
+          1, 'cmd', '/cwd', 'stdout', 'stderr')
+      gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.').AndRaise(error)
+      gclient_scm.gclient_utils.rmtree(self.base_path)
+      gclient_scm.os.path.exists(self.root_dir).AndReturn(True)
+      gclient_scm.scm.SVN.Capture(['--version', '--quiet'], None
+          ).AndReturn('1.5.1')
+      gclient_scm.scm.SVN.RunAndGetFileList(
+          options.verbose,
+          ['checkout', self.url, self.base_path, '--force',
+           '--ignore-externals'],
+          cwd=self.root_dir,
+          file_list=[])
+  
+      gclient_scm.scm.SVN._CaptureInfo([], self.base_path+'/.'
+          ).AndReturn({'Revision': 100})
+  
+      self.mox.ReplayAll()
+      scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
+                              relpath=self.relpath)
+      file_list = []
+      scm.update(options, None, file_list)
+      self.checkstdout('_____ Conflicting directory found in %s. Removing.\n'
+                       % self.base_path)
+    finally:
+      gclient_scm.os.environ = old_environ
 
   def testUpdateGitSvn(self):
     options = self.Options(verbose=True)
@@ -1297,7 +1320,6 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
     gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
     gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')
                                ).AndReturn(False)
-
     self.mox.StubOutWithMock(gclient_scm.GitWrapper, '_Clone', True)
     # pylint: disable=E1120
     gclient_scm.GitWrapper._Clone('refs/remotes/origin/master', self.url,
@@ -1326,7 +1348,6 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
     gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
     gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')
                                ).AndReturn(False)
-
     self.mox.StubOutWithMock(gclient_scm.GitWrapper, '_Clone', True)
     # pylint: disable=E1120
     gclient_scm.GitWrapper._Clone(

tests/gclient_smoketest.py

@@ -21,7 +21,11 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ROOT_DIR)
 
 from testing_support.fake_repos import join, write
-from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive
+from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive, \
+                                       FakeRepoSkiaDEPS
+
+import gclient_utils
+import scm as gclient_scm
 
 import subprocess2
 
@@ -761,6 +765,21 @@ class GClientSmokeSVN(GClientSmokeBase):
     self.checkBlock(res[0],
                     ['running', 'running', 'running'])
 
+  def testUnversionedRepository(self):
+    # Check that gclient automatically deletes crippled SVN repositories.
+    if not self.enabled:
+      return
+    self.gclient(['config', self.svn_base + 'trunk/src/'])
+    cmd = ['sync', '--jobs', '1', '--delete_unversioned_trees', '--reset']
+    self.assertEquals(0, self.gclient(cmd)[-1])
+    third_party = join(self.root_dir, 'src', 'third_party')
+    subprocess2.check_call(['svn', 'propset', '-q', 'svn:ignore', 'foo', '.'],
+                           cwd=third_party)
+
+    # Cripple src/third_party/foo and make sure gclient still succeeds.
+    gclient_utils.rmtree(join(third_party, 'foo', '.svn'))
+    self.assertEquals(0, self.gclient(cmd)[-1])
+
 
 class GClientSmokeSVNTransitive(GClientSmokeBase):
   FAKE_REPOS_CLASS = FakeRepoTransitive
@@ -1323,6 +1342,159 @@ class GClientSmokeBoth(GClientSmokeBase):
     self.assertEquals(sorted(entries), sorted(expected))
 
 
+class SkiaDEPSTransitionSmokeTest(GClientSmokeBase):
+  """Simulate the behavior of bisect bots as they transition across the Skia
+  DEPS change."""
+
+  FAKE_REPOS_CLASS = FakeRepoSkiaDEPS
+
+  def setUp(self):
+    super(SkiaDEPSTransitionSmokeTest, self).setUp()
+    self.enabled = self.FAKE_REPOS.set_up_git() and self.FAKE_REPOS.set_up_svn()
+
+  def testSkiaDEPSChangeSVN(self):
+    if not self.enabled:
+      return
+
+    # Create an initial checkout:
+    # - Single checkout at the root.
+    # - Multiple checkouts in a shared subdirectory.
+    self.gclient(['config', '--spec',
+        'solutions=['
+        '{"name": "src",'
+        ' "url": "' + self.svn_base + 'trunk/src/",'
+        '}]'])
+
+    checkout_path = os.path.join(self.root_dir, 'src')
+    skia = os.path.join(checkout_path, 'third_party', 'skia')
+    skia_gyp = os.path.join(skia, 'gyp')
+    skia_include = os.path.join(skia, 'include')
+    skia_src = os.path.join(skia, 'src')
+
+    gyp_svn_url = self.svn_base + 'skia/gyp'
+    include_svn_url = self.svn_base + 'skia/include'
+    src_svn_url = self.svn_base + 'skia/src'
+    skia_git_url = self.git_base + 'repo_1'
+
+    # Initial sync. Verify that we get the expected checkout.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@2'])
+    self.assertEqual(res[2], 0, 'Initial sync failed.')
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_gyp)['URL'],
+                     gyp_svn_url)
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_include)['URL'],
+                     include_svn_url)
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_src)['URL'],
+                     src_svn_url)
+
+    # Verify that the sync succeeds. Verify that we have the  expected merged
+    # checkout.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@3'])
+    self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia), skia_git_url)
+
+    # Sync again. Verify that we still have the expected merged checkout.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@3'])
+    self.assertEqual(res[2], 0, 'Subsequent sync failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia), skia_git_url)
+
+    # Sync back to the original DEPS. Verify that we get the original structure.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@2'])
+    self.assertEqual(res[2], 0, 'Reverse sync failed.')
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_gyp)['URL'],
+                     gyp_svn_url)
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_include)['URL'],
+                     include_svn_url)
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_src)['URL'],
+                     src_svn_url)
+
+    # Sync again. Verify that we still have the original structure.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision', 'src@2'])
+    self.assertEqual(res[2], 0, 'Subsequent sync #2 failed.')
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_gyp)['URL'],
+                     gyp_svn_url)
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_include)['URL'],
+                     include_svn_url)
+    self.assertEqual(gclient_scm.SVN.CaptureLocalInfo([], skia_src)['URL'],
+                     src_svn_url)
+
+  def testSkiaDEPSChangeGit(self):
+    if not self.enabled:
+      return
+
+    # Create an initial checkout:
+    # - Single checkout at the root.
+    # - Multiple checkouts in a shared subdirectory.
+    self.gclient(['config', '--spec',
+        'solutions=['
+        '{"name": "src",'
+        ' "url": "' + self.git_base + 'repo_2",'
+        '}]'])
+
+    checkout_path = os.path.join(self.root_dir, 'src')
+    skia = os.path.join(checkout_path, 'third_party', 'skia')
+    skia_gyp = os.path.join(skia, 'gyp')
+    skia_include = os.path.join(skia, 'include')
+    skia_src = os.path.join(skia, 'src')
+
+    gyp_git_url = self.git_base + 'repo_3'
+    include_git_url = self.git_base + 'repo_4'
+    src_git_url = self.git_base + 'repo_5'
+    skia_git_url = self.FAKE_REPOS.git_base + 'repo_1'
+
+    pre_hash = self.githash('repo_2', 1)
+    post_hash = self.githash('repo_2', 2)
+
+    # Initial sync. Verify that we get the expected checkout.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision',
+                        'src@%s' % pre_hash])
+    self.assertEqual(res[2], 0, 'Initial sync failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_gyp), gyp_git_url)
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_include), include_git_url)
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_src), src_git_url)
+
+    # Verify that the sync succeeds. Verify that we have the  expected merged
+    # checkout.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision',
+                        'src@%s' % post_hash])
+    self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia), skia_git_url)
+
+    # Sync again. Verify that we still have the expected merged checkout.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision',
+                        'src@%s' % post_hash])
+    self.assertEqual(res[2], 0, 'Subsequent sync failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia), skia_git_url)
+
+    # Sync back to the original DEPS. Verify that we get the original structure.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision',
+                        'src@%s' % pre_hash])
+    self.assertEqual(res[2], 0, 'Reverse sync failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_gyp), gyp_git_url)
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_include), include_git_url)
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_src), src_git_url)
+
+    # Sync again. Verify that we still have the original structure.
+    res = self.gclient(['sync', '--deps', 'mac', '--revision',
+                        'src@%s' % pre_hash])
+    self.assertEqual(res[2], 0, 'Subsequent sync #2 failed.')
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_gyp), gyp_git_url)
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_include), include_git_url)
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             skia_src), src_git_url)
+
+
 class GClientSmokeFromCheckout(GClientSmokeBase):
   # WebKit abuses this. It has a .gclient and a DEPS from a checkout.
   def setUp(self):