Allow gclient clone in non-empty directories

Add an option in DEPS files to clone a project into a temp dir
and then copy into expected final dir.  This allows checking out a
git repo into a folder which is non-empty. It is useful for projects
that are embedded in src/ but want to specify the revision of
src/ in the embedded project (such as android private).

BUG=165280

Review URL: https://chromiumcodereview.appspot.com/19359002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@212720 0039d316-1c4b-4281-b951-d872f2087c98

gclient_scm.py

@@ -10,6 +10,7 @@ import os
 import posixpath
 import re
 import sys
+import tempfile
 import threading
 import time
 
@@ -350,10 +351,9 @@ class GitWrapper(SCMWrapper):
       # hash is also a tag, only make a distinction at checkout
       rev_type = "hash"
 
-    if not os.path.exists(self.checkout_path) or (
-        os.path.isdir(self.checkout_path) and
-        not os.listdir(self.checkout_path)):
-      gclient_utils.safe_makedirs(os.path.dirname(self.checkout_path))
+    if (not os.path.exists(self.checkout_path) or
+        (os.path.isdir(self.checkout_path) and
+         not os.path.exists(os.path.join(self.checkout_path, '.git')))):
       self._Clone(revision, url, options)
       self.UpdateSubmoduleConfig()
       if file_list is not None:
@@ -408,19 +408,6 @@ class GitWrapper(SCMWrapper):
     if return_early:
       return
 
-    if not self._IsValidGitRepo():
-      # .git directory is hosed for some reason, set it back up.
-      print('_____ %s/.git is corrupted, rebuilding' % self.relpath)
-      self._Run(['init'], options)
-      self._Run(['remote', 'set-url', 'origin', url], options)
-
-    if not self._HasHead():
-      # Previous checkout was aborted before branches could be created in repo,
-      # so we need to reconstruct them here.
-      self._Run(['-c', 'core.deltaBaseCacheLimit=2g', 'pull', 'origin',
-                 'master'], options)
-      self._FetchAndReset(revision, file_list, options)
-
     cur_branch = self._GetCurrentBranch()
 
     # Cases:
@@ -859,47 +846,45 @@ class GitWrapper(SCMWrapper):
       print('')
     template_path = os.path.join(
         os.path.dirname(THIS_FILE_PATH), 'git-templates')
-    clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress',
-                 '--template=%s' % template_path]
+    clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--no-checkout',
+                 '--progress', '--template=%s' % template_path]
     if self.cache_dir:
       clone_cmd.append('--shared')
-    if revision.startswith('refs/heads/'):
-      clone_cmd.extend(['-b', revision.replace('refs/heads/', '')])
-      detach_head = False
-    else:
-      detach_head = True
     if options.verbose:
       clone_cmd.append('--verbose')
-    clone_cmd.extend([url, self.checkout_path])
-
+    clone_cmd.append(url)
     # If the parent directory does not exist, Git clone on Windows will not
     # create it, so we need to do it manually.
     parent_dir = os.path.dirname(self.checkout_path)
-    if not os.path.exists(parent_dir):
-      gclient_utils.safe_makedirs(parent_dir)
-
-    for _ in range(3):
-      try:
-        self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
-        break
-      except subprocess2.CalledProcessError, e:
-        # Too bad we don't have access to the actual output yet.
-        # We should check for "transfer closed with NNN bytes remaining to
-        # read". In the meantime, just make sure .git exists.
-        if (e.returncode == 128 and
-            os.path.exists(os.path.join(self.checkout_path, '.git'))):
+    gclient_utils.safe_makedirs(parent_dir)
+    tmp_dir = tempfile.mkdtemp(
+        prefix='_gclient_%s_' % os.path.basename(self.checkout_path),
+        dir=parent_dir)
+    try:
+      clone_cmd.append(tmp_dir)
+      for i in xrange(3):
+        try:
+          self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
+          break
+        except subprocess2.CalledProcessError as e:
+          gclient_utils.rmtree(os.path.join(tmp_dir, '.git'))
+          if e.returncode != 128 or i == 2:
+            raise
           print(str(e))
           print('Retrying...')
-          continue
-        raise e
-
-    # Update the "branch-heads" remote-tracking branches, since we might need it
-    # to checkout a specific revision below.
-    self._UpdateBranchHeads(options, fetch=True)
-
-    if detach_head:
+      gclient_utils.safe_makedirs(self.checkout_path)
+      os.rename(os.path.join(tmp_dir, '.git'),
+                os.path.join(self.checkout_path, '.git'))
+    finally:
+      if os.listdir(tmp_dir):
+        print('\n_____ removing non-empty tmp dir %s' % tmp_dir)
+      gclient_utils.rmtree(tmp_dir)
+    if revision.startswith('refs/heads/'):
+      self._Run(
+          ['checkout', '--quiet', revision.replace('refs/heads/', '')], options)
+    else:
       # Squelch git's very verbose detached HEAD warning and use our own
-      self._Capture(['checkout', '--quiet', '%s' % revision])
+      self._Run(['checkout', '--quiet', revision], options)
       print(
         ('Checked out %s to a detached HEAD. Before making any commits\n'
          'in this repo, you should use \'git checkout <branch>\' to switch to\n'
@@ -977,28 +962,6 @@ class GitWrapper(SCMWrapper):
       # whitespace between projects when syncing.
       print('')
 
-  def _IsValidGitRepo(self):
-    """Returns if the directory is a valid git repository.
-
-    Checks if git status works.
-    """
-    try:
-      self._Capture(['status'])
-      return True
-    except subprocess2.CalledProcessError:
-      return False
-
-  def _HasHead(self):
-    """Returns True if any commit is checked out.
-
-    This is done by checking if rev-parse HEAD works in the current repository.
-    """
-    try:
-      self._GetCurrentBranch()
-      return True
-    except subprocess2.CalledProcessError:
-      return False
-
   @staticmethod
   def _CheckMinVersion(min_version):
     (ok, current_version) = scm.GIT.AssertVersion(min_version)

tests/gclient_scm_test.py

@@ -8,7 +8,6 @@
 # pylint: disable=E1103
 
 # Import before super_mox to keep valid references.
-from os import rename
 from shutil import rmtree
 from subprocess import Popen, PIPE, STDOUT
 
@@ -1026,21 +1025,6 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     self.assertRaisesError(exception, scm.update, options, (), [])
     sys.stdout.close()
 
-  def testUpdateNotGit(self):
-    if not self.enabled:
-      return
-    options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
-    git_path = join(self.base_path, '.git')
-    rename(git_path, git_path + 'foo')
-    exception = ('\n____ . at refs/heads/master\n'
-                 '\tPath is not a git repo. No .git dir.\n'
-                 '\tTo resolve:\n'
-                 '\t\trm -rf .\n'
-                 '\tAnd run gclient sync again\n')
-    self.assertRaisesError(exception, scm.update, options, (), [])
-
   def testRevinfo(self):
     if not self.enabled:
       return

tests/gclient_smoketest.py

@@ -839,8 +839,12 @@ class GClientSmokeGIT(GClientSmokeBase):
     # TODO(maruel): safesync.
     self.gclient(['config', self.git_base + 'repo_1', '--name', 'src'])
     # Test unversioned checkout.
-    self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'],
-                      ['running', 'running', 'running', 'running', 'running'])
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '1'],
+        ['running', ('running', self.root_dir + '/src'),
+         'running', ('running', self.root_dir + '/src/repo2'),
+         'running', ('running', self.root_dir + '/src/repo2/repo_renamed'),
+         'running', 'running'])
     # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must
     # add sync parsing to get the list of updated files.
     tree = self.mangle_git_tree(('repo_1@2', 'src'),
@@ -856,10 +860,13 @@ class GClientSmokeGIT(GClientSmokeBase):
 
     # Test incremental versioned sync: sync backward.
     diffdir = os.path.join(self.root_dir, 'src', 'repo2', 'repo_renamed')
-    self.parseGclient(['sync', '--jobs', '1', '--revision',
-                       'src@' + self.githash('repo_1', 1),
-                       '--deps', 'mac', '--delete_unversioned_trees'],
-                       ['running', 'running', ('running', diffdir), 'deleting'])
+    self.parseGclient(
+        ['sync', '--jobs', '1', '--revision',
+        'src@' + self.githash('repo_1', 1),
+        '--deps', 'mac', '--delete_unversioned_trees'],
+        ['running', ('running', self.root_dir + '/src/repo2/repo3'),
+         'running', ('running', self.root_dir + '/src/repo4'),
+         ('running', diffdir), 'deleting'])
     tree = self.mangle_git_tree(('repo_1@1', 'src'),
                                 ('repo_2@2', 'src/repo2'),
                                 ('repo_3@1', 'src/repo2/repo3'),
@@ -869,8 +876,10 @@ class GClientSmokeGIT(GClientSmokeBase):
     # Test incremental sync: delete-unversioned_trees isn't there.
     expect3 = ('running', os.path.join(self.root_dir, 'src', 'repo2', 'repo3'))
     expect4 = ('running', os.path.join(self.root_dir, 'src', 'repo4'))
-    self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'],
-        ['running', 'running', 'running', expect3, expect4])
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '1'],
+        ['running', ('running', self.root_dir + '/src/repo2/repo_renamed'),
+         'running', 'running', expect3, expect4])
     tree = self.mangle_git_tree(('repo_1@2', 'src'),
                                 ('repo_2@1', 'src/repo2'),
                                 ('repo_3@1', 'src/repo2/repo3'),
@@ -888,9 +897,12 @@ class GClientSmokeGIT(GClientSmokeBase):
     self.parseGclient(
         ['sync', '--deps', 'mac', '--jobs', '1',
          '--revision', 'invalid@' + self.githash('repo_1', 1)],
-        ['running', 'running', 'running', 'running', 'running'],
+        ['running', ('running', self.root_dir + '/src'),
+         'running', ('running', self.root_dir + '/src/repo2'),
+         'running', ('running', self.root_dir + '/src/repo2/repo_renamed'),
+         'running', 'running'],
         'Please fix your script, having invalid --revision flags '
-            'will soon considered an error.\n')
+        'will soon considered an error.\n')
     tree = self.mangle_git_tree(('repo_1@2', 'src'),
                                 ('repo_2@1', 'src/repo2'),
                                 ('repo_3@2', 'src/repo2/repo_renamed'))
@@ -903,9 +915,13 @@ class GClientSmokeGIT(GClientSmokeBase):
       return
     # When no solution name is provided, gclient uses the first solution listed.
     self.gclient(['config', self.git_base + 'repo_1', '--name', 'src'])
-    self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1',
-                       '--revision', self.githash('repo_1', 1)],
-                      ['running', 'running', 'running', 'running'])
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '1',
+         '--revision', self.githash('repo_1', 1)],
+        ['running', ('running', self.root_dir + '/src'),
+         'running', ('running', self.root_dir + '/src/repo2'),
+         'running', ('running', self.root_dir + '/src/repo2/repo3'),
+         'running', ('running', self.root_dir + '/src/repo4')])
     tree = self.mangle_git_tree(('repo_1@1', 'src'),
                                 ('repo_2@2', 'src/repo2'),
                                 ('repo_3@1', 'src/repo2/repo3'),
@@ -918,9 +934,13 @@ class GClientSmokeGIT(GClientSmokeBase):
     # TODO(maruel): safesync.
     self.gclient(['config', self.git_base + 'repo_1', '--name', 'src'])
     # Test unversioned checkout.
-    self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'],
-                      ['running', 'running', 'running', 'running', 'running'],
-                      untangle=True)
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '8'],
+        ['running', ('running', self.root_dir + '/src'),
+         'running', ('running', self.root_dir + '/src/repo2'),
+         'running', ('running', self.root_dir + '/src/repo2/repo_renamed'),
+         'running', 'running'],
+        untangle=True)
     # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must
     # add sync parsing to get the list of updated files.
     tree = self.mangle_git_tree(('repo_1@2', 'src'),
@@ -940,7 +960,9 @@ class GClientSmokeGIT(GClientSmokeBase):
     self.parseGclient(
         ['sync', '--revision', 'src@' + self.githash('repo_1', 1),
           '--deps', 'mac', '--delete_unversioned_trees', '--jobs', '8'],
-        ['running', 'running', expect3, 'deleting'],
+        ['running', ('running', self.root_dir + '/src/repo4'),
+         'running', ('running', self.root_dir + '/src/repo2/repo3'),
+         expect3, 'deleting'],
         untangle=True)
     tree = self.mangle_git_tree(('repo_1@1', 'src'),
                                 ('repo_2@2', 'src/repo2'),
@@ -951,11 +973,11 @@ class GClientSmokeGIT(GClientSmokeBase):
     # Test incremental sync: delete-unversioned_trees isn't there.
     expect4 = os.path.join(self.root_dir, 'src', 'repo2', 'repo3')
     expect5 = os.path.join(self.root_dir, 'src', 'repo4')
-    self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'],
-                      ['running', 'running', 'running',
-                       ('running', expect4),
-                       ('running', expect5)],
-                      untangle=True)
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '8'],
+        ['running', ('running', self.root_dir + '/src/repo2/repo_renamed'),
+         'running', 'running', ('running', expect4), ('running', expect5)],
+        untangle=True)
     tree = self.mangle_git_tree(('repo_1@2', 'src'),
                                 ('repo_2@1', 'src/repo2'),
                                 ('repo_3@1', 'src/repo2/repo3'),
@@ -1074,12 +1096,16 @@ class GClientSmokeBoth(GClientSmokeBase):
         '{"name": "src-git",'
         '"url": "' + self.git_base + 'repo_1"}]'])
     self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'],
-        ['running', 'running', 'running',
+        ['running',
+         'running', ('running', self.root_dir + '/src-git'),
+         'running',
          # This is due to the way svn update is called for a single
          # file when File() is used in a DEPS file.
          ('running', self.root_dir + '/src/file/other'),
-         'running', 'running', 'running', 'running', 'running', 'running',
-         'running', 'running'])
+         'running', 'running', 'running',
+         'running', ('running', self.root_dir + '/src/repo2'),
+         'running', ('running', self.root_dir + '/src/repo2/repo_renamed'),
+         'running', 'running', 'running'])
     tree = self.mangle_git_tree(('repo_1@2', 'src-git'),
                                 ('repo_2@1', 'src/repo2'),
                                 ('repo_3@2', 'src/repo2/repo_renamed'))
@@ -1110,7 +1136,7 @@ class GClientSmokeBoth(GClientSmokeBase):
     self.checkString('', stderr)
     self.assertEquals(0, returncode)
     results = self.splitBlock(stdout)
-    self.assertEquals(12, len(results))
+    self.assertEquals(15, len(results))
     tree = self.mangle_git_tree(('repo_1@2', 'src-git'),
                                 ('repo_2@1', 'src/repo2'),
                                 ('repo_3@2', 'src/repo2/repo_renamed'))
@@ -1137,8 +1163,12 @@ class GClientSmokeBoth(GClientSmokeBase):
     self.parseGclient(
         ['sync', '--deps', 'mac', '--jobs', '1', '--revision', '1',
          '-r', 'src-git@' + self.githash('repo_1', 1)],
-        ['running', 'running', 'running', 'running',
-         'running', 'running', 'running', 'running'],
+        ['running',
+         'running', ('running', self.root_dir + '/src-git'),
+         'running', 'running', 'running',
+         'running', ('running', self.root_dir + '/src/repo2'),
+         'running', ('running', self.root_dir + '/src/repo2/repo3'),
+         'running', ('running', self.root_dir + '/src/repo4')],
         expected_stderr=
             'You must specify the full solution name like --revision src@1\n'
             'when you have multiple solutions setup in your .gclient file.\n'