From 1a977bdc2d8fa8f3a479ac8eb5c3d4ae620e0f8f Mon Sep 17 00:00:00 2001 From: Joanna Wang Date: Thu, 2 Jun 2022 21:51:17 +0000 Subject: [PATCH] Update gclient cloning to work with cog. Bug:1330629 Change-Id: I522255528e36d4806eada70c22afee1277a6778d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3668429 Reviewed-by: Josip Sokcevic Reviewed-by: Aravind Vasudevan Commit-Queue: Joanna Wang --- gclient_scm.py | 162 +++++++++++++++++++++--------------- gclient_utils.py | 9 ++ tests/gclient_scm_test.py | 34 ++++++++ tests/gclient_utils_test.py | 14 ++++ 4 files changed, 153 insertions(+), 66 deletions(-) diff --git a/gclient_scm.py b/gclient_scm.py index c51fee811..baa98ab1c 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -394,7 +394,7 @@ class GitWrapper(SCMWrapper): new_patch_rev = c['revisions'][curr_rev]['ref'] patch_revs_to_process.append(new_patch_rev) - # 4. Return the new patch_revs to process. + # 4. Return the new patch_revs to process. return patch_revs_to_process def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options, @@ -1078,85 +1078,115 @@ class GitWrapper(SCMWrapper): leave HEAD detached as it makes future updates simpler -- in this case the user should first create a new branch or switch to an existing branch before making changes in the repo.""" + in_cog_workspace = self._IsCog() + + if self.print_outbuf: + print_stdout = True + filter_fn = None + else: + print_stdout = False + filter_fn = self.filter + if not options.verbose: # git clone doesn't seem to insert a newline properly before printing # to stdout self.Print('') - cfg = gclient_utils.DefaultIndexPackConfig(url) - clone_cmd = cfg + ['clone', '--no-checkout', '--progress'] - if self.cache_dir: - clone_cmd.append('--shared') - if options.verbose: - clone_cmd.append('--verbose') - clone_cmd.append(url) + # If the parent directory does not exist, Git clone on Windows will not # create it, so we need to do it manually. parent_dir = os.path.dirname(self.checkout_path) gclient_utils.safe_makedirs(parent_dir) - template_dir = None - if hasattr(options, 'no_history') and options.no_history: - if gclient_utils.IsGitSha(revision): - # In the case of a subproject, the pinned sha is not necessarily the - # head of the remote branch (so we can't just use --depth=N). Instead, - # we tell git to fetch all the remote objects from SHA..HEAD by means of - # a template git dir which has a 'shallow' file pointing to the sha. - template_dir = tempfile.mkdtemp( - prefix='_gclient_gittmp_%s' % os.path.basename(self.checkout_path), - dir=parent_dir) - self._Run(['init', '--bare', template_dir], options, cwd=self._root_dir) - with open(os.path.join(template_dir, 'shallow'), 'w') as template_file: - template_file.write(revision) - clone_cmd.append('--template=' + template_dir) - else: - # Otherwise, we're just interested in the HEAD. Just use --depth. - clone_cmd.append('--depth=1') + if in_cog_workspace: + clone_cmd = ['citc', 'clone-repo', url, self.checkout_path] + clone_cmd.append( + gclient_utils.ExtractRefName(self.remote, revision) or revision) + try: + self._Run(clone_cmd, + options, + cwd=self._root_dir, + retry=True, + print_stdout=print_stdout, + filter_fn=filter_fn) + self._Run(['-C', self.checkout_path, 'sparse-checkout', 'reapply'], + options, + cwd=self._root_dir, + retry=True, + print_stdout=print_stdout, + filter_fn=filter_fn) + except: + traceback.print_exc(file=self.out_fh) + raise + self._SetFetchConfig(options) + else: + cfg = gclient_utils.DefaultIndexPackConfig(url) + clone_cmd = cfg + ['clone', '--no-checkout', '--progress'] + if self.cache_dir: + clone_cmd.append('--shared') + if options.verbose: + clone_cmd.append('--verbose') + clone_cmd.append(url) + + template_dir = None + if hasattr(options, 'no_history') and options.no_history: + if gclient_utils.IsGitSha(revision): + # In the case of a subproject, the pinned sha is not necessarily the + # head of the remote branch (so we can't just use --depth=N). Instead, + # we tell git to fetch all the remote objects from SHA..HEAD by means + # of a template git dir which has a 'shallow' file pointing to the + # sha. + template_dir = tempfile.mkdtemp(prefix='_gclient_gittmp_%s' % + os.path.basename(self.checkout_path), + dir=parent_dir) + self._Run(['init', '--bare', template_dir], + options, + cwd=self._root_dir) + with open(os.path.join(template_dir, 'shallow'), + 'w') as template_file: + template_file.write(revision) + clone_cmd.append('--template=' + template_dir) + else: + # Otherwise, we're just interested in the HEAD. Just use --depth. + clone_cmd.append('--depth=1') - tmp_dir = tempfile.mkdtemp( - prefix='_gclient_%s_' % os.path.basename(self.checkout_path), - dir=parent_dir) - try: + tmp_dir = tempfile.mkdtemp(prefix='_gclient_%s_' % + os.path.basename(self.checkout_path), + dir=parent_dir) clone_cmd.append(tmp_dir) - if self.print_outbuf: - print_stdout = True - filter_fn = None - else: - print_stdout = False - filter_fn = self.filter - self._Run(clone_cmd, options, cwd=self._root_dir, retry=True, - print_stdout=print_stdout, filter_fn=filter_fn) - gclient_utils.safe_makedirs(self.checkout_path) - gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'), - os.path.join(self.checkout_path, '.git')) - # TODO(https://github.com/git-for-windows/git/issues/2569): Remove once - # fixed. - if sys.platform.startswith('win'): - try: - self._Run(['config', '--unset', 'core.worktree'], options, - cwd=self.checkout_path) - except subprocess2.CalledProcessError: - pass - except: - traceback.print_exc(file=self.out_fh) - raise - finally: - if os.listdir(tmp_dir): - self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) - gclient_utils.rmtree(tmp_dir) - if template_dir: - gclient_utils.rmtree(template_dir) - self._SetFetchConfig(options) - self._Fetch(options, prune=options.force) - revision = self._AutoFetchRef(options, revision) - remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) - self._Checkout(options, ''.join(remote_ref or revision), quiet=True) + + try: + self._Run(clone_cmd, + options, + cwd=self._root_dir, + retry=True, + print_stdout=print_stdout, + filter_fn=filter_fn) + gclient_utils.safe_makedirs(self.checkout_path) + gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'), + os.path.join(self.checkout_path, '.git')) + except: + traceback.print_exc(file=self.out_fh) + raise + finally: + if os.listdir(tmp_dir): + self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) + gclient_utils.rmtree(tmp_dir) + if template_dir: + gclient_utils.rmtree(template_dir) + + self._SetFetchConfig(options) + self._Fetch(options, prune=options.force) + revision = self._AutoFetchRef(options, revision) + remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) + self._Checkout(options, ''.join(remote_ref or revision), quiet=True) + if self._GetCurrentBranch() is None: # Squelch git's very verbose detached HEAD warning and use our own self.Print( - ('Checked out %s to a detached HEAD. Before making any commits\n' - 'in this repo, you should use \'git checkout \' to switch to\n' - 'an existing branch or use \'git checkout %s -b \' to\n' - 'create a new branch for your work.') % (revision, self.remote)) + ('Checked out %s to a detached HEAD. Before making any commits\n' + 'in this repo, you should use \'git checkout \' to switch \n' + 'to an existing branch or use \'git checkout %s -b \' to\n' + 'create a new branch for your work.') % (revision, self.remote)) def _AskForData(self, prompt, options): if options.jobs > 1: diff --git a/gclient_utils.py b/gclient_utils.py index 9179b7647..bbf273d11 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -111,6 +111,15 @@ def SplitUrlRevision(url): return tuple(components) +def ExtractRefName(remote, full_refs_str): + """Returns the ref name if full_refs_str is a valid ref.""" + result = re.compile(r'^refs(\/.+)?\/((%s)|(heads)|(tags))\/(?P.+)' % + remote).match(full_refs_str) + if result: + return result.group('ref_name') + return None + + def IsGitSha(revision): """Returns true if the given string is a valid hex-encoded sha""" return re.match('^[a-fA-F0-9]{6,40}$', revision) is not None diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index 5ef94bc12..42222311d 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -226,6 +226,36 @@ from :3 class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): + @mock.patch('gclient_scm.GitWrapper._IsCog') + @mock.patch('gclient_scm.GitWrapper._Run', return_value=True) + @mock.patch('gclient_scm.GitWrapper._SetFetchConfig') + @mock.patch('gclient_scm.GitWrapper._GetCurrentBranch') + def testCloneInCog(self, mockGetCurrentBranch, mockSetFetchConfig, mockRun, + _mockIsCog): + """Test that we call the correct commands when in a cog workspace.""" + if not self.enabled: + return + options = self.Options() + scm = gclient_scm.GitWrapper(self.url, self.root_dir, self.relpath) + scm._Clone('123123ab', self.url, options) + self.assertEquals(mockRun.mock_calls, [ + mock.call( + ['citc', 'clone-repo', self.url, scm.checkout_path, '123123ab'], + options, + cwd=scm._root_dir, + retry=True, + print_stdout=False, + filter_fn=scm.filter), + mock.call(['-C', scm.checkout_path, 'sparse-checkout', 'reapply'], + options, + cwd=scm._root_dir, + retry=True, + print_stdout=False, + filter_fn=scm.filter), + ]) + mockSetFetchConfig.assert_called_once() + mockGetCurrentBranch.assert_called_once() + def testRevertMissing(self): if not self.enabled: return @@ -1460,6 +1490,10 @@ class GerritChangesTest(fake_repos.FakeReposTestBase): self.assertEqual(self.githash('repo_1', 5), self.gitrevparse(self.root_dir)) +if 'unittest.util' in __import__('sys').modules: + # Show full diff in self.assertEqual. + __import__('sys').modules['unittest.util']._MAX_LENGTH = 999999999 + if __name__ == '__main__': level = logging.DEBUG if '-v' in sys.argv else logging.FATAL logging.basicConfig( diff --git a/tests/gclient_utils_test.py b/tests/gclient_utils_test.py index c7c6d8498..e2638479a 100755 --- a/tests/gclient_utils_test.py +++ b/tests/gclient_utils_test.py @@ -290,6 +290,20 @@ class SplitUrlRevisionTestCase(unittest.TestCase): self.assertEqual(out_url, url) +class ExtracRefNameTest(unittest.TestCase): + def testMatchFound(self): + self.assertEqual( + 'main', gclient_utils.ExtractRefName('origin', + 'refs/remote/origin/main')) + self.assertEqual('1234', + gclient_utils.ExtractRefName('origin', 'refs/tags/1234')) + self.assertEqual( + 'chicken', gclient_utils.ExtractRefName('origin', 'refs/heads/chicken')) + + def testNoMatch(self): + self.assertIsNone(gclient_utils.ExtractRefName('origin', 'abcbbb1234')) + + class GClientUtilsTest(trial_dir.TestCase): def testHardToDelete(self): # Use the fact that tearDown will delete the directory to make it hard to do