From 93d80d868650767ac978301658ada7c37c6973b3 Mon Sep 17 00:00:00 2001 From: Edward Lesmes Date: Tue, 20 Apr 2021 02:55:11 +0000 Subject: [PATCH] Revert "git-cache: Add option to fetch commits." This reverts commit 4c67f856f06693b39eecaa0efb86b2856c3532bb. Reason for revert: crbug.com/1200657 Original change's description: > git-cache: Add option to fetch commits. > > Add option to git cache to fetch commits. > And use it in bot_update and gclient sync to make sure > the needed commits are present on the checkout. > > Change-Id: I9e90da9e3be6e7bacf714b22bf0b735463e655b6 > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2829942 > Reviewed-by: Gavin Mak > Commit-Queue: Edward Lesmes Change-Id: I31a1defb26119bcb46894ab4084d626b65c45fed No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2837413 Auto-Submit: Edward Lesmes Commit-Queue: Rubber Stamper Bot-Commit: Rubber Stamper --- gclient_scm.py | 11 +++--- git_cache.py | 14 ++------ .../bot_update/resources/bot_update.py | 35 +++++++++++-------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/gclient_scm.py b/gclient_scm.py index 6bf0614ffc..8a3410a887 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -399,7 +399,7 @@ class GitWrapper(SCMWrapper): elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev): # Fetch |target_rev| if it's not already available. url, _ = gclient_utils.SplitUrlRevision(self.url) - mirror = self._GetMirror(url, options, target_rev, target_rev) + mirror = self._GetMirror(url, options, target_rev) if mirror: rev_type = 'branch' if target_rev.startswith('refs/') else 'hash' self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev) @@ -506,7 +506,7 @@ class GitWrapper(SCMWrapper): if revision_ref.startswith('refs/branch-heads'): options.with_branch_heads = True - mirror = self._GetMirror(url, options, revision, revision_ref) + mirror = self._GetMirror(url, options, revision_ref) if mirror: url = mirror.mirror_path @@ -946,14 +946,13 @@ class GitWrapper(SCMWrapper): return os.path.join(self._root_dir, 'old_' + self.relpath.replace(os.sep, '_')) + '.git' - def _GetMirror(self, url, options, revision=None, revision_ref=None): + def _GetMirror(self, url, options, revision_ref=None): """Get a git_cache.Mirror object for the argument url.""" if not self.cache_dir: return None mirror_kwargs = { 'print_func': self.filter, - 'refs': [], - 'commits': [], + 'refs': [] } if hasattr(options, 'with_branch_heads') and options.with_branch_heads: mirror_kwargs['refs'].append('refs/branch-heads/*') @@ -963,8 +962,6 @@ class GitWrapper(SCMWrapper): mirror_kwargs['refs'].append('refs/tags/*') elif revision_ref and revision_ref.startswith('refs/tags/'): mirror_kwargs['refs'].append(revision_ref) - if revision and not revision.startswith('refs/'): - mirror_kwargs['commits'].append(revision) return git_cache.Mirror(url, **mirror_kwargs) def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision): diff --git a/git_cache.py b/git_cache.py index 37b6795007..89f7f6d791 100755 --- a/git_cache.py +++ b/git_cache.py @@ -107,10 +107,9 @@ class Mirror(object): regex = r'\+%s:.*' % src.replace('*', r'\*') return ('+%s:%s' % (src, dest), regex) - def __init__(self, url, refs=None, commits=None, print_func=None): + def __init__(self, url, refs=None, print_func=None): self.url = url self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])]) - self.fetch_commits = set(commits or []) self.basedir = self.UrlToCacheDir(url) self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) if print_func: @@ -449,13 +448,6 @@ class Mirror(object): if spec == '+refs/heads/*:refs/heads/*': raise ClobberNeeded() # Corrupted cache. logging.warning('Fetch of %s failed' % spec) - for commit in self.fetch_commits: - self.print('Fetching %s' % commit) - try: - with self.print_duration_of('fetch %s' % commit): - self.RunGit(['fetch', 'origin', commit], cwd=rundir, retry=True) - except subprocess.CalledProcessError: - logging.warning('Fetch of %s failed' % commit) def populate(self, depth=None, @@ -643,8 +635,6 @@ def CMDpopulate(parser, args): help='Only cache 10000 commits of history') parser.add_option('--ref', action='append', help='Specify additional refs to be fetched') - parser.add_option('--commit', action='append', - help='Specify additional commits to be fetched') parser.add_option('--no_bootstrap', '--no-bootstrap', action='store_true', help='Don\'t bootstrap from Google Storage') @@ -667,7 +657,7 @@ def CMDpopulate(parser, args): print('break_locks is no longer used. Please remove its usage.') url = args[0] - mirror = Mirror(url, refs=options.ref, commits=options.commit) + mirror = Mirror(url, refs=options.ref) kwargs = { 'no_fetch_tags': options.no_fetch_tags, 'verbose': options.verbose, diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py index 34c47cb0bd..21323bf93b 100755 --- a/recipes/recipe_modules/bot_update/resources/bot_update.py +++ b/recipes/recipe_modules/bot_update/resources/bot_update.py @@ -708,16 +708,10 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir, cleanup_dir, enforce_fetch): name = sln['name'] url = sln['url'] - - branch, revision = get_target_branch_and_revision(name, url, revisions) - pin = revision if COMMIT_HASH_RE.match(revision) else None - populate_cmd = (['cache', 'populate', '--ignore_locks', '-v', '--cache-dir', git_cache_dir, url, '--reset-fetch-config']) if no_fetch_tags: populate_cmd.extend(['--no-fetch-tags']) - if pin: - populate_cmd.extend(['--commit', pin]) for ref in refs: populate_cmd.extend(['--ref', ref]) @@ -732,21 +726,34 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir, 'GIT_REDACT_COOKIES': 'o,SSO,GSSO_UberProxy,__Secure-GSSO_UberProxy', } + branch, revision = get_target_branch_and_revision(name, url, revisions) + pin = revision if COMMIT_HASH_RE.match(revision) else None + # Step 1: populate/refresh cache, if necessary. - if enforce_fetch or not pin: + if (enforce_fetch + or not pin + or not _has_in_git_cache(pin, refs, git_cache_dir, url)): git(*populate_cmd, env=env) # If cache still doesn't have required pin/refs, try again and fetch pin/refs # directly. - if not _has_in_git_cache(pin, refs, git_cache_dir, url): - for attempt in range(3): + for attempt in range(3): + if _has_in_git_cache(pin, refs, git_cache_dir, url): + break + try: + mirror_dir = git( + 'cache', 'exists', '--quiet', '--cache-dir', git_cache_dir, url).strip() with git_config_if_not_set( 'http.extraheader', 'X-Return-Encrypted-Headers: all'): - git(*populate_cmd, env=env) - if _has_in_git_cache(pin, refs, git_cache_dir, url): - break - print('Some required refs/commits are still not present.') - print('Waiting 60s and trying again.') + if pin: + git('fetch', 'origin', pin, env=env, cwd=mirror_dir) + for ref in refs: + git('fetch', 'origin', '%s:%s' % (ref, ref), + env=env, cwd=mirror_dir) + break + except SubprocessFailed as e: + print('Failed to fetch required commits and refs: %s' % str(e)) + print('Waiting 60s and trying again') time.sleep(60) # Step 2: populate a checkout from local cache. All operations are local.