diff --git a/gclient_scm.py b/gclient_scm.py index 8a3410a88..6bf0614ff 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -399,7 +399,7 @@ class GitWrapper(SCMWrapper): elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev): # Fetch |target_rev| if it's not already available. url, _ = gclient_utils.SplitUrlRevision(self.url) - mirror = self._GetMirror(url, options, target_rev) + mirror = self._GetMirror(url, options, target_rev, target_rev) if mirror: rev_type = 'branch' if target_rev.startswith('refs/') else 'hash' self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev) @@ -506,7 +506,7 @@ class GitWrapper(SCMWrapper): if revision_ref.startswith('refs/branch-heads'): options.with_branch_heads = True - mirror = self._GetMirror(url, options, revision_ref) + mirror = self._GetMirror(url, options, revision, revision_ref) if mirror: url = mirror.mirror_path @@ -946,13 +946,14 @@ class GitWrapper(SCMWrapper): return os.path.join(self._root_dir, 'old_' + self.relpath.replace(os.sep, '_')) + '.git' - def _GetMirror(self, url, options, revision_ref=None): + def _GetMirror(self, url, options, revision=None, revision_ref=None): """Get a git_cache.Mirror object for the argument url.""" if not self.cache_dir: return None mirror_kwargs = { 'print_func': self.filter, - 'refs': [] + 'refs': [], + 'commits': [], } if hasattr(options, 'with_branch_heads') and options.with_branch_heads: mirror_kwargs['refs'].append('refs/branch-heads/*') @@ -962,6 +963,8 @@ class GitWrapper(SCMWrapper): mirror_kwargs['refs'].append('refs/tags/*') elif revision_ref and revision_ref.startswith('refs/tags/'): mirror_kwargs['refs'].append(revision_ref) + if revision and not revision.startswith('refs/'): + mirror_kwargs['commits'].append(revision) return git_cache.Mirror(url, **mirror_kwargs) def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision): diff --git a/git_cache.py b/git_cache.py index 89f7f6d79..37b679500 100755 --- a/git_cache.py +++ b/git_cache.py @@ -107,9 +107,10 @@ class Mirror(object): regex = r'\+%s:.*' % src.replace('*', r'\*') return ('+%s:%s' % (src, dest), regex) - def __init__(self, url, refs=None, print_func=None): + def __init__(self, url, refs=None, commits=None, print_func=None): self.url = url self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])]) + self.fetch_commits = set(commits or []) self.basedir = self.UrlToCacheDir(url) self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) if print_func: @@ -448,6 +449,13 @@ class Mirror(object): if spec == '+refs/heads/*:refs/heads/*': raise ClobberNeeded() # Corrupted cache. logging.warning('Fetch of %s failed' % spec) + for commit in self.fetch_commits: + self.print('Fetching %s' % commit) + try: + with self.print_duration_of('fetch %s' % commit): + self.RunGit(['fetch', 'origin', commit], cwd=rundir, retry=True) + except subprocess.CalledProcessError: + logging.warning('Fetch of %s failed' % commit) def populate(self, depth=None, @@ -635,6 +643,8 @@ def CMDpopulate(parser, args): help='Only cache 10000 commits of history') parser.add_option('--ref', action='append', help='Specify additional refs to be fetched') + parser.add_option('--commit', action='append', + help='Specify additional commits to be fetched') parser.add_option('--no_bootstrap', '--no-bootstrap', action='store_true', help='Don\'t bootstrap from Google Storage') @@ -657,7 +667,7 @@ def CMDpopulate(parser, args): print('break_locks is no longer used. Please remove its usage.') url = args[0] - mirror = Mirror(url, refs=options.ref) + mirror = Mirror(url, refs=options.ref, commits=options.commit) kwargs = { 'no_fetch_tags': options.no_fetch_tags, 'verbose': options.verbose, diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py index 21323bf93..34c47cb0b 100755 --- a/recipes/recipe_modules/bot_update/resources/bot_update.py +++ b/recipes/recipe_modules/bot_update/resources/bot_update.py @@ -708,10 +708,16 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir, cleanup_dir, enforce_fetch): name = sln['name'] url = sln['url'] + + branch, revision = get_target_branch_and_revision(name, url, revisions) + pin = revision if COMMIT_HASH_RE.match(revision) else None + populate_cmd = (['cache', 'populate', '--ignore_locks', '-v', '--cache-dir', git_cache_dir, url, '--reset-fetch-config']) if no_fetch_tags: populate_cmd.extend(['--no-fetch-tags']) + if pin: + populate_cmd.extend(['--commit', pin]) for ref in refs: populate_cmd.extend(['--ref', ref]) @@ -726,34 +732,21 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir, 'GIT_REDACT_COOKIES': 'o,SSO,GSSO_UberProxy,__Secure-GSSO_UberProxy', } - branch, revision = get_target_branch_and_revision(name, url, revisions) - pin = revision if COMMIT_HASH_RE.match(revision) else None - # Step 1: populate/refresh cache, if necessary. - if (enforce_fetch - or not pin - or not _has_in_git_cache(pin, refs, git_cache_dir, url)): + if enforce_fetch or not pin: git(*populate_cmd, env=env) # If cache still doesn't have required pin/refs, try again and fetch pin/refs # directly. - for attempt in range(3): - if _has_in_git_cache(pin, refs, git_cache_dir, url): - break - try: - mirror_dir = git( - 'cache', 'exists', '--quiet', '--cache-dir', git_cache_dir, url).strip() + if not _has_in_git_cache(pin, refs, git_cache_dir, url): + for attempt in range(3): with git_config_if_not_set( 'http.extraheader', 'X-Return-Encrypted-Headers: all'): - if pin: - git('fetch', 'origin', pin, env=env, cwd=mirror_dir) - for ref in refs: - git('fetch', 'origin', '%s:%s' % (ref, ref), - env=env, cwd=mirror_dir) - break - except SubprocessFailed as e: - print('Failed to fetch required commits and refs: %s' % str(e)) - print('Waiting 60s and trying again') + git(*populate_cmd, env=env) + if _has_in_git_cache(pin, refs, git_cache_dir, url): + break + print('Some required refs/commits are still not present.') + print('Waiting 60s and trying again.') time.sleep(60) # Step 2: populate a checkout from local cache. All operations are local.