Reland "git-cache: Add option to fetch commits."

This is a reland of 4c67f856f0
Issues should have been fixed by crrev.com/c/2838026

Original change's description:
> git-cache: Add option to fetch commits.
>
> Add option to git cache to fetch commits.
> And use it in bot_update and gclient sync to make sure
> the needed commits are present on the checkout.
>
> Change-Id: I9e90da9e3be6e7bacf714b22bf0b735463e655b6
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2829942
> Reviewed-by: Gavin Mak <gavinmak@google.com>
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

Change-Id: Ie5a29737f5a75d28bc7c5c2f6cb99ec7f87cd9e8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2841046
Reviewed-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

gclient_scm.py

@@ -399,7 +399,7 @@ class GitWrapper(SCMWrapper):
     elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev):
       # Fetch |target_rev| if it's not already available.
       url, _ = gclient_utils.SplitUrlRevision(self.url)
-      mirror = self._GetMirror(url, options, target_rev)
+      mirror = self._GetMirror(url, options, target_rev, target_rev)
       if mirror:
         rev_type = 'branch' if target_rev.startswith('refs/') else 'hash'
         self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev)
@@ -508,7 +508,7 @@ class GitWrapper(SCMWrapper):
     if revision_ref.startswith('refs/branch-heads'):
       options.with_branch_heads = True
 
-    mirror = self._GetMirror(url, options, revision_ref)
+    mirror = self._GetMirror(url, options, revision, revision_ref)
     if mirror:
       url = mirror.mirror_path
 
@@ -948,13 +948,14 @@ class GitWrapper(SCMWrapper):
     return os.path.join(self._root_dir,
                         'old_' + self.relpath.replace(os.sep, '_')) + '.git'
 
-  def _GetMirror(self, url, options, revision_ref=None):
+  def _GetMirror(self, url, options, revision=None, revision_ref=None):
     """Get a git_cache.Mirror object for the argument url."""
     if not self.cache_dir:
       return None
     mirror_kwargs = {
         'print_func': self.filter,
-        'refs': []
+        'refs': [],
+        'commits': [],
     }
     if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
       mirror_kwargs['refs'].append('refs/branch-heads/*')
@@ -964,6 +965,8 @@ class GitWrapper(SCMWrapper):
       mirror_kwargs['refs'].append('refs/tags/*')
     elif revision_ref and revision_ref.startswith('refs/tags/'):
       mirror_kwargs['refs'].append(revision_ref)
+    if revision and not revision.startswith('refs/'):
+      mirror_kwargs['commits'].append(revision)
     return git_cache.Mirror(url, **mirror_kwargs)
 
   def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision):

git_cache.py

@@ -107,9 +107,10 @@ class Mirror(object):
     regex = r'\+%s:.*' % src.replace('*', r'\*')
     return ('+%s:%s' % (src, dest), regex)
 
-  def __init__(self, url, refs=None, print_func=None):
+  def __init__(self, url, refs=None, commits=None, print_func=None):
     self.url = url
     self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
+    self.fetch_commits = set(commits or [])
     self.basedir = self.UrlToCacheDir(url)
     self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
     if print_func:
@@ -448,6 +449,13 @@ class Mirror(object):
         if spec == '+refs/heads/*:refs/heads/*':
           raise ClobberNeeded()  # Corrupted cache.
         logging.warning('Fetch of %s failed' % spec)
+    for commit in self.fetch_commits:
+      self.print('Fetching %s' % commit)
+      try:
+        with self.print_duration_of('fetch %s' % commit):
+          self.RunGit(['fetch', 'origin', commit], cwd=rundir, retry=True)
+      except subprocess.CalledProcessError:
+        logging.warning('Fetch of %s failed' % commit)
 
   def populate(self,
                depth=None,
@@ -635,6 +643,8 @@ def CMDpopulate(parser, args):
                     help='Only cache 10000 commits of history')
   parser.add_option('--ref', action='append',
                     help='Specify additional refs to be fetched')
+  parser.add_option('--commit', action='append',
+                    help='Specify additional commits to be fetched')
   parser.add_option('--no_bootstrap', '--no-bootstrap',
                     action='store_true',
                     help='Don\'t bootstrap from Google Storage')
@@ -657,7 +667,7 @@ def CMDpopulate(parser, args):
     print('break_locks is no longer used. Please remove its usage.')
   url = args[0]
 
-  mirror = Mirror(url, refs=options.ref)
+  mirror = Mirror(url, refs=options.ref, commits=options.commit)
   kwargs = {
       'no_fetch_tags': options.no_fetch_tags,
       'verbose': options.verbose,

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -708,10 +708,16 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
                   cleanup_dir, enforce_fetch):
   name = sln['name']
   url = sln['url']
+
+  branch, revision = get_target_branch_and_revision(name, url, revisions)
+  pin = revision if COMMIT_HASH_RE.match(revision) else None
+
   populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
                    '--cache-dir', git_cache_dir, url, '--reset-fetch-config'])
   if no_fetch_tags:
     populate_cmd.extend(['--no-fetch-tags'])
+  if pin:
+    populate_cmd.extend(['--commit', pin])
   for ref in refs:
     populate_cmd.extend(['--ref', ref])
 
@@ -726,34 +732,21 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
         'GIT_REDACT_COOKIES': 'o,SSO,GSSO_UberProxy,__Secure-GSSO_UberProxy',
     }
 
-  branch, revision = get_target_branch_and_revision(name, url, revisions)
-  pin = revision if COMMIT_HASH_RE.match(revision) else None
-
   # Step 1: populate/refresh cache, if necessary.
-  if (enforce_fetch
+  if enforce_fetch or not pin:
-      or not pin
-      or not _has_in_git_cache(pin, refs, git_cache_dir, url)):
     git(*populate_cmd, env=env)
 
   # If cache still doesn't have required pin/refs, try again and fetch pin/refs
   # directly.
-  for attempt in range(3):
+  if not _has_in_git_cache(pin, refs, git_cache_dir, url):
-    if _has_in_git_cache(pin, refs, git_cache_dir, url):
+    for attempt in range(3):
-      break
-    try:
-      mirror_dir = git(
-          'cache', 'exists', '--quiet', '--cache-dir', git_cache_dir, url).strip()
       with git_config_if_not_set(
           'http.extraheader', 'X-Return-Encrypted-Headers: all'):
-        if pin:
+        git(*populate_cmd, env=env)
-          git('fetch', 'origin', pin, env=env, cwd=mirror_dir)
+      if _has_in_git_cache(pin, refs, git_cache_dir, url):
-        for ref in refs:
+        break
-          git('fetch', 'origin', '%s:%s' % (ref, ref),
+      print('Some required refs/commits are still not present.')
-              env=env, cwd=mirror_dir)
+      print('Waiting 60s and trying again.')
-      break
-    except SubprocessFailed as e:
-      print('Failed to fetch required commits and refs: %s' % str(e))
-      print('Waiting 60s and trying again')
       time.sleep(60)
 
   # Step 2: populate a checkout from local cache. All operations are local.