git recipe_module: add experimental git-cache support.

BUG=630601
R=phajdan.jr@chromium.org

Review-Url: https://codereview.chromium.org/2173823002

recipe_modules/git/api.py

@@ -16,7 +16,7 @@ class GitApi(recipe_api.RecipeApi):
 
   def __call__(self, *args, **kwargs):
     """Return a git command step."""
-    name = kwargs.pop('name', 'git '+args[0])
+    name = kwargs.pop('name', 'git ' + args[0])
     infra_step = kwargs.pop('infra_step', True)
     if 'cwd' not in kwargs:
       kwargs.setdefault('cwd', self.m.path['checkout'])
@@ -133,8 +133,10 @@ class GitApi(recipe_api.RecipeApi):
                curl_trace_file=None, can_fail_build=True,
                set_got_revision=False, remote_name=None,
                display_fetch_size=None, file_name=None,
-               submodule_update_recursive=True):
-    """Returns an iterable of steps to perform a full git checkout.
+               submodule_update_recursive=True,
+               use_git_cache=False):
+    """Performs a full git checkout and returns sha1 of checked out revision.
+
     Args:
       url (str): url of remote repo to use as upstream
       ref (str): ref to fetch and check out
@@ -157,6 +159,13 @@ class GitApi(recipe_api.RecipeApi):
       file_name (str): optional path to a single file to checkout.
       submodule_update_recursive (bool): if True, updates submodules
           recursively.
+      use_git_cache (bool): if True, git cache will be used for this checkout.
+          WARNING, this is EXPERIMENTAL!!! This wasn't tested with:
+           * submodules
+           * since origin url is modified
+             to a local path, may cause problem with scripts that do
+             "git fetch origin" or "git push origin".
+           * arbitrary refs such refs/whatever/not-fetched-by-default-to-cache
 
     Returns: If the checkout was successful, this returns the commit hash of
       the checked-out-repo. Otherwise this returns None.
@@ -199,6 +208,12 @@ class GitApi(recipe_api.RecipeApi):
         self.resource('git_setup.py'),
         git_setup_args)
 
+    if use_git_cache:
+      self('retry', 'cache', 'fetch', '-c', self.m.path['git_cache'],
+           cwd=dir_path,
+           name='fetch cache',
+           can_fail_build=can_fail_build)
+
     # There are five kinds of refs we can be handed:
     # 0) None. In this case, we default to properties['branch'].
     # 1) A 40-character SHA1 hash.

recipe_modules/git/example.expected/git-cache-checkout.json

@@ -0,0 +1,189 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
+      "--path",
+      "[SLAVE_BUILD]/src",
+      "--url",
+      "https://chromium.googlesource.com/chromium/src.git"
+    ],
+    "name": "git setup"
+  },
+  {
+    "cmd": [
+      "git",
+      "retry",
+      "cache",
+      "fetch",
+      "-c",
+      "[GIT_CACHE]"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "fetch cache"
+  },
+  {
+    "cmd": [
+      "git",
+      "retry",
+      "fetch",
+      "origin",
+      "master",
+      "--recurse-submodules"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git fetch"
+  },
+  {
+    "cmd": [
+      "git",
+      "checkout",
+      "-f",
+      "FETCH_HEAD"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git checkout"
+  },
+  {
+    "cmd": [
+      "git",
+      "rev-parse",
+      "HEAD"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "read revision",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "git",
+      "clean",
+      "-f",
+      "-d",
+      "-x"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git clean"
+  },
+  {
+    "cmd": [
+      "git",
+      "submodule",
+      "sync"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "submodule sync"
+  },
+  {
+    "cmd": [
+      "git",
+      "submodule",
+      "update",
+      "--init",
+      "--recursive"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "submodule update"
+  },
+  {
+    "cmd": [
+      "git",
+      "-c",
+      "foo=bar",
+      "count-objects",
+      "-v"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "count-objects",
+    "stdout": "/path/to/tmp/"
+  },
+  {
+    "cmd": [
+      "git",
+      "config",
+      "--get",
+      "remote.origin.url"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git config remote.origin.url",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@foo@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "git",
+      "show",
+      "HEAD",
+      "--format=%at",
+      "-s"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git show",
+    "stdout": "/path/to/tmp/"
+  },
+  {
+    "cmd": [
+      "git",
+      "fetch",
+      "origin",
+      "--tags"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git fetch tags"
+  },
+  {
+    "cmd": [
+      "git",
+      "status"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git status"
+  },
+  {
+    "cmd": [
+      "git",
+      "status"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git status can_fail_build"
+  },
+  {
+    "cmd": [
+      "git",
+      "status"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git status cannot_fail_build"
+  },
+  {
+    "cmd": [
+      "git",
+      "rebase",
+      "origin/master"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "my repo rebase"
+  },
+  {
+    "cmd": [
+      "git",
+      "bundle",
+      "create",
+      "[SLAVE_BUILD]/all.bundle",
+      "--all"
+    ],
+    "cwd": "[SLAVE_BUILD]/src",
+    "name": "git bundle"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]

recipe_modules/git/example.py

@@ -36,7 +36,8 @@ def RunSteps(api):
       remote_name=api.properties.get('remote_name'),
       display_fetch_size=api.properties.get('display_fetch_size'),
       file_name=api.properties.get('checkout_file_name'),
-      submodule_update_recursive=submodule_update_recursive)
+      submodule_update_recursive=submodule_update_recursive,
+      use_git_cache=api.properties.get('use_git_cache'))
 
   assert retVal == "deadbeef", (
     "expected retVal to be %r but was %r" % ("deadbeef", retVal))
@@ -145,3 +146,7 @@ def GenTests(api):
       api.step_data('git cat-file abcdef12345:TestFile',
                     stdout=api.raw_io.output('TestOutput')) +
       api.properties(revision='abcdef12345', cat_file='TestFile'))
+
+  yield (
+      api.test('git-cache-checkout') +
+      api.properties(use_git_cache=True))