From dcad7498c3c52c4a76b34703da9e4e4cb90ea535 Mon Sep 17 00:00:00 2001
From: Karen Qian <karenqian@google.com>
Date: Fri, 26 Apr 2019 03:11:16 +0000
Subject: [PATCH] Reland "Stop using compression for git cache update
 bootstrap."

This is a reland of 749139969f34788a65dc1e5e27484325035129ce

Original change's description:
> Stop using compression for git cache update bootstrap.
>
> Change git cache to stop zipping the git directory and instead upload directly to GS.
>
> Bug: 943696
> Change-Id: I310fb48ff3d7fd6781914e14e0e17530d5d9b328
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585029
> Commit-Queue: Karen Qian <karenqian@google.com>
> Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>

Bug: 943696
Change-Id: Ifd3a3a07445345ab647572682ebc6423fb7135b6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585272
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Commit-Queue: Karen Qian <karenqian@google.com>
---
 git_cache.py | 53 +++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/git_cache.py b/git_cache.py
index 4a2a23741..6b20a44b1 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -258,6 +258,10 @@ class Mirror(object):
     # Not recognized.
     return None
 
+  @property
+  def _gs_path(self):
+    return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
+
   @classmethod
   def FromPath(cls, path):
     return cls(cls.CacheDirToUrl(path))
@@ -600,28 +604,43 @@ class Mirror(object):
         lockfile.unlock()
 
   def update_bootstrap(self, prune=False):
-    # The files are named <git number>.zip
+    # The folder is <git number>
     gen_number = subprocess.check_output(
         [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
     # Run Garbage Collect to compress packfile.
     self.RunGit(['gc', '--prune=all'])
-    # Creating a temp file and then deleting it ensures we can use this name.
-    _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
-    os.remove(tmp_zipfile)
-    subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
+
     gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
-    gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
-    dest_name = '%s/%s.zip' % (gs_folder, gen_number)
-    gsutil.call('cp', tmp_zipfile, dest_name)
-    os.remove(tmp_zipfile)
-
-    # Remove all other files in the same directory.
-    if prune:
-      _, ls_out, _ = gsutil.check_call('ls', gs_folder)
-      for filename in ls_out.splitlines():
-        if filename == dest_name:
-          continue
-        gsutil.call('rm', filename)
+
+    src_name = self.mirror_path
+    dest_name = '%s/%s' % (self._gs_path, gen_number)
+
+    # check to see if folder already exists in gs
+    _, ls_out, ls_err = gsutil.check_call('ls', dest_name)
+    _, ls_out_ready, ls_err_ready = (
+      gsutil.check_call('ls', dest_name + '.ready'))
+
+    # only printing out errors because the folder/ready file
+    # might not exist yet, so it will error no matter what
+    if ls_err:
+      print('Failed to check GS:\n%s' % (ls_err))
+
+    if ls_err_ready:
+      print('Failed to check GS:\n%s' % (ls_err_ready))
+
+    if not (ls_out == '' and ls_out_ready == ''):
+      return
+
+    gsutil.call('-m', 'cp', '-r', src_name, dest_name)
+
+    #TODO(karenqian): prune old caches
+
+    # create .ready file and upload
+    _, ready_file_name =  tempfile.mkstemp(suffix='.ready')
+    try:
+      gsutil.call('cp', ready_file_name, '%s.ready' % (dest_name))
+    finally:
+      os.remove(ready_file_name)
 
   @staticmethod
   def DeleteTmpPackFiles(path):