In upload_to_google_storage, pass -z argument through to gsutil.

Also fix some latent bugs in the unit tests.

(cherry-picked by pasko@ from https://codereview.chromium.org/1048103002/)

BUG=467005

Review URL: https://codereview.chromium.org/1059723003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@294674 0039d316-1c4b-4281-b951-d872f2087c98

tests/upload_to_google_storage_unittests.py

@@ -48,14 +48,14 @@ class UploadTests(unittest.TestCase):
     filenames = [self.lorem_ipsum]
     output_filename = '%s.sha1'  % self.lorem_ipsum
     code = upload_to_google_storage.upload_to_google_storage(
-        filenames, self.base_url, self.gsutil, True, False, 1, False)
+        filenames, self.base_url, self.gsutil, True, False, 1, False, 'txt')
     self.assertEqual(
         self.gsutil.history,
         [('check_call',
           ('ls', '%s/%s' % (self.base_url, self.lorem_ipsum_sha1))),
          ('check_call',
-          ('cp', '-q', filenames[0], '%s/%s' % (self.base_url,
-                                                self.lorem_ipsum_sha1)))])
+          ('cp', '-z', 'txt', filenames[0],
+           '%s/%s' % (self.base_url, self.lorem_ipsum_sha1)))])
     self.assertTrue(os.path.exists(output_filename))
     self.assertEqual(
         open(output_filename, 'rb').read(),
@@ -70,7 +70,7 @@ class UploadTests(unittest.TestCase):
     self.gsutil.add_expected(0, '', '')
     self.gsutil.add_expected(0, etag_string, '')
     code = upload_to_google_storage.upload_to_google_storage(
-        filenames, self.base_url, self.gsutil, False, False, 1, False)
+        filenames, self.base_url, self.gsutil, False, False, 1, False, None)
     self.assertEqual(
         self.gsutil.history,
         [('check_call',
@@ -100,7 +100,8 @@ class UploadTests(unittest.TestCase):
         False,
         False,
         self.stdout_queue,
-        self.ret_codes)
+        self.ret_codes,
+        None)
     expected_ret_codes = [
       (20,
        'Encountered error on uploading %s to %s/%s\nExpected error message' %
@@ -114,7 +115,7 @@ class UploadTests(unittest.TestCase):
     with open(output_filename, 'wb') as f:
       f.write(fake_hash)  # Fake hash.
     code = upload_to_google_storage.upload_to_google_storage(
-        filenames, self.base_url, self.gsutil, False, False, 1, True)
+        filenames, self.base_url, self.gsutil, False, False, 1, True, None)
     self.assertEqual(
         self.gsutil.history,
         [('check_call',
@@ -122,7 +123,7 @@ class UploadTests(unittest.TestCase):
          ('check_call',
           ('ls', '-L', '%s/%s' % (self.base_url, fake_hash))),
          ('check_call',
-          ('cp', '-q', filenames[0], '%s/%s' % (self.base_url, fake_hash)))])
+          ('cp', filenames[0], '%s/%s' % (self.base_url, fake_hash)))])
     self.assertEqual(
         open(output_filename, 'rb').read(), fake_hash)
     os.remove(output_filename)

upload_to_google_storage.py

@@ -67,7 +67,7 @@ def get_md5_cached(filename):
 
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
-    use_md5, stdout_queue, ret_codes):
+    use_md5, stdout_queue, ret_codes, gzip):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
@@ -92,7 +92,11 @@ def _upload_worker(
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
-    code, _, err = gsutil.check_call('cp', filename, file_url)
+    gsutil_args = ['cp']
+    if gzip:
+      gsutil_args.extend(['-z', gzip])
+    gsutil_args.extend([filename, file_url])
+    code, _, err = gsutil.check_call(*gsutil_args)
     if code != 0:
       ret_codes.put(
           (code,
@@ -129,7 +133,7 @@ def get_targets(args, parser, use_null_terminator):
 
 def upload_to_google_storage(
     input_filenames, base_url, gsutil, force,
-    use_md5, num_threads, skip_hashing):
+    use_md5, num_threads, skip_hashing, gzip):
   # We only want one MD5 calculation happening at a time to avoid HD thrashing.
   md5_lock = threading.Lock()
 
@@ -147,7 +151,7 @@ def upload_to_google_storage(
     t = threading.Thread(
         target=_upload_worker,
         args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
-              force, use_md5, stdout_queue, ret_codes])
+              force, use_md5, stdout_queue, ret_codes, gzip])
     t.daemon = True
     t.start()
     all_threads.append(t)
@@ -223,6 +227,9 @@ def main():
                     help='Use \\0 instead of \\n when parsing '
                     'the file list from stdin.  This is useful if the input '
                     'is coming from "find ... -print0".')
+  parser.add_option('-z', '--gzip', metavar='ext',
+                    help='Gzip files which end in ext. '
+                         'ext is a comma-separated list')
   (options, args) = parser.parse_args()
 
   # Enumerate our inputs.
@@ -244,7 +251,7 @@ def main():
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
-      options.num_threads, options.skip_hashing)
+      options.num_threads, options.skip_hashing, options.gzip)
 
 
 if __name__ == '__main__':