From 364876e580aafe4b24bf8d4c89450b155d784927 Mon Sep 17 00:00:00 2001 From: "pasko@chromium.org" Date: Fri, 3 Apr 2015 14:14:18 +0000 Subject: [PATCH] In upload_to_google_storage, pass -z argument through to gsutil. Also fix some latent bugs in the unit tests. (cherry-picked by pasko@ from https://codereview.chromium.org/1048103002/) BUG=467005 Review URL: https://codereview.chromium.org/1059723003 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@294674 0039d316-1c4b-4281-b951-d872f2087c98 --- tests/upload_to_google_storage_unittests.py | 17 +++++++++-------- upload_to_google_storage.py | 17 ++++++++++++----- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/upload_to_google_storage_unittests.py b/tests/upload_to_google_storage_unittests.py index 24ac6b89c..3bac038d7 100755 --- a/tests/upload_to_google_storage_unittests.py +++ b/tests/upload_to_google_storage_unittests.py @@ -48,14 +48,14 @@ class UploadTests(unittest.TestCase): filenames = [self.lorem_ipsum] output_filename = '%s.sha1' % self.lorem_ipsum code = upload_to_google_storage.upload_to_google_storage( - filenames, self.base_url, self.gsutil, True, False, 1, False) + filenames, self.base_url, self.gsutil, True, False, 1, False, 'txt') self.assertEqual( self.gsutil.history, [('check_call', ('ls', '%s/%s' % (self.base_url, self.lorem_ipsum_sha1))), ('check_call', - ('cp', '-q', filenames[0], '%s/%s' % (self.base_url, - self.lorem_ipsum_sha1)))]) + ('cp', '-z', 'txt', filenames[0], + '%s/%s' % (self.base_url, self.lorem_ipsum_sha1)))]) self.assertTrue(os.path.exists(output_filename)) self.assertEqual( open(output_filename, 'rb').read(), @@ -70,7 +70,7 @@ class UploadTests(unittest.TestCase): self.gsutil.add_expected(0, '', '') self.gsutil.add_expected(0, etag_string, '') code = upload_to_google_storage.upload_to_google_storage( - filenames, self.base_url, self.gsutil, False, False, 1, False) + filenames, self.base_url, self.gsutil, False, False, 1, False, None) self.assertEqual( self.gsutil.history, [('check_call', @@ -100,7 +100,8 @@ class UploadTests(unittest.TestCase): False, False, self.stdout_queue, - self.ret_codes) + self.ret_codes, + None) expected_ret_codes = [ (20, 'Encountered error on uploading %s to %s/%s\nExpected error message' % @@ -114,7 +115,7 @@ class UploadTests(unittest.TestCase): with open(output_filename, 'wb') as f: f.write(fake_hash) # Fake hash. code = upload_to_google_storage.upload_to_google_storage( - filenames, self.base_url, self.gsutil, False, False, 1, True) + filenames, self.base_url, self.gsutil, False, False, 1, True, None) self.assertEqual( self.gsutil.history, [('check_call', @@ -122,7 +123,7 @@ class UploadTests(unittest.TestCase): ('check_call', ('ls', '-L', '%s/%s' % (self.base_url, fake_hash))), ('check_call', - ('cp', '-q', filenames[0], '%s/%s' % (self.base_url, fake_hash)))]) + ('cp', filenames[0], '%s/%s' % (self.base_url, fake_hash)))]) self.assertEqual( open(output_filename, 'rb').read(), fake_hash) os.remove(output_filename) @@ -162,4 +163,4 @@ class UploadTests(unittest.TestCase): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index d88597cef..4cf9d1a6e 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -67,7 +67,7 @@ def get_md5_cached(filename): def _upload_worker( thread_num, upload_queue, base_url, gsutil, md5_lock, force, - use_md5, stdout_queue, ret_codes): + use_md5, stdout_queue, ret_codes, gzip): while True: filename, sha1_sum = upload_queue.get() if not filename: @@ -92,7 +92,11 @@ def _upload_worker( continue stdout_queue.put('%d> Uploading %s...' % ( thread_num, filename)) - code, _, err = gsutil.check_call('cp', filename, file_url) + gsutil_args = ['cp'] + if gzip: + gsutil_args.extend(['-z', gzip]) + gsutil_args.extend([filename, file_url]) + code, _, err = gsutil.check_call(*gsutil_args) if code != 0: ret_codes.put( (code, @@ -129,7 +133,7 @@ def get_targets(args, parser, use_null_terminator): def upload_to_google_storage( input_filenames, base_url, gsutil, force, - use_md5, num_threads, skip_hashing): + use_md5, num_threads, skip_hashing, gzip): # We only want one MD5 calculation happening at a time to avoid HD thrashing. md5_lock = threading.Lock() @@ -147,7 +151,7 @@ def upload_to_google_storage( t = threading.Thread( target=_upload_worker, args=[thread_num, upload_queue, base_url, gsutil, md5_lock, - force, use_md5, stdout_queue, ret_codes]) + force, use_md5, stdout_queue, ret_codes, gzip]) t.daemon = True t.start() all_threads.append(t) @@ -223,6 +227,9 @@ def main(): help='Use \\0 instead of \\n when parsing ' 'the file list from stdin. This is useful if the input ' 'is coming from "find ... -print0".') + parser.add_option('-z', '--gzip', metavar='ext', + help='Gzip files which end in ext. ' + 'ext is a comma-separated list') (options, args) = parser.parse_args() # Enumerate our inputs. @@ -244,7 +251,7 @@ def main(): return upload_to_google_storage( input_filenames, base_url, gsutil, options.force, options.use_md5, - options.num_threads, options.skip_hashing) + options.num_threads, options.skip_hashing, options.gzip) if __name__ == '__main__':