From be46fddf5f268eb31fd8f4afef41061aa3b16a80 Mon Sep 17 00:00:00 2001 From: "hinoka@chromium.org" Date: Wed, 29 Jul 2015 21:11:36 +0000 Subject: [PATCH] Add verification to downloaded files This does one last check to see if the file downloaded by download_from_google_storage.py actually matches its sha1 BUG= Review URL: https://codereview.chromium.org/1252313005 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@296155 0039d316-1c4b-4281-b951-d872f2087c98 --- download_from_google_storage.py | 9 +++++ .../download_from_google_storage_unittests.py | 40 +++++++++++++++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/download_from_google_storage.py b/download_from_google_storage.py index a45f387ce..44cc3a75b 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -227,6 +227,15 @@ def _downloader_worker_thread(thread_num, q, force, base_url, if code != 0: out_q.put('%d> %s' % (thread_num, err)) ret_codes.put((code, err)) + continue + + remote_sha1 = get_sha1(output_filename) + if remote_sha1 != input_sha1_sum: + msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' % + (thread_num, remote_sha1, input_sha1_sum)) + out_q.put(msg) + ret_codes.put((20, msg)) + continue # Set executable bit. if sys.platform == 'cygwin': diff --git a/tests/download_from_google_storage_unittests.py b/tests/download_from_google_storage_unittests.py index a8af63b0c..f87c6a7ed 100755 --- a/tests/download_from_google_storage_unittests.py +++ b/tests/download_from_google_storage_unittests.py @@ -36,8 +36,8 @@ class GsutilMock(object): self.history = [] self.lock = threading.Lock() - def add_expected(self, return_code, out, err): - self.expected.append((return_code, out, err)) + def add_expected(self, return_code, out, err, fn=None): + self.expected.append((return_code, out, err, fn)) def append_history(self, method, args): self.history.append((method, args)) @@ -46,7 +46,10 @@ class GsutilMock(object): with self.lock: self.append_history('call', args) if self.expected: - return self.expected.pop(0)[0] + code, _out, _err, fn = self.expected.pop(0) + if fn: + fn() + return code else: return 0 @@ -54,7 +57,10 @@ class GsutilMock(object): with self.lock: self.append_history('check_call', args) if self.expected: - return self.expected.pop(0) + code, out, err, fn = self.expected.pop(0) + if fn: + fn() + return code, out, err else: return (0, '', '') @@ -257,6 +263,32 @@ class DownloadTests(unittest.TestCase): self.assertEqual(self.gsutil.history, expected_calls) self.assertEqual(code, 101) + def test_corrupt_download(self): + q = Queue.Queue() + out_q = Queue.Queue() + ret_codes = Queue.Queue() + tmp_dir = tempfile.mkdtemp() + sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' + output_filename = os.path.join(tmp_dir, 'lorem_ipsum.txt') + q.put(('7871c8e24da15bad8b0be2c36edc9dc77e37727f', output_filename)) + q.put((None, None)) + def _write_bad_file(): + with open(output_filename, 'w') as f: + f.write('foobar') + self.gsutil.add_expected(0, '', '') + self.gsutil.add_expected(0, '', '', _write_bad_file) + download_from_google_storage._downloader_worker_thread( + 1, q, True, self.base_url, self.gsutil, out_q, ret_codes, True) + self.assertTrue(q.empty()) + msg = ('1> ERROR remote sha1 (%s) does not match expected sha1 (%s).' % + ('8843d7f92416211de9ebb963ff4ce28125932878', sha1_hash)) + self.assertEquals(out_q.get(), '1> Downloading %s...' % output_filename) + self.assertEquals(out_q.get(), msg) + self.assertEquals(ret_codes.get(), (20, msg)) + self.assertTrue(out_q.empty()) + self.assertTrue(ret_codes.empty()) + + def test_download_directory_no_recursive_non_force(self): sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' input_filename = '%s/%s' % (self.base_url, sha1_hash)