diff --git a/download_from_google_storage.py b/download_from_google_storage.py index b186c63ad..d71de9868 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -55,6 +55,11 @@ def GetNormalizedPlatform(): class Gsutil(object): """Call gsutil with some predefined settings. This is a convenience object, and is also immutable.""" + + MAX_TRIES = 5 + RETRY_BASE_DELAY = 5.0 + RETRY_DELAY_MULTIPLE = 1.3 + def __init__(self, path, boto_path=None, timeout=None, version='4.15'): if not os.path.exists(path): raise FileNotFoundError('GSUtil not found in %s' % path) @@ -100,6 +105,18 @@ class Gsutil(object): return (404, out, err) return (code, out, err) + def check_call_with_retries(self, *args): + delay = self.RETRY_BASE_DELAY + for i in xrange(self.MAX_TRIES): + code, out, err = self.check_call(*args) + if not code or i == self.MAX_TRIES - 1: + break + + time.sleep(delay) + delay *= self.RETRY_DELAY_MULTIPLE + + return code, out, err + def check_platform(target): """Checks if any parent directory of target matches (win|mac|linux).""" diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index 4487457bd..83f2f0e16 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -76,7 +76,7 @@ def _upload_worker( file_url = '%s/%s' % (base_url, sha1_sum) if gsutil.check_call('ls', file_url)[0] == 0 and not force: # File exists, check MD5 hash. - _, out, _ = gsutil.check_call('ls', '-L', file_url) + _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url) etag_match = re.search('ETag:\s+([a-z0-9]{32})', out) if etag_match: remote_md5 = etag_match.group(1) @@ -97,7 +97,7 @@ def _upload_worker( if gzip: gsutil_args.extend(['-z', gzip]) gsutil_args.extend([filename, file_url]) - code, _, err = gsutil.check_call(*gsutil_args) + code, _, err = gsutil.check_call_with_retries(*gsutil_args) if code != 0: ret_codes.put( (code, @@ -109,9 +109,9 @@ def _upload_worker( # the download script will check for to preserve the executable bit. if not sys.platform.startswith('win'): if os.stat(filename).st_mode & stat.S_IEXEC: - code, _, err = gsutil.check_call('setmeta', '-h', - 'x-goog-meta-executable:1', file_url) - if code: + code, _, err = gsutil.check_call_with_retries( + 'setmeta', '-h', 'x-goog-meta-executable:1', file_url) + if not code: ret_codes.put( (code, 'Encountered error on setting metadata on %s\n%s' %