Retry up to three times on upload timeout.

BUG=668526

Change-Id: Id49f1c1394a72880410eb06ba7356307fff7e8c5
Reviewed-on: https://chromium-review.googlesource.com/414496
Commit-Queue: Mike Stipicevic <stip@chromium.org>
Reviewed-by: Marc-Antoine Ruel <maruel@chromium.org>

third_party/upload.py

@@ -133,6 +133,7 @@ for vcs in VCS:
   VCS_ABBREVIATIONS.update((alias, vcs['name']) for alias in vcs['aliases'])
 
 UPLOAD_TIMEOUT = 120
+MAX_UPLOAD_ATTEMPTS = 3
 
 
 # The result of parsing Subversion's [auto-props] setting.
@@ -1018,6 +1019,7 @@ class VersionControlSystem(object):
     patches = dict()
     [patches.setdefault(v, k) for k, v in patch_list]
 
+    def uploadAttempt():
       threads = []
       thread_pool = ThreadPool(options.num_upload_threads)
 
@@ -1040,6 +1042,19 @@ class VersionControlSystem(object):
       for t in threads:
         print(t.get(timeout=UPLOAD_TIMEOUT))
 
+    success = False
+    for _ in range(MAX_UPLOAD_ATTEMPTS):
+      try:
+        uploadAttempt()
+        success = True
+        break
+      except multiprocessing.TimeoutError:
+        LOGGER.warning('Timeout error while uploading, retrying...')
+
+    if not success:
+      raise IOError(
+          '%d consecutive timeout errors, aborting!' % MAX_UPLOAD_ATTEMPTS)
+
   def IsImage(self, filename):
     """Returns true if the filename has an image extension."""
     mimetype =  mimetypes.guess_type(filename)[0]