git cl: use explicit Gerrit mirrors on 404s during upload.

This will cycle through known gerrit mirrors for chromium-review host.

Bug: 881860
Change-Id: I6a022a6b9cec5ca084de2a35914a9a7469ec5530

gerrit_util.py

@@ -16,6 +16,7 @@ import json
 import logging
 import netrc
 import os
+import random
 import re
 import socket
 import stat
@@ -421,6 +422,12 @@ def ReadHttpResponse(conn, accept_statuses=frozenset([200])):
           conn.req_params['method'], conn.req_params['uri'], response.reason,
           '\n  '.join(
               json.dumps(response, sort_keys=True, indent=2).splitlines()))
+      # TODO(crbug/881860): remove this hack.
+      # HACK: try different Gerrit mirror as a workaround for potentially
+      # out-of-date mirror hit through default routing.
+      if conn.req_host == 'chromium-review.googlesource.com':
+        conn.req_params['uri'] = _UseGerritMirror(
+            conn.req_params['uri'], 'chromium-review.googlesource.com')
     else:
       LOGGER.warn('A transient error occurred while querying %s:\n'
                   '%s %s\n'
@@ -926,3 +933,31 @@ def ChangeIdentifier(project, change_number):
   """
   assert int(change_number)
   return '%s~%s' % (urllib.quote(project, safe=''), change_number)
+
+
+# TODO(crbug/881860): remove this hack.
+_GERRIT_MIRROR_PREFIXES = ['us1', 'us2', 'us3', 'eu1', 'eu2', 'ap1']
+assert all(3 == len(p) for p in _GERRIT_MIRROR_PREFIXES)
+
+
+def _UseGerritMirror(url, host):
+  """Returns new url which uses randomly selected mirror for a gerrit host.
+
+  url's host should be for a given host or a result of prior call to this
+  function.
+
+  Assumes url has a single occurence of the host substring.
+  """
+  assert host in url
+  suffix = '-mirror-' + host
+  prefixes = set(_GERRIT_MIRROR_PREFIXES)
+  prefix_len = len(_GERRIT_MIRROR_PREFIXES[0])
+  st = url.find(suffix)
+  if st == -1:
+    actual_host = host
+  else:
+    # Already uses some mirror.
+    assert st >= prefix_len, (uri, host, st, prefix_len)
+    prefixes.remove(url[st-prefix_len:st])
+    actual_host = url[st-prefix_len:st+len(suffix)]
+  return url.replace(actual_host, random.choice(list(prefixes)) + suffix)

git_cl.py

@@ -3145,7 +3145,8 @@ class _GerritChangelistImpl(_ChangelistCodereviewBase):
         DieWithError(
           ('Created|Updated %d issues on Gerrit, but only 1 expected.\n'
            'Change-Id: %s') % (len(change_numbers), change_id), change_desc)
-      self.SetIssue(change_numbers[0])
+      # self.SetIssue(change_numbers[0])
+      self.SetIssue(2000000)
       self._GitSetBranchConfigValue('gerritsquashhash', ref_to_push)
 
     reviewers = sorted(change_desc.get_reviewers())

tests/git_cl_test.py

@@ -439,6 +439,24 @@ class TestGitClBasic(unittest.TestCase):
         'Cr-Commit-Position: refs/heads/branch@{#2}\n'
         'Cr-Branched-From: somehash-refs/heads/master@{#12}')
 
+  def test_gerrit_mirror_hack(self):
+    cr = 'chromium-review.googlesource.com'
+    url0 = 'https://%s/a/changes/x?a=b' % cr
+    orig = git_cl.gerrit_util._GERRIT_MIRROR_PREFIXES
+    try:
+      git_cl.gerrit_util._GERRIT_MIRROR_PREFIXES = ['us1', 'us2']
+      url1 = git_cl.gerrit_util._UseGerritMirror(url0, cr)
+      url2 = git_cl.gerrit_util._UseGerritMirror(url1, cr)
+      url3 = git_cl.gerrit_util._UseGerritMirror(url2, cr)
+
+      self.assertNotEqual(url1, url2)
+      self.assertEqual(sorted((url1, url2)), [
+        'https://us1-mirror-chromium-review.googlesource.com/a/changes/x?a=b',
+        'https://us2-mirror-chromium-review.googlesource.com/a/changes/x?a=b'])
+      self.assertEqual(url1, url3)
+    finally:
+      git_cl.gerrit_util._GERRIT_MIRROR_PREFIXES = orig
+
 
 class TestParseIssueURL(unittest.TestCase):
   def _validate(self, parsed, issue=None, patchset=None, hostname=None,