Add `git cl try --retry-failed`

Bug: 985887
Change-Id: I5aab64a21961ea8f57d77f6523f2ddc5f74e78a9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1818624
Commit-Queue: Quinten Yearsley <qyearsley@chromium.org>
Reviewed-by: Andrii Shyshkalov <tandrii@google.com>

git_cl.py

@@ -521,7 +521,7 @@ def fetch_try_jobs(auth_config, changelist, buildbucket_host,
                    patchset=None):
   """Fetches tryjobs from buildbucket.
 
-  Returns a map from build id to build info as a dictionary.
+  Returns a map from build ID to build info as a dictionary.
   """
   assert buildbucket_host
   assert changelist.GetIssue(), 'CL must be uploaded first'
@@ -563,6 +563,60 @@ def fetch_try_jobs(auth_config, changelist, buildbucket_host,
   return builds
 
 
+def _fetch_latest_builds(auth_config, changelist, buildbucket_host):
+  """Fetches builds from the latest patchset that has builds (within
+  the last few patchsets).
+
+  Args:
+    auth_config (auth.AuthConfig): Auth info for Buildbucket
+    changelist (Changelist): The CL to fetch builds for
+    buildbucket_host (str): Buildbucket host, e.g. "cr-buildbucket.appspot.com"
+
+  Returns:
+    A tuple (builds, patchset) where builds is a dict mapping from build ID to
+    build info from Buildbucket, and patchset is the patchset number where
+    those builds came from.
+  """
+  assert buildbucket_host
+  assert changelist.GetIssue(), 'CL must be uploaded first'
+  assert changelist.GetCodereviewServer(), 'CL must be uploaded first'
+  assert changelist.GetMostRecentPatchset()
+  ps = changelist.GetMostRecentPatchset()
+  min_ps = max(1, ps - 5)
+  while ps >= min_ps:
+    builds = fetch_try_jobs(
+        auth_config, changelist, buildbucket_host, patchset=ps)
+    if len(builds):
+      return builds, ps
+    ps -= 1
+  return [], 0
+
+
+def _filter_failed(builds):
+  """Returns a list of buckets/builders that had failed builds.
+
+  Args:
+    builds (dict): Builds, in the format returned by fetch_try_jobs,
+      i.e. a dict mapping build ID to build info dict, which includes
+      the keys status, result, bucket, and builder_name.
+
+  Returns:
+    A dict of bucket to builder to tests (empty list). This is the same format
+    accepted by _trigger_try_jobs and returned by _get_bucket_map.
+  """
+  buckets = collections.defaultdict(dict)
+  for build in builds.values():
+    if build['status'] == 'COMPLETED' and build['result'] == 'FAILURE':
+      project = build['project']
+      bucket = build['bucket']
+      if bucket.startswith('luci.'):
+        # Assume legacy bucket name luci.<project>.<bucket>.
+        bucket = bucket.split('.')[2]
+      builder = _get_builder_from_build(build)
+      buckets[project + '/' + bucket][builder] = []
+  return buckets
+
+
 def print_try_jobs(options, builds):
   """Prints nicely result of fetch_try_jobs."""
   if not builds:
@@ -577,13 +631,7 @@ def print_try_jobs(options, builds):
     try:
       return builder_names_cache[b['id']]
     except KeyError:
-      try:
-        parameters = json.loads(b['parameters_json'])
-        name = parameters['builder_name']
-      except (ValueError, KeyError) as error:
-        print('WARNING: Failed to get builder name for build %s: %s' % (
-              b['id'], error))
-        name = None
+      name = _get_builder_from_build(b)
       builder_names_cache[b['id']] = name
       return name
 
@@ -656,6 +704,18 @@ def print_try_jobs(options, builds):
   print('Total: %d tryjobs' % total)
 
 
+def _get_builder_from_build(build):
+  """Returns a builder name from a BB v1 build info dict."""
+  try:
+    parameters = json.loads(build['parameters_json'])
+    name = parameters['builder_name']
+  except (ValueError, KeyError) as error:
+    print('WARNING: Failed to get builder name for build %s: %s' % (
+          build['id'], error))
+    name = None
+  return name
+
+
 def _ComputeDiffLineRanges(files, upstream_commit):
   """Gets the changed line ranges for each file since upstream_commit.
 
@@ -4704,6 +4764,10 @@ def CMDtry(parser, args):
       '--buildbucket-host', default='cr-buildbucket.appspot.com',
       help='Host of buildbucket. The default host is %default.')
   parser.add_option_group(group)
+  parser.add_option(
+      '-R', '--retry-failed', action='store_true', default=False,
+      help='Retry failed jobs from the latest set of tryjobs. '
+           'Not allowed with --bucket and --bot options.')
   auth.add_auth_options(parser)
   _add_codereview_issue_select_options(parser)
   options, args = parser.parse_args(args)
@@ -4729,10 +4793,30 @@ def CMDtry(parser, args):
   if error_message:
     parser.error('Can\'t trigger tryjobs: %s' % error_message)
 
-  buckets = _get_bucket_map(cl, options, parser)
-  if buckets and any(b.startswith('master.') for b in buckets):
-    print('ERROR: Buildbot masters are not supported.')
-    return 1
+  if options.retry_failed:
+    if options.bot or options.bucket:
+      print('ERROR: The option --retry-failed is not compatible with '
+            '-B, -b, --bucket, or --bot.', file=sys.stderr)
+      return 1
+    print('Searching for failed tryjobs...')
+    builds, patchset = _fetch_latest_builds(
+        auth_config, cl, options.buildbucket_host)
+    if options.verbose:
+      print('Got %d builds in patchset #%d' % (len(builds), patchset))
+    buckets = _filter_failed(builds)
+    if not buckets:
+      print('There are no failed jobs in the latest set of jobs '
+            '(patchset #%d), doing nothing.' % patchset)
+      return 0
+    num_builders = sum(len(builders) for builders in buckets.values())
+    if num_builders > 10:
+      confirm_or_exit('There are %d builders with failed builds.'
+                      % num_builders, action='continue')
+  else:
+    buckets = _get_bucket_map(cl, options, parser)
+    if buckets and any(b.startswith('master.') for b in buckets):
+      print('ERROR: Buildbot masters are not supported.')
+      return 1
 
   # If no bots are listed and we couldn't get a list based on PRESUBMIT files,
   # then we default to triggering a CQ dry run (see http://crbug.com/625697).

tests/git_cl_test.py

@@ -2575,6 +2575,30 @@ class TestGitCl(TestCase):
     self.assertRegexpMatches(sys.stdout.getvalue(), 'Started:')
     self.assertRegexpMatches(sys.stdout.getvalue(), '2 tryjobs')
 
+  def test_filter_failed_none(self):
+    self.assertEqual(git_cl._filter_failed({}), {})
+
+  def test_filter_failed_some(self):
+    builds = {
+      '9000': {
+        'id': '9000',
+        'bucket': 'luci.chromium.try',
+        'project': 'chromium',
+        'created_by': 'user:someone@chromium.org',
+        'created_ts': '147200002222000',
+        'experimental': False,
+        'parameters_json': json.dumps({
+          'builder_name': 'my-bot',
+          'properties': {'category': 'cq'},
+        }),
+        'status': 'COMPLETED',
+        'result': 'FAILURE',
+      }
+    }
+    self.assertEqual(
+      git_cl._filter_failed(builds),
+      {'chromium/try': {'my-bot': []}})
+
   def _mock_gerrit_changes_for_detail_cache(self):
     self.mock(git_cl.Changelist, '_GetGerritHost', lambda _: 'host')