From 983111f039f6d797fd5f0fed26adc5a3d62cc913 Mon Sep 17 00:00:00 2001 From: Quinten Yearsley Date: Thu, 26 Sep 2019 17:18:48 +0000 Subject: [PATCH] Add `git cl try --retry-failed` Bug: 985887 Change-Id: I5aab64a21961ea8f57d77f6523f2ddc5f74e78a9 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1818624 Commit-Queue: Quinten Yearsley Reviewed-by: Andrii Shyshkalov --- git_cl.py | 108 ++++++++++++++++++++++++++++++++++++++----- tests/git_cl_test.py | 24 ++++++++++ 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/git_cl.py b/git_cl.py index 3ddb9d14c..6c06ef403 100755 --- a/git_cl.py +++ b/git_cl.py @@ -521,7 +521,7 @@ def fetch_try_jobs(auth_config, changelist, buildbucket_host, patchset=None): """Fetches tryjobs from buildbucket. - Returns a map from build id to build info as a dictionary. + Returns a map from build ID to build info as a dictionary. """ assert buildbucket_host assert changelist.GetIssue(), 'CL must be uploaded first' @@ -563,6 +563,60 @@ def fetch_try_jobs(auth_config, changelist, buildbucket_host, return builds +def _fetch_latest_builds(auth_config, changelist, buildbucket_host): + """Fetches builds from the latest patchset that has builds (within + the last few patchsets). + + Args: + auth_config (auth.AuthConfig): Auth info for Buildbucket + changelist (Changelist): The CL to fetch builds for + buildbucket_host (str): Buildbucket host, e.g. "cr-buildbucket.appspot.com" + + Returns: + A tuple (builds, patchset) where builds is a dict mapping from build ID to + build info from Buildbucket, and patchset is the patchset number where + those builds came from. + """ + assert buildbucket_host + assert changelist.GetIssue(), 'CL must be uploaded first' + assert changelist.GetCodereviewServer(), 'CL must be uploaded first' + assert changelist.GetMostRecentPatchset() + ps = changelist.GetMostRecentPatchset() + min_ps = max(1, ps - 5) + while ps >= min_ps: + builds = fetch_try_jobs( + auth_config, changelist, buildbucket_host, patchset=ps) + if len(builds): + return builds, ps + ps -= 1 + return [], 0 + + +def _filter_failed(builds): + """Returns a list of buckets/builders that had failed builds. + + Args: + builds (dict): Builds, in the format returned by fetch_try_jobs, + i.e. a dict mapping build ID to build info dict, which includes + the keys status, result, bucket, and builder_name. + + Returns: + A dict of bucket to builder to tests (empty list). This is the same format + accepted by _trigger_try_jobs and returned by _get_bucket_map. + """ + buckets = collections.defaultdict(dict) + for build in builds.values(): + if build['status'] == 'COMPLETED' and build['result'] == 'FAILURE': + project = build['project'] + bucket = build['bucket'] + if bucket.startswith('luci.'): + # Assume legacy bucket name luci... + bucket = bucket.split('.')[2] + builder = _get_builder_from_build(build) + buckets[project + '/' + bucket][builder] = [] + return buckets + + def print_try_jobs(options, builds): """Prints nicely result of fetch_try_jobs.""" if not builds: @@ -577,13 +631,7 @@ def print_try_jobs(options, builds): try: return builder_names_cache[b['id']] except KeyError: - try: - parameters = json.loads(b['parameters_json']) - name = parameters['builder_name'] - except (ValueError, KeyError) as error: - print('WARNING: Failed to get builder name for build %s: %s' % ( - b['id'], error)) - name = None + name = _get_builder_from_build(b) builder_names_cache[b['id']] = name return name @@ -656,6 +704,18 @@ def print_try_jobs(options, builds): print('Total: %d tryjobs' % total) +def _get_builder_from_build(build): + """Returns a builder name from a BB v1 build info dict.""" + try: + parameters = json.loads(build['parameters_json']) + name = parameters['builder_name'] + except (ValueError, KeyError) as error: + print('WARNING: Failed to get builder name for build %s: %s' % ( + build['id'], error)) + name = None + return name + + def _ComputeDiffLineRanges(files, upstream_commit): """Gets the changed line ranges for each file since upstream_commit. @@ -4704,6 +4764,10 @@ def CMDtry(parser, args): '--buildbucket-host', default='cr-buildbucket.appspot.com', help='Host of buildbucket. The default host is %default.') parser.add_option_group(group) + parser.add_option( + '-R', '--retry-failed', action='store_true', default=False, + help='Retry failed jobs from the latest set of tryjobs. ' + 'Not allowed with --bucket and --bot options.') auth.add_auth_options(parser) _add_codereview_issue_select_options(parser) options, args = parser.parse_args(args) @@ -4729,10 +4793,30 @@ def CMDtry(parser, args): if error_message: parser.error('Can\'t trigger tryjobs: %s' % error_message) - buckets = _get_bucket_map(cl, options, parser) - if buckets and any(b.startswith('master.') for b in buckets): - print('ERROR: Buildbot masters are not supported.') - return 1 + if options.retry_failed: + if options.bot or options.bucket: + print('ERROR: The option --retry-failed is not compatible with ' + '-B, -b, --bucket, or --bot.', file=sys.stderr) + return 1 + print('Searching for failed tryjobs...') + builds, patchset = _fetch_latest_builds( + auth_config, cl, options.buildbucket_host) + if options.verbose: + print('Got %d builds in patchset #%d' % (len(builds), patchset)) + buckets = _filter_failed(builds) + if not buckets: + print('There are no failed jobs in the latest set of jobs ' + '(patchset #%d), doing nothing.' % patchset) + return 0 + num_builders = sum(len(builders) for builders in buckets.values()) + if num_builders > 10: + confirm_or_exit('There are %d builders with failed builds.' + % num_builders, action='continue') + else: + buckets = _get_bucket_map(cl, options, parser) + if buckets and any(b.startswith('master.') for b in buckets): + print('ERROR: Buildbot masters are not supported.') + return 1 # If no bots are listed and we couldn't get a list based on PRESUBMIT files, # then we default to triggering a CQ dry run (see http://crbug.com/625697). diff --git a/tests/git_cl_test.py b/tests/git_cl_test.py index a51eddcd6..2ae36b374 100755 --- a/tests/git_cl_test.py +++ b/tests/git_cl_test.py @@ -2575,6 +2575,30 @@ class TestGitCl(TestCase): self.assertRegexpMatches(sys.stdout.getvalue(), 'Started:') self.assertRegexpMatches(sys.stdout.getvalue(), '2 tryjobs') + def test_filter_failed_none(self): + self.assertEqual(git_cl._filter_failed({}), {}) + + def test_filter_failed_some(self): + builds = { + '9000': { + 'id': '9000', + 'bucket': 'luci.chromium.try', + 'project': 'chromium', + 'created_by': 'user:someone@chromium.org', + 'created_ts': '147200002222000', + 'experimental': False, + 'parameters_json': json.dumps({ + 'builder_name': 'my-bot', + 'properties': {'category': 'cq'}, + }), + 'status': 'COMPLETED', + 'result': 'FAILURE', + } + } + self.assertEqual( + git_cl._filter_failed(builds), + {'chromium/try': {'my-bot': []}}) + def _mock_gerrit_changes_for_detail_cache(self): self.mock(git_cl.Changelist, '_GetGerritHost', lambda _: 'host')