From f7e510b2f7ca1e3636c7f769d65e61827ef3cb39 Mon Sep 17 00:00:00 2001 From: Ravi Mistry Date: Mon, 25 Apr 2022 20:28:41 +0000 Subject: [PATCH] Add download-topics support to bot_update Tested by: * `python recipes.py test run` * Tested end-to-end by patching in this unsubmitted change in Skia's recipes with https://skia-review.googlesource.com/c/skia/+/532768 Bug: chromium:1319415 Change-Id: Ia1c9c495ef6482b3fdb494e1c1c9320541bcd0c5 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3602901 Reviewed-by: Gavin Mak Commit-Queue: Ravi Mistry --- recipes/README.recipes.md | 18 +++++++-- recipes/recipe_modules/bot_update/api.py | 5 +++ .../bot_update/resources/bot_update.py | 16 ++++++-- .../bot_update/tests/download_topics.py | 37 +++++++++++++++++++ 4 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 recipes/recipe_modules/bot_update/tests/download_topics.py diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 5222260e2..f9ad11e76 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -19,6 +19,7 @@ **[Recipes](#Recipes)** * [bot_update:examples/full](#recipes-bot_update_examples_full) (Python3 ✅) * [bot_update:tests/do_not_retry_patch_failures_in_cq](#recipes-bot_update_tests_do_not_retry_patch_failures_in_cq) (Python3 ✅) + * [bot_update:tests/download_topics](#recipes-bot_update_tests_download_topics) (Python3 ✅) * [bot_update:tests/ensure_checkout](#recipes-bot_update_tests_ensure_checkout) (Python3 ✅) * [depot_tools:examples/full](#recipes-depot_tools_examples_full) (Python3 ✅) * [gclient:examples/full](#recipes-gclient_examples_full) (Python3 ✅) @@ -59,12 +60,12 @@ Recipe module to ensure a checkout is consistent on a bot. Wrapper for easy calling of bot_update. -— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#523)(self, bot_update_step):** +— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#528)(self, bot_update_step):** Deapplies a patch, taking care of DEPS and solution revisions properly. -— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#81)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, clobber=False, root_solution_revision=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, assert_one_gerrit_change=True, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, \*\*kwargs):** +— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#81)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, clobber=False, root_solution_revision=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, assert_one_gerrit_change=True, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, download_topics=False, \*\*kwargs):** Args: * gclient_config: The gclient configuration to use when running bot_update. @@ -92,8 +93,10 @@ Args: change in self.m.buildbucket.build.input.gerrit_changes, because bot_update module ONLY supports one change. Users may specify a change via tryserver.set_change() and explicitly set this flag False. + * download_topics: If True, gclient downloads and patches locally from all + open Gerrit CLs that have the same topic as the tested patch ref. -— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#500)(self, project_name, gclient_config=None):** +— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#505)(self, project_name, gclient_config=None):** Returns all property names used for storing the checked-out revision of a given project. @@ -109,7 +112,7 @@ Returns (list of str): All properties that'll hold the checked-out revision   **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#49)(self):** -— **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#451)(self, bot_update_json, name):** +— **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#456)(self, bot_update_json, name):** Sets a fixed revision for a single dependency using project revision properties. @@ -1000,6 +1003,13 @@ PYTHON_VERSION_COMPATIBILITY: PY2+3 PYTHON_VERSION_COMPATIBILITY: PY2+3 — **def [RunSteps](/recipes/recipe_modules/bot_update/tests/do_not_retry_patch_failures_in_cq.py#19)(api):** +### *recipes* / [bot\_update:tests/download\_topics](/recipes/recipe_modules/bot_update/tests/download_topics.py) + +[DEPS](/recipes/recipe_modules/bot_update/tests/download_topics.py#9): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/json][recipe_engine/recipe_modules/json] + +PYTHON_VERSION_COMPATIBILITY: PY2+3 + +— **def [RunSteps](/recipes/recipe_modules/bot_update/tests/download_topics.py#16)(api):** ### *recipes* / [bot\_update:tests/ensure\_checkout](/recipes/recipe_modules/bot_update/tests/ensure_checkout.py) [DEPS](/recipes/recipe_modules/bot_update/tests/ensure_checkout.py#9): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/json][recipe_engine/recipe_modules/json] diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py index 503c15627..39b9be075 100644 --- a/recipes/recipe_modules/bot_update/api.py +++ b/recipes/recipe_modules/bot_update/api.py @@ -99,6 +99,7 @@ class BotUpdateApi(recipe_api.RecipeApi): set_output_commit=False, step_test_data=None, enforce_fetch=False, + download_topics=False, **kwargs): """ Args: @@ -127,6 +128,8 @@ class BotUpdateApi(recipe_api.RecipeApi): change in self.m.buildbucket.build.input.gerrit_changes, because bot_update module ONLY supports one change. Users may specify a change via tryserver.set_change() and explicitly set this flag False. + * download_topics: If True, gclient downloads and patches locally from all + open Gerrit CLs that have the same topic as the tested patch ref. """ assert not (ignore_input_commit and set_output_commit) if assert_one_gerrit_change: @@ -274,6 +277,8 @@ class BotUpdateApi(recipe_api.RecipeApi): cmd.append('--with_tags') if gerrit_no_reset: cmd.append('--gerrit_no_reset') + if download_topics: + cmd.append('--download_topics') if enforce_fetch: cmd.append('--enforce_fetch') if no_fetch_tags: diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py index 32c0825a6..c67d13b34 100755 --- a/recipes/recipe_modules/bot_update/resources/bot_update.py +++ b/recipes/recipe_modules/bot_update/resources/bot_update.py @@ -205,7 +205,7 @@ def call(*args, **kwargs): # pragma: no cover try: proc = subprocess.Popen(args, **kwargs) except: - print('\t%s failed to exectute.' % ' '.join(args)) + print('\t%s failed to execute.' % ' '.join(args)) raise observers = [ RepeatingTimer(300, _print_pstree), @@ -414,7 +414,7 @@ def git_config_if_not_set(key, value): def gclient_sync( with_branch_heads, with_tags, revisions, patch_refs, gerrit_reset, - gerrit_rebase_patch_ref): + gerrit_rebase_patch_ref, download_topics=False): args = ['sync', '--verbose', '--reset', '--force', '--nohooks', '--noprehooks', '--delete_unversioned_trees'] if with_branch_heads: @@ -433,6 +433,8 @@ def gclient_sync( args.append('--no-reset-patch-ref') if not gerrit_rebase_patch_ref: args.append('--no-rebase-patch-ref') + if download_topics: + args.append('--download-topics') try: call_gclient(*args) @@ -829,7 +831,8 @@ def emit_json(out_file, did_run, **kwargs): def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, target_cpu, patch_root, patch_refs, gerrit_rebase_patch_ref, no_fetch_tags, refs, git_cache_dir, cleanup_dir, - gerrit_reset, enforce_fetch, experiments): + gerrit_reset, enforce_fetch, experiments, + download_topics=False): # Get a checkout of each solution, without DEPS or hooks. # Calling git directly because there is no way to run Gclient without # invoking DEPS. @@ -863,7 +866,8 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, gc_revisions, patch_refs, gerrit_reset, - gerrit_rebase_patch_ref) + gerrit_rebase_patch_ref, + download_topics) # Now that gclient_sync has finished, we should revert any .DEPS.git so that # presubmit doesn't complain about it being modified. @@ -952,6 +956,9 @@ def parse_args(): help=('Enforce a new fetch to refresh the git cache, even if the ' 'solution revision passed in already exists in the current ' 'git cache.')) + parse.add_option( + '--download_topics', + action='store_true') parse.add_option('--clobber', action='store_true', help='Delete checkout first, always') @@ -1091,6 +1098,7 @@ def checkout(options, git_slns, specs, revisions, step_text): patch_root=options.patch_root, patch_refs=options.patch_refs, gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, + download_topics=options.download_topics, # Control how the fetch step will occur. no_fetch_tags=options.no_fetch_tags, diff --git a/recipes/recipe_modules/bot_update/tests/download_topics.py b/recipes/recipe_modules/bot_update/tests/download_topics.py new file mode 100644 index 000000000..1ae0112b8 --- /dev/null +++ b/recipes/recipe_modules/bot_update/tests/download_topics.py @@ -0,0 +1,37 @@ +# Copyright 2022 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine import post_process + +PYTHON_VERSION_COMPATIBILITY = 'PY2+3' + +DEPS = [ + 'bot_update', + 'gclient', + 'recipe_engine/json', +] + + +def RunSteps(api): + api.gclient.set_config('depot_tools') + api.bot_update.ensure_checkout() + api.bot_update.ensure_checkout(download_topics=True) + + +def GenTests(api): + yield ( + api.test('basic') + + api.post_process(post_process.StatusSuccess) + + api.post_process(post_process.DropExpectation) + ) + + yield ( + api.test('failure') + + api.override_step_data( + 'bot_update', + api.json.output({'did_run': True}), + retcode=1) + + api.post_process(post_process.StatusAnyFailure) + + api.post_process(post_process.DropExpectation) + )