Add download-topics support to bot_update

Tested by:
* `python recipes.py test run`
* Tested end-to-end by patching in this unsubmitted change in Skia's recipes with https://skia-review.googlesource.com/c/skia/+/532768

Bug: chromium:1319415
Change-Id: Ia1c9c495ef6482b3fdb494e1c1c9320541bcd0c5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3602901
Reviewed-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Ravi Mistry <rmistry@chromium.org>

recipes/README.recipes.md

@@ -19,6 +19,7 @@
 **[Recipes](#Recipes)**
   * [bot_update:examples/full](#recipes-bot_update_examples_full) (Python3 ✅)
   * [bot_update:tests/do_not_retry_patch_failures_in_cq](#recipes-bot_update_tests_do_not_retry_patch_failures_in_cq) (Python3 ✅)
+  * [bot_update:tests/download_topics](#recipes-bot_update_tests_download_topics) (Python3 ✅)
   * [bot_update:tests/ensure_checkout](#recipes-bot_update_tests_ensure_checkout) (Python3 ✅)
   * [depot_tools:examples/full](#recipes-depot_tools_examples_full) (Python3 ✅)
   * [gclient:examples/full](#recipes-gclient_examples_full) (Python3 ✅)
@@ -59,12 +60,12 @@ Recipe module to ensure a checkout is consistent on a bot.
 
 Wrapper for easy calling of bot_update.
 
-— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#523)(self, bot_update_step):**
+— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#528)(self, bot_update_step):**
 
 Deapplies a patch, taking care of DEPS and solution revisions properly.
     
 
-— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#81)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, clobber=False, root_solution_revision=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, assert_one_gerrit_change=True, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, \*\*kwargs):**
+— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#81)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, clobber=False, root_solution_revision=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, assert_one_gerrit_change=True, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, download_topics=False, \*\*kwargs):**
 
 Args:
   * gclient_config: The gclient configuration to use when running bot_update.
@@ -92,8 +93,10 @@ Args:
     change in self.m.buildbucket.build.input.gerrit_changes, because
     bot_update module ONLY supports one change. Users may specify a change
     via tryserver.set_change() and explicitly set this flag False.
+  * download_topics: If True, gclient downloads and patches locally from all
+    open Gerrit CLs that have the same topic as the tested patch ref.
 
-— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#500)(self, project_name, gclient_config=None):**
+— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#505)(self, project_name, gclient_config=None):**
 
 Returns all property names used for storing the checked-out revision of
 a given project.
@@ -109,7 +112,7 @@ Returns (list of str): All properties that'll hold the checked-out revision
 
 &emsp; **@property**<br>&mdash; **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#49)(self):**
 
-&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#451)(self, bot_update_json, name):**
+&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#456)(self, bot_update_json, name):**
 
 Sets a fixed revision for a single dependency using project revision
 properties.
@@ -1000,6 +1003,13 @@ PYTHON_VERSION_COMPATIBILITY: PY2+3
 PYTHON_VERSION_COMPATIBILITY: PY2+3
 
 &mdash; **def [RunSteps](/recipes/recipe_modules/bot_update/tests/do_not_retry_patch_failures_in_cq.py#19)(api):**
+### *recipes* / [bot\_update:tests/download\_topics](/recipes/recipe_modules/bot_update/tests/download_topics.py)
+
+[DEPS](/recipes/recipe_modules/bot_update/tests/download_topics.py#9): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/json][recipe_engine/recipe_modules/json]
+
+PYTHON_VERSION_COMPATIBILITY: PY2+3
+
+&mdash; **def [RunSteps](/recipes/recipe_modules/bot_update/tests/download_topics.py#16)(api):**
 ### *recipes* / [bot\_update:tests/ensure\_checkout](/recipes/recipe_modules/bot_update/tests/ensure_checkout.py)
 
 [DEPS](/recipes/recipe_modules/bot_update/tests/ensure_checkout.py#9): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/json][recipe_engine/recipe_modules/json]

recipes/recipe_modules/bot_update/api.py

@@ -99,6 +99,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
                       set_output_commit=False,
                       step_test_data=None,
                       enforce_fetch=False,
+                      download_topics=False,
                       **kwargs):
     """
     Args:
@@ -127,6 +128,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
         change in self.m.buildbucket.build.input.gerrit_changes, because
         bot_update module ONLY supports one change. Users may specify a change
         via tryserver.set_change() and explicitly set this flag False.
+      * download_topics: If True, gclient downloads and patches locally from all
+        open Gerrit CLs that have the same topic as the tested patch ref.
     """
     assert not (ignore_input_commit and set_output_commit)
     if assert_one_gerrit_change:
@@ -274,6 +277,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
       cmd.append('--with_tags')
     if gerrit_no_reset:
       cmd.append('--gerrit_no_reset')
+    if download_topics:
+      cmd.append('--download_topics')
     if enforce_fetch:
       cmd.append('--enforce_fetch')
     if no_fetch_tags:

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -205,7 +205,7 @@ def call(*args, **kwargs):  # pragma: no cover
   try:
     proc = subprocess.Popen(args, **kwargs)
   except:
-    print('\t%s failed to exectute.' % ' '.join(args)) 
+    print('\t%s failed to execute.' % ' '.join(args))
     raise
   observers = [
       RepeatingTimer(300, _print_pstree),
@@ -414,7 +414,7 @@ def git_config_if_not_set(key, value):
 def gclient_sync(
     with_branch_heads, with_tags, revisions,
     patch_refs, gerrit_reset,
-    gerrit_rebase_patch_ref):
+    gerrit_rebase_patch_ref, download_topics=False):
   args = ['sync', '--verbose', '--reset', '--force',
           '--nohooks', '--noprehooks', '--delete_unversioned_trees']
   if with_branch_heads:
@@ -433,6 +433,8 @@ def gclient_sync(
       args.append('--no-reset-patch-ref')
     if not gerrit_rebase_patch_ref:
       args.append('--no-rebase-patch-ref')
+    if download_topics:
+      args.append('--download-topics')
 
   try:
     call_gclient(*args)
@@ -829,7 +831,8 @@ def emit_json(out_file, did_run, **kwargs):
 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     target_cpu, patch_root, patch_refs, gerrit_rebase_patch_ref,
                     no_fetch_tags, refs, git_cache_dir, cleanup_dir,
-                    gerrit_reset, enforce_fetch, experiments):
+                    gerrit_reset, enforce_fetch, experiments,
+                    download_topics=False):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
@@ -863,7 +866,8 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
       gc_revisions,
       patch_refs,
       gerrit_reset,
-      gerrit_rebase_patch_ref)
+      gerrit_rebase_patch_ref,
+      download_topics)
 
   # Now that gclient_sync has finished, we should revert any .DEPS.git so that
   # presubmit doesn't complain about it being modified.
@@ -952,6 +956,9 @@ def parse_args():
       help=('Enforce a new fetch to refresh the git cache, even if the '
             'solution revision passed in already exists in the current '
             'git cache.'))
+  parse.add_option(
+      '--download_topics',
+      action='store_true')
 
   parse.add_option('--clobber', action='store_true',
                    help='Delete checkout first, always')
@@ -1091,6 +1098,7 @@ def checkout(options, git_slns, specs, revisions, step_text):
           patch_root=options.patch_root,
           patch_refs=options.patch_refs,
           gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref,
+          download_topics=options.download_topics,
 
           # Control how the fetch step will occur.
           no_fetch_tags=options.no_fetch_tags,

recipes/recipe_modules/bot_update/tests/download_topics.py

@@ -0,0 +1,37 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from recipe_engine import post_process
+
+PYTHON_VERSION_COMPATIBILITY = 'PY2+3'
+
+DEPS = [
+  'bot_update',
+  'gclient',
+  'recipe_engine/json',
+]
+
+
+def RunSteps(api):
+  api.gclient.set_config('depot_tools')
+  api.bot_update.ensure_checkout()
+  api.bot_update.ensure_checkout(download_topics=True)
+
+
+def GenTests(api):
+  yield (
+      api.test('basic') +
+      api.post_process(post_process.StatusSuccess) +
+      api.post_process(post_process.DropExpectation)
+  )
+
+  yield (
+      api.test('failure') +
+      api.override_step_data(
+          'bot_update',
+          api.json.output({'did_run': True}),
+          retcode=1) +
+      api.post_process(post_process.StatusAnyFailure) +
+      api.post_process(post_process.DropExpectation)
+  )