From 41d9d87e9653e46166db0e97a184c801a28d75e3 Mon Sep 17 00:00:00 2001 From: Robert Iannucci Date: Thu, 14 Dec 2017 18:45:20 -0800 Subject: [PATCH] [recipe_modules/depot_tools] Add on_path helper. This will be used by various things in build and build_internal which "need" a depot_tools on PATH (though I suspect most of those should be refactored to use the depot_tools in chromium/src.git). R=tandrii@chromium.org Bug: 789808 Change-Id: I5d0590ccdc5482bd8e3048ad46a39b5aedacbddb Reviewed-on: https://chromium-review.googlesource.com/828465 Reviewed-by: Vadim Shtayura Reviewed-by: Andrii Shyshkalov Commit-Queue: Robbie Iannucci --- recipes/README.recipes.md | 48 +++++++----- recipes/recipe_modules/bot_update/api.py | 10 +-- .../recipe_modules/depot_tools/__init__.py | 6 ++ recipes/recipe_modules/depot_tools/api.py | 25 +++++++ .../examples/full.expected/basic.json | 12 +++ .../examples/full.expected/basic_luci.json | 75 +++++++++++++++++++ .../examples/full.expected/win.json | 12 +++ .../depot_tools/examples/full.py | 11 ++- 8 files changed, 171 insertions(+), 28 deletions(-) create mode 100644 recipes/recipe_modules/depot_tools/examples/full.expected/basic_luci.json diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 85f3cec92..9cf0d4bae 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -5,7 +5,7 @@ **[Recipe Modules](#Recipe-Modules)** * [bot_update](#recipe_modules-bot_update) — Recipe module to ensure a checkout is consistent on a bot. * [cipd](#recipe_modules-cipd) - * [depot_tools](#recipe_modules-depot_tools) + * [depot_tools](#recipe_modules-depot_tools) — The `depot_tools` module provides safe functions to access paths within the depot_tools repo. * [gclient](#recipe_modules-gclient) * [gerrit](#recipe_modules-gerrit) * [git](#recipe_modules-git) @@ -48,14 +48,14 @@ Recipe module to ensure a checkout is consistent on a bot. Wrapper for easy calling of bot_update. -— **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#54)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):** +— **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#46)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):** -— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#450)(self, bot_update_step):** +— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#442)(self, bot_update_step):** Deapplies a patch, taking care of DEPS and solution revisions properly. -— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, no_shallow=False, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=False, oauth2_json=False, use_site_config_creds=True, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** +— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, no_shallow=False, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=False, oauth2_json=False, use_site_config_creds=True, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** Args: use_site_config_creds: If the oauth2 credentials are in the buildbot @@ -74,7 +74,7 @@ Args: manifest_name: The name of the manifest to upload to LogDog. This must be unique for the whole build. -— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#427)(self, project_name, gclient_config=None):** +— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#419)(self, project_name, gclient_config=None):** Returns all property names used for storing the checked-out revision of a given project. @@ -88,7 +88,7 @@ Args: Returns (list of str): All properties that'll hold the checked-out revision of the given project. An empty list if no such properties exist. -  **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#48)(self):** +  **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#40)(self):** ### *recipe_modules* / [cipd](/recipes/recipe_modules/cipd) [DEPS](/recipes/recipe_modules/cipd/__init__.py#1): [infra\_paths](#recipe_modules-infra_paths), [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step] @@ -193,27 +193,39 @@ parameters will be used. — **def [set\_tag](/recipes/recipe_modules/cipd/api.py#366)(self, package_name, version, tags):** ### *recipe_modules* / [depot\_tools](/recipes/recipe_modules/depot_tools) -[DEPS](/recipes/recipe_modules/depot_tools/__init__.py#1): [recipe\_engine/platform][recipe_engine/recipe_modules/platform] +[DEPS](/recipes/recipe_modules/depot_tools/__init__.py#5): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime] -#### **class [DepotToolsApi](/recipes/recipe_modules/depot_tools/api.py#7)([RecipeApi][recipe_engine/wkt/RecipeApi]):** +The `depot_tools` module provides safe functions to access paths within +the depot_tools repo. -  **@property**
— **def [cros\_path](/recipes/recipe_modules/depot_tools/api.py#21)(self):** +#### **class [DepotToolsApi](/recipes/recipe_modules/depot_tools/api.py#12)([RecipeApi][recipe_engine/wkt/RecipeApi]):** -  **@property**
— **def [download\_from\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#8)(self):** +  **@property**
— **def [cros\_path](/recipes/recipe_modules/depot_tools/api.py#26)(self):** -  **@property**
— **def [gn\_py\_path](/recipes/recipe_modules/depot_tools/api.py#25)(self):** +  **@property**
— **def [download\_from\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#13)(self):** -  **@property**
— **def [gsutil\_py\_path](/recipes/recipe_modules/depot_tools/api.py#31)(self):** +  **@property**
— **def [gn\_py\_path](/recipes/recipe_modules/depot_tools/api.py#30)(self):** -  **@property**
— **def [ninja\_path](/recipes/recipe_modules/depot_tools/api.py#35)(self):** +  **@property**
— **def [gsutil\_py\_path](/recipes/recipe_modules/depot_tools/api.py#36)(self):** -  **@property**
— **def [presubmit\_support\_py\_path](/recipes/recipe_modules/depot_tools/api.py#40)(self):** +  **@property**
— **def [ninja\_path](/recipes/recipe_modules/depot_tools/api.py#40)(self):** -  **@property**
— **def [root](/recipes/recipe_modules/depot_tools/api.py#16)(self):** +  **@contextlib.contextmanager**
— **def [on\_path](/recipes/recipe_modules/depot_tools/api.py#49)(self):** + +Use this context manager to put depot_tools on $PATH. + +Example: + + with api.depot_tools.on_path(): + # run some steps + +  **@property**
— **def [presubmit\_support\_py\_path](/recipes/recipe_modules/depot_tools/api.py#45)(self):** + +  **@property**
— **def [root](/recipes/recipe_modules/depot_tools/api.py#21)(self):** Returns (Path): The "depot_tools" root directory. -  **@property**
— **def [upload\_to\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#12)(self):** +  **@property**
— **def [upload\_to\_google\_storage\_path](/recipes/recipe_modules/depot_tools/api.py#17)(self):** ### *recipe_modules* / [gclient](/recipes/recipe_modules/gclient) [DEPS](/recipes/recipe_modules/gclient/__init__.py#1): [infra\_paths](#recipe_modules-infra_paths), [tryserver](#recipe_modules-tryserver), [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step] @@ -767,9 +779,9 @@ like checkout or compile), and some of these tests have failed. — **def [RunSteps](/recipes/recipe_modules/cipd/examples/platform_suffix.py#22)(api, arch_override, bits_override, expect_error):** ### *recipes* / [depot\_tools:examples/full](/recipes/recipe_modules/depot_tools/examples/full.py) -[DEPS](/recipes/recipe_modules/depot_tools/examples/full.py#5): [depot\_tools](#recipe_modules-depot_tools), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step] +[DEPS](/recipes/recipe_modules/depot_tools/examples/full.py#5): [depot\_tools](#recipe_modules-depot_tools), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime], [recipe\_engine/step][recipe_engine/recipe_modules/step] -— **def [RunSteps](/recipes/recipe_modules/depot_tools/examples/full.py#13)(api):** +— **def [RunSteps](/recipes/recipe_modules/depot_tools/examples/full.py#14)(api):** ### *recipes* / [fetch\_end\_to\_end\_test](/recipes/recipes/fetch_end_to_end_test.py) [DEPS](/recipes/recipes/fetch_end_to_end_test.py#5): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step] diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py index b6412de14..8076fc18d 100644 --- a/recipes/recipe_modules/bot_update/api.py +++ b/recipes/recipe_modules/bot_update/api.py @@ -34,15 +34,7 @@ class BotUpdateApi(recipe_api.RecipeApi): bot_update_path = self.resource('bot_update.py') kwargs.setdefault('infra_step', True) - # On buildbot we have to put this on the FRONT of path, to combat the - # 'automatic' depot_tools. However, on LUCI, there is no automatic - # depot_tools, so it's safer to put it at the END of path, where it won't - # accidentally override e.g. python, vpython, etc. - key = 'env_prefixes' - if self.m.runtime.is_luci: - key = 'env_suffixes' - - with self.m.context(**{key: {'PATH': [self.m.depot_tools.root]}}): + with self.m.depot_tools.on_path(): return self.m.python(name, bot_update_path, cmd, **kwargs) @property diff --git a/recipes/recipe_modules/depot_tools/__init__.py b/recipes/recipe_modules/depot_tools/__init__.py index 8e3441f83..ffadd7b64 100644 --- a/recipes/recipe_modules/depot_tools/__init__.py +++ b/recipes/recipe_modules/depot_tools/__init__.py @@ -1,3 +1,9 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + DEPS = [ + 'recipe_engine/context', 'recipe_engine/platform', + 'recipe_engine/runtime', ] diff --git a/recipes/recipe_modules/depot_tools/api.py b/recipes/recipe_modules/depot_tools/api.py index 16863ad54..e24e61482 100644 --- a/recipes/recipe_modules/depot_tools/api.py +++ b/recipes/recipe_modules/depot_tools/api.py @@ -2,6 +2,11 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""The `depot_tools` module provides safe functions to access paths within +the depot_tools repo.""" + +import contextlib + from recipe_engine import recipe_api class DepotToolsApi(recipe_api.RecipeApi): @@ -40,3 +45,23 @@ class DepotToolsApi(recipe_api.RecipeApi): @property def presubmit_support_py_path(self): return self.package_repo_resource('presubmit_support.py') + + @contextlib.contextmanager + def on_path(self): + """Use this context manager to put depot_tools on $PATH. + + Example: + + with api.depot_tools.on_path(): + # run some steps + """ + # On buildbot we have to put this on the FRONT of path, to combat the + # 'automatic' depot_tools. However, on LUCI, there is no automatic + # depot_tools, so it's safer to put it at the END of path, where it won't + # accidentally override e.g. python, vpython, etc. + key = 'env_prefixes' + if self.m.runtime.is_luci: + key = 'env_suffixes' + + with self.m.context(**{key: {'PATH': [self.root]}}): + yield diff --git a/recipes/recipe_modules/depot_tools/examples/full.expected/basic.json b/recipes/recipe_modules/depot_tools/examples/full.expected/basic.json index 1cc3f560d..c0a95f627 100644 --- a/recipes/recipe_modules/depot_tools/examples/full.expected/basic.json +++ b/recipes/recipe_modules/depot_tools/examples/full.expected/basic.json @@ -55,6 +55,18 @@ ], "name": "presubmit_support_py_path" }, + { + "cmd": [ + "echo", + "$PATH" + ], + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "name": "on_path" + }, { "name": "$result", "recipe_result": null, diff --git a/recipes/recipe_modules/depot_tools/examples/full.expected/basic_luci.json b/recipes/recipe_modules/depot_tools/examples/full.expected/basic_luci.json new file mode 100644 index 000000000..0f1ff7539 --- /dev/null +++ b/recipes/recipe_modules/depot_tools/examples/full.expected/basic_luci.json @@ -0,0 +1,75 @@ +[ + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]" + ], + "name": "root" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/download_from_google_storage.py" + ], + "name": "download_from_google_storage" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/upload_to_google_storage.py" + ], + "name": "upload_to_google_storage" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/cros" + ], + "name": "cros" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/gn.py" + ], + "name": "gn_py_path" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py" + ], + "name": "gsutil_py_path" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/ninja" + ], + "name": "ninja_path" + }, + { + "cmd": [ + "ls", + "RECIPE_PACKAGE_REPO[depot_tools]/presubmit_support.py" + ], + "name": "presubmit_support_py_path" + }, + { + "cmd": [ + "echo", + "$PATH" + ], + "env_suffixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "name": "on_path" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/depot_tools/examples/full.expected/win.json b/recipes/recipe_modules/depot_tools/examples/full.expected/win.json index e7f4b94f6..94ee93708 100644 --- a/recipes/recipe_modules/depot_tools/examples/full.expected/win.json +++ b/recipes/recipe_modules/depot_tools/examples/full.expected/win.json @@ -55,6 +55,18 @@ ], "name": "presubmit_support_py_path" }, + { + "cmd": [ + "echo", + "$PATH" + ], + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "name": "on_path" + }, { "name": "$result", "recipe_result": null, diff --git a/recipes/recipe_modules/depot_tools/examples/full.py b/recipes/recipe_modules/depot_tools/examples/full.py index 57c6d66f8..71e5c1249 100644 --- a/recipes/recipe_modules/depot_tools/examples/full.py +++ b/recipes/recipe_modules/depot_tools/examples/full.py @@ -4,9 +4,10 @@ DEPS = [ 'depot_tools', - 'recipe_engine/step', 'recipe_engine/path', 'recipe_engine/platform', + 'recipe_engine/runtime', + 'recipe_engine/step', ] @@ -36,8 +37,16 @@ def RunSteps(api): 'presubmit_support_py_path', ['ls', api.depot_tools.presubmit_support_py_path]) + with api.depot_tools.on_path(): + api.step('on_path', ['echo', '$PATH']) + def GenTests(api): yield api.test('basic') + yield ( + api.test('basic_luci') + + api.runtime(is_experimental=False, is_luci=True) + ) + yield api.test('win') + api.platform('win', 32)