From 3869a16107f7ea28323f0aa4845511623ed59335 Mon Sep 17 00:00:00 2001 From: Josip Sokcevic Date: Fri, 18 Aug 2023 19:45:45 +0000 Subject: [PATCH] [recipe] Support stripping dep prefix It's the only roll that uses use_relative_path = False. Instead of fully supporting that case, we will ask user to provide prefix that should be removed when gitlinks are added. Change-Id: I1e39d0ce145a246e5b69b7cf64e6b9471bcb1aea Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4794721 Commit-Queue: Josip Sokcevic Reviewed-by: Keybo Qian --- recipes/README.recipes.md | 9 +- recipes/recipe_modules/gclient/api.py | 29 +++-- .../gclient/examples/full.expected/basic.json | 66 ++++++++++ .../examples/full.expected/revision.json | 117 ++++++++++++++++++ .../examples/full.expected/tryserver.json | 117 ++++++++++++++++++ .../recipe_modules/gclient/examples/full.py | 14 ++- 6 files changed, 340 insertions(+), 12 deletions(-) diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 06e16ca76c..3c4f6ec45d 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -228,7 +228,7 @@ Args: — **def [resolve\_revision](/recipes/recipe_modules/gclient/api.py#161)(self, revision):** -— **def [roll\_deps](/recipes/recipe_modules/gclient/api.py#428)(self, deps_path, dep_updates, test_data=None):** +— **def [roll\_deps](/recipes/recipe_modules/gclient/api.py#428)(self, deps_path, dep_updates, strip_prefix_for_gitlink=None, test_data=None):** Updates DEPS file to desired revisions, and returns all requried file changes. @@ -237,14 +237,17 @@ Args: deps_path - Path to DEPS file that will be modified. dep_updates - A map of dependencies to update (key = dependency name, value = revision). + strip_prefix_for_gitlink - Prefix that will be removed when adding + gitlinks. This is only useful for repositories + that use use_relative_path = True. That's + currently only chromium/src. Returns: A map of all files that need to be modified (key = file path, value = file content) in addition to DEPS file itself. Note: that git submodules (gitlinks) are treated as files and content is a commit hash. - Note: deps_path is not added to returned map since the repo relative path - is not known. + Note: we expect DEPS to be in the root of the project. — **def [runhooks](/recipes/recipe_modules/gclient/api.py#285)(self, args=None, name='runhooks', \*\*kwargs):** diff --git a/recipes/recipe_modules/gclient/api.py b/recipes/recipe_modules/gclient/api.py index a791fd4018..49a36d6db1 100644 --- a/recipes/recipe_modules/gclient/api.py +++ b/recipes/recipe_modules/gclient/api.py @@ -425,7 +425,11 @@ class GclientApi(recipe_api.RecipeApi): def DepsDiffException(self): return DepsDiffException - def roll_deps(self, deps_path, dep_updates, test_data=None): + def roll_deps(self, + deps_path, + dep_updates, + strip_prefix_for_gitlink=None, + test_data=None): """Updates DEPS file to desired revisions, and returns all requried file changes. @@ -433,22 +437,24 @@ class GclientApi(recipe_api.RecipeApi): deps_path - Path to DEPS file that will be modified. dep_updates - A map of dependencies to update (key = dependency name, value = revision). + strip_prefix_for_gitlink - Prefix that will be removed when adding + gitlinks. This is only useful for repositories + that use use_relative_path = True. That's + currently only chromium/src. Returns: A map of all files that need to be modified (key = file path, value = file content) in addition to DEPS file itself. Note: that git submodules (gitlinks) are treated as files and content is a commit hash. - Note: deps_path is not added to returned map since the repo relative path - is not known. + Note: we expect DEPS to be in the root of the project. """ + deps_content = self.m.file.read_text('Read DEPS file', deps_path, test_data) update_gitlink = False dep_updates_args = [] file_changes = {} - deps_contents = self.m.file.read_text('Read DEPS file', deps_path, - test_data) - lines = deps_contents.split('\n') + lines = deps_content.split('\n') for line in lines: if line.startswith('git_dependencies = '): if 'DEPS' not in line: @@ -460,8 +466,17 @@ class GclientApi(recipe_api.RecipeApi): dep_updates_args.extend(['-r', f'{dep}@{rev}']) if update_gitlink: # Add gitlink updates to file changes. - file_changes[dep] = rev.encode('UTF-8') + gitlink_path = dep + if strip_prefix_for_gitlink and \ + gitlink_path.startswith(strip_prefix_for_gitlink): + # strip src/ from path + gitlink_path = gitlink_path[len(strip_prefix_for_gitlink):] + + file_changes[gitlink_path] = rev.encode('UTF-8') # Apply the updates to the local DEPS files. self.m.gclient('setdep', ['setdep', '--deps-file', deps_path] + dep_updates_args) + + updated_deps = self.m.file.read_raw('Read modified DEPS', deps_path) + file_changes['DEPS'] = updated_deps return file_changes diff --git a/recipes/recipe_modules/gclient/examples/full.expected/basic.json b/recipes/recipe_modules/gclient/examples/full.expected/basic.json index 50c00fb690..9cdfa7c254 100644 --- a/recipes/recipe_modules/gclient/examples/full.expected/basic.json +++ b/recipes/recipe_modules/gclient/examples/full.expected/basic.json @@ -37,6 +37,72 @@ "infra_step": true, "name": "gclient setdep" }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "foo", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Read modified DEPS" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "\ngit_dependencies = 'SYNC'\n", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Read DEPS file (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@@@@", + "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@git_dependencies = 'SYNC'@@@", + "@@@STEP_LOG_END@\ngit_dependencies = 'SYNC'\n@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_REPO[depot_tools]/gclient.py", + "setdep", + "--deps-file", + "\ngit_dependencies = 'SYNC'\n", + "-r", + "src/bar@2222222222222222222222222222222222222222" + ], + "env_suffixes": { + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "gclient setdep (2)" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "\ngit_dependencies = 'SYNC'\n", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Read modified DEPS (2)" + }, { "cmd": [ "vpython3", diff --git a/recipes/recipe_modules/gclient/examples/full.expected/revision.json b/recipes/recipe_modules/gclient/examples/full.expected/revision.json index fd293cfe47..1903239b0a 100644 --- a/recipes/recipe_modules/gclient/examples/full.expected/revision.json +++ b/recipes/recipe_modules/gclient/examples/full.expected/revision.json @@ -64,6 +64,123 @@ }, "name": "gclient setdep" }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "foo", + "/path/to/tmp/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Read modified DEPS" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "\ngit_dependencies = 'SYNC'\n", + "/path/to/tmp/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Read DEPS file (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@@@@", + "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@git_dependencies = 'SYNC'@@@", + "@@@STEP_LOG_END@\ngit_dependencies = 'SYNC'\n@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_REPO[depot_tools]/gclient.py", + "setdep", + "--deps-file", + "\ngit_dependencies = 'SYNC'\n", + "-r", + "src/bar@2222222222222222222222222222222222222222" + ], + "env": { + "DEPOT_TOOLS_REPORT_BUILD": "project/ci/builder/8945511751514863184" + }, + "env_suffixes": { + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gclient setdep (2)" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "\ngit_dependencies = 'SYNC'\n", + "/path/to/tmp/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Read modified DEPS (2)" + }, { "cmd": [ "vpython3", diff --git a/recipes/recipe_modules/gclient/examples/full.expected/tryserver.json b/recipes/recipe_modules/gclient/examples/full.expected/tryserver.json index 01d9862ff3..8dd24a2c8e 100644 --- a/recipes/recipe_modules/gclient/examples/full.expected/tryserver.json +++ b/recipes/recipe_modules/gclient/examples/full.expected/tryserver.json @@ -64,6 +64,123 @@ }, "name": "gclient setdep" }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "foo", + "/path/to/tmp/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Read modified DEPS" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "\ngit_dependencies = 'SYNC'\n", + "/path/to/tmp/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Read DEPS file (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@@@@", + "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@git_dependencies = 'SYNC'@@@", + "@@@STEP_LOG_END@\ngit_dependencies = 'SYNC'\n@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_REPO[depot_tools]/gclient.py", + "setdep", + "--deps-file", + "\ngit_dependencies = 'SYNC'\n", + "-r", + "src/bar@2222222222222222222222222222222222222222" + ], + "env": { + "DEPOT_TOOLS_REPORT_BUILD": "project/try/builder/8945511751514863184" + }, + "env_suffixes": { + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gclient setdep (2)" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "\ngit_dependencies = 'SYNC'\n", + "/path/to/tmp/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "project:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Read modified DEPS (2)" + }, { "cmd": [ "vpython3", diff --git a/recipes/recipe_modules/gclient/examples/full.py b/recipes/recipe_modules/gclient/examples/full.py index aa2b12845c..bddcf2e1c4 100644 --- a/recipes/recipe_modules/gclient/examples/full.py +++ b/recipes/recipe_modules/gclient/examples/full.py @@ -75,8 +75,18 @@ def RunSteps(api): file_changes = api.gclient.roll_deps( 'foo', {'foo': '2222222222222222222222222222222222222222'}, test_data=DEPS_CONTENT) - api.assertions.assertEqual( - file_changes, {'foo': b'2222222222222222222222222222222222222222'}) + + api.assertions.assertEqual(file_changes['foo'], + b'2222222222222222222222222222222222222222') + + file_changes = api.gclient.roll_deps( + DEPS_CONTENT, {'src/bar': '2222222222222222222222222222222222222222'}, + strip_prefix_for_gitlink='src/', + test_data=DEPS_CONTENT) + + # We expect src/ to be stripped from src/bar. + api.assertions.assertEqual(file_changes['bar'], + b'2222222222222222222222222222222222222222') for config_name in TEST_CONFIGS: api.gclient.make_config(config_name)