[recipe] Support stripping dep prefix

It's the only roll that uses use_relative_path = False. Instead of fully
supporting that case, we will ask user to provide prefix that should be
removed when gitlinks are added.

Change-Id: I1e39d0ce145a246e5b69b7cf64e6b9471bcb1aea
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4794721
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Keybo Qian <keybo@google.com>

recipes/README.recipes.md

@@ -228,7 +228,7 @@ Args:
 
 — **def [resolve\_revision](/recipes/recipe_modules/gclient/api.py#161)(self, revision):**
 
-— **def [roll\_deps](/recipes/recipe_modules/gclient/api.py#428)(self, deps_path, dep_updates, test_data=None):**
+— **def [roll\_deps](/recipes/recipe_modules/gclient/api.py#428)(self, deps_path, dep_updates, strip_prefix_for_gitlink=None, test_data=None):**
 
 Updates DEPS file to desired revisions, and returns all requried file
 changes.
@@ -237,14 +237,17 @@ Args:
   deps_path - Path to DEPS file that will be modified.
   dep_updates - A map of dependencies to update (key = dependency name,
                 value = revision).
+  strip_prefix_for_gitlink - Prefix that will be removed when adding
+                             gitlinks. This is only useful for repositories
+                             that use use_relative_path = True. That's
+                             currently only chromium/src.
 
 Returns:
   A map of all files that need to be modified (key = file path, value = file
   content) in addition to DEPS file itself.
   Note: that git submodules (gitlinks) are treated as files and content is a
   commit hash.
-  Note: deps_path is not added to returned map since the repo relative path
-  is not known.
+  Note: we expect DEPS to be in the root of the project.
 
 — **def [runhooks](/recipes/recipe_modules/gclient/api.py#285)(self, args=None, name='runhooks', \*\*kwargs):**
 

recipes/recipe_modules/gclient/api.py

@@ -425,7 +425,11 @@ class GclientApi(recipe_api.RecipeApi):
   def DepsDiffException(self):
     return DepsDiffException
 
-  def roll_deps(self, deps_path, dep_updates, test_data=None):
+  def roll_deps(self,
+                deps_path,
+                dep_updates,
+                strip_prefix_for_gitlink=None,
+                test_data=None):
     """Updates DEPS file to desired revisions, and returns all requried file
     changes.
 
@@ -433,22 +437,24 @@ class GclientApi(recipe_api.RecipeApi):
       deps_path - Path to DEPS file that will be modified.
       dep_updates - A map of dependencies to update (key = dependency name,
                     value = revision).
+      strip_prefix_for_gitlink - Prefix that will be removed when adding
+                                 gitlinks. This is only useful for repositories
+                                 that use use_relative_path = True. That's
+                                 currently only chromium/src.
 
     Returns:
       A map of all files that need to be modified (key = file path, value = file
       content) in addition to DEPS file itself.
       Note: that git submodules (gitlinks) are treated as files and content is a
       commit hash.
-      Note: deps_path is not added to returned map since the repo relative path
-      is not known.
+      Note: we expect DEPS to be in the root of the project.
     """
+    deps_content = self.m.file.read_text('Read DEPS file', deps_path, test_data)
     update_gitlink = False
     dep_updates_args = []
     file_changes = {}
 
-    deps_contents = self.m.file.read_text('Read DEPS file', deps_path,
-                                          test_data)
-    lines = deps_contents.split('\n')
+    lines = deps_content.split('\n')
     for line in lines:
       if line.startswith('git_dependencies = '):
         if 'DEPS' not in line:
@@ -460,8 +466,17 @@ class GclientApi(recipe_api.RecipeApi):
       dep_updates_args.extend(['-r', f'{dep}@{rev}'])
       if update_gitlink:
         # Add gitlink updates to file changes.
-        file_changes[dep] = rev.encode('UTF-8')
+        gitlink_path = dep
+        if strip_prefix_for_gitlink and \
+            gitlink_path.startswith(strip_prefix_for_gitlink):
+          # strip src/ from path
+          gitlink_path = gitlink_path[len(strip_prefix_for_gitlink):]
+
+        file_changes[gitlink_path] = rev.encode('UTF-8')
     # Apply the updates to the local DEPS files.
     self.m.gclient('setdep',
                    ['setdep', '--deps-file', deps_path] + dep_updates_args)
+
+    updated_deps = self.m.file.read_raw('Read modified DEPS', deps_path)
+    file_changes['DEPS'] = updated_deps
     return file_changes

recipes/recipe_modules/gclient/examples/full.expected/basic.json

@@ -37,6 +37,72 @@
     "infra_step": true,
     "name": "gclient setdep"
   },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "foo",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Read modified DEPS"
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "\ngit_dependencies = 'SYNC'\n",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Read DEPS file (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@@@@",
+      "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@git_dependencies = 'SYNC'@@@",
+      "@@@STEP_LOG_END@\ngit_dependencies = 'SYNC'\n@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_REPO[depot_tools]/gclient.py",
+      "setdep",
+      "--deps-file",
+      "\ngit_dependencies = 'SYNC'\n",
+      "-r",
+      "src/bar@2222222222222222222222222222222222222222"
+    ],
+    "env_suffixes": {
+      "PATH": [
+        "RECIPE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "name": "gclient setdep (2)"
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "\ngit_dependencies = 'SYNC'\n",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "name": "Read modified DEPS (2)"
+  },
   {
     "cmd": [
       "vpython3",

recipes/recipe_modules/gclient/examples/full.expected/revision.json

@@ -64,6 +64,123 @@
     },
     "name": "gclient setdep"
   },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "foo",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:ci"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "Read modified DEPS"
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "\ngit_dependencies = 'SYNC'\n",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:ci"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "Read DEPS file (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@@@@",
+      "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@git_dependencies = 'SYNC'@@@",
+      "@@@STEP_LOG_END@\ngit_dependencies = 'SYNC'\n@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_REPO[depot_tools]/gclient.py",
+      "setdep",
+      "--deps-file",
+      "\ngit_dependencies = 'SYNC'\n",
+      "-r",
+      "src/bar@2222222222222222222222222222222222222222"
+    ],
+    "env": {
+      "DEPOT_TOOLS_REPORT_BUILD": "project/ci/builder/8945511751514863184"
+    },
+    "env_suffixes": {
+      "PATH": [
+        "RECIPE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:ci"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "gclient setdep (2)"
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "\ngit_dependencies = 'SYNC'\n",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:ci"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "Read modified DEPS (2)"
+  },
   {
     "cmd": [
       "vpython3",

recipes/recipe_modules/gclient/examples/full.expected/tryserver.json

@@ -64,6 +64,123 @@
     },
     "name": "gclient setdep"
   },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "foo",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:try"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "Read modified DEPS"
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "\ngit_dependencies = 'SYNC'\n",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:try"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "Read DEPS file (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@@@@",
+      "@@@STEP_LOG_LINE@\ngit_dependencies = 'SYNC'\n@git_dependencies = 'SYNC'@@@",
+      "@@@STEP_LOG_END@\ngit_dependencies = 'SYNC'\n@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_REPO[depot_tools]/gclient.py",
+      "setdep",
+      "--deps-file",
+      "\ngit_dependencies = 'SYNC'\n",
+      "-r",
+      "src/bar@2222222222222222222222222222222222222222"
+    ],
+    "env": {
+      "DEPOT_TOOLS_REPORT_BUILD": "project/try/builder/8945511751514863184"
+    },
+    "env_suffixes": {
+      "PATH": [
+        "RECIPE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:try"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "gclient setdep (2)"
+  },
+  {
+    "cmd": [
+      "vpython3",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "copy",
+      "\ngit_dependencies = 'SYNC'\n",
+      "/path/to/tmp/"
+    ],
+    "infra_step": true,
+    "luci_context": {
+      "realm": {
+        "name": "project:try"
+      },
+      "resultdb": {
+        "current_invocation": {
+          "name": "invocations/build:8945511751514863184",
+          "update_token": "token"
+        },
+        "hostname": "rdbhost"
+      }
+    },
+    "name": "Read modified DEPS (2)"
+  },
   {
     "cmd": [
       "vpython3",

recipes/recipe_modules/gclient/examples/full.py

@@ -75,8 +75,18 @@ def RunSteps(api):
   file_changes = api.gclient.roll_deps(
       'foo', {'foo': '2222222222222222222222222222222222222222'},
       test_data=DEPS_CONTENT)
-  api.assertions.assertEqual(
-      file_changes, {'foo': b'2222222222222222222222222222222222222222'})
+
+  api.assertions.assertEqual(file_changes['foo'],
+                             b'2222222222222222222222222222222222222222')
+
+  file_changes = api.gclient.roll_deps(
+      DEPS_CONTENT, {'src/bar': '2222222222222222222222222222222222222222'},
+      strip_prefix_for_gitlink='src/',
+      test_data=DEPS_CONTENT)
+
+  # We expect src/ to be stripped from src/bar.
+  api.assertions.assertEqual(file_changes['bar'],
+                             b'2222222222222222222222222222222222222222')
 
   for config_name in TEST_CONFIGS:
     api.gclient.make_config(config_name)