diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 69754eaae6..368a229ff5 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -42,22 +42,22 @@ ### *recipe_modules* / [bot\_update](/recipes/recipe_modules/bot_update) -[DEPS](/recipes/recipe_modules/bot_update/__init__.py#1): [depot\_tools](#recipe_modules-depot_tools), [gclient](#recipe_modules-gclient), [gerrit](#recipe_modules-gerrit), [tryserver](#recipe_modules-tryserver), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime], [recipe\_engine/source\_manifest][recipe_engine/recipe_modules/source_manifest], [recipe\_engine/step][recipe_engine/recipe_modules/step] +[DEPS](/recipes/recipe_modules/bot_update/__init__.py#1): [depot\_tools](#recipe_modules-depot_tools), [gclient](#recipe_modules-gclient), [gerrit](#recipe_modules-gerrit), [gitiles](#recipe_modules-gitiles), [tryserver](#recipe_modules-tryserver), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime], [recipe\_engine/source\_manifest][recipe_engine/recipe_modules/source_manifest], [recipe\_engine/step][recipe_engine/recipe_modules/step] Recipe module to ensure a checkout is consistent on a bot. #### **class [BotUpdateApi](/recipes/recipe_modules/bot_update/api.py#12)([RecipeApi][recipe_engine/wkt/RecipeApi]):** -— **def [\_\_call\_\_](/recipes/recipe_modules/bot_update/api.py#43)(self, name, cmd, \*\*kwargs):** +— **def [\_\_call\_\_](/recipes/recipe_modules/bot_update/api.py#37)(self, name, cmd, \*\*kwargs):** Wrapper for easy calling of bot_update. -— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#393)(self, bot_update_step):** +— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#427)(self, bot_update_step):** Deapplies a patch, taking care of DEPS and solution revisions properly. -— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#56)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, patch_refs=None, \*\*kwargs):** +— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#78)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, patch_refs=None, \*\*kwargs):** Args: gclient_config: The gclient configuration to use when running bot_update. @@ -68,7 +68,7 @@ Args: manifest_name: The name of the manifest to upload to LogDog. This must be unique for the whole build. -— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#370)(self, project_name, gclient_config=None):** +— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#404)(self, project_name, gclient_config=None):** Returns all property names used for storing the checked-out revision of a given project. @@ -82,9 +82,9 @@ Args: Returns (list of str): All properties that'll hold the checked-out revision of the given project. An empty list if no such properties exist. -— **def [initialize](/recipes/recipe_modules/bot_update/api.py#33)(self):** +— **def [initialize](/recipes/recipe_modules/bot_update/api.py#30)(self):** -  **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#52)(self):** +  **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#46)(self):** ### *recipe_modules* / [cipd](/recipes/recipe_modules/cipd) [DEPS](/recipes/recipe_modules/cipd/__init__.py#1): [infra\_paths](#recipe_modules-infra_paths), [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step] diff --git a/recipes/recipe_modules/bot_update/__init__.py b/recipes/recipe_modules/bot_update/__init__.py index 805b6e5c6b..7927b97ee4 100644 --- a/recipes/recipe_modules/bot_update/__init__.py +++ b/recipes/recipe_modules/bot_update/__init__.py @@ -2,6 +2,7 @@ DEPS = [ 'depot_tools', 'gclient', 'gerrit', + 'gitiles', 'recipe_engine/buildbucket', 'recipe_engine/context', 'recipe_engine/json', @@ -33,8 +34,6 @@ PROPERTIES = { # Common fields for both systems. 'deps_revision_overrides': Property(default={}), 'fail_patch': Property(default=None, kind=str), - 'parent_got_revision': Property(default=None), - 'revision': Property(default=None), '$depot_tools/bot_update': Property( help='Properties specific to bot_update module.', diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py index 2622ca4fa6..f08c9944df 100644 --- a/recipes/recipe_modules/bot_update/api.py +++ b/recipes/recipe_modules/bot_update/api.py @@ -11,9 +11,8 @@ from recipe_engine import recipe_api class BotUpdateApi(recipe_api.RecipeApi): - def __init__(self, properties, patch_issue, patch_set, - repository, patch_repository_url, patch_ref, - patch_gerrit_url, revision, parent_got_revision, + def __init__(self, properties, patch_issue, patch_set, repository, + patch_repository_url, patch_ref, patch_gerrit_url, deps_revision_overrides, fail_patch, *args, **kwargs): self._apply_patch_on_gclient = properties.get( 'apply_patch_on_gclient', True) @@ -22,8 +21,6 @@ class BotUpdateApi(recipe_api.RecipeApi): self._repository = repository or patch_repository_url self._gerrit_ref = patch_ref self._gerrit = patch_gerrit_url - self._revision = revision - self._parent_got_revision = parent_got_revision self._deps_revision_overrides = deps_revision_overrides self._fail_patch = fail_patch @@ -31,12 +28,9 @@ class BotUpdateApi(recipe_api.RecipeApi): super(BotUpdateApi, self).__init__(*args, **kwargs) def initialize(self): - build_input = self.m.buildbucket.build.input - if (self._revision is None and build_input.HasField('gitiles_commit')): - self._revision = build_input.gitiles_commit.id - - if self._repository is None and len(build_input.gerrit_changes) == 1: - cl = build_input.gerrit_changes[0] + changes = self.m.buildbucket.build.input.gerrit_changes + if self._repository is None and len(changes) == 1: + cl = changes[0] host = re.sub(r'([^\.]+)-review(\.googlesource\.com)', r'\1\2', cl.host) self._repository = 'https://%s/%s' % (host, cl.project) @@ -53,6 +47,34 @@ class BotUpdateApi(recipe_api.RecipeApi): def last_returned_properties(self): return self._last_returned_properties + def _get_commit_repo_path(self, commit, gclient_config): + """Returns local path to the repo that the commit is associated with. + + The commit must be a self.m.buildbucket.common_pb2.GitilesCommit. + If commit does not specify any repo, returns name of the first solution. + + Raises an InfraFailure if the commit specifies a repo unexpected by gclient. + """ + assert gclient_config.solutions, 'gclient_config.solutions is empty' + + # if repo is not specified, choose the first solution. + if not (commit.host and commit.project): + return gclient_config.solutions[0].name + assert commit.host and commit.project + + repo_url = self.m.gitiles.unparse_repo_url(commit.host, commit.project) + repo_path = self.m.gclient.get_repo_path( + repo_url, gclient_config=gclient_config) + if not repo_path: + raise self.m.step.InfraFailure( + 'invalid (host, project) pair in ' + 'buildbucket.build.input.gitiles_commit: ' + '(%r, %r) does not match any of configured gclient solutions ' + 'and not present in gclient.c.repo_path_map' % ( + commit.host, commit.project)) + + return repo_path + def ensure_checkout(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, @@ -133,20 +155,32 @@ class BotUpdateApi(recipe_api.RecipeApi): for solution in cfg.solutions: if solution.revision: revisions[solution.name] = solution.revision - elif solution == cfg.solutions[0]: - # TODO(machenbach): We should explicitly pass HEAD for ALL solutions - # that don't specify anything else. - revisions[solution.name] = ( - self._parent_got_revision or - self._revision or - 'HEAD') + + # Apply input gitiles_commit, if any. + input_commit = self.m.buildbucket.build.input.gitiles_commit + if input_commit.id or input_commit.ref: + repo_path = self._get_commit_repo_path(input_commit, cfg) + # Note: this is not entirely correct. build.input.gitiles_commit + # definition says "The Gitiles commit to run against.". + # However, here we ignore it if the config specified a revision. + # This is necessary because existing builders rely on this behavior, e.g. + # they want to force refs/heads/master at the config level. + revisions[repo_path] = ( + revisions.get(repo_path) or input_commit.id or input_commit.ref) + + # Guarantee that first solution has a revision. + # TODO(machenbach): We should explicitly pass HEAD for ALL solutions + # that don't specify anything else. + first_sol = cfg.solutions[0].name + revisions[first_sol] = revisions.get(first_sol) or 'HEAD' + if cfg.revisions: # Only update with non-empty values. Some recipe might otherwise # overwrite the HEAD default with an empty string. revisions.update( (k, v) for k, v in cfg.revisions.iteritems() if v) if cfg.solutions and root_solution_revision: - revisions[cfg.solutions[0].name] = root_solution_revision + revisions[first_sol] = root_solution_revision # Allow for overrides required to bisect into rolls. revisions.update(self._deps_revision_overrides) diff --git a/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci with invalid repo.json b/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci with invalid repo.json new file mode 100644 index 0000000000..75f6e7cadf --- /dev/null +++ b/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci with invalid repo.json @@ -0,0 +1,8 @@ +[ + { + "name": "$result", + "reason": "invalid (host, project) pair in buildbucket.build.input.gitiles_commit: (u'chromium.googlesource.com', u'trash') does not match any of configured gclient solutions and not present in gclient.c.repo_path_map", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci without repo.json b/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci without repo.json new file mode 100644 index 0000000000..840f40586b --- /dev/null +++ b/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci without repo.json @@ -0,0 +1,55 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'v8', 'url': 'https://chromium.googlesource.com/v8/v8'}]", + "--patch_root", + "v8", + "--revision_mapping_file", + "{}", + "--git-cache-dir", + "[GIT_CACHE]", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "v8@2d72510e447ab60a9728aeea2362d8be2cbd7789" + ], + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"v8\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {}, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"v8\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {}, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"v8\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {}, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/bot_update/examples/buildbucket.py b/recipes/recipe_modules/bot_update/examples/buildbucket.py index 073391f63f..2579e711c0 100644 --- a/recipes/recipe_modules/bot_update/examples/buildbucket.py +++ b/recipes/recipe_modules/bot_update/examples/buildbucket.py @@ -23,6 +23,25 @@ def GenTests(api): git_repo='https://chromium.googlesource.com/v8/v8', revision='2d72510e447ab60a9728aeea2362d8be2cbd7789') + yield api.test('ci with invalid repo') + api.buildbucket.ci_build( + 'v8', 'ci', 'builder', + git_repo='https://chromium.googlesource.com/trash', + revision='2d72510e447ab60a9728aeea2362d8be2cbd7789') + + yield api.test('ci without repo') + api.buildbucket.build( + api.buildbucket.build_pb2.Build( + builder=api.buildbucket.build_pb2.BuilderID( + project='v8', + bucket='ci', + builder='builder', + ), + input=api.buildbucket.build_pb2.Build.Input( + gitiles_commit=api.buildbucket.common_pb2.GitilesCommit( + id='2d72510e447ab60a9728aeea2362d8be2cbd7789', + ), + ), + )) + yield api.test('try') + api.buildbucket.try_build( 'v8', 'try', 'builder', gerrit_host='chromium-review.googlesource.com')