diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py index e9a261036..45f4e19af 100755 --- a/recipes/recipe_modules/bot_update/resources/bot_update.py +++ b/recipes/recipe_modules/bot_update/resources/bot_update.py @@ -61,6 +61,9 @@ COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s*(.*)') COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' COMMIT_ORIGINAL_POSITION_FOOTER_KEY = 'Cr-Original-Commit-Position' +# Regular expression to parse gclient's revinfo entries. +REVINFO_RE = re.compile(r'^([^:]+):\s+([^@]+)@(.+)$') + # Copied from scripts/recipes/chromium.py. GOT_REVISION_MAPPINGS = { CHROMIUM_SRC_URL: { @@ -381,7 +384,12 @@ def gclient_sync( with_branch_heads, with_tags, revisions, patch_refs, gerrit_reset, gerrit_rebase_patch_ref): + # We just need to allocate a filename. + fd, gclient_output_file = tempfile.mkstemp(suffix='.json') + os.close(fd) + args = ['sync', '--verbose', '--reset', '--force', + '--output-json', gclient_output_file, '--nohooks', '--noprehooks', '--delete_unversioned_trees'] if with_branch_heads: args += ['--with_branch_heads'] @@ -409,6 +417,15 @@ def gclient_sync( raise PatchFailed(e.message, e.code, e.output) # Throw a GclientSyncFailed exception so we can catch this independently. raise GclientSyncFailed(e.message, e.code, e.output) + else: + with open(gclient_output_file) as f: + return json.load(f) + finally: + os.remove(gclient_output_file) + + +def gclient_revinfo(): + return call_gclient('revinfo', '-a') or '' def normalize_git_url(url): @@ -436,18 +453,18 @@ def normalize_git_url(url): def create_manifest(): - revinfo = call_gclient( - 'revinfo', '-a', '--ignore-dep-type', 'cipd', '--output-json', '-') - if not revinfo: - return {} - return { - path: { - 'repository': info['url'], - 'revision': info['rev'], - } - for path, info in json.loads(revinfo).items() - if info['rev'] is not None - } + manifest = {} + output = gclient_revinfo() + for line in output.strip().splitlines(): + match = REVINFO_RE.match(line.strip()) + if match: + manifest[match.group(1)] = { + 'repository': match.group(2), + 'revision': match.group(3), + } + else: + print("WARNING: Couldn't match revinfo line:\n%s" % line) + return manifest def get_commit_message_footer_map(message): @@ -756,22 +773,28 @@ def get_commit_position(git_path, revision='HEAD'): return None -def parse_got_revision(manifest, got_revision_mapping): +def parse_got_revision(gclient_output, got_revision_mapping): """Translate git gclient revision mapping to build properties.""" properties = {} - manifest = { + solutions_output = { # Make sure path always ends with a single slash. - '%s/' % path.rstrip('/'): info - for path, info in manifest.items() + '%s/' % path.rstrip('/') : solution_output for path, solution_output + in gclient_output['solutions'].items() } for property_name, dir_name in got_revision_mapping.items(): # Make sure dir_name always ends with a single slash. dir_name = '%s/' % dir_name.rstrip('/') - if dir_name not in manifest: + if dir_name not in solutions_output: continue - info = manifest[dir_name] - revision = git('rev-parse', 'HEAD', cwd=dir_name).strip() - commit_position = get_commit_position(dir_name) + solution_output = solutions_output[dir_name] + if solution_output.get('scm') is None: + # This is an ignored DEPS, so the output got_revision should be 'None'. + revision = commit_position = None + else: + # Since we are using .DEPS.git, everything had better be git. + assert solution_output.get('scm') == 'git' + revision = git('rev-parse', 'HEAD', cwd=dir_name).strip() + commit_position = get_commit_position(dir_name) properties[property_name] = revision if commit_position: @@ -820,7 +843,7 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, # Let gclient do the DEPS syncing. # The branch-head refspec is a special case because it's possible Chrome # src, which contains the branch-head refspecs, is DEPSed in. - gclient_sync( + gclient_output = gclient_sync( BRANCH_HEADS_REFSPEC in refs, TAGS_REFSPEC in refs, gc_revisions, @@ -839,6 +862,8 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, gclient_configure(solutions, target_os, target_os_only, target_cpu, git_cache_dir) + return gclient_output + def parse_revisions(revisions, root): """Turn a list of revision specs into a nice dictionary. @@ -1061,12 +1086,12 @@ def checkout(options, git_slns, specs, revisions, step_text): git_cache_dir=options.git_cache_dir, cleanup_dir=options.cleanup_dir, gerrit_reset=not options.gerrit_no_reset) - ensure_checkout(**checkout_parameters) + gclient_output = ensure_checkout(**checkout_parameters) should_delete_dirty_file = True except GclientSyncFailed: print('We failed gclient sync, lets delete the checkout and retry.') ensure_no_checkout(dir_names, options.cleanup_dir) - ensure_checkout(**checkout_parameters) + gclient_output = ensure_checkout(**checkout_parameters) should_delete_dirty_file = True except PatchFailed as e: # Tell recipes information such as root, got_revision, etc. @@ -1100,8 +1125,7 @@ def checkout(options, git_slns, specs, revisions, step_text): if not revision_mapping: revision_mapping['got_revision'] = first_sln - manifest = create_manifest() - got_revisions = parse_got_revision(manifest, revision_mapping) + got_revisions = parse_got_revision(gclient_output, revision_mapping) if not got_revisions: # TODO(hinoka): We should probably bail out here, but in the interest @@ -1118,7 +1142,7 @@ def checkout(options, git_slns, specs, revisions, step_text): step_text=step_text, fixed_revisions=revisions, properties=got_revisions, - manifest=manifest) + manifest=create_manifest()) def print_debug_info(): diff --git a/tests/bot_update_coverage_test.py b/tests/bot_update_coverage_test.py index 99824db11..811a7c2ea 100755 --- a/tests/bot_update_coverage_test.py +++ b/tests/bot_update_coverage_test.py @@ -5,6 +5,7 @@ import codecs import copy +import json import os import sys import unittest @@ -86,11 +87,21 @@ class MockedCall(object): class MockedGclientSync(): - """A class producing a callable instance of gclient sync.""" + """A class producing a callable instance of gclient sync. + + Because for bot_update, gclient sync also emits an output json file, we need + a callable object that can understand where the output json file is going, and + emit a (albite) fake file for bot_update to consume. + """ def __init__(self, fake_filesystem): + self.output = {} + self.fake_filesystem = fake_filesystem self.records = [] def __call__(self, *args, **_): + output_json_index = args.index('--output-json') + 1 + with self.fake_filesystem.open(args[output_json_index], 'w') as f: + json.dump(self.output, f) self.records.append(args)