# Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from recipe_engine import recipe_api class RevisionResolver(object): """Resolves the revision based on build properties.""" def resolve(self, properties): # pragma: no cover raise NotImplementedError() class RevisionFallbackChain(RevisionResolver): """Specify that a given project's sync revision follows the fallback chain.""" def __init__(self, default=None): self._default = default def resolve(self, properties): """Resolve the revision via the revision fallback chain. If the given revision was set using the revision_fallback_chain() function, this function will follow the chain, looking at relevant build properties until it finds one set or reaches the end of the chain and returns the default. If the given revision was not set using revision_fallback_chain(), this function just returns it as-is. """ return (properties.get('parent_got_revision') or properties.get('orig_revision') or properties.get('revision') or self._default) def jsonish_to_python(spec, is_top=False): """Turn a json spec into a python parsable object. This exists because Gclient specs, while resembling json, is actually ingested using a python "eval()". Therefore a bit of plumming is required to turn our newly constructed Gclient spec into a gclient-readable spec. """ ret = '' if is_top: # We're the 'top' level, so treat this dict as a suite. ret = '\n'.join( '%s = %s' % (k, jsonish_to_python(spec[k])) for k in sorted(spec) ) else: if isinstance(spec, dict): ret += '{' ret += ', '.join( "%s: %s" % (repr(str(k)), jsonish_to_python(spec[k])) for k in sorted(spec) ) ret += '}' elif isinstance(spec, list): ret += '[' ret += ', '.join(jsonish_to_python(x) for x in spec) ret += ']' elif isinstance(spec, basestring): ret = repr(str(spec)) else: ret = repr(spec) return ret class GclientApi(recipe_api.RecipeApi): # Singleton object to indicate to checkout() that we should run a revert if # we detect that we're on the tryserver. RevertOnTryserver = object() def __init__(self, **kwargs): super(GclientApi, self).__init__(**kwargs) self.USE_MIRROR = None self._spec_alias = None def __call__(self, name, cmd, infra_step=True, **kwargs): """Wrapper for easy calling of gclient steps.""" assert isinstance(cmd, (list, tuple)) prefix = 'gclient ' if self.spec_alias: prefix = ('[spec: %s] ' % self.spec_alias) + prefix with self.m.context( env_suffixes={'PATH': [self.repo_resource()]}): return self.m.python(prefix + name, self.repo_resource('gclient.py'), cmd, infra_step=infra_step, **kwargs) @property def use_mirror(self): """Indicates if gclient will use mirrors in its configuration.""" if self.USE_MIRROR is None: self.USE_MIRROR = self.m.properties.get('use_mirror', True) return self.USE_MIRROR @use_mirror.setter def use_mirror(self, val): # pragma: no cover self.USE_MIRROR = val @property def spec_alias(self): """Optional name for the current spec for step naming.""" return self._spec_alias @spec_alias.setter def spec_alias(self, name): self._spec_alias = name @spec_alias.deleter def spec_alias(self): self._spec_alias = None def get_config_defaults(self): return { 'USE_MIRROR': self.use_mirror, 'CACHE_DIR': self.m.infra_paths.default_git_cache_dir, } @staticmethod def config_to_pythonish(cfg): return jsonish_to_python(cfg.as_jsonish(), True) # TODO(machenbach): Remove this method when the old mapping is deprecated. @staticmethod def got_revision_reverse_mapping(cfg): """Returns the merged got_revision_reverse_mapping. Returns (dict): A mapping from property name -> project name. It merges the values of the deprecated got_revision_mapping and the new got_revision_reverse_mapping. """ rev_map = cfg.got_revision_mapping.as_jsonish() reverse_rev_map = cfg.got_revision_reverse_mapping.as_jsonish() combined_length = len(rev_map) + len(reverse_rev_map) reverse_rev_map.update({v: k for k, v in rev_map.iteritems()}) # Make sure we never have duplicate values in the old map. assert combined_length == len(reverse_rev_map) return reverse_rev_map def resolve_revision(self, revision): if hasattr(revision, 'resolve'): return revision.resolve(self.m.properties) return revision def sync(self, cfg, extra_sync_flags=None, **kwargs): revisions = [] self.set_patch_repo_revision(gclient_config=cfg) for i, s in enumerate(cfg.solutions): if i == 0 and s.revision is None: s.revision = RevisionFallbackChain() if s.revision is not None and s.revision != '': fixed_revision = self.resolve_revision(s.revision) if fixed_revision: revisions.extend(['--revision', '%s@%s' % (s.name, fixed_revision)]) for name, revision in sorted(cfg.revisions.items()): fixed_revision = self.resolve_revision(revision) if fixed_revision: revisions.extend(['--revision', '%s@%s' % (name, fixed_revision)]) test_data_paths = set(self.got_revision_reverse_mapping(cfg).values() + [s.name for s in cfg.solutions]) step_test_data = lambda: ( self.test_api.output_json(test_data_paths)) try: # clean() isn't used because the gclient sync flags passed in checkout() # do much the same thing, and they're more correct than doing a separate # 'gclient revert' because it makes sure the other args are correct when # a repo was deleted and needs to be re-cloned (notably # --with_branch_heads), whereas 'revert' uses default args for clone # operations. # # TODO(mmoss): To be like current official builders, this step could # just delete the whole /build/ directory and start each # build from scratch. That might be the least bad solution, at least # until we have a reliable gclient method to produce a pristine working # dir for git-based builds (e.g. maybe some combination of 'git # reset/clean -fx' and removing the 'out' directory). j = '-j2' if self.m.platform.is_win else '-j8' args = ['sync', '--verbose', '--nohooks', j, '--reset', '--force', '--upstream', '--no-nag-max', '--with_branch_heads', '--with_tags'] args.extend(extra_sync_flags or []) if cfg.delete_unversioned_trees: args.append('--delete_unversioned_trees') self('sync', args + revisions + ['--output-json', self.m.json.output()], step_test_data=step_test_data, **kwargs) finally: result = self.m.step.active_result solutions = result.json.output['solutions'] for propname, path in sorted( self.got_revision_reverse_mapping(cfg).iteritems()): # gclient json paths always end with a slash info = solutions.get(path + '/') or solutions.get(path) if info: result.presentation.properties[propname] = info['revision'] return result def inject_parent_got_revision(self, gclient_config=None, override=False): """Match gclient config to build revisions obtained from build_properties. Args: gclient_config (gclient config object) - The config to manipulate. A value of None manipulates the module's built-in config (self.c). override (bool) - If True, will forcibly set revision and custom_vars even if the config already contains values for them. """ cfg = gclient_config or self.c for prop, custom_var in cfg.parent_got_revision_mapping.iteritems(): val = str(self.m.properties.get(prop, '')) # TODO(infra): Fix coverage. if val: # pragma: no cover # Special case for 'src', inject into solutions[0] if custom_var is None: # This is not covered because we are deprecating this feature and # it is no longer used by the public recipes. if cfg.solutions[0].revision is None or override: # pragma: no cover cfg.solutions[0].revision = val else: if custom_var not in cfg.solutions[0].custom_vars or override: cfg.solutions[0].custom_vars[custom_var] = val def checkout(self, gclient_config=None, revert=RevertOnTryserver, inject_parent_got_revision=True, extra_sync_flags=None, **kwargs): """Return a step generator function for gclient checkouts.""" cfg = gclient_config or self.c assert cfg.complete() if revert is self.RevertOnTryserver: revert = self.m.tryserver.is_tryserver if inject_parent_got_revision: self.inject_parent_got_revision(cfg, override=True) self('setup', ['config', '--spec', self.config_to_pythonish(cfg)], **kwargs) sync_step = None try: sync_step = self.sync(cfg, extra_sync_flags=extra_sync_flags, **kwargs) cfg_cmds = [ ('user.name', 'local_bot'), ('user.email', 'local_bot@example.com'), ] for var, val in cfg_cmds: name = 'recurse (git config %s)' % var self(name, ['recurse', 'git', 'config', var, val], **kwargs) finally: cwd = self.m.context.cwd or self.m.path['start_dir'] if 'checkout' not in self.m.path: self.m.path['checkout'] = cwd.join( *cfg.solutions[0].name.split(self.m.path.sep)) return sync_step def runhooks(self, args=None, name='runhooks', **kwargs): args = args or [] assert isinstance(args, (list, tuple)) with self.m.context(cwd=(self.m.context.cwd or self.m.path['checkout'])): return self(name, ['runhooks'] + list(args), infra_step=False, **kwargs) def break_locks(self): """Remove all index.lock files. If a previous run of git crashed, bot was reset, etc... we might end up with leftover index.lock files. """ self.m.python.inline( 'cleanup index.lock', """ import os, sys build_path = sys.argv[1] if os.path.exists(build_path): for (path, dir, files) in os.walk(build_path): for cur_file in files: if cur_file.endswith('index.lock'): path_to_file = os.path.join(path, cur_file) print 'deleting %s' % path_to_file os.remove(path_to_file) """, args=[self.m.path['start_dir']], infra_step=True, ) def get_gerrit_patch_root(self, gclient_config=None): """Returns local path to the repo where gerrit patch will be applied. If there is no patch, returns None. If patch is specified, but such repo is not found among configured solutions or repo_path_map, returns name of the first solution. This is done solely for backward compatibility with existing tests. Please do not rely on this logic in new code. Instead, properly map a repository to a local path using repo_path_map. TODO(nodir): remove this. Update all recipe tests to specify a git_repo matching the recipe. """ cfg = gclient_config or self.c repo_url = self.m.tryserver.gerrit_change_repo_url if not repo_url: return None root = self.get_repo_path(repo_url, gclient_config=cfg) # This is wrong, but that's what a ton of recipe tests expect today root = root or cfg.solutions[0].name return root def _canonicalize_repo_url(self, repo_url): """Attempts to make repo_url canonical. Supports Gitiles URL.""" return self.m.gitiles.canonicalize_repo_url(repo_url) def get_repo_path(self, repo_url, gclient_config=None): """Returns local path to the repo checkout given its url. Consults cfg.repo_path_map and fallbacks to urls in configured solutions. Returns None if not found. """ rel_path = self._get_repo_path(repo_url, gclient_config=gclient_config) if rel_path: return self.m.path.join(*rel_path.split('/')) return None def _get_repo_path(self, repo_url, gclient_config=None): repo_url = self._canonicalize_repo_url(repo_url) cfg = gclient_config or self.c rel_path, _ = cfg.repo_path_map.get(repo_url, ('', '')) if rel_path: return rel_path # repo_path_map keys may be non-canonical. for key, (rel_path, _) in cfg.repo_path_map.iteritems(): if self._canonicalize_repo_url(key) == repo_url: return rel_path for s in cfg.solutions: if self._canonicalize_repo_url(s.url) == repo_url: return s.name return None def set_patch_repo_revision(self, gclient_config=None): """Updates config revision corresponding to patch_project. Useful for bot_update only, as this is the only consumer of gclient's config revision map. This doesn't overwrite the revision if it was already set. """ cfg = gclient_config or self.c repo_url = self.m.tryserver.gerrit_change_repo_url path, revision = cfg.repo_path_map.get(repo_url, (None, None)) if path and revision and path not in cfg.revisions: cfg.revisions[path] = revision