You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
353 lines
13 KiB
Python
353 lines
13 KiB
Python
# Copyright 2013 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
from recipe_engine import recipe_api
|
|
|
|
|
|
class RevisionResolver(object):
|
|
"""Resolves the revision based on build properties."""
|
|
|
|
def resolve(self, properties): # pragma: no cover
|
|
raise NotImplementedError()
|
|
|
|
|
|
class RevisionFallbackChain(RevisionResolver):
|
|
"""Specify that a given project's sync revision follows the fallback chain."""
|
|
def __init__(self, default=None):
|
|
self._default = default
|
|
|
|
def resolve(self, properties):
|
|
"""Resolve the revision via the revision fallback chain.
|
|
|
|
If the given revision was set using the revision_fallback_chain() function,
|
|
this function will follow the chain, looking at relevant build properties
|
|
until it finds one set or reaches the end of the chain and returns the
|
|
default. If the given revision was not set using revision_fallback_chain(),
|
|
this function just returns it as-is.
|
|
"""
|
|
return (properties.get('parent_got_revision') or
|
|
properties.get('orig_revision') or
|
|
properties.get('revision') or
|
|
self._default)
|
|
|
|
|
|
def jsonish_to_python(spec, is_top=False):
|
|
"""Turn a json spec into a python parsable object.
|
|
|
|
This exists because Gclient specs, while resembling json, is actually
|
|
ingested using a python "eval()". Therefore a bit of plumming is required
|
|
to turn our newly constructed Gclient spec into a gclient-readable spec.
|
|
"""
|
|
ret = ''
|
|
if is_top: # We're the 'top' level, so treat this dict as a suite.
|
|
ret = '\n'.join(
|
|
'%s = %s' % (k, jsonish_to_python(spec[k])) for k in sorted(spec)
|
|
)
|
|
else:
|
|
if isinstance(spec, dict):
|
|
ret += '{'
|
|
ret += ', '.join(
|
|
"%s: %s" % (repr(str(k)), jsonish_to_python(spec[k]))
|
|
for k in sorted(spec)
|
|
)
|
|
ret += '}'
|
|
elif isinstance(spec, list):
|
|
ret += '['
|
|
ret += ', '.join(jsonish_to_python(x) for x in spec)
|
|
ret += ']'
|
|
elif isinstance(spec, basestring):
|
|
ret = repr(str(spec))
|
|
else:
|
|
ret = repr(spec)
|
|
return ret
|
|
|
|
class GclientApi(recipe_api.RecipeApi):
|
|
# Singleton object to indicate to checkout() that we should run a revert if
|
|
# we detect that we're on the tryserver.
|
|
RevertOnTryserver = object()
|
|
|
|
def __init__(self, **kwargs):
|
|
super(GclientApi, self).__init__(**kwargs)
|
|
self.USE_MIRROR = None
|
|
self._spec_alias = None
|
|
|
|
def __call__(self, name, cmd, infra_step=True, **kwargs):
|
|
"""Wrapper for easy calling of gclient steps."""
|
|
assert isinstance(cmd, (list, tuple))
|
|
prefix = 'gclient '
|
|
if self.spec_alias:
|
|
prefix = ('[spec: %s] ' % self.spec_alias) + prefix
|
|
|
|
# TODO(phajdan.jr): create a helper for adding to PATH.
|
|
env = self.m.context.env
|
|
env.setdefault('PATH', '%(PATH)s')
|
|
env['PATH'] = self.m.path.pathsep.join([
|
|
env['PATH'], str(self._module.PACKAGE_REPO_ROOT)])
|
|
|
|
with self.m.context(env=env):
|
|
return self.m.python(prefix + name,
|
|
self.package_repo_resource('gclient.py'),
|
|
cmd,
|
|
infra_step=infra_step,
|
|
**kwargs)
|
|
|
|
@property
|
|
def use_mirror(self):
|
|
"""Indicates if gclient will use mirrors in its configuration."""
|
|
if self.USE_MIRROR is None:
|
|
self.USE_MIRROR = self.m.properties.get('use_mirror', True)
|
|
return self.USE_MIRROR
|
|
|
|
@use_mirror.setter
|
|
def use_mirror(self, val): # pragma: no cover
|
|
self.USE_MIRROR = val
|
|
|
|
@property
|
|
def spec_alias(self):
|
|
"""Optional name for the current spec for step naming."""
|
|
return self._spec_alias
|
|
|
|
@spec_alias.setter
|
|
def spec_alias(self, name):
|
|
self._spec_alias = name
|
|
|
|
@spec_alias.deleter
|
|
def spec_alias(self):
|
|
self._spec_alias = None
|
|
|
|
def get_config_defaults(self):
|
|
return {
|
|
'USE_MIRROR': self.use_mirror,
|
|
'CACHE_DIR': self.m.infra_paths.default_git_cache_dir,
|
|
}
|
|
|
|
@staticmethod
|
|
def config_to_pythonish(cfg):
|
|
return jsonish_to_python(cfg.as_jsonish(), True)
|
|
|
|
# TODO(machenbach): Remove this method when the old mapping is deprecated.
|
|
@staticmethod
|
|
def got_revision_reverse_mapping(cfg):
|
|
"""Returns the merged got_revision_reverse_mapping.
|
|
|
|
Returns (dict): A mapping from property name -> project name. It merges the
|
|
values of the deprecated got_revision_mapping and the new
|
|
got_revision_reverse_mapping.
|
|
"""
|
|
rev_map = cfg.got_revision_mapping.as_jsonish()
|
|
reverse_rev_map = cfg.got_revision_reverse_mapping.as_jsonish()
|
|
combined_length = len(rev_map) + len(reverse_rev_map)
|
|
reverse_rev_map.update({v: k for k, v in rev_map.iteritems()})
|
|
|
|
# Make sure we never have duplicate values in the old map.
|
|
assert combined_length == len(reverse_rev_map)
|
|
return reverse_rev_map
|
|
|
|
def resolve_revision(self, revision):
|
|
if hasattr(revision, 'resolve'):
|
|
return revision.resolve(self.m.properties)
|
|
return revision
|
|
|
|
def sync(self, cfg, extra_sync_flags=None, **kwargs):
|
|
revisions = []
|
|
self.set_patch_project_revision(self.m.properties.get('patch_project'), cfg)
|
|
for i, s in enumerate(cfg.solutions):
|
|
if i == 0 and s.revision is None:
|
|
s.revision = RevisionFallbackChain()
|
|
|
|
if s.revision is not None and s.revision != '':
|
|
fixed_revision = self.resolve_revision(s.revision)
|
|
if fixed_revision:
|
|
revisions.extend(['--revision', '%s@%s' % (s.name, fixed_revision)])
|
|
|
|
for name, revision in sorted(cfg.revisions.items()):
|
|
fixed_revision = self.resolve_revision(revision)
|
|
if fixed_revision:
|
|
revisions.extend(['--revision', '%s@%s' % (name, fixed_revision)])
|
|
|
|
test_data_paths = set(self.got_revision_reverse_mapping(cfg).values() +
|
|
[s.name for s in cfg.solutions])
|
|
step_test_data = lambda: (
|
|
self.test_api.output_json(test_data_paths))
|
|
try:
|
|
# clean() isn't used because the gclient sync flags passed in checkout()
|
|
# do much the same thing, and they're more correct than doing a separate
|
|
# 'gclient revert' because it makes sure the other args are correct when
|
|
# a repo was deleted and needs to be re-cloned (notably
|
|
# --with_branch_heads), whereas 'revert' uses default args for clone
|
|
# operations.
|
|
#
|
|
# TODO(mmoss): To be like current official builders, this step could
|
|
# just delete the whole <slave_name>/build/ directory and start each
|
|
# build from scratch. That might be the least bad solution, at least
|
|
# until we have a reliable gclient method to produce a pristine working
|
|
# dir for git-based builds (e.g. maybe some combination of 'git
|
|
# reset/clean -fx' and removing the 'out' directory).
|
|
j = '-j2' if self.m.platform.is_win else '-j8'
|
|
args = ['sync', '--verbose', '--nohooks', j, '--reset', '--force',
|
|
'--upstream', '--no-nag-max', '--with_branch_heads',
|
|
'--with_tags']
|
|
args.extend(extra_sync_flags or [])
|
|
if cfg.delete_unversioned_trees:
|
|
args.append('--delete_unversioned_trees')
|
|
self('sync', args + revisions +
|
|
['--output-json', self.m.json.output()],
|
|
step_test_data=step_test_data,
|
|
**kwargs)
|
|
finally:
|
|
result = self.m.step.active_result
|
|
solutions = result.json.output['solutions']
|
|
for propname, path in sorted(
|
|
self.got_revision_reverse_mapping(cfg).iteritems()):
|
|
# gclient json paths always end with a slash
|
|
info = solutions.get(path + '/') or solutions.get(path)
|
|
if info:
|
|
result.presentation.properties[propname] = info['revision']
|
|
|
|
return result
|
|
|
|
def inject_parent_got_revision(self, gclient_config=None, override=False):
|
|
"""Match gclient config to build revisions obtained from build_properties.
|
|
|
|
Args:
|
|
gclient_config (gclient config object) - The config to manipulate. A value
|
|
of None manipulates the module's built-in config (self.c).
|
|
override (bool) - If True, will forcibly set revision and custom_vars
|
|
even if the config already contains values for them.
|
|
"""
|
|
cfg = gclient_config or self.c
|
|
|
|
for prop, custom_var in cfg.parent_got_revision_mapping.iteritems():
|
|
val = str(self.m.properties.get(prop, ''))
|
|
# TODO(infra): Fix coverage.
|
|
if val: # pragma: no cover
|
|
# Special case for 'src', inject into solutions[0]
|
|
if custom_var is None:
|
|
# This is not covered because we are deprecating this feature and
|
|
# it is no longer used by the public recipes.
|
|
if cfg.solutions[0].revision is None or override: # pragma: no cover
|
|
cfg.solutions[0].revision = val
|
|
else:
|
|
if custom_var not in cfg.solutions[0].custom_vars or override:
|
|
cfg.solutions[0].custom_vars[custom_var] = val
|
|
|
|
def checkout(self, gclient_config=None, revert=RevertOnTryserver,
|
|
inject_parent_got_revision=True, extra_sync_flags=None,
|
|
**kwargs):
|
|
"""Return a step generator function for gclient checkouts."""
|
|
cfg = gclient_config or self.c
|
|
assert cfg.complete()
|
|
|
|
if revert is self.RevertOnTryserver:
|
|
revert = self.m.tryserver.is_tryserver
|
|
|
|
if inject_parent_got_revision:
|
|
self.inject_parent_got_revision(cfg, override=True)
|
|
|
|
self('setup', ['config', '--spec', self.config_to_pythonish(cfg)], **kwargs)
|
|
|
|
sync_step = None
|
|
try:
|
|
sync_step = self.sync(cfg, extra_sync_flags=extra_sync_flags, **kwargs)
|
|
|
|
cfg_cmds = [
|
|
('user.name', 'local_bot'),
|
|
('user.email', 'local_bot@example.com'),
|
|
]
|
|
for var, val in cfg_cmds:
|
|
name = 'recurse (git config %s)' % var
|
|
self(name, ['recurse', 'git', 'config', var, val], **kwargs)
|
|
finally:
|
|
cwd = kwargs.get('cwd', self.m.path['start_dir'])
|
|
if 'checkout' not in self.m.path:
|
|
self.m.path['checkout'] = cwd.join(
|
|
*cfg.solutions[0].name.split(self.m.path.sep))
|
|
|
|
return sync_step
|
|
|
|
def runhooks(self, args=None, name='runhooks', **kwargs):
|
|
args = args or []
|
|
assert isinstance(args, (list, tuple))
|
|
with self.m.context(cwd=(self.m.context.cwd or self.m.path['checkout'])):
|
|
return self(name, ['runhooks'] + list(args), infra_step=False, **kwargs)
|
|
|
|
@property
|
|
def is_blink_mode(self):
|
|
""" Indicates wether the caller is to use the Blink config rather than the
|
|
Chromium config. This may happen for one of two reasons:
|
|
1. The builder is configured to always use TOT Blink. (factory property
|
|
top_of_tree_blink=True)
|
|
2. A try job comes in that applies to the Blink tree. (patch_project is
|
|
blink)
|
|
"""
|
|
return (
|
|
self.m.properties.get('top_of_tree_blink') or
|
|
self.m.properties.get('patch_project') == 'blink')
|
|
|
|
def break_locks(self):
|
|
"""Remove all index.lock files. If a previous run of git crashed, bot was
|
|
reset, etc... we might end up with leftover index.lock files.
|
|
"""
|
|
self.m.python.inline(
|
|
'cleanup index.lock',
|
|
"""
|
|
import os, sys
|
|
|
|
build_path = sys.argv[1]
|
|
if os.path.exists(build_path):
|
|
for (path, dir, files) in os.walk(build_path):
|
|
for cur_file in files:
|
|
if cur_file.endswith('index.lock'):
|
|
path_to_file = os.path.join(path, cur_file)
|
|
print 'deleting %s' % path_to_file
|
|
os.remove(path_to_file)
|
|
""",
|
|
args=[self.m.path['start_dir']],
|
|
infra_step=True,
|
|
)
|
|
|
|
def calculate_patch_root(self, patch_project, gclient_config=None,
|
|
patch_repo=None):
|
|
"""Returns path where a patch should be applied to based patch_project.
|
|
|
|
Maps the patch's repo to a path of directories relative to checkout's root,
|
|
which describe where to place the patch. If no mapping is found for the
|
|
repo url, falls back to trying to find a mapping for the old-style
|
|
"patch_project".
|
|
|
|
For now, considers only first solution (c.solutions[0]), but in theory can
|
|
be extended to all of them.
|
|
|
|
See patch_projects and repo_path_map solution config property.
|
|
|
|
Returns:
|
|
Relative path, including solution's root.
|
|
If patch_project is not given or not recognized, it'll be just first
|
|
solution root.
|
|
"""
|
|
cfg = gclient_config or self.c
|
|
root, _ = cfg.repo_path_map.get(patch_repo, ('', ''))
|
|
if not root:
|
|
root, _ = cfg.patch_projects.get(patch_project, ('', ''))
|
|
if not root:
|
|
# Failure case - assume patch is for first solution, as this is what most
|
|
# projects rely on.
|
|
return cfg.solutions[0].name
|
|
# Note, that c.patch_projects contains patch roots as
|
|
# slash(/)-separated path, which are roots of the respective project repos
|
|
# and include actual solution name in them.
|
|
return self.m.path.join(*root.split('/'))
|
|
|
|
def set_patch_project_revision(self, patch_project, gclient_config=None):
|
|
"""Updates config revision corresponding to patch_project.
|
|
|
|
Useful for bot_update only, as this is the only consumer of gclient's config
|
|
revision map. This doesn't overwrite the revision if it was already set.
|
|
"""
|
|
assert patch_project is None or isinstance(patch_project, basestring)
|
|
cfg = gclient_config or self.c
|
|
path, revision = cfg.patch_projects.get(patch_project, (None, None))
|
|
if path and revision and path not in cfg.revisions:
|
|
cfg.revisions[path] = revision
|