aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert recent gclient.py changes to fix Chrome releases.

Browse Source 
Reason for revert: This is breaking official release scripts, which have long
used "None" URL functionality to allow gclient commands to run against
"local" DEPS (buildspec) files. Please do not remove this behavior.

This reverts the following commits:
  ebdd0db493b20f0abeab8960e6ea0ceb7c6b379a: "gclient: Remove URLs from hierarchy."
  54a5c2ba8ac2f9b8f4a32fe79913f13545e4aab9: "gclient: Refactor PrintRevInfo"
  083eb25f9acbe034db94a1bd5c1659125b6ebf98: "gclient: Don't allow URL to be None."

BUG=846194

TBR=agable@chromium.org,ehmaldonado@chromium.org

Change-Id: Ibdd5581889bd4afd86474199c7b64555f01bbbca
Reviewed-on: https://chromium-review.googlesource.com/1070893
Commit-Queue: Michael Moss <mmoss@chromium.org>
Reviewed-by: Michael Moss <mmoss@chromium.org>
changes/93/1070893/5
Michael Moss 7 years ago committed by Commit Bot
parent 1f156687a2
commit 4e9b50ab86
4 changed files with 249 additions and 166 deletions
Show Stats Download Patch File Download Diff File

283
gclient.py
Unescape Escape View File

@ -241,7 +241,7 @@ class DependencySettings(object):
"""Immutable configuration settings.""" """Immutable configuration settings."""
def __init__( def __init__(
self, parent, raw_url, url, managed, custom_deps, custom_vars, self, parent, raw_url, url, managed, custom_deps, custom_vars,
custom_hooks, deps_file, relative, condition): custom_hooks, deps_file, should_process, relative, condition):
# These are not mutable: # These are not mutable:
self._parent = parent self._parent = parent
self._deps_file = deps_file self._deps_file = deps_file
@ -250,9 +250,13 @@ class DependencySettings(object):
# The condition as string (or None). Useful to keep e.g. for flatten. # The condition as string (or None). Useful to keep e.g. for flatten.
self._condition = condition self._condition = condition
# 'managed' determines whether or not this dependency is synced/updated by # 'managed' determines whether or not this dependency is synced/updated by
# gclient after gclient checks it out initially. The user specifies # gclient after gclient checks it out initially. The difference between
# 'managed' via the --unmanaged command-line flag or a .gclient config. # 'managed' and 'should_process' is that the user specifies 'managed' via
# the --unmanaged command-line flag or a .gclient config, where
# 'should_process' is dynamically set by gclient if it goes over its
# recursion limit and controls gclient's behavior so it does not misbehave.
self._managed = managed self._managed = managed
self._should_process = should_process
# If this is a recursed-upon sub-dependency, and the parent has # If this is a recursed-upon sub-dependency, and the parent has
# use_relative_paths set, then this dependency should check out its own # use_relative_paths set, then this dependency should check out its own
# dependencies relative to that parent's path for this, rather than # dependencies relative to that parent's path for this, rather than
@ -267,10 +271,15 @@ class DependencySettings(object):
self._custom_deps = custom_deps or {} self._custom_deps = custom_deps or {}
self._custom_hooks = custom_hooks or [] self._custom_hooks = custom_hooks or []
assert self.url is not None, (self.name, self.url) # Post process the url to remove trailing slashes.
if isinstance(self.url, basestring):
# urls are sometime incorrectly written as proto://host/path/@rev. Replace # urls are sometime incorrectly written as proto://host/path/@rev. Replace
# it to proto://host/path@rev. # it to proto://host/path@rev.
self.set_url(self.url.replace('/@', '@')) self.set_url(self.url.replace('/@', '@'))
elif not isinstance(self.url, (None.__class__)):
raise gclient_utils.Error(
('dependency url must be either string or None, '
'instead of %s') % self.url.__class__.__name__)
# Make any deps_file path platform-appropriate. # Make any deps_file path platform-appropriate.
if self._deps_file: if self._deps_file:
@ -297,6 +306,11 @@ class DependencySettings(object):
return self or GClient(None, None) return self or GClient(None, None)
return self.parent.root return self.parent.root
@property
def should_process(self):
"""True if this dependency should be processed, i.e. checked out."""
return self._should_process
@property @property
def custom_vars(self): def custom_vars(self):
return self._custom_vars.copy() return self._custom_vars.copy()
@ -352,12 +366,12 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
"""Object that represents a dependency checkout.""" """Object that represents a dependency checkout."""
def __init__(self, parent, name, raw_url, url, managed, custom_deps, def __init__(self, parent, name, raw_url, url, managed, custom_deps,
custom_vars, custom_hooks, deps_file, relative, condition, custom_vars, custom_hooks, deps_file, should_process,
print_outbuf=False): relative, condition, print_outbuf=False):
gclient_utils.WorkItem.__init__(self, name) gclient_utils.WorkItem.__init__(self, name)
DependencySettings.__init__( DependencySettings.__init__(
self, parent, raw_url, url, managed, custom_deps, custom_vars, self, parent, raw_url, url, managed, custom_deps, custom_vars,
custom_hooks, deps_file, relative, condition) custom_hooks, deps_file, should_process, relative, condition)
# This is in both .gclient and DEPS files: # This is in both .gclient and DEPS files:
self._deps_hooks = [] self._deps_hooks = []
@ -409,7 +423,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
# None. # None.
self.recursedeps = None self.recursedeps = None
self._OverrideUrl()
# This is inherited from WorkItem. We want the URL to be a resource. # This is inherited from WorkItem. We want the URL to be a resource.
if self.url and isinstance(self.url, basestring):
# The url is usually given to gclient either as https://blah@123 # The url is usually given to gclient either as https://blah@123
# or just https://blah. The @123 portion is irrelevant. # or just https://blah. The @123 portion is irrelevant.
self.resources.append(self.url.split('@')[0]) self.resources.append(self.url.split('@')[0])
@ -421,8 +437,40 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
if not self.name and self.parent: if not self.name and self.parent:
raise gclient_utils.Error('Dependency without name') raise gclient_utils.Error('Dependency without name')
def _OverrideUrl(self):
"""Resolves the parsed url from the parent hierarchy."""
parsed_url = self.get_custom_deps(self._name, self.url)
if parsed_url != self.url:
logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self._name,
self.url, parsed_url)
self.set_url(parsed_url)
elif isinstance(self.url, basestring):
parsed_url = urlparse.urlparse(self.url)
if (not parsed_url[0] and
not re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2])):
path = parsed_url[2]
if not path.startswith('/'):
raise gclient_utils.Error(
'relative DEPS entry \'%s\' must begin with a slash' % self.url)
# A relative url. Get the parent url, strip from the last '/'
# (equivalent to unix basename), and append the relative url.
parent_url = self.parent.url
parsed_url = parent_url[:parent_url.rfind('/')] + self.url
logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self.name,
self.url, parsed_url)
self.set_url(parsed_url)
elif self.url is None:
logging.info('Dependency(%s)._OverrideUrl(None) -> None', self._name)
else:
raise gclient_utils.Error('Unknown url type')
def PinToActualRevision(self): def PinToActualRevision(self):
"""Updates self.url and self.raw_url to the revision checked out on disk.""" """Updates self.url and self.raw_url to the revision checked out on disk."""
if self.url is None:
return
url = raw_url = None url = raw_url = None
scm = self.CreateSCM() scm = self.CreateSCM()
if os.path.isdir(scm.checkout_path): if os.path.isdir(scm.checkout_path):
@ -438,7 +486,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
condition_part = ([' "condition": %r,' % self.condition] condition_part = ([' "condition": %r,' % self.condition]
if self.condition else []) if self.condition else [])
s.extend([ s.extend([
' # %s' % self.hierarchy(), ' # %s' % self.hierarchy(include_url=False),
' "%s": {' % (self.name,), ' "%s": {' % (self.name,),
' "url": "%s",' % (self.raw_url,), ' "url": "%s",' % (self.raw_url,),
] + condition_part + [ ] + condition_part + [
@ -472,7 +520,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
if self.name: if self.name:
requirements |= set( requirements |= set(
obj.name for obj in self.root.subtree() obj.name for obj in self.root.subtree(False)
if (obj is not self if (obj is not self
and obj.name and and obj.name and
self.name.startswith(posixpath.join(obj.name, '')))) self.name.startswith(posixpath.join(obj.name, ''))))
@ -511,11 +559,15 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
raise gclient_utils.Error( raise gclient_utils.Error(
'The same name "%s" appears multiple times in the deps section' % 'The same name "%s" appears multiple times in the deps section' %
self.name) self.name)
if not self.should_process:
# Return early, no need to set requirements.
return True
# This require a full tree traversal with locks. # This require a full tree traversal with locks.
siblings = [d for d in self.root.subtree() if d.name == self.name] siblings = [d for d in self.root.subtree(False) if d.name == self.name]
for sibling in siblings: for sibling in siblings:
if self.url != sibling.url: # Allow to have only one to be None or ''.
if self.url != sibling.url and bool(self.url) == bool(sibling.url):
raise gclient_utils.Error( raise gclient_utils.Error(
('Dependency %s specified more than once:\n' ('Dependency %s specified more than once:\n'
' %s [%s]\n' ' %s [%s]\n'
@ -565,30 +617,11 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
return deps return deps
def FormatUrl(self, name, url):
custom_deps_url = self.get_custom_deps(name, url)
if url != custom_deps_url:
return custom_deps_url
if url is None:
return None
if not isinstance(url, basestring):
raise gclient_utils.Error(
('dependency url must be either string or None, '
'instead of %s') % self.url.__class__.__name__)
url = url.format(**self.get_vars())
# For relative URLs, strip the parent url (self.url) from the last '/'
# and append the relative url.
if url[0] == '/':
url = self.url[:self.url.rfind('/')] + url
return url
def _deps_to_objects(self, deps, use_relative_paths): def _deps_to_objects(self, deps, use_relative_paths):
"""Convert a deps dict to a dict of Dependency objects.""" """Convert a deps dict to a dict of Dependency objects."""
if not self.recursion_limit:
return []
deps_to_add = [] deps_to_add = []
for name, dep_value in deps.iteritems(): for name, dep_value in deps.iteritems():
should_process = self.recursion_limit and self.should_process
deps_file = self.deps_file deps_file = self.deps_file
if self.recursedeps is not None: if self.recursedeps is not None:
ent = self.recursedeps.get(name) ent = self.recursedeps.get(name)
@ -598,32 +631,31 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
continue continue
condition = dep_value.get('condition') condition = dep_value.get('condition')
dep_type = dep_value.get('dep_type', 'git') dep_type = dep_value.get('dep_type')
should_process = True if condition and not self._get_option('process_all_deps', False):
if condition and not self.get_option('process_all_deps', False): should_process = should_process and gclient_eval.EvaluateCondition(
should_process = gclient_eval.EvaluateCondition(
condition, self.get_vars()) condition, self.get_vars())
if not should_process:
continue
if dep_type == 'cipd': if dep_type == 'cipd':
cipd_root = self.GetCipdRoot() cipd_root = self.GetCipdRoot()
for package in dep_value.get('packages', []): for package in dep_value.get('packages', []):
package['version'] = package['version'].format(**self.get_vars()) if 'version' in package:
# Matches version to vars value.
raw_version = package['version']
version = raw_version.format(**self.get_vars())
package['version'] = version
deps_to_add.append( deps_to_add.append(
CipdDependency( CipdDependency(
self, name, package, cipd_root, self.custom_vars, self, name, package, cipd_root, self.custom_vars,
use_relative_paths, condition)) should_process, use_relative_paths, condition))
else: else:
raw_url = dep_value.get('url') raw_url = dep_value.get('url')
url = self.FormatUrl(name, raw_url) url = raw_url.format(**self.get_vars()) if raw_url else None
if url:
deps_to_add.append( deps_to_add.append(
GitDependency( GitDependency(
self, name, raw_url, url, None, None, self.custom_vars, None, self, name, raw_url, url, None, None, self.custom_vars, None,
deps_file, use_relative_paths, condition)) deps_file, should_process, use_relative_paths, condition))
deps_to_add.sort(key=lambda x: x.name) deps_to_add.sort(key=lambda x: x.name)
return deps_to_add return deps_to_add
@ -660,7 +692,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
try: try:
local_scope = gclient_eval.Parse( local_scope = gclient_eval.Parse(
deps_content, expand_vars, deps_content, expand_vars,
self.get_option('validate_syntax', False), self._get_option('validate_syntax', False),
filepath, self.get_vars()) filepath, self.get_vars())
except SyntaxError as e: except SyntaxError as e:
gclient_utils.SyntaxErrorToError(filepath, e) gclient_utils.SyntaxErrorToError(filepath, e)
@ -766,8 +798,11 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
self.add_dependencies_and_close(deps_to_add, hooks_to_run) self.add_dependencies_and_close(deps_to_add, hooks_to_run)
logging.info('ParseDepsFile(%s) done' % self.name) logging.info('ParseDepsFile(%s) done' % self.name)
def get_option(self, attr, default=None): def _get_option(self, attr, default):
return getattr(self.root._options, attr, default) obj = self
while not hasattr(obj, '_options'):
obj = obj.parent
return getattr(obj._options, attr, default)
def add_dependencies_and_close(self, deps_to_add, hooks): def add_dependencies_and_close(self, deps_to_add, hooks):
"""Adds the dependencies, hooks and mark the parsing as done.""" """Adds the dependencies, hooks and mark the parsing as done."""
@ -793,6 +828,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
# Don't enforce this for custom_deps. # Don't enforce this for custom_deps.
if dep.name in self._custom_deps: if dep.name in self._custom_deps:
continue continue
if isinstance(dep.url, basestring):
parsed_url = urlparse.urlparse(dep.url) parsed_url = urlparse.urlparse(dep.url)
if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts: if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
bad_deps.append(dep) bad_deps.append(dep)
@ -837,6 +873,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
"""Runs |command| then parse the DEPS file.""" """Runs |command| then parse the DEPS file."""
logging.info('Dependency(%s).run()' % self.name) logging.info('Dependency(%s).run()' % self.name)
assert self._file_list == [] assert self._file_list == []
if not self.should_process:
return
# When running runhooks, there's no need to consult the SCM. # When running runhooks, there's no need to consult the SCM.
# All known hooks are expected to run unconditionally regardless of working # All known hooks are expected to run unconditionally regardless of working
# copy state, so skip the SCM status check. # copy state, so skip the SCM status check.
@ -886,6 +924,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
self.RunPreDepsHooks() self.RunPreDepsHooks()
# Parse the dependencies of this dependency. # Parse the dependencies of this dependency.
for s in self.dependencies: for s in self.dependencies:
if s.should_process:
work_queue.enqueue(s) work_queue.enqueue(s)
if command == 'recurse': if command == 'recurse':
@ -896,7 +935,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
# Pass in the SCM type as an env variable. Make sure we don't put # Pass in the SCM type as an env variable. Make sure we don't put
# unicode strings in the environment. # unicode strings in the environment.
env = os.environ.copy() env = os.environ.copy()
if scm:
env['GCLIENT_SCM'] = str(scm) env['GCLIENT_SCM'] = str(scm)
if self.url:
env['GCLIENT_URL'] = str(self.url) env['GCLIENT_URL'] = str(self.url)
env['GCLIENT_DEP_PATH'] = str(self.name) env['GCLIENT_DEP_PATH'] = str(self.name)
if options.prepend_dir and scm == 'git': if options.prepend_dir and scm == 'git':
@ -928,7 +969,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
print_stdout = True print_stdout = True
filter_fn = None filter_fn = None
if os.path.isdir(cwd): if self.url is None:
print('Skipped omitted dependency %s' % cwd, file=sys.stderr)
elif os.path.isdir(cwd):
try: try:
gclient_utils.CheckCallAndFilter( gclient_utils.CheckCallAndFilter(
args, cwd=cwd, env=env, print_stdout=print_stdout, args, cwd=cwd, env=env, print_stdout=print_stdout,
@ -972,7 +1015,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
RunOnDeps() must have been called before to load the DEPS. RunOnDeps() must have been called before to load the DEPS.
""" """
result = [] result = []
if not self.recursion_limit: if not self.should_process or not self.recursion_limit:
# Don't run the hook when it is above recursion_limit. # Don't run the hook when it is above recursion_limit.
return result return result
# If "--force" was specified, run all hooks regardless of what files have # If "--force" was specified, run all hooks regardless of what files have
@ -1017,13 +1060,14 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
return None return None
return self.root.GetCipdRoot() return self.root.GetCipdRoot()
def subtree(self): def subtree(self, include_all):
"""Breadth first recursion excluding root node.""" """Breadth first recursion excluding root node."""
dependencies = self.dependencies dependencies = self.dependencies
for d in dependencies: for d in dependencies:
if d.should_process or include_all:
yield d yield d
for d in dependencies: for d in dependencies:
for i in d.subtree(): for i in d.subtree(include_all):
yield i yield i
@gclient_utils.lockedmethod @gclient_utils.lockedmethod
@ -1101,7 +1145,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
def __str__(self): def __str__(self):
out = [] out = []
for i in ('name', 'url', 'custom_deps', for i in ('name', 'url', 'custom_deps',
'custom_vars', 'deps_hooks', 'file_list', 'custom_vars', 'deps_hooks', 'file_list', 'should_process',
'processed', 'hooks_ran', 'deps_parsed', 'requirements', 'processed', 'hooks_ran', 'deps_parsed', 'requirements',
'allowed_hosts'): 'allowed_hosts'):
# First try the native property if it exists. # First try the native property if it exists.
@ -1120,12 +1164,16 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
def __repr__(self): def __repr__(self):
return '%s: %s' % (self.name, self.url) return '%s: %s' % (self.name, self.url)
def hierarchy(self): def hierarchy(self, include_url=True):
"""Returns a human-readable hierarchical reference to a Dependency.""" """Returns a human-readable hierarchical reference to a Dependency."""
out = self.name def format_name(d):
if include_url:
return '%s(%s)' % (d.name, d.url)
return d.name
out = format_name(self)
i = self.parent i = self.parent
while i and i.name: while i and i.name:
out = '%s -> %s' % (i.name, out) out = '%s -> %s' % (format_name(i), out)
i = i.parent i = i.parent
return out return out
@ -1245,8 +1293,8 @@ solutions = %(solution_list)s
# Do not change previous behavior. Only solution level and immediate DEPS # Do not change previous behavior. Only solution level and immediate DEPS
# are processed. # are processed.
self._recursion_limit = 2 self._recursion_limit = 2
GitDependency.__init__(self, None, None, '', '', True, None, None, None, GitDependency.__init__(self, None, None, None, None, True, None, None, None,
'unused', None, None, True) 'unused', True, None, None, True)
self._options = options self._options = options
if options.deps_os: if options.deps_os:
enforced_os = options.deps_os.split(',') enforced_os = options.deps_os.split(',')
@ -1264,7 +1312,7 @@ solutions = %(solution_list)s
"""Verify that the config matches the state of the existing checked-out """Verify that the config matches the state of the existing checked-out
solutions.""" solutions."""
for dep in self.dependencies: for dep in self.dependencies:
if dep.managed: if dep.managed and dep.url:
scm = dep.CreateSCM() scm = dep.CreateSCM()
actual_url = scm.GetActualRemoteURL(self._options) actual_url = scm.GetActualRemoteURL(self._options)
if actual_url and not scm.DoesRemoteURLMatch(self._options): if actual_url and not scm.DoesRemoteURLMatch(self._options):
@ -1336,7 +1384,6 @@ it or fix the checkout.
deps_to_add = [] deps_to_add = []
for s in config_dict.get('solutions', []): for s in config_dict.get('solutions', []):
try: try:
if s['url']:
deps_to_add.append(GitDependency( deps_to_add.append(GitDependency(
self, s['name'], s['url'], s['url'], self, s['name'], s['url'], s['url'],
s.get('managed', True), s.get('managed', True),
@ -1344,6 +1391,7 @@ it or fix the checkout.
s.get('custom_vars', {}), s.get('custom_vars', {}),
s.get('custom_hooks', []), s.get('custom_hooks', []),
s.get('deps_file', 'DEPS'), s.get('deps_file', 'DEPS'),
True,
None, None,
None, None,
True)) True))
@ -1351,9 +1399,6 @@ it or fix the checkout.
raise gclient_utils.Error('Invalid .gclient file. Solution is ' raise gclient_utils.Error('Invalid .gclient file. Solution is '
'incomplete: %s' % s) 'incomplete: %s' % s)
self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', [])) self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', []))
if not self.dependencies:
raise gclient_utils.Error('No solution specified')
logging.info('SetConfig() done') logging.info('SetConfig() done')
def SaveConfig(self): def SaveConfig(self):
@ -1413,7 +1458,7 @@ it or fix the checkout.
# Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
# makes testing a bit too fun. # makes testing a bit too fun.
result = 'entries = {\n' result = 'entries = {\n'
for entry in self.root.subtree(): for entry in self.root.subtree(False):
result += ' %s: %s,\n' % (pprint.pformat(entry.name), result += ' %s: %s,\n' % (pprint.pformat(entry.name),
pprint.pformat(entry.url)) pprint.pformat(entry.url))
result += '}\n' result += '}\n'
@ -1481,6 +1526,9 @@ it or fix the checkout.
command: The command to use (e.g., 'status' or 'diff') command: The command to use (e.g., 'status' or 'diff')
args: list of str - extra arguments to add to the command line. args: list of str - extra arguments to add to the command line.
""" """
if not self.dependencies:
raise gclient_utils.Error('No solution specified')
revision_overrides = {} revision_overrides = {}
patch_refs = {} patch_refs = {}
# It's unnecessary to check for revision overrides for 'recurse'. # It's unnecessary to check for revision overrides for 'recurse'.
@ -1505,6 +1553,7 @@ it or fix the checkout.
self._options.jobs, pm, ignore_requirements=ignore_requirements, self._options.jobs, pm, ignore_requirements=ignore_requirements,
verbose=self._options.verbose) verbose=self._options.verbose)
for s in self.dependencies: for s in self.dependencies:
if s.should_process:
work_queue.enqueue(s) work_queue.enqueue(s)
work_queue.flush(revision_overrides, command, args, options=self._options, work_queue.flush(revision_overrides, command, args, options=self._options,
patch_refs=patch_refs) patch_refs=patch_refs)
@ -1542,7 +1591,7 @@ it or fix the checkout.
# Notify the user if there is an orphaned entry in their working copy. # Notify the user if there is an orphaned entry in their working copy.
# Only delete the directory if there are no changes in it, and # Only delete the directory if there are no changes in it, and
# delete_unversioned_trees is set to true. # delete_unversioned_trees is set to true.
entries = [i.name for i in self.root.subtree()] entries = [i.name for i in self.root.subtree(False) if i.url]
full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep)) full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep))
for e in entries] for e in entries]
@ -1639,59 +1688,68 @@ it or fix the checkout.
work_queue = gclient_utils.ExecutionQueue( work_queue = gclient_utils.ExecutionQueue(
self._options.jobs, None, False, verbose=self._options.verbose) self._options.jobs, None, False, verbose=self._options.verbose)
for s in self.dependencies: for s in self.dependencies:
if s.should_process:
work_queue.enqueue(s) work_queue.enqueue(s)
work_queue.flush({}, None, [], options=self._options, patch_refs=None) work_queue.flush({}, None, [], options=self._options, patch_refs=None)
def ShouldPrint(dep): def ShouldPrintRevision(dep):
return (not self._options.filter return (not self._options.filter
or dep.FuzzyMatchUrl(self._options.filter)) or dep.FuzzyMatchUrl(self._options.filter))
for dep in self.subtree():
if self._options.snapshot or self._options.actual:
dep.PinToActualRevision()
if self._options.snapshot: if self._options.snapshot:
json_output = [ json_output = []
{ # First level at .gclient
for d in self.dependencies:
entries = {}
def GrabDeps(dep):
"""Recursively grab dependencies."""
for d in dep.dependencies:
d.PinToActualRevision()
if ShouldPrintRevision(d):
entries[d.name] = d.url
GrabDeps(d)
GrabDeps(d)
json_output.append({
'name': d.name, 'name': d.name,
'solution_url': d.url, 'solution_url': d.url,
'deps_file': d.deps_file, 'deps_file': d.deps_file,
'managed': d.managed, 'managed': d.managed,
'custom_deps': { 'custom_deps': entries,
subdep.name: subdep.url })
for subdep in d.subtree() if self._options.output_json == '-':
if ShouldPrint(subdep) print(json.dumps(json_output, indent=2, separators=(',', ': ')))
},
}
for d in self.dependencies
if ShouldPrint(d)
]
output = json.dumps(json_output, indent=2, separators=(',', ': '))
if not self._options.output_json:
output = self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': output}
elif self._options.output_json: elif self._options.output_json:
with open(self._options.output_json, 'w') as f:
json.dump(json_output, f)
else:
# Print the snapshot configuration file
print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {
'solution_list': pprint.pformat(json_output, indent=2),
})
else:
entries = {}
for d in self.root.subtree(False):
if self._options.actual:
d.PinToActualRevision()
if ShouldPrintRevision(d):
entries[d.name] = d.url
if self._options.output_json:
json_output = { json_output = {
d.name: { name: {
'url': d.url.split('@')[0], 'url': rev.split('@')[0] if rev else None,
'rev': d.url.split('@')[1] if '@' in d.url else None, 'rev': rev.split('@')[1] if rev and '@' in rev else None,
} }
for d in self.subtree() for name, rev in entries.iteritems()
if ShouldPrint(d)
} }
output = json.dumps(json_output, indent=2, separators=(',', ': ')) if self._options.output_json == '-':
print(json.dumps(json_output, indent=2, separators=(',', ': ')))
else: else:
output = '\n'.join(
'%s: %s' % (d.name, d.url)
for d in self.subtree()
if ShouldPrint(d)
)
if self._options.output_json and self._options.output_json != '-':
with open(self._options.output_json, 'w') as f: with open(self._options.output_json, 'w') as f:
f.write(output) json.dump(json_output, f)
else: else:
print(output) keys = sorted(entries.keys())
for x in keys:
print('%s: %s' % (x, entries[x]))
logging.info(str(self)) logging.info(str(self))
def ParseDepsFile(self, expand_vars=None): def ParseDepsFile(self, expand_vars=None):
@ -1749,14 +1807,14 @@ class CipdDependency(Dependency):
def __init__( def __init__(
self, parent, name, dep_value, cipd_root, self, parent, name, dep_value, cipd_root,
custom_vars, relative, condition): custom_vars, should_process, relative, condition):
package = dep_value['package'] package = dep_value['package']
version = dep_value['version'] version = dep_value['version']
url = urlparse.urljoin( url = urlparse.urljoin(
cipd_root.service_url, '%s@%s' % (package, version)) cipd_root.service_url, '%s@%s' % (package, version))
super(CipdDependency, self).__init__( super(CipdDependency, self).__init__(
parent, name + ':' + package, url, url, None, None, custom_vars, parent, name + ':' + package, url, url, None, None, custom_vars,
None, None, relative, condition) None, None, should_process, relative, condition)
if relative: if relative:
# TODO(jbudorick): Implement relative if necessary. # TODO(jbudorick): Implement relative if necessary.
raise gclient_utils.Error( raise gclient_utils.Error(
@ -1773,6 +1831,8 @@ class CipdDependency(Dependency):
patch_refs): patch_refs):
"""Runs |command| then parse the DEPS file.""" """Runs |command| then parse the DEPS file."""
logging.info('CipdDependency(%s).run()' % self.name) logging.info('CipdDependency(%s).run()' % self.name)
if not self.should_process:
return
self._CreatePackageIfNecessary() self._CreatePackageIfNecessary()
super(CipdDependency, self).run(revision_overrides, command, args, super(CipdDependency, self).run(revision_overrides, command, args,
work_queue, options, patch_refs) work_queue, options, patch_refs)
@ -1816,7 +1876,7 @@ class CipdDependency(Dependency):
condition_part = ([' "condition": %r,' % self.condition] condition_part = ([' "condition": %r,' % self.condition]
if self.condition else []) if self.condition else [])
s.extend([ s.extend([
' # %s' % self.hierarchy(), ' # %s' % self.hierarchy(include_url=False),
' "%s": {' % (self.name.split(':')[0],), ' "%s": {' % (self.name.split(':')[0],),
' "packages": [', ' "packages": [',
]) ])
@ -1935,6 +1995,9 @@ class Flattener(object):
Arguments: Arguments:
dep (Dependency): dependency to process dep (Dependency): dependency to process
""" """
if dep.url is None:
return
# Make sure the revision is always fully specified (a hash), # Make sure the revision is always fully specified (a hash),
# as opposed to refs or tags which might change. Similarly, # as opposed to refs or tags which might change. Similarly,
# shortened shas might become ambiguous; make sure to always # shortened shas might become ambiguous; make sure to always
@ -1972,6 +2035,7 @@ class Flattener(object):
deps_path = os.path.join(self._client.root_dir, dep.name, deps_file) deps_path = os.path.join(self._client.root_dir, dep.name, deps_file)
if not os.path.exists(deps_path): if not os.path.exists(deps_path):
return return
assert dep.url
self._deps_files.add((dep.url, deps_file, dep.hierarchy_data())) self._deps_files.add((dep.url, deps_file, dep.hierarchy_data()))
for dep in self._deps.itervalues(): for dep in self._deps.itervalues():
add_deps_file(dep) add_deps_file(dep)
@ -1997,6 +2061,7 @@ class Flattener(object):
""" """
assert dep.name not in self._deps or self._deps.get(dep.name) == dep, ( assert dep.name not in self._deps or self._deps.get(dep.name) == dep, (
dep.name, self._deps.get(dep.name)) dep.name, self._deps.get(dep.name))
if dep.url:
self._deps[dep.name] = dep self._deps[dep.name] = dep
def _flatten_dep(self, dep): def _flatten_dep(self, dep):
@ -2014,7 +2079,7 @@ class Flattener(object):
# Only include vars explicitly listed in the DEPS files or gclient solution, # Only include vars explicitly listed in the DEPS files or gclient solution,
# not automatic, local overrides (i.e. not all of dep.get_vars()). # not automatic, local overrides (i.e. not all of dep.get_vars()).
hierarchy = dep.hierarchy() hierarchy = dep.hierarchy(include_url=False)
for key, value in dep._vars.iteritems(): for key, value in dep._vars.iteritems():
# Make sure there are no conflicting variables. It is fine however # Make sure there are no conflicting variables. It is fine however
# to use same variable name, as long as the value is consistent. # to use same variable name, as long as the value is consistent.
@ -2128,7 +2193,7 @@ def _DepsOsToLines(deps_os):
condition_part = ([' "condition": %r,' % dep.condition] condition_part = ([' "condition": %r,' % dep.condition]
if dep.condition else []) if dep.condition else [])
s.extend([ s.extend([
' # %s' % dep.hierarchy(), ' # %s' % dep.hierarchy(include_url=False),
' "%s": {' % (name,), ' "%s": {' % (name,),
' "url": "%s",' % (dep.raw_url,), ' "url": "%s",' % (dep.raw_url,),
] + condition_part + [ ] + condition_part + [
@ -2147,7 +2212,7 @@ def _HooksToLines(name, hooks):
s = ['%s = [' % name] s = ['%s = [' % name]
for dep, hook in hooks: for dep, hook in hooks:
s.extend([ s.extend([
' # %s' % dep.hierarchy(), ' # %s' % dep.hierarchy(include_url=False),
' {', ' {',
]) ])
if hook.name is not None: if hook.name is not None:
@ -2176,7 +2241,7 @@ def _HooksOsToLines(hooks_os):
s.append(' "%s": [' % hook_os) s.append(' "%s": [' % hook_os)
for dep, hook in os_hooks: for dep, hook in os_hooks:
s.extend([ s.extend([
' # %s' % dep.hierarchy(), ' # %s' % dep.hierarchy(include_url=False),
' {', ' {',
]) ])
if hook.name is not None: if hook.name is not None:
@ -2508,12 +2573,16 @@ def CMDsync(parser, args):
ret = client.RunOnDeps('update', args) ret = client.RunOnDeps('update', args)
if options.output_json: if options.output_json:
slns = {} slns = {}
for d in client.subtree(): for d in client.subtree(True):
normed = d.name.replace('\\', '/').rstrip('/') + '/' normed = d.name.replace('\\', '/').rstrip('/') + '/'
if normed in slns and not d.should_process:
# If an unprocessed dependency would override an existing dependency,
# ignore it.
continue
slns[normed] = { slns[normed] = {
'revision': d.got_revision, 'revision': d.got_revision,
'scm': d.used_scm.name if d.used_scm else None, 'scm': d.used_scm.name if d.used_scm else None,
'url': str(d.url), 'url': str(d.url) if d.url else None,
} }
with open(options.output_json, 'wb') as f: with open(options.output_json, 'wb') as f:
json.dump({'solutions': slns}, f) json.dump({'solutions': slns}, f)

2
recipes/trigger_recipe_roller.txt
Unescape Escape View File

@ -9,4 +9,4 @@ As the CI needs of the browser grew, Batty, the Build and Test Yeti, got
a new friend: a new friend:
The End. The End!

31
tests/gclient_smoketest.py
Unescape Escape View File

@ -246,14 +246,9 @@ class GClientSmoke(GClientSmokeBase):
']\n' ']\n'
'cache_dir = None\n') % self.git_base) 'cache_dir = None\n') % self.git_base)
os.remove(p) test(['config', '--spec', '["blah blah"]'], '["blah blah"]')
results = self.gclient(['config', '--spec', '["blah blah"]'])
self.assertEqual(('', 'Error: No solution specified\n', 1), results)
results = self.gclient(['config', '--spec',
'solutions=[{"name": "./", "url": None}]'])
self.assertEqual(('', 'Error: No solution specified\n', 1), results)
os.remove(p)
results = self.gclient(['config', 'foo', 'faa', 'fuu']) results = self.gclient(['config', 'foo', 'faa', 'fuu'])
err = ('Usage: gclient.py config [options] [url]\n\n' err = ('Usage: gclient.py config [options] [url]\n\n'
'gclient.py: error: Inconsistent arguments. Use either --spec or one' 'gclient.py: error: Inconsistent arguments. Use either --spec or one'
@ -261,6 +256,24 @@ class GClientSmoke(GClientSmokeBase):
self.check(('', err, 2), results) self.check(('', err, 2), results)
self.assertFalse(os.path.exists(join(self.root_dir, '.gclient'))) self.assertFalse(os.path.exists(join(self.root_dir, '.gclient')))
def testSolutionNone(self):
results = self.gclient(['config', '--spec',
'solutions=[{"name": "./", "url": None}]'])
self.check(('', '', 0), results)
results = self.gclient(['sync'])
self.check(('', '', 0), results)
self.assertTree({})
results = self.gclient(['revinfo'])
self.check(('./: None\n', '', 0), results)
self.check(('', '', 0), self.gclient(['diff']))
self.assertTree({})
self.check(('', '', 0), self.gclient(['pack']))
self.check(('', '', 0), self.gclient(['revert']))
self.assertTree({})
self.check(('', '', 0), self.gclient(['runhooks']))
self.assertTree({})
self.check(('', '', 0), self.gclient(['status']))
def testDifferentTopLevelDirectory(self): def testDifferentTopLevelDirectory(self):
# Check that even if the .gclient file does not mention the directory src # Check that even if the .gclient file does not mention the directory src
# itself, but it is included via dependencies, the .gclient file is used. # itself, but it is included via dependencies, the .gclient file is used.
@ -796,10 +809,8 @@ class GClientSmokeGIT(GClientSmokeBase):
with open(output_json) as f: with open(output_json) as f:
output_json = json.load(f) output_json = json.load(f)
self.maxDiff = None
out = [{ out = [{
'solution_url': '%srepo_1@%s' % ( 'solution_url': self.git_base + 'repo_1',
self.git_base, self.githash('repo_1', 2)),
'managed': True, 'managed': True,
'name': 'src', 'name': 'src',
'deps_file': 'DEPS', 'deps_file': 'DEPS',

21
tests/gclient_test.py
Unescape Escape View File

@ -202,7 +202,7 @@ class GclientTest(trial_dir.TestCase):
url = 'proto://host/path/@revision' url = 'proto://host/path/@revision'
d = gclient.Dependency( d = gclient.Dependency(
None, 'name', url, url, None, None, None, None, 'name', url, url, None, None, None,
None, '', False, None, True) None, '', True, False, None, True)
self.assertEquals('proto://host/path@revision', d.url) self.assertEquals('proto://host/path@revision', d.url)
def testStr(self): def testStr(self):
@ -213,17 +213,17 @@ class GclientTest(trial_dir.TestCase):
[ [
gclient.Dependency( gclient.Dependency(
obj, 'foo', 'svn://example.com/foo', 'svn://example.com/foo', None, obj, 'foo', 'svn://example.com/foo', 'svn://example.com/foo', None,
None, None, None, 'DEPS', False, None, True), None, None, None, 'DEPS', True, False, None, True),
gclient.Dependency( gclient.Dependency(
obj, 'bar', 'svn://example.com/bar', 'svn://example.com/bar', None, obj, 'bar', 'svn://example.com/bar', 'svn://example.com/bar', None,
None, None, None, 'DEPS', False, None, True), None, None, None, 'DEPS', True, False, None, True),
], ],
[]) [])
obj.dependencies[0].add_dependencies_and_close( obj.dependencies[0].add_dependencies_and_close(
[ [
gclient.Dependency( gclient.Dependency(
obj.dependencies[0], 'foo/dir1', 'svn://example.com/foo/dir1', obj.dependencies[0], 'foo/dir1', 'svn://example.com/foo/dir1',
'svn://example.com/foo/dir1', None, None, None, None, 'DEPS', 'svn://example.com/foo/dir1', None, None, None, None, 'DEPS', True,
False, None, True), False, None, True),
], ],
[]) [])
@ -232,7 +232,7 @@ class GclientTest(trial_dir.TestCase):
# pylint: disable=protected-access # pylint: disable=protected-access
obj.dependencies[0]._file_list.append('foo') obj.dependencies[0]._file_list.append('foo')
str_obj = str(obj) str_obj = str(obj)
self.assertEquals(230, len(str_obj), '%d\n%s' % (len(str_obj), str_obj)) self.assertEquals(322, len(str_obj), '%d\n%s' % (len(str_obj), str_obj))
def testHooks(self): def testHooks(self):
topdir = self.root_dir topdir = self.root_dir
@ -543,12 +543,14 @@ class GclientTest(trial_dir.TestCase):
('foo/rel', 'svn://example.com/rel'), ('foo/rel', 'svn://example.com/rel'),
], self._get_processed()) ], self._get_processed())
self.assertEqual(4, len(sol.dependencies)) self.assertEqual(6, len(sol.dependencies))
self.assertEqual([ self.assertEqual([
('foo/bar', 'svn://example.com/override'), ('foo/bar', 'svn://example.com/override'),
('foo/baz', 'svn://example.com/baz'), ('foo/baz', 'svn://example.com/baz'),
('foo/new', 'svn://example.com/new'), ('foo/new', 'svn://example.com/new'),
('foo/rel', 'svn://example.com/rel'), ('foo/rel', 'svn://example.com/rel'),
('foo/skip', None),
('foo/skip2', None),
], [(dep.name, dep.url) for dep in sol.dependencies]) ], [(dep.name, dep.url) for dep in sol.dependencies])
def testDepsOsOverrideDepsInDepsFile(self): def testDepsOsOverrideDepsInDepsFile(self):
@ -1129,7 +1131,8 @@ class GclientTest(trial_dir.TestCase):
[ [
gclient.Dependency( gclient.Dependency(
obj, 'foo', 'svn://example.com/foo', 'svn://example.com/foo', None, obj, 'foo', 'svn://example.com/foo', 'svn://example.com/foo', None,
None, None, None, 'DEPS', False, None, True), None, None, None, 'DEPS', True,
False, None, True),
], ],
[]) [])
obj.dependencies[0].add_dependencies_and_close( obj.dependencies[0].add_dependencies_and_close(
@ -1137,12 +1140,12 @@ class GclientTest(trial_dir.TestCase):
gclient.CipdDependency(obj.dependencies[0], 'foo', gclient.CipdDependency(obj.dependencies[0], 'foo',
{'package': 'foo_package', {'package': 'foo_package',
'version': 'foo_version'}, 'version': 'foo_version'},
cipd_root, None, False, cipd_root, None, True, False,
'fake_condition'), 'fake_condition'),
gclient.CipdDependency(obj.dependencies[0], 'foo', gclient.CipdDependency(obj.dependencies[0], 'foo',
{'package': 'bar_package', {'package': 'bar_package',
'version': 'bar_version'}, 'version': 'bar_version'},
cipd_root, None, False, cipd_root, None, True, False,
'fake_condition'), 'fake_condition'),
], ],
[]) [])

Write Preview
Loading…
Cancel
Save