aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gclient flatten: preserve variable placeholders (reland #1)

Browse Source 
This is an exact reland of https://chromium-review.googlesource.com/583617 .

One of the main use cases is making it clear which revision hashes
need to be changed together. The way it's usually done is one variable
referenced several times. With this CL, we preserve the references
from original DEPS, as opposed to evaluating them and losing some info.

This CL actually makes Var() emit a variable placeholder
instead of its value, and adds support for these placeholders
to gclient.

One of possible next steps might be to deprecate Var().

Bug: 570091, 748486
Change-Id: Id47e3771b7163149a4cd427b84f84ece52772f34
Reviewed-on: https://chromium-review.googlesource.com/586594
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Reviewed-by: William Hesse <whesse@google.com>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Paweł Hajdan Jr. <phajdan.jr@chromium.org>
changes/94/586594/5
Paweł Hajdan, Jr 8 years ago committed by Commit Bot
parent 5aa5cd76f0
commit fc6196b306
5 changed files with 61 additions and 48 deletions
Show Stats Download Patch File Download Diff File

61
gclient.py
Unescape Escape View File

@ -194,7 +194,8 @@ class Hook(object):
not gclient_eval.EvaluateCondition(self._condition, self._variables)): not gclient_eval.EvaluateCondition(self._condition, self._variables)):
return return
cmd = list(self._action) cmd = [arg.format(**self._variables) for arg in self._action]
if cmd[0] == 'python': if cmd[0] == 'python':
# If the hook specified "python" as the first item, the action is a # If the hook specified "python" as the first item, the action is a
# Python script. Run it by starting a new copy of the same # Python script. Run it by starting a new copy of the same
@ -221,32 +222,16 @@ class Hook(object):
gclient_utils.CommandToStr(cmd), elapsed_time)) gclient_utils.CommandToStr(cmd), elapsed_time))
class GClientKeywords(object): class DependencySettings(object):
class VarImpl(object):
def __init__(self, custom_vars, local_scope):
self._custom_vars = custom_vars
self._local_scope = local_scope
def Lookup(self, var_name):
"""Implements the Var syntax."""
if var_name in self._custom_vars:
return self._custom_vars[var_name]
elif var_name in self._local_scope.get("vars", {}):
return self._local_scope["vars"][var_name]
raise gclient_utils.Error("Var is not defined: %s" % var_name)
class DependencySettings(GClientKeywords):
"""Immutable configuration settings.""" """Immutable configuration settings."""
def __init__( def __init__(
self, parent, url, managed, custom_deps, custom_vars, self, parent, raw_url, url, managed, custom_deps, custom_vars,
custom_hooks, deps_file, should_process, relative, custom_hooks, deps_file, should_process, relative,
condition, condition_value): condition, condition_value):
GClientKeywords.__init__(self)
# These are not mutable: # These are not mutable:
self._parent = parent self._parent = parent
self._deps_file = deps_file self._deps_file = deps_file
self._raw_url = raw_url
self._url = url self._url = url
# The condition as string (or None). Useful to keep e.g. for flatten. # The condition as string (or None). Useful to keep e.g. for flatten.
self._condition = condition self._condition = condition
@ -324,8 +309,14 @@ class DependencySettings(GClientKeywords):
def custom_hooks(self): def custom_hooks(self):
return self._custom_hooks[:] return self._custom_hooks[:]
@property
def raw_url(self):
"""URL before variable expansion."""
return self._raw_url
@property @property
def url(self): def url(self):
"""URL after variable expansion."""
return self._url return self._url
@property @property
@ -354,12 +345,12 @@ class DependencySettings(GClientKeywords):
class Dependency(gclient_utils.WorkItem, DependencySettings): class Dependency(gclient_utils.WorkItem, DependencySettings):
"""Object that represents a dependency checkout.""" """Object that represents a dependency checkout."""
def __init__(self, parent, name, url, managed, custom_deps, def __init__(self, parent, name, raw_url, url, managed, custom_deps,
custom_vars, custom_hooks, deps_file, should_process, custom_vars, custom_hooks, deps_file, should_process,
relative, condition, condition_value): relative, condition, condition_value):
gclient_utils.WorkItem.__init__(self, name) gclient_utils.WorkItem.__init__(self, name)
DependencySettings.__init__( DependencySettings.__init__(
self, parent, url, managed, custom_deps, custom_vars, self, parent, raw_url, url, managed, custom_deps, custom_vars,
custom_hooks, deps_file, should_process, relative, custom_hooks, deps_file, should_process, relative,
condition, condition_value) condition, condition_value)
@ -636,21 +627,24 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
condition = None condition = None
condition_value = True condition_value = True
if isinstance(dep_value, basestring): if isinstance(dep_value, basestring):
url = dep_value raw_url = dep_value
else: else:
# This should be guaranteed by schema checking in gclient_eval. # This should be guaranteed by schema checking in gclient_eval.
assert isinstance(dep_value, collections.Mapping) assert isinstance(dep_value, collections.Mapping)
url = dep_value['url'] raw_url = dep_value['url']
# Take into account should_process metadata set by MergeWithOsDeps. # Take into account should_process metadata set by MergeWithOsDeps.
should_process = (should_process and should_process = (should_process and
dep_value.get('should_process', True)) dep_value.get('should_process', True))
condition = dep_value.get('condition') condition = dep_value.get('condition')
url = raw_url.format(**self.get_vars())
if condition: if condition:
condition_value = gclient_eval.EvaluateCondition( condition_value = gclient_eval.EvaluateCondition(
condition, self.get_vars()) condition, self.get_vars())
should_process = should_process and condition_value should_process = should_process and condition_value
deps_to_add.append(Dependency( deps_to_add.append(Dependency(
self, name, url, None, None, self.custom_vars, None, self, name, raw_url, url, None, None, self.custom_vars, None,
deps_file, should_process, use_relative_paths, condition, deps_file, should_process, use_relative_paths, condition,
condition_value)) condition_value))
deps_to_add.sort(key=lambda x: x.name) deps_to_add.sort(key=lambda x: x.name)
@ -685,10 +679,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
local_scope = {} local_scope = {}
if deps_content: if deps_content:
# One thing is unintuitive, vars = {} must happen before Var() use.
var = self.VarImpl(self.custom_vars, local_scope)
global_scope = { global_scope = {
'Var': var.Lookup, 'Var': lambda var_name: '{%s}' % var_name,
'deps_os': {}, 'deps_os': {},
} }
# Eval the content. # Eval the content.
@ -734,7 +726,10 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
elif self._relative: elif self._relative:
rel_prefix = os.path.dirname(self.name) rel_prefix = os.path.dirname(self.name)
deps = local_scope.get('deps', {}) deps = {}
for key, value in local_scope.get('deps', {}).iteritems():
deps[key.format(**self.get_vars())] = value
if 'recursion' in local_scope: if 'recursion' in local_scope:
self.recursion_override = local_scope.get('recursion') self.recursion_override = local_scope.get('recursion')
logging.warning( logging.warning(
@ -1203,7 +1198,7 @@ solutions = [
# Do not change previous behavior. Only solution level and immediate DEPS # Do not change previous behavior. Only solution level and immediate DEPS
# are processed. # are processed.
self._recursion_limit = 2 self._recursion_limit = 2
Dependency.__init__(self, None, None, None, True, None, None, None, Dependency.__init__(self, None, None, None, None, True, None, None, None,
'unused', True, None, None, True) 'unused', True, None, None, True)
self._options = options self._options = options
if options.deps_os: if options.deps_os:
@ -1286,7 +1281,7 @@ it or fix the checkout.
for s in config_dict.get('solutions', []): for s in config_dict.get('solutions', []):
try: try:
deps_to_add.append(Dependency( deps_to_add.append(Dependency(
self, s['name'], s['url'], self, s['name'], s['url'], s['url'],
s.get('managed', True), s.get('managed', True),
s.get('custom_deps', {}), s.get('custom_deps', {}),
s.get('custom_vars', {}), s.get('custom_vars', {}),
@ -1738,7 +1733,7 @@ class Flattener(object):
continue continue
scm = gclient_scm.CreateSCM( scm = gclient_scm.CreateSCM(
dep.parsed_url, self._client.root_dir, dep.name, dep.outbuf) dep.parsed_url, self._client.root_dir, dep.name, dep.outbuf)
dep._parsed_url = dep._url = '%s@%s' % ( dep._parsed_url = dep._raw_url = dep._url = '%s@%s' % (
url, scm.revinfo(self._client._options, [], None)) url, scm.revinfo(self._client._options, [], None))
self._deps_string = '\n'.join( self._deps_string = '\n'.join(
@ -1875,7 +1870,7 @@ def _DepsToLines(deps):
s.extend([ s.extend([
' # %s' % dep.hierarchy(include_url=False), ' # %s' % dep.hierarchy(include_url=False),
' "%s": {' % (name,), ' "%s": {' % (name,),
' "url": "%s",' % (dep.url,), ' "url": "%s",' % (dep.raw_url,),
] + condition_part + [ ] + condition_part + [
' },', ' },',
'', '',

6
testing_support/fake_repos.py
Unescape Escape View File

@ -458,6 +458,8 @@ pre_deps_hooks = [
'DEPS': """ 'DEPS': """
vars = { vars = {
'DummyVariable': 'repo', 'DummyVariable': 'repo',
'git_base': '%(git_base)s',
'hook1_contents': 'git_hooked1',
} }
gclient_gn_args_file = 'src/gclient.args' gclient_gn_args_file = 'src/gclient.args'
gclient_gn_args = ['DummyVariable'] gclient_gn_args = ['DummyVariable']
@ -466,7 +468,7 @@ allowed_hosts = [
] ]
deps = { deps = {
'src/repo2': { 'src/repo2': {
'url': '%(git_base)srepo_2@%(hash)s', 'url': Var('git_base') + 'repo_2@%(hash)s',
'condition': 'True', 'condition': 'True',
}, },
'src/repo4': { 'src/repo4': {
@ -493,7 +495,7 @@ hooks = [
{ {
'pattern': '.', 'pattern': '.',
'action': ['python', '-c', 'action': ['python', '-c',
'open(\\'src/git_hooked1\\', \\'w\\').write(\\'git_hooked1\\')'], 'open(\\'src/git_hooked1\\', \\'w\\').write(\\'{hook1_contents}\\')'],
}, },
{ {
# Should not be run. # Should not be run.

4
tests/gclient_eval_unittest.py
Unescape Escape View File

@ -103,7 +103,7 @@ class ExecTest(unittest.TestCase):
def test_var(self): def test_var(self):
local_scope = {} local_scope = {}
global_scope = { global_scope = {
'Var': gclient.GClientKeywords.VarImpl({}, local_scope).Lookup, 'Var': lambda var_name: '{%s}' % var_name,
} }
gclient_eval.Exec('\n'.join([ gclient_eval.Exec('\n'.join([
'vars = {', 'vars = {',
@ -115,7 +115,7 @@ class ExecTest(unittest.TestCase):
]), global_scope, local_scope, '<string>') ]), global_scope, local_scope, '<string>')
self.assertEqual({ self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]), 'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'abarb')]), 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),
}, local_scope) }, local_scope)

23
tests/gclient_smoketest.py
Unescape Escape View File

@ -578,6 +578,7 @@ class GClientSmokeGIT(GClientSmokeBase):
with open(output_deps) as f: with open(output_deps) as f:
deps_contents = f.read() deps_contents = f.read()
self.maxDiff = None
self.assertEqual([ self.assertEqual([
'gclient_gn_args_file = "src/gclient.args"', 'gclient_gn_args_file = "src/gclient.args"',
'gclient_gn_args = [\'DummyVariable\']', 'gclient_gn_args = [\'DummyVariable\']',
@ -598,7 +599,7 @@ class GClientSmokeGIT(GClientSmokeBase):
'', '',
' # src -> src/repo2', ' # src -> src/repo2',
' "src/repo2": {', ' "src/repo2": {',
' "url": "git://127.0.0.1:20000/git/repo_2@%s",' % ( ' "url": "{git_base}repo_2@%s",' % (
self.githash('repo_2', 1)[:7]), self.githash('repo_2', 1)[:7]),
' "condition": "True",', ' "condition": "True",',
' },', ' },',
@ -661,7 +662,8 @@ class GClientSmokeGIT(GClientSmokeBase):
' "action": [', ' "action": [',
' "python",', ' "python",',
' "-c",', ' "-c",',
' "open(\'src/git_hooked1\', \'w\').write(\'git_hooked1\')",', ' "open(\'src/git_hooked1\', \'w\')'
'.write(\'{hook1_contents}\')",',
' ]', ' ]',
' },', ' },',
'', '',
@ -700,6 +702,12 @@ class GClientSmokeGIT(GClientSmokeBase):
' # src', ' # src',
' "DummyVariable": \'repo\',', ' "DummyVariable": \'repo\',',
'', '',
' # src',
' "git_base": \'git://127.0.0.1:20000/git/\',',
'',
' # src',
' "hook1_contents": \'git_hooked1\',',
'',
'}', '}',
'', '',
], deps_contents.splitlines()) ], deps_contents.splitlines())
@ -741,7 +749,7 @@ class GClientSmokeGIT(GClientSmokeBase):
'', '',
' # src -> src/repo2', ' # src -> src/repo2',
' "src/repo2": {', ' "src/repo2": {',
' "url": "git://127.0.0.1:20000/git/repo_2@%s",' % ( ' "url": "{git_base}repo_2@%s",' % (
self.githash('repo_2', 1)[:7]), self.githash('repo_2', 1)[:7]),
' "condition": "True",', ' "condition": "True",',
' },', ' },',
@ -805,7 +813,8 @@ class GClientSmokeGIT(GClientSmokeBase):
' "action": [', ' "action": [',
' "python",', ' "python",',
' "-c",', ' "-c",',
' "open(\'src/git_hooked1\', \'w\').write(\'git_hooked1\')",', ' "open(\'src/git_hooked1\', \'w\')'
'.write(\'{hook1_contents}\')",',
' ]', ' ]',
' },', ' },',
'', '',
@ -844,6 +853,12 @@ class GClientSmokeGIT(GClientSmokeBase):
' # src', ' # src',
' "DummyVariable": \'repo\',', ' "DummyVariable": \'repo\',',
'', '',
' # src',
' "git_base": \'git://127.0.0.1:20000/git/\',',
'',
' # src',
' "hook1_contents": \'git_hooked1\',',
'',
'}', '}',
'', '',
], deps_contents.splitlines()) ], deps_contents.splitlines())

15
tests/gclient_test.py
Unescape Escape View File

@ -200,8 +200,9 @@ class GclientTest(trial_dir.TestCase):
def testAutofix(self): def testAutofix(self):
# Invalid urls causes pain when specifying requirements. Make sure it's # Invalid urls causes pain when specifying requirements. Make sure it's
# auto-fixed. # auto-fixed.
url = 'proto://host/path/@revision'
d = gclient.Dependency( d = gclient.Dependency(
None, 'name', 'proto://host/path/@revision', None, None, None, None, 'name', url, url, None, None, None,
None, '', True, False, None, True) None, '', True, False, None, True)
self.assertEquals('proto://host/path@revision', d.url) self.assertEquals('proto://host/path@revision', d.url)
@ -212,17 +213,17 @@ class GclientTest(trial_dir.TestCase):
obj.add_dependencies_and_close( obj.add_dependencies_and_close(
[ [
gclient.Dependency( gclient.Dependency(
obj, 'foo', 'url', None, None, None, None, 'DEPS', True, False, obj, 'foo', 'raw_url', 'url', None, None, None, None, 'DEPS', True,
None, True), False, None, True),
gclient.Dependency( gclient.Dependency(
obj, 'bar', 'url', None, None, None, None, 'DEPS', True, False, obj, 'bar', 'raw_url', 'url', None, None, None, None, 'DEPS', True,
None, True), False, None, True),
], ],
[]) [])
obj.dependencies[0].add_dependencies_and_close( obj.dependencies[0].add_dependencies_and_close(
[ [
gclient.Dependency( gclient.Dependency(
obj.dependencies[0], 'foo/dir1', 'url', None, None, None, obj.dependencies[0], 'foo/dir1', 'raw_url', 'url', None, None, None,
None, 'DEPS', True, False, None, True), None, 'DEPS', True, False, None, True),
], ],
[]) [])
@ -620,7 +621,7 @@ class GclientTest(trial_dir.TestCase):
def testLateOverride(self): def testLateOverride(self):
"""Verifies expected behavior of LateOverride.""" """Verifies expected behavior of LateOverride."""
url = "git@github.com:dart-lang/spark.git" url = "git@github.com:dart-lang/spark.git"
d = gclient.Dependency(None, 'name', 'url', d = gclient.Dependency(None, 'name', 'raw_url', 'url',
None, None, None, None, '', True, False, None, True) None, None, None, None, '', True, False, None, True)
late_url = d.LateOverride(url) late_url = d.LateOverride(url)
self.assertEquals(url, late_url) self.assertEquals(url, late_url)

Write Preview
Loading…
Cancel
Save