aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gclient: Remove syntax validation flags.

Browse Source 
Change-Id: I7215335eb4592cba80d31595b3bac75f55372dee
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2083589
Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
changes/89/2083589/4
Edward Lemur 5 years ago committed by LUCI CQ
parent a3b6fd06f9
commit 67cabcd71f
4 changed files with 181 additions and 256 deletions
Show Stats Download Patch File Download Diff File

10
gclient.py
Unescape Escape View File

@ -716,8 +716,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
if deps_content: if deps_content:
try: try:
local_scope = gclient_eval.Parse( local_scope = gclient_eval.Parse(
deps_content, self._get_option('validate_syntax', False), deps_content, filepath, self.get_vars(), self.get_builtin_vars())
filepath, self.get_vars(), self.get_builtin_vars())
except SyntaxError as e: except SyntaxError as e:
gclient_utils.SyntaxErrorToError(filepath, e) gclient_utils.SyntaxErrorToError(filepath, e)
@ -2698,12 +2697,6 @@ def CMDsync(parser, args):
help='DEPRECATED: This is a no-op.') help='DEPRECATED: This is a no-op.')
parser.add_option('-m', '--manually_grab_svn_rev', action='store_true', parser.add_option('-m', '--manually_grab_svn_rev', action='store_true',
help='DEPRECATED: This is a no-op.') help='DEPRECATED: This is a no-op.')
# TODO(phajdan.jr): Remove validation options once default (crbug/570091).
parser.add_option('--validate-syntax', action='store_true', default=True,
help='Validate the .gclient and DEPS syntax')
parser.add_option('--disable-syntax-validation', action='store_false',
dest='validate_syntax',
help='Disable validation of .gclient and DEPS syntax.')
parser.add_option('--no-rebase-patch-ref', action='store_false', parser.add_option('--no-rebase-patch-ref', action='store_false',
dest='rebase_patch_ref', default=True, dest='rebase_patch_ref', default=True,
help='Bypass rebase of the patch ref after checkout.') help='Bypass rebase of the patch ref after checkout.')
@ -2745,7 +2738,6 @@ CMDupdate = CMDsync
def CMDvalidate(parser, args): def CMDvalidate(parser, args):
"""Validates the .gclient and DEPS syntax.""" """Validates the .gclient and DEPS syntax."""
options, args = parser.parse_args(args) options, args = parser.parse_args(args)
options.validate_syntax = True
client = GClient.LoadCurrentConfig(options) client = GClient.LoadCurrentConfig(options)
rv = client.RunOnDeps('validate', args) rv = client.RunOnDeps('validate', args)
if rv == 0: if rv == 0:

52
gclient_eval.py
Unescape Escape View File

@ -372,44 +372,10 @@ def Exec(content, filename='<unknown>', vars_override=None, builtin_vars=None):
value = _gclient_eval(node, filename, vars_dict) value = _gclient_eval(node, filename, vars_dict)
local_scope.SetNode(name, value, node) local_scope.SetNode(name, value, node)
return _GCLIENT_SCHEMA.validate(local_scope) try:
return _GCLIENT_SCHEMA.validate(local_scope)
except schema.SchemaError as e:
def ExecLegacy(content, filename='<unknown>', vars_override=None, raise gclient_utils.Error(str(e))
builtin_vars=None):
"""Executes a DEPS file |content| using exec."""
local_scope = {}
global_scope = {'Var': lambda var_name: '{%s}' % var_name}
# If we use 'exec' directly, it complains that 'Parse' contains a nested
# function with free variables.
# This is because on versions of Python < 2.7.9, "exec(a, b, c)" not the same
# as "exec a in b, c" (See https://bugs.python.org/issue21591).
eval(compile(content, filename, 'exec'), global_scope, local_scope)
vars_dict = {}
vars_dict.update(local_scope.get('vars', {}))
if builtin_vars:
vars_dict.update(builtin_vars)
if vars_override:
vars_dict.update({k: v for k, v in vars_override.items() if k in vars_dict})
if not vars_dict:
return local_scope
def _DeepFormat(node):
if isinstance(node, basestring):
return node.format(**vars_dict)
elif isinstance(node, dict):
return {k.format(**vars_dict): _DeepFormat(v) for k, v in node.items()}
elif isinstance(node, list):
return [_DeepFormat(elem) for elem in node]
elif isinstance(node, tuple):
return tuple(_DeepFormat(elem) for elem in node)
else:
return node
return _DeepFormat(local_scope)
def _StandardizeDeps(deps_dict, vars_dict): def _StandardizeDeps(deps_dict, vars_dict):
@ -477,8 +443,7 @@ def UpdateCondition(info_dict, op, new_condition):
del info_dict['condition'] del info_dict['condition']
def Parse(content, validate_syntax, filename, vars_override=None, def Parse(content, filename, vars_override=None, builtin_vars=None):
builtin_vars=None):
"""Parses DEPS strings. """Parses DEPS strings.
Executes the Python-like string stored in content, resulting in a Python Executes the Python-like string stored in content, resulting in a Python
@ -487,8 +452,6 @@ def Parse(content, validate_syntax, filename, vars_override=None,
Args: Args:
content: str. DEPS file stored as a string. content: str. DEPS file stored as a string.
validate_syntax: bool. Whether syntax should be validated using the schema
defined above.
filename: str. The name of the DEPS file, or a string describing the source filename: str. The name of the DEPS file, or a string describing the source
of the content, e.g. '<string>', '<unknown>'. of the content, e.g. '<string>', '<unknown>'.
vars_override: dict, optional. A dictionary with overrides for the variables vars_override: dict, optional. A dictionary with overrides for the variables
@ -500,10 +463,7 @@ def Parse(content, validate_syntax, filename, vars_override=None,
A Python dict with the parsed contents of the DEPS file, as specified by the A Python dict with the parsed contents of the DEPS file, as specified by the
schema above. schema above.
""" """
if validate_syntax: result = Exec(content, filename, vars_override, builtin_vars)
result = Exec(content, filename, vars_override, builtin_vars)
else:
result = ExecLegacy(content, filename, vars_override, builtin_vars)
vars_dict = result.get('vars', {}) vars_dict = result.get('vars', {})
if 'deps' in result: if 'deps' in result:

370
tests/gclient_eval_unittest.py
Unescape Escape View File

@ -20,6 +20,7 @@ metrics.DISABLE_METRICS_COLLECTION = True
import gclient import gclient
import gclient_eval import gclient_eval
import gclient_utils
class GClientEvalTest(unittest.TestCase): class GClientEvalTest(unittest.TestCase):
@ -103,7 +104,7 @@ class ExecTest(unittest.TestCase):
'invalid assignment: overrides var \'a\'', str(cm.exception)) 'invalid assignment: overrides var \'a\'', str(cm.exception))
def test_schema_wrong_type(self): def test_schema_wrong_type(self):
with self.assertRaises(schema.SchemaError): with self.assertRaises(gclient_utils.Error):
gclient_eval.Exec('include_rules = {}') gclient_eval.Exec('include_rules = {}')
def test_recursedeps_list(self): def test_recursedeps_list(self):
@ -787,7 +788,7 @@ class RevisionTest(unittest.TestCase):
class ParseTest(unittest.TestCase): class ParseTest(unittest.TestCase):
def callParse(self, validate_syntax=True, vars_override=None): def callParse(self, vars_override=None):
return gclient_eval.Parse('\n'.join([ return gclient_eval.Parse('\n'.join([
'vars = {', 'vars = {',
' "foo": "bar",', ' "foo": "bar",',
@ -795,7 +796,7 @@ class ParseTest(unittest.TestCase):
'deps = {', 'deps = {',
' "a_dep": "a{foo}b",', ' "a_dep": "a{foo}b",',
'}', '}',
]), validate_syntax, '<unknown>', vars_override) ]), '<unknown>', vars_override)
def test_supports_vars_inside_vars(self): def test_supports_vars_inside_vars(self):
deps_file = '\n'.join([ deps_file = '\n'.join([
@ -810,16 +811,14 @@ class ParseTest(unittest.TestCase):
' },', ' },',
'}', '}',
]) ])
for validate_syntax in False, True: local_scope = gclient_eval.Parse(deps_file, '<unknown>', None)
local_scope = gclient_eval.Parse( self.assertEqual({
deps_file, validate_syntax, '<unknown>', None) 'vars': {'foo': 'bar',
self.assertEqual({ 'baz': '"bar" == "bar"'},
'vars': {'foo': 'bar', 'deps': {'src/baz': {'url': 'baz_url',
'baz': '"bar" == "bar"'}, 'dep_type': 'git',
'deps': {'src/baz': {'url': 'baz_url', 'condition': 'baz'}},
'dep_type': 'git', }, local_scope)
'condition': 'baz'}},
}, local_scope)
def test_has_builtin_vars(self): def test_has_builtin_vars(self):
builtin_vars = {'builtin_var': 'foo'} builtin_vars = {'builtin_var': 'foo'}
@ -828,13 +827,11 @@ class ParseTest(unittest.TestCase):
' "a_dep": "a{builtin_var}b",', ' "a_dep": "a{builtin_var}b",',
'}', '}',
]) ])
for validate_syntax in False, True: local_scope = gclient_eval.Parse(deps_file, '<unknown>', None, builtin_vars)
local_scope = gclient_eval.Parse( self.assertEqual({
deps_file, validate_syntax, '<unknown>', None, builtin_vars) 'deps': {'a_dep': {'url': 'afoob',
self.assertEqual({ 'dep_type': 'git'}},
'deps': {'a_dep': {'url': 'afoob', }, local_scope)
'dep_type': 'git'}},
}, local_scope)
def test_declaring_builtin_var_has_no_effect(self): def test_declaring_builtin_var_has_no_effect(self):
builtin_vars = {'builtin_var': 'foo'} builtin_vars = {'builtin_var': 'foo'}
@ -846,14 +843,12 @@ class ParseTest(unittest.TestCase):
' "a_dep": "a{builtin_var}b",', ' "a_dep": "a{builtin_var}b",',
'}', '}',
]) ])
for validate_syntax in False, True: local_scope = gclient_eval.Parse(deps_file, '<unknown>', None, builtin_vars)
local_scope = gclient_eval.Parse( self.assertEqual({
deps_file, validate_syntax, '<unknown>', None, builtin_vars) 'vars': {'builtin_var': 'bar'},
self.assertEqual({ 'deps': {'a_dep': {'url': 'afoob',
'vars': {'builtin_var': 'bar'}, 'dep_type': 'git'}},
'deps': {'a_dep': {'url': 'afoob', }, local_scope)
'dep_type': 'git'}},
}, local_scope)
def test_override_builtin_var(self): def test_override_builtin_var(self):
builtin_vars = {'builtin_var': 'foo'} builtin_vars = {'builtin_var': 'foo'}
@ -863,32 +858,28 @@ class ParseTest(unittest.TestCase):
' "a_dep": "a{builtin_var}b",', ' "a_dep": "a{builtin_var}b",',
'}', '}',
]) ])
for validate_syntax in False, True: local_scope = gclient_eval.Parse(
local_scope = gclient_eval.Parse( deps_file, '<unknown>', vars_override, builtin_vars)
deps_file, validate_syntax, '<unknown>', vars_override, builtin_vars) self.assertEqual({
self.assertEqual({ 'deps': {'a_dep': {'url': 'aoverrideb',
'deps': {'a_dep': {'url': 'aoverrideb', 'dep_type': 'git'}},
'dep_type': 'git'}}, }, local_scope, str(local_scope))
}, local_scope, str(local_scope))
def test_expands_vars(self): def test_expands_vars(self):
for validate_syntax in True, False: local_scope = self.callParse()
local_scope = self.callParse(validate_syntax=validate_syntax) self.assertEqual({
self.assertEqual({ 'vars': {'foo': 'bar'},
'vars': {'foo': 'bar'}, 'deps': {'a_dep': {'url': 'abarb',
'deps': {'a_dep': {'url': 'abarb', 'dep_type': 'git'}},
'dep_type': 'git'}}, }, local_scope)
}, local_scope)
def test_overrides_vars(self): def test_overrides_vars(self):
for validate_syntax in True, False: local_scope = self.callParse(vars_override={'foo': 'baz'})
local_scope = self.callParse(validate_syntax=validate_syntax, self.assertEqual({
vars_override={'foo': 'baz'}) 'vars': {'foo': 'bar'},
self.assertEqual({ 'deps': {'a_dep': {'url': 'abazb',
'vars': {'foo': 'bar'}, 'dep_type': 'git'}},
'deps': {'a_dep': {'url': 'abazb', }, local_scope)
'dep_type': 'git'}},
}, local_scope)
def test_no_extra_vars(self): def test_no_extra_vars(self):
deps_file = '\n'.join([ deps_file = '\n'.join([
@ -901,101 +892,131 @@ class ParseTest(unittest.TestCase):
]) ])
with self.assertRaises(KeyError) as cm: with self.assertRaises(KeyError) as cm:
gclient_eval.Parse( gclient_eval.Parse(deps_file, '<unknown>', {'baz': 'lalala'})
deps_file, True, '<unknown>', {'baz': 'lalala'})
self.assertIn('baz was used as a variable, but was not declared', self.assertIn('baz was used as a variable, but was not declared',
str(cm.exception)) str(cm.exception))
with self.assertRaises(KeyError) as cm:
gclient_eval.Parse(
deps_file, False, '<unknown>', {'baz': 'lalala'})
self.assertIn('baz', str(cm.exception))
def test_standardizes_deps_string_dep(self): def test_standardizes_deps_string_dep(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'deps = {',
'deps = {', ' "a_dep": "a_url@a_rev",',
' "a_dep": "a_url@a_rev",', '}',
'}', ]), '<unknown>')
]), validate_syntax, '<unknown>') self.assertEqual({
self.assertEqual({ 'deps': {'a_dep': {'url': 'a_url@a_rev',
'deps': {'a_dep': {'url': 'a_url@a_rev', 'dep_type': 'git'}},
'dep_type': 'git'}}, }, local_scope)
}, local_scope)
def test_standardizes_deps_dict_dep(self): def test_standardizes_deps_dict_dep(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'deps = {',
'deps = {', ' "a_dep": {',
' "a_dep": {', ' "url": "a_url@a_rev",',
' "url": "a_url@a_rev",', ' "condition": "checkout_android",',
' "condition": "checkout_android",', ' },',
' },', '}',
'}', ]), '<unknown>')
]), validate_syntax, '<unknown>') self.assertEqual({
self.assertEqual({ 'deps': {'a_dep': {'url': 'a_url@a_rev',
'deps': {'a_dep': {'url': 'a_url@a_rev', 'dep_type': 'git',
'dep_type': 'git', 'condition': 'checkout_android'}},
'condition': 'checkout_android'}}, }, local_scope)
}, local_scope)
def test_ignores_none_in_deps_os(self): def test_ignores_none_in_deps_os(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'deps = {',
'deps = {', ' "a_dep": "a_url@a_rev",',
' "a_dep": "a_url@a_rev",', '}',
'}', 'deps_os = {',
'deps_os = {', ' "mac": {',
' "mac": {', ' "a_dep": None,',
' "a_dep": None,', ' },',
' },', '}',
'}', ]), '<unknown>')
]), validate_syntax, '<unknown>') self.assertEqual({
self.assertEqual({ 'deps': {'a_dep': {'url': 'a_url@a_rev',
'deps': {'a_dep': {'url': 'a_url@a_rev', 'dep_type': 'git'}},
'dep_type': 'git'}}, }, local_scope)
}, local_scope)
def test_merges_deps_os_extra_dep(self): def test_merges_deps_os_extra_dep(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'deps = {',
'deps = {', ' "a_dep": "a_url@a_rev",',
' "a_dep": "a_url@a_rev",', '}',
'}', 'deps_os = {',
'deps_os = {', ' "mac": {',
' "mac": {', ' "b_dep": "b_url@b_rev"',
' "b_dep": "b_url@b_rev"', ' },',
' },', '}',
'}', ]), '<unknown>')
]), validate_syntax, '<unknown>') self.assertEqual({
self.assertEqual({ 'deps': {'a_dep': {'url': 'a_url@a_rev',
'deps': {'a_dep': {'url': 'a_url@a_rev', 'dep_type': 'git'},
'dep_type': 'git'}, 'b_dep': {'url': 'b_url@b_rev',
'b_dep': {'url': 'b_url@b_rev', 'dep_type': 'git',
'dep_type': 'git', 'condition': 'checkout_mac'}},
'condition': 'checkout_mac'}}, }, local_scope)
}, local_scope)
def test_merges_deps_os_existing_dep_with_no_condition(self): def test_merges_deps_os_existing_dep_with_no_condition(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'deps = {',
'deps = {', ' "a_dep": "a_url@a_rev",',
' "a_dep": "a_url@a_rev",', '}',
'}', 'deps_os = {',
'deps_os = {', ' "mac": {',
' "mac": {', ' "a_dep": "a_url@a_rev"',
' "a_dep": "a_url@a_rev"', ' },',
' },', '}',
'}', ]), '<unknown>')
]), validate_syntax, '<unknown>') self.assertEqual({
self.assertEqual({ 'deps': {'a_dep': {'url': 'a_url@a_rev',
'deps': {'a_dep': {'url': 'a_url@a_rev', 'dep_type': 'git'}},
'dep_type': 'git'}}, }, local_scope)
}, local_scope)
def test_merges_deps_os_existing_dep_with_condition(self): def test_merges_deps_os_existing_dep_with_condition(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'deps = {',
' "a_dep": {',
' "url": "a_url@a_rev",',
' "condition": "some_condition",',
' },',
'}',
'deps_os = {',
' "mac": {',
' "a_dep": "a_url@a_rev"',
' },',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {
'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': '(checkout_mac) or (some_condition)'},
},
}, local_scope)
def test_merges_deps_os_multiple_os(self):
local_scope = gclient_eval.Parse('\n'.join([
'deps_os = {',
' "win": {'
' "a_dep": "a_url@a_rev"',
' },',
' "mac": {',
' "a_dep": "a_url@a_rev"',
' },',
'}',
]), '<unknown>')
self.assertEqual({
'deps': {
'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': '(checkout_mac) or (checkout_win)'},
},
}, local_scope)
def test_fails_to_merge_same_dep_with_different_revisions(self):
with self.assertRaises(gclient_eval.gclient_utils.Error) as cm:
gclient_eval.Parse('\n'.join([
'deps = {', 'deps = {',
' "a_dep": {', ' "a_dep": {',
' "url": "a_url@a_rev",', ' "url": "a_url@a_rev",',
@ -1004,76 +1025,31 @@ class ParseTest(unittest.TestCase):
'}', '}',
'deps_os = {', 'deps_os = {',
' "mac": {', ' "mac": {',
' "a_dep": "a_url@a_rev"', ' "a_dep": "a_url@b_rev"',
' },', ' },',
'}', '}',
]), validate_syntax, '<unknown>') ]), '<unknown>')
self.assertEqual({ self.assertIn('conflicts with existing deps', str(cm.exception))
'deps': {
'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': '(checkout_mac) or (some_condition)'},
},
}, local_scope)
def test_merges_deps_os_multiple_os(self):
for validate_syntax in True, False:
local_scope = gclient_eval.Parse('\n'.join([
'deps_os = {',
' "win": {'
' "a_dep": "a_url@a_rev"',
' },',
' "mac": {',
' "a_dep": "a_url@a_rev"',
' },',
'}',
]), validate_syntax, '<unknown>')
self.assertEqual({
'deps': {
'a_dep': {'url': 'a_url@a_rev',
'dep_type': 'git',
'condition': '(checkout_mac) or (checkout_win)'},
},
}, local_scope)
def test_fails_to_merge_same_dep_with_different_revisions(self):
for validate_syntax in True, False:
with self.assertRaises(gclient_eval.gclient_utils.Error) as cm:
gclient_eval.Parse('\n'.join([
'deps = {',
' "a_dep": {',
' "url": "a_url@a_rev",',
' "condition": "some_condition",',
' },',
'}',
'deps_os = {',
' "mac": {',
' "a_dep": "a_url@b_rev"',
' },',
'}',
]), validate_syntax, '<unknown>')
self.assertIn('conflicts with existing deps', str(cm.exception))
def test_merges_hooks_os(self): def test_merges_hooks_os(self):
for validate_syntax in True, False: local_scope = gclient_eval.Parse('\n'.join([
local_scope = gclient_eval.Parse('\n'.join([ 'hooks = [',
'hooks = [', ' {',
' {', ' "action": ["a", "action"],',
' "action": ["a", "action"],', ' },',
' },', ']',
']', 'hooks_os = {',
'hooks_os = {', ' "mac": [',
' "mac": [', ' {',
' {', ' "action": ["b", "action"]',
' "action": ["b", "action"]', ' },',
' },', ' ]',
' ]', '}',
'}', ]), '<unknown>')
]), validate_syntax, '<unknown>') self.assertEqual({
self.assertEqual({ "hooks": [{"action": ["a", "action"]},
"hooks": [{"action": ["a", "action"]}, {"action": ["b", "action"], "condition": "checkout_mac"}],
{"action": ["b", "action"], "condition": "checkout_mac"}], }, local_scope)
}, local_scope)

5
tests/gclient_test.py
Unescape Escape View File

@ -1002,7 +1002,6 @@ class GclientTest(trial_dir.TestCase):
'}') '}')
options, _ = gclient.OptionParser().parse_args([]) options, _ = gclient.OptionParser().parse_args([])
options.ignore_dep_type = 'git' options.ignore_dep_type = 'git'
options.validate_syntax = True
obj = gclient.GClient.LoadCurrentConfig(options) obj = gclient.GClient.LoadCurrentConfig(options)
self.assertEqual(1, len(obj.dependencies)) self.assertEqual(1, len(obj.dependencies))
@ -1132,7 +1131,7 @@ class GclientTest(trial_dir.TestCase):
obj.RunOnDeps('None', []) obj.RunOnDeps('None', [])
self.fail() self.fail()
except gclient_utils.Error as e: except gclient_utils.Error as e:
self.assertIn('allowed_hosts must be', str(e)) self.assertIn('Key \'allowed_hosts\' error:', str(e))
finally: finally:
self._get_processed() self._get_processed()
@ -1160,7 +1159,6 @@ class GclientTest(trial_dir.TestCase):
' }\n' ' }\n'
'}') '}')
options, _ = gclient.OptionParser().parse_args([]) options, _ = gclient.OptionParser().parse_args([])
options.validate_syntax = True
obj = gclient.GClient.LoadCurrentConfig(options) obj = gclient.GClient.LoadCurrentConfig(options)
self.assertEqual(1, len(obj.dependencies)) self.assertEqual(1, len(obj.dependencies))
@ -1202,7 +1200,6 @@ class GclientTest(trial_dir.TestCase):
'}') '}')
options, _ = gclient.OptionParser().parse_args([]) options, _ = gclient.OptionParser().parse_args([])
options.ignore_dep_type = 'cipd' options.ignore_dep_type = 'cipd'
options.validate_syntax = True
obj = gclient.GClient.LoadCurrentConfig(options) obj = gclient.GClient.LoadCurrentConfig(options)
self.assertEqual(1, len(obj.dependencies)) self.assertEqual(1, len(obj.dependencies))

Write Preview
Loading…
Cancel
Save