diff --git a/gclient.py b/gclient.py index 239ff28bf..c4bf01176 100755 --- a/gclient.py +++ b/gclient.py @@ -736,7 +736,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): deps_to_add.sort(key=lambda x: x.name) return deps_to_add - def ParseDepsFile(self, expand_vars=True): + def ParseDepsFile(self): """Parses the DEPS file for this dependency.""" assert not self.deps_parsed assert not self.dependencies @@ -765,15 +765,15 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): local_scope = {} if deps_content: + # Eval the content. try: - vars_override = {} - if self.parent: - vars_override = self.parent.get_vars() - vars_override.update(self.get_vars()) - local_scope = gclient_eval.Parse( - deps_content, expand_vars, - self._get_option('validate_syntax', False), - filepath, vars_override) + if self._get_option('validate_syntax', False): + local_scope = gclient_eval.Exec(deps_content, filepath) + else: + global_scope = { + 'Var': lambda var_name: '{%s}' % var_name, + } + exec(deps_content, global_scope, local_scope) except SyntaxError as e: gclient_utils.SyntaxErrorToError(filepath, e) @@ -988,7 +988,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): file_list[i] = file_list[i][1:] # Always parse the DEPS file. - self.ParseDepsFile(expand_vars=(command != 'flatten')) + self.ParseDepsFile() self._run_is_done(file_list or [], parsed_url) if command in ('update', 'revert') and not options.noprehooks: self.RunPreDepsHooks() @@ -1864,7 +1864,7 @@ it or fix the checkout. print('%s: %s' % (x, entries[x])) logging.info(str(self)) - def ParseDepsFile(self, expand_vars=None): + def ParseDepsFile(self): """No DEPS to parse for a .gclient file.""" raise gclient_utils.Error('Internal error') @@ -1973,7 +1973,7 @@ class CipdDependency(Dependency): self._cipd_package = self._cipd_root.add_package( self._cipd_subdir, self._package_name, self._package_version) - def ParseDepsFile(self, expand_vars=None): + def ParseDepsFile(self): """CIPD dependencies are not currently allowed to have nested deps.""" self.add_dependencies_and_close([], []) @@ -2930,9 +2930,7 @@ def CMDsetdep(parser, args): 'DEPS file %s does not exist.' % options.deps_file) with open(options.deps_file) as f: contents = f.read() - local_scope = gclient_eval.Parse( - contents, expand_vars=True, validate_syntax=True, - filename=options.deps_file) + local_scope = gclient_eval.Exec(contents) for var in options.vars: name, _, value = var.partition('=') diff --git a/gclient_eval.py b/gclient_eval.py index 0ebb1c528..081c96fdb 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -37,7 +37,7 @@ class _NodeDict(collections.MutableMapping): def GetNode(self, key): return self.data[key][1] - def SetNode(self, key, value, node): + def _SetNode(self, key, value, node): self.data[key] = (value, node) @@ -183,7 +183,7 @@ _GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({ })) -def _gclient_eval(node_or_string, vars_dict, expand_vars, filename): +def _gclient_eval(node_or_string, filename=''): """Safely evaluates a single expression. Returns the result.""" _allowed_names = {'None': None, 'True': True, 'False': False} if isinstance(node_or_string, basestring): @@ -192,15 +192,7 @@ def _gclient_eval(node_or_string, vars_dict, expand_vars, filename): node_or_string = node_or_string.body def _convert(node): if isinstance(node, ast.Str): - if not expand_vars: - return node.s - try: - return node.s.format(**vars_dict) - except KeyError as e: - raise ValueError( - '%s was used as a variable, but was not declared in the vars dict ' - '(file %r, line %s)' % ( - e.message, filename, getattr(node, 'lineno', ''))) + return node.s elif isinstance(node, ast.Num): return node.n elif isinstance(node, ast.Tuple): @@ -230,18 +222,7 @@ def _gclient_eval(node_or_string, vars_dict, expand_vars, filename): raise ValueError( 'Var\'s argument must be a variable name (file %r, line %s)' % ( filename, getattr(node, 'lineno', ''))) - if not expand_vars: - return '{%s}' % arg - if vars_dict is None: - raise ValueError( - 'vars must be declared before Var can be used (file %r, line %s)' - % (filename, getattr(node, 'lineno', ''))) - if arg not in vars_dict: - raise ValueError( - '%s was used as a variable, but was not declared in the vars dict ' - '(file %r, line %s)' % ( - arg, filename, getattr(node, 'lineno', ''))) - return vars_dict[arg] + return '{%s}' % arg elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): return _convert(node.left) + _convert(node.right) elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod): @@ -254,19 +235,13 @@ def _gclient_eval(node_or_string, vars_dict, expand_vars, filename): return _convert(node_or_string) -def Exec(content, expand_vars, filename='', vars_override=None): - """Safely execs a set of assignments.""" +def Exec(content, filename=''): + """Safely execs a set of assignments. Mutates |local_scope|.""" node_or_string = ast.parse(content, filename=filename, mode='exec') if isinstance(node_or_string, ast.Expression): node_or_string = node_or_string.body - tokens = { - token[2]: list(token) - for token in tokenize.generate_tokens( - cStringIO.StringIO(content).readline) - } - local_scope = _NodeDict({}, tokens) - vars_dict = {} + defined_variables = set() def _visit_in_module(node): if isinstance(node, ast.Assign): if len(node.targets) != 1: @@ -278,22 +253,15 @@ def Exec(content, expand_vars, filename='', vars_override=None): raise ValueError( 'invalid assignment: target should be a name (file %r, line %s)' % ( filename, getattr(node, 'lineno', ''))) - value = _gclient_eval(node.value, vars_dict, expand_vars, filename) + value = _gclient_eval(node.value, filename=filename) - if target.id in local_scope: + if target.id in defined_variables: raise ValueError( 'invalid assignment: overrides var %r (file %r, line %s)' % ( target.id, filename, getattr(node, 'lineno', ''))) - if target.id == 'vars': - vars_dict.update(value) - if vars_override: - vars_dict.update({ - k: v - for k, v in vars_override.iteritems() - if k in vars_dict}) - - local_scope.SetNode(target.id, value, node.value) + defined_variables.add(target.id) + return target.id, (value, node.value) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -301,8 +269,15 @@ def Exec(content, expand_vars, filename='', vars_override=None): getattr(node, 'lineno', ''))) if isinstance(node_or_string, ast.Module): + data = [] for stmt in node_or_string.body: - _visit_in_module(stmt) + data.append(_visit_in_module(stmt)) + tokens = { + token[2]: list(token) + for token in tokenize.generate_tokens( + cStringIO.StringIO(content).readline) + } + local_scope = _NodeDict(data, tokens) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -314,70 +289,6 @@ def Exec(content, expand_vars, filename='', vars_override=None): return _GCLIENT_SCHEMA.validate(local_scope) -def Parse(content, expand_vars, validate_syntax, filename, vars_override=None): - """Parses DEPS strings. - - Executes the Python-like string stored in content, resulting in a Python - dictionary specifyied by the schema above. Supports syntax validation and - variable expansion. - - Args: - content: str. DEPS file stored as a string. - expand_vars: bool. Whether variables should be expanded to their values. - validate_syntax: bool. Whether syntax should be validated using the schema - defined above. - filename: str. The name of the DEPS file, or a string describing the source - of the content, e.g. '', ''. - vars_override: dict, optional. A dictionary with overrides for the variables - defined by the DEPS file. - - Returns: - A Python dict with the parsed contents of the DEPS file, as specified by the - schema above. - """ - # TODO(ehmaldonado): Make validate_syntax = True the only case - if validate_syntax: - return Exec(content, expand_vars, filename, vars_override) - - local_scope = {} - global_scope = {'Var': lambda var_name: '{%s}' % var_name} - - # If we use 'exec' directly, it complains that 'Parse' contains a nested - # function with free variables. - # This is because on versions of Python < 2.7.9, "exec(a, b, c)" not the same - # as "exec a in b, c" (See https://bugs.python.org/issue21591). - eval(compile(content, filename, 'exec'), global_scope, local_scope) - - if 'vars' not in local_scope or not expand_vars: - return local_scope - - vars_dict = {} - vars_dict.update(local_scope['vars']) - if vars_override: - vars_dict.update({ - k: v - for k, v in vars_override.iteritems() - if k in vars_dict - }) - - def _DeepFormat(node): - if isinstance(node, basestring): - return node.format(**vars_dict) - elif isinstance(node, dict): - return { - k.format(**vars_dict): _DeepFormat(v) - for k, v in node.iteritems() - } - elif isinstance(node, list): - return [_DeepFormat(elem) for elem in node] - elif isinstance(node, tuple): - return tuple(_DeepFormat(elem) for elem in node) - else: - return node - - return _DeepFormat(local_scope) - - def EvaluateCondition(condition, variables, referenced_variables=None): """Safely evaluates a boolean condition. Returns the result.""" if not referenced_variables: @@ -505,7 +416,7 @@ def SetVar(gclient_dict, var_name, value): "The vars entry for %s has no formatting information." % var_name) _UpdateAstString(tokens, node, value) - gclient_dict['vars'].SetNode(var_name, value, node) + gclient_dict['vars']._SetNode(var_name, value, node) def SetCIPD(gclient_dict, dep_name, package_name, new_version): @@ -539,7 +450,7 @@ def SetCIPD(gclient_dict, dep_name, package_name, new_version): new_version = 'version:' + new_version _UpdateAstString(tokens, node, new_version) - packages[0].SetNode('version', new_version, node) + packages[0]._SetNode('version', new_version, node) def SetRevision(gclient_dict, dep_name, new_revision): @@ -566,9 +477,8 @@ def SetRevision(gclient_dict, dep_name, new_revision): SetVar(gclient_dict, node.args[0].s, new_revision) else: _UpdateAstString(tokens, node, new_revision) - value = _gclient_eval(dep_node, gclient_dict.get('vars', None), - expand_vars=True, filename='') - dep_dict.SetNode(dep_key, value, dep_node) + value = _gclient_eval(dep_node) + dep_dict._SetNode(dep_key, value, dep_node) if isinstance(gclient_dict['deps'][dep_name], _NodeDict): _UpdateRevision(gclient_dict['deps'][dep_name], 'url') diff --git a/tests/gclient_eval_unittest.py b/tests/gclient_eval_unittest.py index 850edec92..7f841d692 100755 --- a/tests/gclient_eval_unittest.py +++ b/tests/gclient_eval_unittest.py @@ -20,15 +20,6 @@ import gclient_eval _SAMPLE_DEPS_FILE = textwrap.dedent("""\ -vars = { - 'git_repo': 'https://example.com/repo.git', - # Some comment with bad indentation - 'dep_2_rev': '1ced', - # Some more comments with bad indentation - # and trailing whitespaces """ + """ - 'dep_3_rev': '5p1e5', -} - deps = { 'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef', # Some comment @@ -54,89 +45,59 @@ deps = { 'dep_type': 'cipd', }, } -""") + +vars = { + 'git_repo': 'https://example.com/repo.git', + # Some comment with bad indentation + 'dep_2_rev': '1ced', + # Some more comments + # 1 + # 2 + # 3 + 'dep_3_rev': '5p1e5', +}""") class GClientEvalTest(unittest.TestCase): def test_str(self): - self.assertEqual( - 'foo', - gclient_eval._gclient_eval('"foo"', None, False, '')) + self.assertEqual('foo', gclient_eval._gclient_eval('"foo"')) def test_tuple(self): - self.assertEqual( - ('a', 'b'), - gclient_eval._gclient_eval('("a", "b")', None, False, '')) + self.assertEqual(('a', 'b'), gclient_eval._gclient_eval('("a", "b")')) def test_list(self): - self.assertEqual( - ['a', 'b'], - gclient_eval._gclient_eval('["a", "b"]', None, False, '')) + self.assertEqual(['a', 'b'], gclient_eval._gclient_eval('["a", "b"]')) def test_dict(self): - self.assertEqual( - {'a': 'b'}, - gclient_eval._gclient_eval('{"a": "b"}', None, False, '')) + self.assertEqual({'a': 'b'}, gclient_eval._gclient_eval('{"a": "b"}')) def test_name_safe(self): - self.assertEqual( - True, - gclient_eval._gclient_eval('True', None, False, '')) + self.assertEqual(True, gclient_eval._gclient_eval('True')) def test_name_unsafe(self): with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('UnsafeName', None, False, '') + gclient_eval._gclient_eval('UnsafeName') self.assertIn('invalid name \'UnsafeName\'', str(cm.exception)) - def test_invalid_call(self): - with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('Foo("bar")', None, False, '') - self.assertIn('Var is the only allowed function', str(cm.exception)) - def test_call(self): self.assertEqual( '{bar}', - gclient_eval._gclient_eval('Var("bar")', None, False, '')) - - def test_expands_vars(self): - self.assertEqual( - 'foo', - gclient_eval._gclient_eval('Var("bar")', {'bar': 'foo'}, True, - '')) - - def test_expands_vars_with_braces(self): - self.assertEqual( - 'foo', - gclient_eval._gclient_eval('"{bar}"', {'bar': 'foo'}, True, - '')) - - def test_invalid_var(self): - with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('"{bar}"', {}, True, '') - self.assertIn('bar was used as a variable, but was not declared', - str(cm.exception)) + gclient_eval._gclient_eval('Var("bar")')) def test_plus(self): - self.assertEqual( - 'foo', - gclient_eval._gclient_eval('"f" + "o" + "o"', None, False, - '')) + self.assertEqual('foo', gclient_eval._gclient_eval('"f" + "o" + "o"')) def test_format(self): - self.assertEqual( - 'foo', - gclient_eval._gclient_eval('"%s" % "foo"', None, False, '')) + self.assertEqual('foo', gclient_eval._gclient_eval('"%s" % "foo"')) def test_not_expression(self): with self.assertRaises(SyntaxError) as cm: - gclient_eval._gclient_eval( - 'def foo():\n pass', None, False, '') + gclient_eval._gclient_eval('def foo():\n pass') self.assertIn('invalid syntax', str(cm.exception)) def test_not_whitelisted(self): with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval( - '[x for x in [1, 2, 3]]', None, False, '') + gclient_eval._gclient_eval('[x for x in [1, 2, 3]]') self.assertIn( 'unexpected AST node: <_ast.ListComp object', str(cm.exception)) @@ -144,32 +105,31 @@ class GClientEvalTest(unittest.TestCase): for test_case in itertools.permutations(range(4)): input_data = ['{'] + ['"%s": "%s",' % (n, n) for n in test_case] + ['}'] expected = [(str(n), str(n)) for n in test_case] - result = gclient_eval._gclient_eval( - ''.join(input_data), None, False, '') + result = gclient_eval._gclient_eval(''.join(input_data)) self.assertEqual(expected, result.items()) class ExecTest(unittest.TestCase): def test_multiple_assignment(self): with self.assertRaises(ValueError) as cm: - gclient_eval.Exec('a, b, c = "a", "b", "c"', expand_vars=True) + gclient_eval.Exec('a, b, c = "a", "b", "c"') self.assertIn( 'invalid assignment: target should be a name', str(cm.exception)) def test_override(self): with self.assertRaises(ValueError) as cm: - gclient_eval.Exec('a = "a"\na = "x"', expand_vars=True) + gclient_eval.Exec('a = "a"\na = "x"') self.assertIn( 'invalid assignment: overrides var \'a\'', str(cm.exception)) def test_schema_wrong_type(self): with self.assertRaises(schema.SchemaError): - gclient_eval.Exec('include_rules = {}', expand_vars=True) + gclient_eval.Exec('include_rules = {}', '') def test_recursedeps_list(self): local_scope = gclient_eval.Exec( 'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]', - expand_vars=True) + '') self.assertEqual( {'recursedeps': [['src/third_party/angle', 'DEPS.chromium']]}, local_scope) @@ -182,71 +142,16 @@ class ExecTest(unittest.TestCase): 'deps = {', ' "a_dep": "a" + Var("foo") + "b",', '}', - ]), expand_vars=True) - self.assertEqual({ - 'vars': collections.OrderedDict([('foo', 'bar')]), - 'deps': collections.OrderedDict([('a_dep', 'abarb')]), - }, local_scope) - - def test_braces_var(self): - local_scope = gclient_eval.Exec('\n'.join([ - 'vars = {', - ' "foo": "bar",', - '}', - 'deps = {', - ' "a_dep": "a{foo}b",', - '}', - ]), expand_vars=True) - self.assertEqual({ - 'vars': collections.OrderedDict([('foo', 'bar')]), - 'deps': collections.OrderedDict([('a_dep', 'abarb')]), - }, local_scope) - - def test_var_unexpanded(self): - local_scope = gclient_eval.Exec('\n'.join([ - 'vars = {', - ' "foo": "bar",', - '}', - 'deps = {', - ' "a_dep": "a" + Var("foo") + "b",', - '}', - ]), expand_vars=False) + ]), '') self.assertEqual({ 'vars': collections.OrderedDict([('foo', 'bar')]), 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]), }, local_scope) def test_empty_deps(self): - local_scope = gclient_eval.Exec('deps = {}', expand_vars=True) + local_scope = gclient_eval.Exec('deps = {}', '') self.assertEqual({'deps': {}}, local_scope) - def test_overrides_vars(self): - local_scope = gclient_eval.Exec('\n'.join([ - 'vars = {', - ' "foo": "bar",', - '}', - 'deps = {', - ' "a_dep": "a{foo}b",', - '}', - ]), expand_vars=True, vars_override={'foo': 'baz'}) - self.assertEqual({ - 'vars': collections.OrderedDict([('foo', 'bar')]), - 'deps': collections.OrderedDict([('a_dep', 'abazb')]), - }, local_scope) - - def test_doesnt_override_undeclared_vars(self): - with self.assertRaises(ValueError) as cm: - gclient_eval.Exec('\n'.join([ - 'vars = {', - ' "foo": "bar",', - '}', - 'deps = {', - ' "a_dep": "a{baz}b",', - '}', - ]), expand_vars=True, vars_override={'baz': 'lalala'}) - self.assertIn('baz was used as a variable, but was not declared', - str(cm.exception)) - class EvaluateConditionTest(unittest.TestCase): def test_true(self): @@ -299,8 +204,8 @@ class EvaluateConditionTest(unittest.TestCase): class SetVarTest(unittest.TestCase): - def test_sets_var(self): - local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, expand_vars=True) + def testSetVar(self): + local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE) gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee') result = gclient_eval.RenderDEPSFile(local_scope) @@ -311,8 +216,8 @@ class SetVarTest(unittest.TestCase): class SetCipdTest(unittest.TestCase): - def test_sets_cipd(self): - local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, expand_vars=True) + def testSetCIPD(self): + local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE) gclient_eval.SetCIPD( local_scope, 'src/cipd/package', 'another/cipd/package', '6.789') @@ -323,23 +228,23 @@ class SetCipdTest(unittest.TestCase): class SetRevisionTest(unittest.TestCase): def setUp(self): - self.local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, expand_vars=True) + self.local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE) - def test_sets_revision(self): + def testSetRevision(self): gclient_eval.SetRevision( self.local_scope, 'src/dep', 'deadfeed') result = gclient_eval.RenderDEPSFile(self.local_scope) self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed')) - def test_sets_revision_inside_dict(self): + def testSetRevisionInUrl(self): gclient_eval.SetRevision( self.local_scope, 'src/dep_3', '0ff1ce') result = gclient_eval.RenderDEPSFile(self.local_scope) self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce')) - def test_sets_revision_in_vars(self): + def testSetRevisionInVars(self): gclient_eval.SetRevision( self.local_scope, 'src/android/dep_2', 'c0ffee') result = gclient_eval.RenderDEPSFile(self.local_scope) @@ -347,68 +252,6 @@ class SetRevisionTest(unittest.TestCase): self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee')) -class ParseTest(unittest.TestCase): - def callParse(self, expand_vars=True, validate_syntax=True, - vars_override=None): - return gclient_eval.Parse('\n'.join([ - 'vars = {', - ' "foo": "bar",', - '}', - 'deps = {', - ' "a_dep": "a{foo}b",', - '}', - ]), expand_vars, validate_syntax, '', vars_override) - - def test_expands_vars(self): - for validate_syntax in True, False: - local_scope = self.callParse(validate_syntax=validate_syntax) - self.assertEqual({ - 'vars': collections.OrderedDict([('foo', 'bar')]), - 'deps': collections.OrderedDict([('a_dep', 'abarb')]), - }, local_scope) - - def test_no_expands_vars(self): - for validate_syntax in True, False: - local_scope = self.callParse(expand_vars=False, - validate_syntax=validate_syntax) - self.assertEqual({ - 'vars': collections.OrderedDict([('foo', 'bar')]), - 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]), - }, local_scope) - - def test_overrides_vars(self): - for validate_syntax in True, False: - local_scope = self.callParse(validate_syntax=validate_syntax, - vars_override={'foo': 'baz'}) - self.assertEqual({ - 'vars': collections.OrderedDict([('foo', 'bar')]), - 'deps': collections.OrderedDict([('a_dep', 'abazb')]), - }, local_scope) - - def test_no_extra_vars(self): - deps_file = '\n'.join([ - 'vars = {', - ' "foo": "bar",', - '}', - 'deps = {', - ' "a_dep": "a{baz}b",', - '}', - ]) - - with self.assertRaises(ValueError) as cm: - gclient_eval.Parse( - deps_file, expand_vars=True, validate_syntax=True, - filename='', vars_override={'baz': 'lalala'}) - self.assertIn('baz was used as a variable, but was not declared', - str(cm.exception)) - - with self.assertRaises(KeyError) as cm: - gclient_eval.Parse( - deps_file, expand_vars=True, validate_syntax=False, - filename='', vars_override={'baz': 'lalala'}) - self.assertIn('baz', str(cm.exception)) - - if __name__ == '__main__': level = logging.DEBUG if '-v' in sys.argv else logging.FATAL logging.basicConfig(