From beec006697a11600e254ac6c69e98f76d555306b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Hajdan=2C=20Jr?= Date: Wed, 10 May 2017 21:51:05 +0200 Subject: [PATCH] gclient validate: add schema checking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: 570091 Change-Id: I1297f817f2e3d791c22b256de40f12c0c23dceb5 Reviewed-on: https://chromium-review.googlesource.com/500807 Commit-Queue: Paweł Hajdan Jr. Reviewed-by: Dirk Pranke --- gclient_eval.py | 72 ++++ tests/gclient_eval_unittest.py | 26 +- third_party/schema/.editorconfig | 15 + third_party/schema/.gitignore | 174 +++++++++ third_party/schema/.travis.yml | 37 ++ third_party/schema/LICENSE-MIT | 19 + third_party/schema/MANIFEST.in | 1 + third_party/schema/README.chromium | 12 + third_party/schema/README.rst | 382 ++++++++++++++++++++ third_party/schema/__init__.py | 1 + third_party/schema/schema.py | 338 ++++++++++++++++++ third_party/schema/setup.cfg | 5 + third_party/schema/setup.py | 30 ++ third_party/schema/test_schema.py | 556 +++++++++++++++++++++++++++++ third_party/schema/tox.ini | 33 ++ 15 files changed, 1693 insertions(+), 8 deletions(-) create mode 100644 third_party/schema/.editorconfig create mode 100644 third_party/schema/.gitignore create mode 100644 third_party/schema/.travis.yml create mode 100644 third_party/schema/LICENSE-MIT create mode 100644 third_party/schema/MANIFEST.in create mode 100644 third_party/schema/README.chromium create mode 100644 third_party/schema/README.rst create mode 100644 third_party/schema/__init__.py create mode 100644 third_party/schema/schema.py create mode 100644 third_party/schema/setup.cfg create mode 100644 third_party/schema/setup.py create mode 100644 third_party/schema/test_schema.py create mode 100644 third_party/schema/tox.ini diff --git a/gclient_eval.py b/gclient_eval.py index 7364a9542..889eaeece 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -4,6 +4,76 @@ import ast +from third_party import schema + + +# See https://github.com/keleshev/schema for docs how to configure schema. +_GCLIENT_HOOKS_SCHEMA = [{ + # Hook action: list of command-line arguments to invoke. + 'action': [basestring], + + # Name of the hook. Doesn't affect operation. + schema.Optional('name'): basestring, + + # Hook pattern (regex). Originally intended to limit some hooks to run + # only when files matching the pattern have changed. In practice, with git, + # gclient runs all the hooks regardless of this field. + schema.Optional('pattern'): basestring, +}] + +_GCLIENT_SCHEMA = schema.Schema({ + # List of host names from which dependencies are allowed (whitelist). + # NOTE: when not present, all hosts are allowed. + # NOTE: scoped to current DEPS file, not recursive. + schema.Optional('allowed_hosts'): [basestring], + + # Mapping from paths to repo and revision to check out under that path. + # Applying this mapping to the on-disk checkout is the main purpose + # of gclient, and also why the config file is called DEPS. + # + # The following functions are allowed: + # + # File(): specifies to expect to checkout a file instead of a directory + # From(): used to fetch a dependency definition from another DEPS file + # Var(): allows variable substitution (either from 'vars' dict below, + # or command-line override) + schema.Optional('deps'): {schema.Optional(basestring): basestring}, + + # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux'). + schema.Optional('deps_os'): {basestring: {basestring: basestring}}, + + # Hooks executed after gclient sync (unless suppressed), or explicitly + # on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details. + # Also see 'pre_deps_hooks'. + schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA, + + # Rules which #includes are allowed in the directory. + # Also see 'skip_child_includes' and 'specific_include_rules'. + schema.Optional('include_rules'): [basestring], + + # Hooks executed before processing DEPS. See 'hooks' for more details. + schema.Optional('pre_deps_hooks'): _GCLIENT_HOOKS_SCHEMA, + + # Whitelists deps for which recursion should be enabled. + schema.Optional('recursedeps'): [ + schema.Or(basestring, (basestring, basestring)) + ], + + # Blacklists directories for checking 'include_rules'. + schema.Optional('skip_child_includes'): [basestring], + + # Mapping from paths to include rules specific for that path. + # See 'include_rules' for more details. + schema.Optional('specific_include_rules'): {basestring: [basestring]}, + + # For recursed-upon sub-dependencies, check out their own dependencies + # relative to the paren't path, rather than relative to the .gclient file. + schema.Optional('use_relative_paths'): bool, + + # Variables that can be referenced using Var() - see 'deps'. + schema.Optional('vars'): {basestring: basestring}, +}) + def _gclient_eval(node_or_string, global_scope, filename=''): """Safely evaluates a single expression. Returns the result.""" @@ -140,3 +210,5 @@ def Check(content, path, global_scope, expected_scope): result_scope = _gclient_exec(content, global_scope, filename=path) compare(expected_scope, result_scope, '', result_scope) + + _GCLIENT_SCHEMA.validate(result_scope) diff --git a/tests/gclient_eval_unittest.py b/tests/gclient_eval_unittest.py index adde6a720..11fca5f44 100755 --- a/tests/gclient_eval_unittest.py +++ b/tests/gclient_eval_unittest.py @@ -10,6 +10,8 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from third_party import schema + import gclient_eval @@ -75,13 +77,12 @@ class GClientExecTest(unittest.TestCase): class CheckTest(unittest.TestCase): TEST_CODE=""" -list_var = ["a", "b", "c"] +include_rules = ["a", "b", "c"] -dict_var = {"a": "1", "b": "2", "c": "3"} +vars = {"a": "1", "b": "2", "c": "3"} -nested_var = { - "list": ["a", "b", "c"], - "dict": {"a": "1", "b": "2", "c": "3"} +deps_os = { + "linux": {"a": "1", "b": "2", "c": "3"} }""" def setUp(self): @@ -92,20 +93,29 @@ nested_var = { gclient_eval.Check(self.TEST_CODE, '', {}, self.expected) def test_fail_list(self): - self.expected['list_var'][0] = 'x' + self.expected['include_rules'][0] = 'x' with self.assertRaises(gclient_eval.CheckFailure): gclient_eval.Check(self.TEST_CODE, '', {}, self.expected) def test_fail_dict(self): - self.expected['dict_var']['a'] = 'x' + self.expected['vars']['a'] = 'x' with self.assertRaises(gclient_eval.CheckFailure): gclient_eval.Check(self.TEST_CODE, '', {}, self.expected) def test_fail_nested(self): - self.expected['nested_var']['dict']['c'] = 'x' + self.expected['deps_os']['linux']['c'] = 'x' with self.assertRaises(gclient_eval.CheckFailure): gclient_eval.Check(self.TEST_CODE, '', {}, self.expected) + def test_schema_unknown_key(self): + with self.assertRaises(schema.SchemaWrongKeyError): + gclient_eval.Check('foo = "bar"', '', {}, {'foo': 'bar'}) + + def test_schema_wrong_type(self): + with self.assertRaises(schema.SchemaError): + gclient_eval.Check( + 'include_rules = {}', '', {}, {'include_rules': {}}) + if __name__ == '__main__': level = logging.DEBUG if '-v' in sys.argv else logging.FATAL diff --git a/third_party/schema/.editorconfig b/third_party/schema/.editorconfig new file mode 100644 index 000000000..76cea3a78 --- /dev/null +++ b/third_party/schema/.editorconfig @@ -0,0 +1,15 @@ +# EditorConfig file: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# 4 space indentation +[*.py] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true diff --git a/third_party/schema/.gitignore b/third_party/schema/.gitignore new file mode 100644 index 000000000..fda81fbec --- /dev/null +++ b/third_party/schema/.gitignore @@ -0,0 +1,174 @@ +*.py[co] + +# Vim +*.swp +*.swo + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +#Translations +*.mo + +#Mr Developer +.mr.developer.cfg + +# Sphinx +docs/_* + +# Created by https://www.gitignore.io/api/ython,python,osx,pycharm + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + + +### OSX ### +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio + +*.iml + +## Directory-based project format: +.idea/ +# if you remove the above rule, at least ignore the following: + +# User-specific stuff: +# .idea/workspace.xml +# .idea/tasks.xml +# .idea/dictionaries + +# Sensitive or high-churn files: +# .idea/dataSources.ids +# .idea/dataSources.xml +# .idea/sqlDataSources.xml +# .idea/dynamic.xml +# .idea/uiDesigner.xml + +# Gradle: +# .idea/gradle.xml +# .idea/libraries + +# Mongo Explorer plugin: +# .idea/mongoSettings.xml + +## File-based project format: +*.ipr +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties diff --git a/third_party/schema/.travis.yml b/third_party/schema/.travis.yml new file mode 100644 index 000000000..44c1f65fb --- /dev/null +++ b/third_party/schema/.travis.yml @@ -0,0 +1,37 @@ +# use new container-based infrastructure +sudo: false + +language: python + +addons: + apt: + sources: + - deadsnakes + packages: + - python3.5 + +cache: + pip: true + directories: + - .tox + +install: pip install codecov tox + +env: + - TOX_ENV=py26 + - TOX_ENV=py27 + - TOX_ENV=py33 + - TOX_ENV=py34 + - TOX_ENV=py35 + - TOX_ENV=pypy + - TOX_ENV=pypy3 + - TOX_ENV=coverage + - TOX_ENV=pep8 + +script: + - tox -e $TOX_ENV + +# publish coverage only after a successful build +after_success: + - codecov + diff --git a/third_party/schema/LICENSE-MIT b/third_party/schema/LICENSE-MIT new file mode 100644 index 000000000..3b2eb5cee --- /dev/null +++ b/third_party/schema/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright (c) 2012 Vladimir Keleshev, + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/schema/MANIFEST.in b/third_party/schema/MANIFEST.in new file mode 100644 index 000000000..c9834633e --- /dev/null +++ b/third_party/schema/MANIFEST.in @@ -0,0 +1 @@ +include README.rst LICENSE-MIT *.py diff --git a/third_party/schema/README.chromium b/third_party/schema/README.chromium new file mode 100644 index 000000000..d1aabf794 --- /dev/null +++ b/third_party/schema/README.chromium @@ -0,0 +1,12 @@ +Name: schema +URL: https://github.com/keleshev/schema +Version: 0.6.6 +License: MIT + +Description: +schema is a library for validating Python data structures, such as those +obtained from config-files, forms, external services or command-line parsing, +converted from JSON/YAML (or something else) to Python data-types. + +Modifications: +None. diff --git a/third_party/schema/README.rst b/third_party/schema/README.rst new file mode 100644 index 000000000..0bcb0f7fd --- /dev/null +++ b/third_party/schema/README.rst @@ -0,0 +1,382 @@ +Schema validation just got Pythonic +=============================================================================== + +**schema** is a library for validating Python data structures, such as those +obtained from config-files, forms, external services or command-line +parsing, converted from JSON/YAML (or something else) to Python data-types. + + +.. image:: https://secure.travis-ci.org/keleshev/schema.png?branch=master + :target: https://travis-ci.org/keleshev/schema + +.. image:: https://img.shields.io/codecov/c/github/keleshev/schema.svg + :target: http://codecov.io/github/keleshev/schema + +Example +---------------------------------------------------------------------------- + +Here is a quick example to get a feeling of **schema**, validating a list of +entries with personal information: + +.. code:: python + + >>> from schema import Schema, And, Use, Optional + + >>> schema = Schema([{'name': And(str, len), + ... 'age': And(Use(int), lambda n: 18 <= n <= 99), + ... Optional('sex'): And(str, Use(str.lower), + ... lambda s: s in ('male', 'female'))}]) + + >>> data = [{'name': 'Sue', 'age': '28', 'sex': 'FEMALE'}, + ... {'name': 'Sam', 'age': '42'}, + ... {'name': 'Sacha', 'age': '20', 'sex': 'Male'}] + + >>> validated = schema.validate(data) + + >>> assert validated == [{'name': 'Sue', 'age': 28, 'sex': 'female'}, + ... {'name': 'Sam', 'age': 42}, + ... {'name': 'Sacha', 'age' : 20, 'sex': 'male'}] + + +If data is valid, ``Schema.validate`` will return the validated data +(optionally converted with `Use` calls, see below). + +If data is invalid, ``Schema`` will raise ``SchemaError`` exception. + + +Installation +------------------------------------------------------------------------------- + +Use `pip `_ or easy_install:: + + pip install schema + +Alternatively, you can just drop ``schema.py`` file into your project—it is +self-contained. + +- **schema** is tested with Python 2.6, 2.7, 3.2, 3.3 and PyPy. +- **schema** follows `semantic versioning `_. + +How ``Schema`` validates data +------------------------------------------------------------------------------- + +Types +~~~~~ + +If ``Schema(...)`` encounters a type (such as ``int``, ``str``, ``object``, +etc.), it will check if the corresponding piece of data is an instance of that type, +otherwise it will raise ``SchemaError``. + +.. code:: python + + >>> from schema import Schema + + >>> Schema(int).validate(123) + 123 + + >>> Schema(int).validate('123') + Traceback (most recent call last): + ... + SchemaUnexpectedTypeError: '123' should be instance of 'int' + + >>> Schema(object).validate('hai') + 'hai' + +Callables +~~~~~~~~~ + +If ``Schema(...)`` encounters a callable (function, class, or object with +``__call__`` method) it will call it, and if its return value evaluates to +``True`` it will continue validating, else—it will raise ``SchemaError``. + +.. code:: python + + >>> import os + + >>> Schema(os.path.exists).validate('./') + './' + + >>> Schema(os.path.exists).validate('./non-existent/') + Traceback (most recent call last): + ... + SchemaError: exists('./non-existent/') should evaluate to True + + >>> Schema(lambda n: n > 0).validate(123) + 123 + + >>> Schema(lambda n: n > 0).validate(-12) + Traceback (most recent call last): + ... + SchemaError: (-12) should evaluate to True + +"Validatables" +~~~~~~~~~~~~~~ + +If ``Schema(...)`` encounters an object with method ``validate`` it will run +this method on corresponding data as ``data = obj.validate(data)``. This method +may raise ``SchemaError`` exception, which will tell ``Schema`` that that piece +of data is invalid, otherwise—it will continue validating. + +An example of "validatable" is ``Regex``, that tries to match a string or a +buffer with the given regular expression (itself as a string, buffer or +compiled regex ``SRE_Pattern``): + +.. code:: python + + >>> from schema import Regex + >>> import re + + >>> Regex(r'^foo').validate('foobar') + 'foobar' + + >>> Regex(r'^[A-Z]+$', flags=re.I).validate('those-dashes-dont-match') + Traceback (most recent call last): + ... + SchemaError: Regex('^[A-Z]+$', flags=re.IGNORECASE) does not match 'those-dashes-dont-match' + +For a more general case, you can use ``Use`` for creating such objects. +``Use`` helps to use a function or type to convert a value while validating it: + +.. code:: python + + >>> from schema import Use + + >>> Schema(Use(int)).validate('123') + 123 + + >>> Schema(Use(lambda f: open(f, 'a'))).validate('LICENSE-MIT') + + +Dropping the details, ``Use`` is basically: + +.. code:: python + + class Use(object): + + def __init__(self, callable_): + self._callable = callable_ + + def validate(self, data): + try: + return self._callable(data) + except Exception as e: + raise SchemaError('%r raised %r' % (self._callable.__name__, e)) + +Now you can write your own validation-aware classes and data types. + +Lists, similar containers +~~~~~~~~~~~~~~~~~~~~~~~~~ + +If ``Schema(...)`` encounters an instance of ``list``, ``tuple``, ``set`` or +``frozenset``, it will validate contents of corresponding data container +against schemas listed inside that container: + + +.. code:: python + + >>> Schema([1, 0]).validate([1, 1, 0, 1]) + [1, 1, 0, 1] + + >>> Schema((int, float)).validate((5, 7, 8, 'not int or float here')) + Traceback (most recent call last): + ... + SchemaError: Or(, ) did not validate 'not int or float here' + 'not int or float here' should be instance of 'float' + +Dictionaries +~~~~~~~~~~~~ + +If ``Schema(...)`` encounters an instance of ``dict``, it will validate data +key-value pairs: + +.. code:: python + + >>> d = Schema({'name': str, + ... 'age': lambda n: 18 <= n <= 99}).validate({'name': 'Sue', 'age': 28}) + + >>> assert d == {'name': 'Sue', 'age': 28} + +You can specify keys as schemas too: + +.. code:: python + + >>> schema = Schema({str: int, # string keys should have integer values + ... int: None}) # int keys should be always None + + >>> data = schema.validate({'key1': 1, 'key2': 2, + ... 10: None, 20: None}) + + >>> schema.validate({'key1': 1, + ... 10: 'not None here'}) + Traceback (most recent call last): + ... + SchemaError: Key '10' error: + None does not match 'not None here' + +This is useful if you want to check certain key-values, but don't care +about other: + +.. code:: python + + >>> schema = Schema({'': int, + ... '': Use(open), + ... str: object}) # don't care about other str keys + + >>> data = schema.validate({'': 10, + ... '': 'README.rst', + ... '--verbose': True}) + +You can mark a key as optional as follows: + +.. code:: python + + >>> from schema import Optional + >>> Schema({'name': str, + ... Optional('occupation'): str}).validate({'name': 'Sam'}) + {'name': 'Sam'} + +``Optional`` keys can also carry a ``default``, to be used when no key in the +data matches: + +.. code:: python + + >>> from schema import Optional + >>> Schema({Optional('color', default='blue'): str, + ... str: str}).validate({'texture': 'furry'} + ... ) == {'color': 'blue', 'texture': 'furry'} + True + +Defaults are used verbatim, not passed through any validators specified in the +value. + +**schema** has classes ``And`` and ``Or`` that help validating several schemas +for the same data: + +.. code:: python + + >>> from schema import And, Or + + >>> Schema({'age': And(int, lambda n: 0 < n < 99)}).validate({'age': 7}) + {'age': 7} + + >>> Schema({'password': And(str, lambda s: len(s) > 6)}).validate({'password': 'hai'}) + Traceback (most recent call last): + ... + SchemaError: Key 'password' error: + ('hai') should evaluate to True + + >>> Schema(And(Or(int, float), lambda x: x > 0)).validate(3.1415) + 3.1415 + +Extra Keys +~~~~~~~~~~ + +The ``Schema(...)`` parameter ``ignore_extra_keys`` causes validation to ignore extra keys in a dictionary, and also to not return them after validating. + +.. code:: python + + >>> schema = Schema({'name': str}, ignore_extra_keys=True) + >>> schema.validate({'name': 'Sam', 'age': '42'}) + {'name': 'Sam'} + +If you would like any extra keys returned, use ``object: object`` as one of the key/value pairs, which will match any key and any value. +Otherwise, extra keys will raise a ``SchemaError``. + +User-friendly error reporting +------------------------------------------------------------------------------- + +You can pass a keyword argument ``error`` to any of validatable classes +(such as ``Schema``, ``And``, ``Or``, ``Regex``, ``Use``) to report this error +instead of a built-in one. + +.. code:: python + + >>> Schema(Use(int, error='Invalid year')).validate('XVII') + Traceback (most recent call last): + ... + SchemaError: Invalid year + +You can see all errors that occurred by accessing exception's ``exc.autos`` +for auto-generated error messages, and ``exc.errors`` for errors +which had ``error`` text passed to them. + +You can exit with ``sys.exit(exc.code)`` if you want to show the messages +to the user without traceback. ``error`` messages are given precedence in that +case. + +A JSON API example +------------------------------------------------------------------------------- + +Here is a quick example: validation of +`create a gist `_ +request from github API. + +.. code:: python + + >>> gist = '''{"description": "the description for this gist", + ... "public": true, + ... "files": { + ... "file1.txt": {"content": "String file contents"}, + ... "other.txt": {"content": "Another file contents"}}}''' + + >>> from schema import Schema, And, Use, Optional + + >>> import json + + >>> gist_schema = Schema(And(Use(json.loads), # first convert from JSON + ... # use basestring since json returns unicode + ... {Optional('description'): basestring, + ... 'public': bool, + ... 'files': {basestring: {'content': basestring}}})) + + >>> gist = gist_schema.validate(gist) + + # gist: + {u'description': u'the description for this gist', + u'files': {u'file1.txt': {u'content': u'String file contents'}, + u'other.txt': {u'content': u'Another file contents'}}, + u'public': True} + +Using **schema** with `docopt `_ +------------------------------------------------------------------------------- + +Assume you are using **docopt** with the following usage-pattern: + + Usage: my_program.py [--count=N] ... + +and you would like to validate that ```` are readable, and that +```` exists, and that ``--count`` is either integer from 0 to 5, or +``None``. + +Assuming **docopt** returns the following dict: + +.. code:: python + + >>> args = {'': ['LICENSE-MIT', 'setup.py'], + ... '': '../', + ... '--count': '3'} + +this is how you validate it using ``schema``: + +.. code:: python + + >>> from schema import Schema, And, Or, Use + >>> import os + + >>> s = Schema({'': [Use(open)], + ... '': os.path.exists, + ... '--count': Or(None, And(Use(int), lambda n: 0 < n < 5))}) + + >>> args = s.validate(args) + + >>> args[''] + [, ] + + >>> args[''] + '../' + + >>> args['--count'] + 3 + +As you can see, **schema** validated data successfully, opened files and +converted ``'3'`` to ``int``. diff --git a/third_party/schema/__init__.py b/third_party/schema/__init__.py new file mode 100644 index 000000000..23e80ae81 --- /dev/null +++ b/third_party/schema/__init__.py @@ -0,0 +1 @@ +from .schema import * diff --git a/third_party/schema/schema.py b/third_party/schema/schema.py new file mode 100644 index 000000000..51019062e --- /dev/null +++ b/third_party/schema/schema.py @@ -0,0 +1,338 @@ +"""schema is a library for validating Python data structures, such as those +obtained from config-files, forms, external services or command-line +parsing, converted from JSON/YAML (or something else) to Python data-types.""" + +import re + +__version__ = '0.6.6' +__all__ = ['Schema', + 'And', 'Or', 'Regex', 'Optional', 'Use', + 'SchemaError', + 'SchemaWrongKeyError', + 'SchemaMissingKeyError', + 'SchemaUnexpectedTypeError'] + + +class SchemaError(Exception): + """Error during Schema validation.""" + + def __init__(self, autos, errors=None): + self.autos = autos if type(autos) is list else [autos] + self.errors = errors if type(errors) is list else [errors] + Exception.__init__(self, self.code) + + @property + def code(self): + """ + Removes duplicates values in auto and error list. + parameters. + """ + def uniq(seq): + """ + Utility function that removes duplicate. + """ + seen = set() + seen_add = seen.add + # This way removes duplicates while preserving the order. + return [x for x in seq if x not in seen and not seen_add(x)] + data_set = uniq(i for i in self.autos if i is not None) + error_list = uniq(i for i in self.errors if i is not None) + if error_list: + return '\n'.join(error_list) + return '\n'.join(data_set) + + +class SchemaWrongKeyError(SchemaError): + """Error Should be raised when an unexpected key is detected within the + data set being.""" + pass + + +class SchemaMissingKeyError(SchemaError): + """Error should be raised when a mandatory key is not found within the + data set being vaidated""" + pass + + +class SchemaUnexpectedTypeError(SchemaError): + """Error should be raised when a type mismatch is detected within the + data set being validated.""" + pass + + +class And(object): + """ + Utility function to combine validation directives in AND Boolean fashion. + """ + def __init__(self, *args, **kw): + self._args = args + assert set(kw).issubset(['error', 'schema', 'ignore_extra_keys']) + self._error = kw.get('error') + self._ignore_extra_keys = kw.get('ignore_extra_keys', False) + # You can pass your inherited Schema class. + self._schema = kw.get('schema', Schema) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, + ', '.join(repr(a) for a in self._args)) + + def validate(self, data): + """ + Validate data using defined sub schema/expressions ensuring all + values are valid. + :param data: to be validated with sub defined schemas. + :return: returns validated data + """ + for s in [self._schema(s, error=self._error, + ignore_extra_keys=self._ignore_extra_keys) + for s in self._args]: + data = s.validate(data) + return data + + +class Or(And): + """Utility function to combine validation directives in a OR Boolean + fashion.""" + def validate(self, data): + """ + Validate data using sub defined schema/expressions ensuring at least + one value is valid. + :param data: data to be validated by provided schema. + :return: return validated data if not validation + """ + x = SchemaError([], []) + for s in [self._schema(s, error=self._error, + ignore_extra_keys=self._ignore_extra_keys) + for s in self._args]: + try: + return s.validate(data) + except SchemaError as _x: + x = _x + raise SchemaError(['%r did not validate %r' % (self, data)] + x.autos, + [self._error.format(data) if self._error else None] + + x.errors) + + +class Regex(object): + """ + Enables schema.py to validate string using regular expressions. + """ + # Map all flags bits to a more readable description + NAMES = ['re.ASCII', 're.DEBUG', 're.VERBOSE', 're.UNICODE', 're.DOTALL', + 're.MULTILINE', 're.LOCALE', 're.IGNORECASE', 're.TEMPLATE'] + + def __init__(self, pattern_str, flags=0, error=None): + self._pattern_str = pattern_str + flags_list = [Regex.NAMES[i] for i, f in # Name for each bit + enumerate('{0:09b}'.format(flags)) if f != '0'] + + if flags_list: + self._flags_names = ', flags=' + '|'.join(flags_list) + else: + self._flags_names = '' + + self._pattern = re.compile(pattern_str, flags=flags) + self._error = error + + def __repr__(self): + return '%s(%r%s)' % ( + self.__class__.__name__, self._pattern_str, self._flags_names + ) + + def validate(self, data): + """ + Validated data using defined regex. + :param data: data to be validated + :return: return validated data. + """ + e = self._error + + try: + if self._pattern.search(data): + return data + else: + raise SchemaError('%r does not match %r' % (self, data), e) + except TypeError: + raise SchemaError('%r is not string nor buffer' % data, e) + + +class Use(object): + """ + For more general use cases, you can use the Use class to transform + the data while it is being validate. + """ + def __init__(self, callable_, error=None): + assert callable(callable_) + self._callable = callable_ + self._error = error + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self._callable) + + def validate(self, data): + try: + return self._callable(data) + except SchemaError as x: + raise SchemaError([None] + x.autos, + [self._error.format(data) + if self._error else None] + x.errors) + except BaseException as x: + f = _callable_str(self._callable) + raise SchemaError('%s(%r) raised %r' % (f, data, x), + self._error.format(data) + if self._error else None) + + +COMPARABLE, CALLABLE, VALIDATOR, TYPE, DICT, ITERABLE = range(6) + + +def _priority(s): + """Return priority for a given object.""" + if type(s) in (list, tuple, set, frozenset): + return ITERABLE + if type(s) is dict: + return DICT + if issubclass(type(s), type): + return TYPE + if hasattr(s, 'validate'): + return VALIDATOR + if callable(s): + return CALLABLE + else: + return COMPARABLE + + +class Schema(object): + """ + Entry point of the library, use this class to instantiate validation + schema for the data that will be validated. + """ + def __init__(self, schema, error=None, ignore_extra_keys=False): + self._schema = schema + self._error = error + self._ignore_extra_keys = ignore_extra_keys + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self._schema) + + @staticmethod + def _dict_key_priority(s): + """Return priority for a given key object.""" + if isinstance(s, Optional): + return _priority(s._schema) + 0.5 + return _priority(s) + + def validate(self, data): + Schema = self.__class__ + s = self._schema + e = self._error + i = self._ignore_extra_keys + flavor = _priority(s) + if flavor == ITERABLE: + data = Schema(type(s), error=e).validate(data) + o = Or(*s, error=e, schema=Schema, ignore_extra_keys=i) + return type(data)(o.validate(d) for d in data) + if flavor == DICT: + data = Schema(dict, error=e).validate(data) + new = type(data)() # new - is a dict of the validated values + coverage = set() # matched schema keys + # for each key and value find a schema entry matching them, if any + sorted_skeys = sorted(s, key=self._dict_key_priority) + for key, value in data.items(): + for skey in sorted_skeys: + svalue = s[skey] + try: + nkey = Schema(skey, error=e).validate(key) + except SchemaError: + pass + else: + try: + nvalue = Schema(svalue, error=e, + ignore_extra_keys=i).validate(value) + except SchemaError as x: + k = "Key '%s' error:" % nkey + raise SchemaError([k] + x.autos, [e] + x.errors) + else: + new[nkey] = nvalue + coverage.add(skey) + break + required = set(k for k in s if type(k) is not Optional) + if not required.issubset(coverage): + missing_keys = required - coverage + s_missing_keys = \ + ', '.join(repr(k) for k in sorted(missing_keys, key=repr)) + raise \ + SchemaMissingKeyError('Missing keys: ' + s_missing_keys, e) + if not self._ignore_extra_keys and (len(new) != len(data)): + wrong_keys = set(data.keys()) - set(new.keys()) + s_wrong_keys = \ + ', '.join(repr(k) for k in sorted(wrong_keys, key=repr)) + raise \ + SchemaWrongKeyError( + 'Wrong keys %s in %r' % (s_wrong_keys, data), + e.format(data) if e else None) + + # Apply default-having optionals that haven't been used: + defaults = set(k for k in s if type(k) is Optional and + hasattr(k, 'default')) - coverage + for default in defaults: + new[default.key] = default.default + + return new + if flavor == TYPE: + if isinstance(data, s): + return data + else: + raise SchemaUnexpectedTypeError( + '%r should be instance of %r' % (data, s.__name__), + e.format(data) if e else None) + if flavor == VALIDATOR: + try: + return s.validate(data) + except SchemaError as x: + raise SchemaError([None] + x.autos, [e] + x.errors) + except BaseException as x: + raise SchemaError( + '%r.validate(%r) raised %r' % (s, data, x), + self._error.format(data) if self._error else None) + if flavor == CALLABLE: + f = _callable_str(s) + try: + if s(data): + return data + except SchemaError as x: + raise SchemaError([None] + x.autos, [e] + x.errors) + except BaseException as x: + raise SchemaError( + '%s(%r) raised %r' % (f, data, x), + self._error.format(data) if self._error else None) + raise SchemaError('%s(%r) should evaluate to True' % (f, data), e) + if s == data: + return data + else: + raise SchemaError('%r does not match %r' % (s, data), + e.format(data) if e else None) + + +class Optional(Schema): + """Marker for an optional part of the validation Schema.""" + _MARKER = object() + + def __init__(self, *args, **kwargs): + default = kwargs.pop('default', self._MARKER) + super(Optional, self).__init__(*args, **kwargs) + if default is not self._MARKER: + # See if I can come up with a static key to use for myself: + if _priority(self._schema) != COMPARABLE: + raise TypeError( + 'Optional keys with defaults must have simple, ' + 'predictable values, like literal strings or ints. ' + '"%r" is too complex.' % (self._schema,)) + self.default = default + self.key = self._schema + + +def _callable_str(callable_): + if hasattr(callable_, '__name__'): + return callable_.__name__ + return str(callable_) diff --git a/third_party/schema/setup.cfg b/third_party/schema/setup.cfg new file mode 100644 index 000000000..b3b58088d --- /dev/null +++ b/third_party/schema/setup.cfg @@ -0,0 +1,5 @@ +[wheel] +universal = 1 + +[semantic_release] +version_variable = schema.py:__version__ diff --git a/third_party/schema/setup.py b/third_party/schema/setup.py new file mode 100644 index 000000000..93bad57ef --- /dev/null +++ b/third_party/schema/setup.py @@ -0,0 +1,30 @@ +from setuptools import setup + +import codecs +import schema + + +setup( + name=schema.__name__, + version=schema.__version__, + author="Vladimir Keleshev", + author_email="vladimir@keleshev.com", + description="Simple data validation library", + license="MIT", + keywords="schema json validation", + url="https://github.com/keleshev/schema", + py_modules=['schema'], + long_description=codecs.open('README.rst', 'r', 'utf-8').read(), + classifiers=[ + "Development Status :: 3 - Alpha", + "Topic :: Utilities", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: Implementation :: PyPy", + "License :: OSI Approved :: MIT License", + ], +) diff --git a/third_party/schema/test_schema.py b/third_party/schema/test_schema.py new file mode 100644 index 000000000..d040b83c1 --- /dev/null +++ b/third_party/schema/test_schema.py @@ -0,0 +1,556 @@ +from __future__ import with_statement +from collections import defaultdict, namedtuple +from operator import methodcaller +import os +import re +import sys +import copy + +from pytest import raises + +from schema import (Schema, Use, And, Or, Regex, Optional, + SchemaError, SchemaWrongKeyError, + SchemaMissingKeyError, SchemaUnexpectedTypeError) + +if sys.version_info[0] == 3: + basestring = str # Python 3 does not have basestring + unicode = str # Python 3 does not have unicode + + +SE = raises(SchemaError) + + +def ve(_): + raise ValueError() + + +def se(_): + raise SchemaError('first auto', 'first error') + + +def test_schema(): + + assert Schema(1).validate(1) == 1 + with SE: Schema(1).validate(9) + + assert Schema(int).validate(1) == 1 + with SE: Schema(int).validate('1') + assert Schema(Use(int)).validate('1') == 1 + with SE: Schema(int).validate(int) + + assert Schema(str).validate('hai') == 'hai' + with SE: Schema(str).validate(1) + assert Schema(Use(str)).validate(1) == '1' + + assert Schema(list).validate(['a', 1]) == ['a', 1] + assert Schema(dict).validate({'a': 1}) == {'a': 1} + with SE: Schema(dict).validate(['a', 1]) + + assert Schema(lambda n: 0 < n < 5).validate(3) == 3 + with SE: Schema(lambda n: 0 < n < 5).validate(-1) + + +def test_validate_file(): + assert Schema( + Use(open)).validate('LICENSE-MIT').read().startswith('Copyright') + with SE: Schema(Use(open)).validate('NON-EXISTENT') + assert Schema(os.path.exists).validate('.') == '.' + with SE: Schema(os.path.exists).validate('./non-existent/') + assert Schema(os.path.isfile).validate('LICENSE-MIT') == 'LICENSE-MIT' + with SE: Schema(os.path.isfile).validate('NON-EXISTENT') + + +def test_and(): + assert And(int, lambda n: 0 < n < 5).validate(3) == 3 + with SE: And(int, lambda n: 0 < n < 5).validate(3.33) + assert And(Use(int), lambda n: 0 < n < 5).validate(3.33) == 3 + with SE: And(Use(int), lambda n: 0 < n < 5).validate('3.33') + + +def test_or(): + assert Or(int, dict).validate(5) == 5 + assert Or(int, dict).validate({}) == {} + with SE: Or(int, dict).validate('hai') + assert Or(int).validate(4) + with SE: Or().validate(2) + + +def test_regex(): + # Simple case: validate string + assert Regex(r'foo').validate('afoot') == 'afoot' + with SE: Regex(r'bar').validate('afoot') + + # More complex case: validate string + assert Regex(r'^[a-z]+$').validate('letters') == 'letters' + with SE: + Regex(r'^[a-z]+$').validate('letters + spaces') == 'letters + spaces' + + # Validate dict key + assert (Schema({Regex(r'^foo'): str}) + .validate({'fookey': 'value'}) == {'fookey': 'value'}) + with SE: Schema({Regex(r'^foo'): str}).validate({'barkey': 'value'}) + + # Validate dict value + assert (Schema({str: Regex(r'^foo')}).validate({'key': 'foovalue'}) == + {'key': 'foovalue'}) + with SE: Schema({str: Regex(r'^foo')}).validate({'key': 'barvalue'}) + + # Error if the value does not have a buffer interface + with SE: Regex(r'bar').validate(1) + with SE: Regex(r'bar').validate({}) + with SE: Regex(r'bar').validate([]) + with SE: Regex(r'bar').validate(None) + + # Validate that the pattern has a buffer interface + assert Regex(re.compile(r'foo')).validate('foo') == 'foo' + assert Regex(unicode('foo')).validate('foo') == 'foo' + with raises(TypeError): Regex(1).validate('bar') + with raises(TypeError): Regex({}).validate('bar') + with raises(TypeError): Regex([]).validate('bar') + with raises(TypeError): Regex(None).validate('bar') + + +def test_validate_list(): + assert Schema([1, 0]).validate([1, 0, 1, 1]) == [1, 0, 1, 1] + assert Schema([1, 0]).validate([]) == [] + with SE: Schema([1, 0]).validate(0) + with SE: Schema([1, 0]).validate([2]) + assert And([1, 0], lambda l: len(l) > 2).validate([0, 1, 0]) == [0, 1, 0] + with SE: And([1, 0], lambda l: len(l) > 2).validate([0, 1]) + + +def test_list_tuple_set_frozenset(): + assert Schema([int]).validate([1, 2]) + with SE: Schema([int]).validate(['1', 2]) + assert Schema(set([int])).validate(set([1, 2])) == set([1, 2]) + with SE: Schema(set([int])).validate([1, 2]) # not a set + with SE: Schema(set([int])).validate(['1', 2]) + assert Schema(tuple([int])).validate(tuple([1, 2])) == tuple([1, 2]) + with SE: Schema(tuple([int])).validate([1, 2]) # not a set + + +def test_strictly(): + assert Schema(int).validate(1) == 1 + with SE: Schema(int).validate('1') + + +def test_dict(): + assert Schema({'key': 5}).validate({'key': 5}) == {'key': 5} + with SE: Schema({'key': 5}).validate({'key': 'x'}) + with SE: Schema({'key': 5}).validate(['key', 5]) + assert Schema({'key': int}).validate({'key': 5}) == {'key': 5} + assert Schema({'n': int, 'f': float}).validate( + {'n': 5, 'f': 3.14}) == {'n': 5, 'f': 3.14} + with SE: Schema({'n': int, 'f': float}).validate( + {'n': 3.14, 'f': 5}) + with SE: + try: + Schema({}).validate({'abc': None, 1: None}) + except SchemaWrongKeyError as e: + assert e.args[0].startswith("Wrong keys 'abc', 1 in") + raise + with SE: + try: + Schema({'key': 5}).validate({}) + except SchemaMissingKeyError as e: + assert e.args[0] == "Missing keys: 'key'" + raise + with SE: + try: + Schema({'key': 5}).validate({'n': 5}) + except SchemaMissingKeyError as e: + assert e.args[0] == "Missing keys: 'key'" + raise + with SE: + try: + Schema({}).validate({'n': 5}) + except SchemaWrongKeyError as e: + assert e.args[0] == "Wrong keys 'n' in {'n': 5}" + raise + with SE: + try: + Schema({'key': 5}).validate({'key': 5, 'bad': 5}) + except SchemaWrongKeyError as e: + assert e.args[0] in ["Wrong keys 'bad' in {'key': 5, 'bad': 5}", + "Wrong keys 'bad' in {'bad': 5, 'key': 5}"] + raise + with SE: + try: + Schema({}).validate({'a': 5, 'b': 5}) + except SchemaError as e: + assert e.args[0] in ["Wrong keys 'a', 'b' in {'a': 5, 'b': 5}", + "Wrong keys 'a', 'b' in {'b': 5, 'a': 5}"] + raise + + with SE: + try: + Schema({int: int}).validate({'': ''}) + except SchemaUnexpectedTypeError as e: + assert e.args[0] in ["'' should be instance of 'int'"] + + +def test_dict_keys(): + assert Schema({str: int}).validate( + {'a': 1, 'b': 2}) == {'a': 1, 'b': 2} + with SE: Schema({str: int}).validate({1: 1, 'b': 2}) + assert Schema({Use(str): Use(int)}).validate( + {1: 3.14, 3.14: 1}) == {'1': 3, '3.14': 1} + + +def test_ignore_extra_keys(): + assert Schema({'key': 5}, ignore_extra_keys=True).validate( + {'key': 5, 'bad': 4}) == {'key': 5} + assert Schema({'key': 5, 'dk': {'a': 'a'}}, ignore_extra_keys=True).validate( + {'key': 5, 'bad': 'b', 'dk': {'a': 'a', 'bad': 'b'}}) == \ + {'key': 5, 'dk': {'a': 'a'}} + assert Schema([{'key': 'v'}], ignore_extra_keys=True).validate( + [{'key': 'v', 'bad': 'bad'}]) == [{'key': 'v'}] + assert Schema([{'key': 'v'}], ignore_extra_keys=True).validate( + [{'key': 'v', 'bad': 'bad'}]) == [{'key': 'v'}] + + +def test_ignore_extra_keys_validation_and_return_keys(): + assert Schema({'key': 5, object: object}, ignore_extra_keys=True).validate( + {'key': 5, 'bad': 4}) == {'key': 5, 'bad': 4} + assert Schema({'key': 5, 'dk': {'a': 'a', object: object}}, + ignore_extra_keys=True).validate( + {'key': 5, 'dk': {'a': 'a', 'bad': 'b'}}) == \ + {'key': 5, 'dk': {'a': 'a', 'bad': 'b'}} + + +def test_dict_optional_keys(): + with SE: Schema({'a': 1, 'b': 2}).validate({'a': 1}) + assert Schema({'a': 1, Optional('b'): 2}).validate({'a': 1}) == {'a': 1} + assert Schema({'a': 1, Optional('b'): 2}).validate( + {'a': 1, 'b': 2}) == {'a': 1, 'b': 2} + # Make sure Optionals are favored over types: + assert Schema({basestring: 1, + Optional('b'): 2}).validate({'a': 1, 'b': 2}) == {'a': 1, 'b': 2} + + +def test_dict_optional_defaults(): + # Optionals fill out their defaults: + assert Schema({Optional('a', default=1): 11, + Optional('b', default=2): 22}).validate({'a': 11}) == {'a': 11, 'b': 2} + + # Optionals take precedence over types. Here, the "a" is served by the + # Optional: + assert Schema({Optional('a', default=1): 11, + basestring: 22}).validate({'b': 22}) == {'a': 1, 'b': 22} + + with raises(TypeError): + Optional(And(str, Use(int)), default=7) + + +def test_dict_subtypes(): + d = defaultdict(int, key=1) + v = Schema({'key': 1}).validate(d) + assert v == d + assert isinstance(v, defaultdict) + # Please add tests for Counter and OrderedDict once support for Python2.6 + # is dropped! + + +def test_dict_key_error(): + try: + Schema({'k': int}).validate({'k': 'x'}) + except SchemaError as e: + assert e.code == "Key 'k' error:\n'x' should be instance of 'int'" + try: + Schema({'k': {'k2': int}}).validate({'k': {'k2': 'x'}}) + except SchemaError as e: + code = "Key 'k' error:\nKey 'k2' error:\n'x' should be instance of 'int'" + assert e.code == code + try: + Schema({'k': {'k2': int}}, error='k2 should be int').validate({'k': {'k2': 'x'}}) + except SchemaError as e: + assert e.code == 'k2 should be int' + + +def test_complex(): + s = Schema({'': And([Use(open)], lambda l: len(l)), + '': os.path.exists, + Optional('--count'): And(int, lambda n: 0 <= n <= 5)}) + data = s.validate({'': ['./LICENSE-MIT'], '': './'}) + assert len(data) == 2 + assert len(data['']) == 1 + assert data[''][0].read().startswith('Copyright') + assert data[''] == './' + + +def test_nice_errors(): + try: + Schema(int, error='should be integer').validate('x') + except SchemaError as e: + assert e.errors == ['should be integer'] + try: + Schema(Use(float), error='should be a number').validate('x') + except SchemaError as e: + assert e.code == 'should be a number' + try: + Schema({Optional('i'): Use(int, error='should be a number')}).validate({'i': 'x'}) + except SchemaError as e: + assert e.code == 'should be a number' + + +def test_use_error_handling(): + try: + Use(ve).validate('x') + except SchemaError as e: + assert e.autos == ["ve('x') raised ValueError()"] + assert e.errors == [None] + try: + Use(ve, error='should not raise').validate('x') + except SchemaError as e: + assert e.autos == ["ve('x') raised ValueError()"] + assert e.errors == ['should not raise'] + try: + Use(se).validate('x') + except SchemaError as e: + assert e.autos == [None, 'first auto'] + assert e.errors == [None, 'first error'] + try: + Use(se, error='second error').validate('x') + except SchemaError as e: + assert e.autos == [None, 'first auto'] + assert e.errors == ['second error', 'first error'] + + +def test_or_error_handling(): + try: + Or(ve).validate('x') + except SchemaError as e: + assert e.autos[0].startswith('Or(') + assert e.autos[0].endswith(") did not validate 'x'") + assert e.autos[1] == "ve('x') raised ValueError()" + assert len(e.autos) == 2 + assert e.errors == [None, None] + try: + Or(ve, error='should not raise').validate('x') + except SchemaError as e: + assert e.autos[0].startswith('Or(') + assert e.autos[0].endswith(") did not validate 'x'") + assert e.autos[1] == "ve('x') raised ValueError()" + assert len(e.autos) == 2 + assert e.errors == ['should not raise', 'should not raise'] + try: + Or('o').validate('x') + except SchemaError as e: + assert e.autos == ["Or('o') did not validate 'x'", + "'o' does not match 'x'"] + assert e.errors == [None, None] + try: + Or('o', error='second error').validate('x') + except SchemaError as e: + assert e.autos == ["Or('o') did not validate 'x'", + "'o' does not match 'x'"] + assert e.errors == ['second error', 'second error'] + + +def test_and_error_handling(): + try: + And(ve).validate('x') + except SchemaError as e: + assert e.autos == ["ve('x') raised ValueError()"] + assert e.errors == [None] + try: + And(ve, error='should not raise').validate('x') + except SchemaError as e: + assert e.autos == ["ve('x') raised ValueError()"] + assert e.errors == ['should not raise'] + try: + And(str, se).validate('x') + except SchemaError as e: + assert e.autos == [None, 'first auto'] + assert e.errors == [None, 'first error'] + try: + And(str, se, error='second error').validate('x') + except SchemaError as e: + assert e.autos == [None, 'first auto'] + assert e.errors == ['second error', 'first error'] + + +def test_schema_error_handling(): + try: + Schema(Use(ve)).validate('x') + except SchemaError as e: + assert e.autos == [None, "ve('x') raised ValueError()"] + assert e.errors == [None, None] + try: + Schema(Use(ve), error='should not raise').validate('x') + except SchemaError as e: + assert e.autos == [None, "ve('x') raised ValueError()"] + assert e.errors == ['should not raise', None] + try: + Schema(Use(se)).validate('x') + except SchemaError as e: + assert e.autos == [None, None, 'first auto'] + assert e.errors == [None, None, 'first error'] + try: + Schema(Use(se), error='second error').validate('x') + except SchemaError as e: + assert e.autos == [None, None, 'first auto'] + assert e.errors == ['second error', None, 'first error'] + + +def test_use_json(): + import json + gist_schema = Schema(And(Use(json.loads), # first convert from JSON + {Optional('description'): basestring, + 'public': bool, + 'files': {basestring: {'content': basestring}}})) + gist = '''{"description": "the description for this gist", + "public": true, + "files": { + "file1.txt": {"content": "String file contents"}, + "other.txt": {"content": "Another file contents"}}}''' + assert gist_schema.validate(gist) + + +def test_error_reporting(): + s = Schema({'': [Use(open, error=' should be readable')], + '': And(os.path.exists, error=' should exist'), + '--count': Or(None, And(Use(int), lambda n: 0 < n < 5), + error='--count should be integer 0 < n < 5')}, + error='Error:') + s.validate({'': [], '': './', '--count': 3}) + + try: + s.validate({'': [], '': './', '--count': '10'}) + except SchemaError as e: + assert e.code == 'Error:\n--count should be integer 0 < n < 5' + try: + s.validate({'': [], '': './hai', '--count': '2'}) + except SchemaError as e: + assert e.code == 'Error:\n should exist' + try: + s.validate({'': ['hai'], '': './', '--count': '2'}) + except SchemaError as e: + assert e.code == 'Error:\n should be readable' + + +def test_schema_repr(): # what about repr with `error`s? + schema = Schema([Or(None, And(str, Use(float)))]) + repr_ = "Schema([Or(None, And(, Use()))])" + # in Python 3 repr contains , not + assert repr(schema).replace('class', 'type') == repr_ + + +def test_validate_object(): + schema = Schema({object: str}) + assert schema.validate({42: 'str'}) == {42: 'str'} + with SE: schema.validate({42: 777}) + + +def test_issue_9_prioritized_key_comparison(): + validate = Schema({'key': 42, object: 42}).validate + assert validate({'key': 42, 777: 42}) == {'key': 42, 777: 42} + + +def test_issue_9_prioritized_key_comparison_in_dicts(): + # http://stackoverflow.com/questions/14588098/docopt-schema-validation + s = Schema({'ID': Use(int, error='ID should be an int'), + 'FILE': Or(None, Use(open, error='FILE should be readable')), + Optional(str): object}) + data = {'ID': 10, 'FILE': None, 'other': 'other', 'other2': 'other2'} + assert s.validate(data) == data + data = {'ID': 10, 'FILE': None} + assert s.validate(data) == data + + +def test_missing_keys_exception_with_non_str_dict_keys(): + s = Schema({And(str, Use(str.lower), 'name'): And(str, len)}) + with SE: s.validate(dict()) + with SE: + try: + Schema({1: 'x'}).validate(dict()) + except SchemaMissingKeyError as e: + assert e.args[0] == "Missing keys: 1" + raise + + +def test_issue_56_cant_rely_on_callables_to_have_name(): + s = Schema(methodcaller('endswith', '.csv')) + assert s.validate('test.csv') == 'test.csv' + with SE: + try: + s.validate('test.py') + except SchemaError as e: + assert "operator.methodcaller" in e.args[0] + raise + + +def test_exception_handling_with_bad_validators(): + BadValidator = namedtuple("BadValidator", ["validate"]) + s = Schema(BadValidator("haha")) + with SE: + try: + s.validate("test") + except SchemaError as e: + assert "TypeError" in e.args[0] + raise + + +def test_issue_83_iterable_validation_return_type(): + TestSetType = type("TestSetType", (set,), dict()) + data = TestSetType(["test", "strings"]) + s = Schema(set([str])) + assert isinstance(s.validate(data), TestSetType) + + +def test_optional_key_convert_failed_randomly_while_with_another_optional_object(): + """ + In this test, created_at string "2015-10-10 00:00:00" is expected to be converted + to a datetime instance. + - it works when the schema is + + s = Schema({ + 'created_at': _datetime_validator, + Optional(basestring): object, + }) + + - but when wrapping the key 'created_at' with Optional, it fails randomly + :return: + """ + import datetime + fmt = '%Y-%m-%d %H:%M:%S' + _datetime_validator = Or(None, Use(lambda i: datetime.datetime.strptime(i, fmt))) + # FIXME given tests enough + for i in range(1024): + s = Schema({ + Optional('created_at'): _datetime_validator, + Optional('updated_at'): _datetime_validator, + Optional('birth'): _datetime_validator, + Optional(basestring): object, + }) + data = { + 'created_at': '2015-10-10 00:00:00' + } + validated_data = s.validate(data) + # is expected to be converted to a datetime instance, but fails randomly + # (most of the time) + assert isinstance(validated_data['created_at'], datetime.datetime) + # assert isinstance(validated_data['created_at'], basestring) + + +def test_copy(): + s1 = SchemaError('a', None) + s2 = copy.deepcopy(s1) + assert s1 is not s2 + assert type(s1) is type(s2) + + +def test_inheritance(): + def convert(data): + if isinstance(data, int): + return data + 1 + return data + + class MySchema(Schema): + def validate(self, data): + return super(MySchema, self).validate(convert(data)) + + s = {'k': int, 'd': {'k': int, 'l': [{'l': [int]}]}} + v = {'k': 1, 'd': {'k': 2, 'l': [{'l': [3, 4, 5]}]}} + d = MySchema(s).validate(v) + assert d['k'] == 2 and d['d']['k'] == 3 and d['d']['l'][0]['l'] == [4, 5, 6] diff --git a/third_party/schema/tox.ini b/third_party/schema/tox.ini new file mode 100644 index 000000000..bc858eddb --- /dev/null +++ b/third_party/schema/tox.ini @@ -0,0 +1,33 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests in +# multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip +# install tox" and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py32, py33, py34, py35, pypy, pypy3, coverage, pep8 + +[testenv] +commands = py.test +deps = pytest + + +[testenv:py27] +commands = py.test --doctest-glob=README.rst # test documentation +deps = pytest + +[testenv:pep8] +# pep8 disabled for E701 (multiple statements on one line) and E126 (continuation line over-indented for hanging indent) +commands = flake8 --max-line-length=90 --show-source -v --count --ignore=E701,E126 +deps = flake8 + +[testenv:coverage] +#TODO: how to force this on py27? +commands = coverage erase + py.test --doctest-glob=README.rst --cov schema + coverage report -m +deps = pytest + pytest-cov + coverage + +[flake8] +exclude=.venv,.git,.tox