# HG changeset patch # User Ted Mielczarek # Date 1533552333 14400 # Mon Aug 06 06:45:33 2018 -0400 # Node ID a077bf122c05f9643ad66280d6b395b1751eff36 # Parent 5ca6befe8b6f5d64addfd0145ad0ad423d4373b5 bug 1461992 - update vendored copy of voluptuous to 0.11.5. r=gps voluptuous 0.11.1 added support for a `description` argument for Required and Optional objects, which is useful for adding descriptions in the schema that we can persist when converting it to json-schema format. This patch vendors the current version of voluptuous, which is 0.11.5. MozReview-Commit-ID: 2qt1KE8MPYR Differential Revision: https://phabricator.services.mozilla.com/D2839 diff --git a/Pipfile b/Pipfile --- a/Pipfile +++ b/Pipfile @@ -11,9 +11,9 @@ blessings = "==1.7" jsmin = "==2.1.0" json-e = "==2.5.0" pipenv = "==2018.5.18" pytest = "==3.6.2" python-hglib = "==2.4" requests = "==2.9.1" six = "==1.10.0" virtualenv = "==15.2.0" -voluptuous = "==0.10.5" +voluptuous = "==0.11.5" diff --git a/Pipfile.lock b/Pipfile.lock --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,12 +1,12 @@ { "_meta": { "hash": { - "sha256": "609a35f65e9a4c07e0e1473ec982c6b5028622e9a795b6cfb8555ad8574804f3" + "sha256": "f718e0b6ec2c030d4becf157f8ca0fd1b2f32ca277d5d3d2407a2dee33119441" }, "pipfile-spec": 6, "requires": {}, "sources": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true @@ -64,21 +64,21 @@ "hashes": [ "sha256:f9114a25ed4b575395fbb2daa1183c5b781a647b387fdf28596220bb114673e8" ], "index": "pypi", "version": "==2.5.0" }, "more-itertools": { "hashes": [ - "sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8", - "sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3", - "sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0" + "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", + "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", + "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" ], - "version": "==4.2.0" + "version": "==4.3.0" }, "pipenv": { "hashes": [ "sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece", "sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741" ], "index": "pypi", "version": "==2018.5.18" @@ -141,16 +141,17 @@ "hashes": [ "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7", "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8" ], "version": "==0.3.0" }, "voluptuous": { "hashes": [ - "sha256:7a7466f8dc3666a292d186d1d871a47bf2120836ccb900d5ba904674957a2396" + "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1", + "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef" ], "index": "pypi", - "version": "==0.10.5" + "version": "==0.11.5" } }, "develop": {} } diff --git a/third_party/python/more-itertools/MANIFEST.in b/third_party/python/more-itertools/MANIFEST.in --- a/third_party/python/more-itertools/MANIFEST.in +++ b/third_party/python/more-itertools/MANIFEST.in @@ -1,8 +1,9 @@ include README.rst include LICENSE include docs/*.rst include docs/Makefile include docs/make.bat include docs/conf.py +include docs/_static/* include fabfile.py include tox.ini diff --git a/third_party/python/more-itertools/PKG-INFO b/third_party/python/more-itertools/PKG-INFO --- a/third_party/python/more-itertools/PKG-INFO +++ b/third_party/python/more-itertools/PKG-INFO @@ -1,11 +1,11 @@ Metadata-Version: 1.1 Name: more-itertools -Version: 4.2.0 +Version: 4.3.0 Summary: More routines for operating on iterables, beyond itertools Home-page: https://github.com/erikrose/more-itertools Author: Erik Rose Author-email: erikrose@grinchcentral.com License: MIT Description: ============== More Itertools ============== @@ -13,16 +13,111 @@ Description: ============== .. image:: https://coveralls.io/repos/github/erikrose/more-itertools/badge.svg?branch=master :target: https://coveralls.io/github/erikrose/more-itertools?branch=master Python's ``itertools`` library is a gem - you can compose elegant solutions for a variety of problems with the functions it provides. In ``more-itertools`` we collect additional building blocks, recipes, and routines for working with Python iterables. + ---- + + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Grouping | `chunked `_, | + | | `sliced `_, | + | | `distribute `_, | + | | `divide `_, | + | | `split_at `_, | + | | `split_before `_, | + | | `split_after `_, | + | | `bucket `_, | + | | `grouper `_, | + | | `partition `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Lookahead and lookback | `spy `_, | + | | `peekable `_, | + | | `seekable `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Windowing | `windowed `_, | + | | `stagger `_, | + | | `pairwise `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Augmenting | `count_cycle `_, | + | | `intersperse `_, | + | | `padded `_, | + | | `adjacent `_, | + | | `groupby_transform `_, | + | | `padnone `_, | + | | `ncycles `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Combining | `collapse `_, | + | | `sort_together `_, | + | | `interleave `_, | + | | `interleave_longest `_, | + | | `collate `_, | + | | `zip_offset `_, | + | | `dotproduct `_, | + | | `flatten `_, | + | | `roundrobin `_, | + | | `prepend `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Summarizing | `ilen `_, | + | | `first `_, | + | | `last `_, | + | | `one `_, | + | | `unique_to_each `_, | + | | `locate `_, | + | | `rlocate `_, | + | | `consecutive_groups `_, | + | | `exactly_n `_, | + | | `run_length `_, | + | | `map_reduce `_, | + | | `all_equal `_, | + | | `first_true `_, | + | | `nth `_, | + | | `quantify `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Selecting | `islice_extended `_, | + | | `strip `_, | + | | `lstrip `_, | + | | `rstrip `_, | + | | `take `_, | + | | `tail `_, | + | | `unique_everseen `_, | + | | `unique_justseen `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Combinatorics | `distinct_permutations `_, | + | | `circular_shifts `_, | + | | `powerset `_, | + | | `random_product `_, | + | | `random_permutation `_, | + | | `random_combination `_, | + | | `random_combination_with_replacement `_, | + | | `nth_combination `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Wrapping | `always_iterable `_, | + | | `consumer `_, | + | | `with_iter `_, | + | | `iter_except `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Others | `replace `_, | + | | `numeric_range `_, | + | | `always_reversible `_, | + | | `side_effect `_, | + | | `iterate `_, | + | | `difference `_, | + | | `make_decorator `_, | + | | `SequenceView `_, | + | | `consume `_, | + | | `accumulate `_, | + | | `tabulate `_, | + | | `repeatfunc `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + Getting started =============== To get started, install the library with `pip `_: .. code-block:: shell pip install more-itertools @@ -67,16 +162,30 @@ Description: ============== repository. Thanks for contributing! Version History =============== + 4.3.0 + ----- + + * New itertools: + * last (thanks to tmshn) + * replace (thanks to pylang) + * rlocate (thanks to jferard and pylang) + + * Improvements to existing itertools: + * locate can now search for multiple items + + * Other changes: + * The docs now include a nice table of tools (thanks MSeifert04) + 4.2.0 ----- * New itertools: * map_reduce (thanks to pylang) * prepend (from the `Python 3.7 docs `_) * Improvements to existing itertools: diff --git a/third_party/python/more-itertools/README.rst b/third_party/python/more-itertools/README.rst --- a/third_party/python/more-itertools/README.rst +++ b/third_party/python/more-itertools/README.rst @@ -5,16 +5,111 @@ More Itertools .. image:: https://coveralls.io/repos/github/erikrose/more-itertools/badge.svg?branch=master :target: https://coveralls.io/github/erikrose/more-itertools?branch=master Python's ``itertools`` library is a gem - you can compose elegant solutions for a variety of problems with the functions it provides. In ``more-itertools`` we collect additional building blocks, recipes, and routines for working with Python iterables. +---- + ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Grouping | `chunked `_, | +| | `sliced `_, | +| | `distribute `_, | +| | `divide `_, | +| | `split_at `_, | +| | `split_before `_, | +| | `split_after `_, | +| | `bucket `_, | +| | `grouper `_, | +| | `partition `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Lookahead and lookback | `spy `_, | +| | `peekable `_, | +| | `seekable `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Windowing | `windowed `_, | +| | `stagger `_, | +| | `pairwise `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Augmenting | `count_cycle `_, | +| | `intersperse `_, | +| | `padded `_, | +| | `adjacent `_, | +| | `groupby_transform `_, | +| | `padnone `_, | +| | `ncycles `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Combining | `collapse `_, | +| | `sort_together `_, | +| | `interleave `_, | +| | `interleave_longest `_, | +| | `collate `_, | +| | `zip_offset `_, | +| | `dotproduct `_, | +| | `flatten `_, | +| | `roundrobin `_, | +| | `prepend `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Summarizing | `ilen `_, | +| | `first `_, | +| | `last `_, | +| | `one `_, | +| | `unique_to_each `_, | +| | `locate `_, | +| | `rlocate `_, | +| | `consecutive_groups `_, | +| | `exactly_n `_, | +| | `run_length `_, | +| | `map_reduce `_, | +| | `all_equal `_, | +| | `first_true `_, | +| | `nth `_, | +| | `quantify `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Selecting | `islice_extended `_, | +| | `strip `_, | +| | `lstrip `_, | +| | `rstrip `_, | +| | `take `_, | +| | `tail `_, | +| | `unique_everseen `_, | +| | `unique_justseen `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Combinatorics | `distinct_permutations `_, | +| | `circular_shifts `_, | +| | `powerset `_, | +| | `random_product `_, | +| | `random_permutation `_, | +| | `random_combination `_, | +| | `random_combination_with_replacement `_, | +| | `nth_combination `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Wrapping | `always_iterable `_, | +| | `consumer `_, | +| | `with_iter `_, | +| | `iter_except `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Others | `replace `_, | +| | `numeric_range `_, | +| | `always_reversible `_, | +| | `side_effect `_, | +| | `iterate `_, | +| | `difference `_, | +| | `make_decorator `_, | +| | `SequenceView `_, | +| | `consume `_, | +| | `accumulate `_, | +| | `tabulate `_, | +| | `repeatfunc `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + Getting started =============== To get started, install the library with `pip `_: .. code-block:: shell pip install more-itertools diff --git a/third_party/python/more-itertools/docs/_static/theme_overrides.css b/third_party/python/more-itertools/docs/_static/theme_overrides.css new file mode 100644 --- /dev/null +++ b/third_party/python/more-itertools/docs/_static/theme_overrides.css @@ -0,0 +1,14 @@ +/* https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html */ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} diff --git a/third_party/python/more-itertools/docs/api.rst b/third_party/python/more-itertools/docs/api.rst --- a/third_party/python/more-itertools/docs/api.rst +++ b/third_party/python/more-itertools/docs/api.rst @@ -119,19 +119,21 @@ Summarizing These tools return summarized or aggregated data from an iterable. ---- **New itertools** .. autofunction:: ilen .. autofunction:: first(iterable[, default]) +.. autofunction:: last(iterable[, default]) .. autofunction:: one .. autofunction:: unique_to_each .. autofunction:: locate(iterable, pred=bool) +.. autofunction:: rlocate(iterable, pred=bool) .. autofunction:: consecutive_groups(iterable, ordering=lambda x: x) .. autofunction:: exactly_n(iterable, n, predicate=bool) .. autoclass:: run_length .. autofunction:: map_reduce ---- **Itertools recipes** @@ -211,16 +213,17 @@ consume iterables. .. autofunction:: iter_except Others ====== **New itertools** +.. autofunction:: replace .. autofunction:: numeric_range(start, stop, step) .. autofunction:: always_reversible .. autofunction:: side_effect .. autofunction:: iterate .. autofunction:: difference(iterable, func=operator.sub) .. autofunction:: make_decorator .. autoclass:: SequenceView diff --git a/third_party/python/more-itertools/docs/conf.py b/third_party/python/more-itertools/docs/conf.py --- a/third_party/python/more-itertools/docs/conf.py +++ b/third_party/python/more-itertools/docs/conf.py @@ -45,17 +45,17 @@ master_doc = 'index' project = u'more-itertools' copyright = u'2012, Erik Rose' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '4.2.0' +version = '4.3.0' # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some @@ -119,16 +119,21 @@ html_theme_path = [sphinx_rtd_theme.get_ # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_context = { + # https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html + 'css_files': ['_static/theme_overrides.css'], +} + # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True diff --git a/third_party/python/more-itertools/docs/versions.rst b/third_party/python/more-itertools/docs/versions.rst --- a/third_party/python/more-itertools/docs/versions.rst +++ b/third_party/python/more-itertools/docs/versions.rst @@ -1,14 +1,28 @@ =============== Version History =============== .. automodule:: more_itertools +4.3.0 +----- + +* New itertools: + * :func:`last` (thanks to tmshn) + * :func:`replace` (thanks to pylang) + * :func:`rlocate` (thanks to jferard and pylang) + +* Improvements to existing itertools: + * :func:`locate` can now search for multiple items + +* Other changes: + * The docs now include a nice table of tools (thanks MSeifert04) + 4.2.0 ----- * New itertools: * :func:`map_reduce` (thanks to pylang) * :func:`prepend` (from the `Python 3.7 docs `_) * Improvements to existing itertools: diff --git a/third_party/python/more-itertools/more_itertools/more.py b/third_party/python/more-itertools/more_itertools/more.py --- a/third_party/python/more-itertools/more_itertools/more.py +++ b/third_party/python/more-itertools/more_itertools/more.py @@ -7,16 +7,17 @@ from itertools import ( chain, compress, count, cycle, dropwhile, groupby, islice, repeat, + starmap, takewhile, tee ) from operator import itemgetter, lt, gt, sub from sys import maxsize, version_info try: from collections.abc import Sequence except ImportError: @@ -47,24 +48,27 @@ from .recipes import consume, flatten, t 'first', 'groupby_transform', 'ilen', 'interleave_longest', 'interleave', 'intersperse', 'islice_extended', 'iterate', + 'last', 'locate', 'lstrip', 'make_decorator', 'map_reduce', 'numeric_range', 'one', 'padded', 'peekable', + 'replace', + 'rlocate', 'rstrip', 'run_length', 'seekable', 'SequenceView', 'side_effect', 'sliced', 'sort_together', 'split_at', @@ -131,16 +135,42 @@ def first(iterable, default=_marker): # want to do something different with flow control when I raise the # exception, and it's weird to explicitly catch StopIteration. if default is _marker: raise ValueError('first() was called on an empty iterable, and no ' 'default value was provided.') return default +def last(iterable, default=_marker): + """Return the last item of *iterable*, or *default* if *iterable* is + empty. + + >>> last([0, 1, 2, 3]) + 3 + >>> last([], 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + """ + try: + try: + # Try to access the last item directly + return iterable[-1] + except (TypeError, AttributeError, KeyError): + # If not slice-able, iterate entirely using length-1 deque + return deque(iterable, maxlen=1)[0] + except IndexError: # If the iterable was empty + if default is _marker: + raise ValueError('last() was called on an empty iterable, and no ' + 'default value was provided.') + return default + + class peekable(object): """Wrap an iterator to allow lookahead and prepending elements. Call :meth:`peek` on the result to get the value that will be returned by :func:`next`. This won't advance the iterator: >>> p = peekable(['a', 'b']) >>> p.peek() @@ -1430,38 +1460,37 @@ def count_cycle(iterable, n=None): """ iterable = tuple(iterable) if not iterable: return iter(()) counter = count() if n is None else range(n) return ((i, item) for i in counter for item in iterable) -def locate(iterable, pred=bool): +def locate(iterable, pred=bool, window_size=None): """Yield the index of each item in *iterable* for which *pred* returns ``True``. *pred* defaults to :func:`bool`, which will select truthy items: >>> list(locate([0, 1, 1, 0, 1, 0, 0])) [1, 2, 4] Set *pred* to a custom function to, e.g., find the indexes for a particular - item: + item. >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) [1, 3] - Use with :func:`windowed` to find the indexes of a sub-sequence: - - >>> from more_itertools import windowed + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> sub = [1, 2, 3] - >>> pred = lambda w: w == tuple(sub) # windowed() returns tuples - >>> list(locate(windowed(iterable, len(sub)), pred=pred)) + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(locate(iterable, pred=pred, window_size=3)) [1, 5, 9] Use with :func:`seekable` to find indexes and then retrieve the associated items: >>> from itertools import count >>> from more_itertools import seekable >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) @@ -1469,17 +1498,24 @@ def locate(iterable, pred=bool): >>> pred = lambda x: x > 100 >>> indexes = locate(it, pred=pred) >>> i = next(indexes) >>> it.seek(i) >>> next(it) 106 """ - return compress(count(), map(pred, iterable)) + if window_size is None: + return compress(count(), map(pred, iterable)) + + if window_size < 1: + raise ValueError('window size must be at least 1') + + it = windowed(iterable, window_size, fillvalue=_marker) + return compress(count(), starmap(pred, it)) def lstrip(iterable, pred): """Yield the items from *iterable*, but strip any from the beginning for which *pred* returns ``True``. For example, to remove a set of items from the start of an iterable: @@ -2027,17 +2063,17 @@ def map_reduce(iterable, keyfunc, valuef >>> keyfunc = lambda x: x.upper() >>> valuefunc = lambda x: 1 >>> reducefunc = sum >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) >>> sorted(result.items()) [('A', 1), ('B', 2), ('C', 3)] You may want to filter the input iterable before applying the map/reduce - proecdure: + procedure: >>> all_items = range(30) >>> items = [x for x in all_items if 10 <= x <= 20] # Filter >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 >>> categories = map_reduce(items, keyfunc=keyfunc) >>> sorted(categories.items()) [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) @@ -2061,8 +2097,115 @@ def map_reduce(iterable, keyfunc, valuef ret[key].append(value) if reducefunc is not None: for key, value_list in ret.items(): ret[key] = reducefunc(value_list) ret.default_factory = None return ret + + +def rlocate(iterable, pred=bool, window_size=None): + """Yield the index of each item in *iterable* for which *pred* returns + ``True``, starting from the right and moving left. + + *pred* defaults to :func:`bool`, which will select truthy items: + + >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 + [4, 2, 1] + + Set *pred* to a custom function to, e.g., find the indexes for a particular + item: + + >>> iterable = iter('abcb') + >>> pred = lambda x: x == 'b' + >>> list(rlocate(iterable, pred)) + [3, 1] + + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(rlocate(iterable, pred=pred, window_size=3)) + [9, 5, 1] + + Beware, this function won't return anything for infinite iterables. + If *iterable* is reversible, ``rlocate`` will reverse it and search from + the right. Otherwise, it will search from the left and return the results + in reverse order. + + See :func:`locate` to for other example applications. + + """ + if window_size is None: + try: + len_iter = len(iterable) + return ( + len_iter - i - 1 for i in locate(reversed(iterable), pred) + ) + except TypeError: + pass + + return reversed(list(locate(iterable, pred, window_size))) + + +def replace(iterable, pred, substitutes, count=None, window_size=1): + """Yield the items from *iterable*, replacing the items for which *pred* + returns ``True`` with the items from the iterable *substitutes*. + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] + >>> pred = lambda x: x == 0 + >>> substitutes = (2, 3) + >>> list(replace(iterable, pred, substitutes)) + [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] + + If *count* is given, the number of replacements will be limited: + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] + >>> pred = lambda x: x == 0 + >>> substitutes = [None] + >>> list(replace(iterable, pred, substitutes, count=2)) + [1, 1, None, 1, 1, None, 1, 1, 0] + + Use *window_size* to control the number of items passed as arguments to + *pred*. This allows for locating and replacing subsequences. + + >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] + >>> window_size = 3 + >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred + >>> substitutes = [3, 4] # Splice in these items + >>> list(replace(iterable, pred, substitutes, window_size=window_size)) + [3, 4, 5, 3, 4, 5] + + """ + if window_size < 1: + raise ValueError('window_size must be at least 1') + + # Save the substitutes iterable, since it's used more than once + substitutes = tuple(substitutes) + + # Add padding such that the number of windows matches the length of the + # iterable + it = chain(iterable, [_marker] * (window_size - 1)) + windows = windowed(it, window_size) + + n = 0 + for w in windows: + # If the current window matches our predicate (and we haven't hit + # our maximum number of replacements), splice in the substitutes + # and then consume the following windows that overlap with this one. + # For example, if the iterable is (0, 1, 2, 3, 4...) + # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... + # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) + if pred(*w): + if (count is None) or (n < count): + n += 1 + for s in substitutes: + yield s + consume(windows, window_size - 1) + continue + + # If there was no match (or we've reached the replacement limit), + # yield the first item from the window. + if w and (w[0] is not _marker): + yield w[0] diff --git a/third_party/python/more-itertools/more_itertools/tests/test_more.py b/third_party/python/more-itertools/more_itertools/tests/test_more.py --- a/third_party/python/more-itertools/more_itertools/tests/test_more.py +++ b/third_party/python/more-itertools/more_itertools/tests/test_more.py @@ -1,10 +1,11 @@ from __future__ import division, print_function, unicode_literals +from collections import OrderedDict from decimal import Decimal from doctest import DocTestSuite from fractions import Fraction from functools import partial, reduce from heapq import merge from io import StringIO from itertools import ( chain, @@ -109,16 +110,100 @@ class FirstTests(TestCase): """It should raise StopIteration for empty iterables.""" self.assertRaises(ValueError, lambda: mi.first([])) def test_default(self): """It should return the provided default arg for empty iterables.""" self.assertEqual(mi.first([], 'boo'), 'boo') +class IterOnlyRange: + """User-defined iterable class which only support __iter__. + + It is not specified to inherit ``object``, so indexing on a instance will + raise an ``AttributeError`` rather than ``TypeError`` in Python 2. + + >>> r = IterOnlyRange(5) + >>> r[0] + AttributeError: IterOnlyRange instance has no attribute '__getitem__' + + Note: In Python 3, ``TypeError`` will be raised because ``object`` is + inherited implicitly by default. + + >>> r[0] + TypeError: 'IterOnlyRange' object does not support indexing + """ + def __init__(self, n): + """Set the length of the range.""" + self.n = n + + def __iter__(self): + """Works same as range().""" + return iter(range(self.n)) + + +class LastTests(TestCase): + """Tests for ``last()``""" + + def test_many_nonsliceable(self): + """Test that it works on many-item non-slice-able iterables.""" + # Also try it on a generator expression to make sure it works on + # whatever those return, across Python versions. + self.assertEqual(mi.last(x for x in range(4)), 3) + + def test_one_nonsliceable(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.last(x for x in range(1)), 0) + + def test_empty_stop_iteration_nonsliceable(self): + """It should raise ValueError for empty non-slice-able iterables.""" + self.assertRaises(ValueError, lambda: mi.last(x for x in range(0))) + + def test_default_nonsliceable(self): + """It should return the provided default arg for empty non-slice-able + iterables. + """ + self.assertEqual(mi.last((x for x in range(0)), 'boo'), 'boo') + + def test_many_sliceable(self): + """Test that it works on many-item slice-able iterables.""" + self.assertEqual(mi.last([0, 1, 2, 3]), 3) + + def test_one_sliceable(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.last([3]), 3) + + def test_empty_stop_iteration_sliceable(self): + """It should raise ValueError for empty slice-able iterables.""" + self.assertRaises(ValueError, lambda: mi.last([])) + + def test_default_sliceable(self): + """It should return the provided default arg for empty slice-able + iterables. + """ + self.assertEqual(mi.last([], 'boo'), 'boo') + + def test_dict(self): + """last(dic) and last(dic.keys()) should return same result.""" + dic = {'a': 1, 'b': 2, 'c': 3} + self.assertEqual(mi.last(dic), mi.last(dic.keys())) + + def test_ordereddict(self): + """last(dic) should return the last key.""" + od = OrderedDict() + od['a'] = 1 + od['b'] = 2 + od['c'] = 3 + self.assertEqual(mi.last(od), 'c') + + def test_customrange(self): + """It should work on custom class where [] raises AttributeError.""" + self.assertEqual(mi.last(IterOnlyRange(5)), 4) + + class PeekableTests(TestCase): """Tests for ``peekable()`` behavor not incidentally covered by testing ``collate()`` """ def test_peek_default(self): """Make sure passing a default into ``peek()`` works.""" p = mi.peekable([]) @@ -1457,16 +1542,36 @@ class LocateTests(TestCase): def test_custom_pred(self): iterable = ['0', 1, 1, '0', 1, '0', '0'] pred = lambda x: x == '0' actual = list(mi.locate(iterable, pred)) expected = [0, 3, 5, 6] self.assertEqual(actual, expected) + def test_window_size(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda *args: args == ('0', 1) + actual = list(mi.locate(iterable, pred, window_size=2)) + expected = [0, 3] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = [1, 2, 3, 4] + pred = lambda a, b, c, d, e: True + actual = list(mi.locate(iterable, pred, window_size=5)) + expected = [0] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = [1, 2, 3, 4] + pred = lambda: True + with self.assertRaises(ValueError): + list(mi.locate(iterable, pred, window_size=0)) + class StripFunctionTests(TestCase): def test_hashable(self): iterable = list('www.example.com') pred = lambda x: x in set('cmowz.') self.assertEqual(list(mi.lstrip(iterable, pred)), list('example.com')) self.assertEqual(list(mi.rstrip(iterable, pred)), list('www.example')) @@ -1841,8 +1946,129 @@ class MapReduceTests(TestCase): ) expected = [(0, 0), (1, 6), (2, 4)] self.assertEqual(actual, expected) def test_ret(self): d = mi.map_reduce([1, 0, 2, 0, 1, 0], bool) self.assertEqual(d, {False: [0, 0, 0], True: [1, 2, 1]}) self.assertRaises(KeyError, lambda: d[None].append(1)) + + +class RlocateTests(TestCase): + def test_default_pred(self): + iterable = [0, 1, 1, 0, 1, 0, 0] + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it)) + expected = [4, 2, 1] + self.assertEqual(actual, expected) + + def test_no_matches(self): + iterable = [0, 0, 0] + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it)) + expected = [] + self.assertEqual(actual, expected) + + def test_custom_pred(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda x: x == '0' + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it, pred)) + expected = [6, 5, 3, 0] + self.assertEqual(actual, expected) + + def test_efficient_reversal(self): + iterable = range(10 ** 10) # Is efficiently reversible + target = 10 ** 10 - 2 + pred = lambda x: x == target # Find-able from the right + actual = next(mi.rlocate(iterable, pred)) + self.assertEqual(actual, target) + + def test_window_size(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda *args: args == ('0', 1) + for it in (iterable, iter(iterable)): + actual = list(mi.rlocate(it, pred, window_size=2)) + expected = [3, 0] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = [1, 2, 3, 4] + pred = lambda a, b, c, d, e: True + for it in (iterable, iter(iterable)): + actual = list(mi.rlocate(iterable, pred, window_size=5)) + expected = [0] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = [1, 2, 3, 4] + pred = lambda: True + for it in (iterable, iter(iterable)): + with self.assertRaises(ValueError): + list(mi.locate(iterable, pred, window_size=0)) + + +class ReplaceTests(TestCase): + def test_basic(self): + iterable = range(10) + pred = lambda x: x % 2 == 0 + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes)) + expected = [1, 3, 5, 7, 9] + self.assertEqual(actual, expected) + + def test_count(self): + iterable = range(10) + pred = lambda x: x % 2 == 0 + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, count=4)) + expected = [1, 3, 5, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size(self): + iterable = range(10) + pred = lambda *args: args == (0, 1, 2) + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, window_size=3)) + expected = [3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size_end(self): + iterable = range(10) + pred = lambda *args: args == (7, 8, 9) + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, window_size=3)) + expected = [0, 1, 2, 3, 4, 5, 6] + self.assertEqual(actual, expected) + + def test_window_size_count(self): + iterable = range(10) + pred = lambda *args: (args == (0, 1, 2)) or (args == (7, 8, 9)) + substitutes = [] + actual = list( + mi.replace(iterable, pred, substitutes, count=1, window_size=3) + ) + expected = [3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = range(4) + pred = lambda a, b, c, d, e: True + substitutes = [5, 6, 7] + actual = list(mi.replace(iterable, pred, substitutes, window_size=5)) + expected = [5, 6, 7] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = range(10) + pred = lambda *args: True + substitutes = [] + with self.assertRaises(ValueError): + list(mi.replace(iterable, pred, substitutes, window_size=0)) + + def test_iterable_substitutes(self): + iterable = range(5) + pred = lambda x: x % 2 == 0 + substitutes = iter('__') + actual = list(mi.replace(iterable, pred, substitutes)) + expected = ['_', '_', 1, '_', '_', 3, '_', '_'] + self.assertEqual(actual, expected) diff --git a/third_party/python/more-itertools/more_itertools/tests/test_recipes.py b/third_party/python/more-itertools/more_itertools/tests/test_recipes.py --- a/third_party/python/more-itertools/more_itertools/tests/test_recipes.py +++ b/third_party/python/more-itertools/more_itertools/tests/test_recipes.py @@ -585,16 +585,25 @@ class NthCombinationTests(TestCase): actual = mi.nth_combination(iterable, r, index) self.assertEqual(actual, expected) def test_long(self): actual = mi.nth_combination(range(180), 4, 2000000) expected = (2, 12, 35, 126) self.assertEqual(actual, expected) + def test_invalid_r(self): + for r in (-1, 3): + with self.assertRaises(ValueError): + mi.nth_combination([], r, 0) + + def test_invalid_index(self): + with self.assertRaises(IndexError): + mi.nth_combination('abcdefg', 3, -36) + class PrependTests(TestCase): def test_basic(self): value = 'a' iterator = iter('bcdefg') actual = list(mi.prepend(value, iterator)) expected = list('abcdefg') self.assertEqual(actual, expected) diff --git a/third_party/python/more-itertools/setup.py b/third_party/python/more-itertools/setup.py --- a/third_party/python/more-itertools/setup.py +++ b/third_party/python/more-itertools/setup.py @@ -23,17 +23,17 @@ def get_long_description(): version_history = sub(r':func:`([a-zA-Z0-9._]+)`', r'\1', version_history) ret = readme + '\n\n' + version_history return ret setup( name='more-itertools', - version='4.2.0', + version='4.3.0', description='More routines for operating on iterables, beyond itertools', long_description=get_long_description(), author='Erik Rose', author_email='erikrose@grinchcentral.com', license='MIT', packages=find_packages(exclude=['ez_setup']), install_requires=['six>=1.0.0,<2.0.0'], test_suite='more_itertools.tests', diff --git a/third_party/python/voluptuous/CHANGELOG.md b/third_party/python/voluptuous/CHANGELOG.md --- a/third_party/python/voluptuous/CHANGELOG.md +++ b/third_party/python/voluptuous/CHANGELOG.md @@ -1,11 +1,40 @@ # Changelog -## [Unreleased] +## [0.11.0] + +**Changes**: + +- [#293](https://github.com/alecthomas/voluptuous/pull/293): Support Python 3.6. +- [#294](https://github.com/alecthomas/voluptuous/pull/294): Drop support for Python 2.6, 3.1 and 3.2. +- [#318](https://github.com/alecthomas/voluptuous/pull/318): Allow to use nested schema and allow any validator to be compiled. +- [#324](https://github.com/alecthomas/voluptuous/pull/324): + Default values MUST now pass validation just as any regular value. This is a backward incompatible change if a schema uses default values that don't pass validation against the specified schema. +- [#328](https://github.com/alecthomas/voluptuous/pull/328): + Modify `__lt__` in Marker class to allow comparison with non Marker objects, such as str and int. + +**New**: + +- [#307](https://github.com/alecthomas/voluptuous/pull/307): Add description field to `Marker` instances. +- [#311](https://github.com/alecthomas/voluptuous/pull/311): Add `Schema.infer` method for basic schema inference. +- [#314](https://github.com/alecthomas/voluptuous/pull/314): Add `SomeOf` validator. + +**Fixes**: + +- [#279](https://github.com/alecthomas/voluptuous/pull/279): + Treat Python 2 old-style classes like types when validating. +- [#280](https://github.com/alecthomas/voluptuous/pull/280): Make + `IsDir()`, `IsFile()` and `PathExists()` consistent between different Python versions. +- [#290](https://github.com/alecthomas/voluptuous/pull/290): Use absolute imports to avoid import conflicts. +- [#291](https://github.com/alecthomas/voluptuous/pull/291): Fix `Coerce` validator to catch `decimal.InvalidOperation`. +- [#298](https://github.com/alecthomas/voluptuous/pull/298): Make `Schema([])` usage consistent with `Schema({})`. +- [#303](https://github.com/alecthomas/voluptuous/pull/303): Allow partial validation when using validate decorator. +- [#316](https://github.com/alecthomas/voluptuous/pull/316): Make `Schema.__eq__` deterministic. +- [#319](https://github.com/alecthomas/voluptuous/pull/319): Replace implementation of `Maybe(s)` with `Any(None, s)` to allow it to be compiled. ## [0.10.5] - [#278](https://github.com/alecthomas/voluptuous/pull/278): Unicode translation to python 2 issue fixed. ## [0.10.2] diff --git a/third_party/python/voluptuous/PKG-INFO b/third_party/python/voluptuous/PKG-INFO --- a/third_party/python/voluptuous/PKG-INFO +++ b/third_party/python/voluptuous/PKG-INFO @@ -1,666 +1,744 @@ -Metadata-Version: 1.1 +Metadata-Version: 2.1 Name: voluptuous -Version: 0.10.5 -Summary: Voluptuous is a Python data validation library +Version: 0.11.5 +Summary: # Voluptuous is a Python data validation library Home-page: https://github.com/alecthomas/voluptuous Author: Alec Thomas Author-email: alec@swapoff.org License: BSD Download-URL: https://pypi.python.org/pypi/voluptuous -Description: Voluptuous is a Python data validation library - ============================================== +Description: # Voluptuous is a Python data validation library - |Build Status| |Coverage Status| |Gitter chat| + [![Build Status](https://travis-ci.org/alecthomas/voluptuous.png)](https://travis-ci.org/alecthomas/voluptuous) + [![Coverage Status](https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master)](https://coveralls.io/github/alecthomas/voluptuous?branch=master) [![Gitter chat](https://badges.gitter.im/alecthomas.png)](https://gitter.im/alecthomas/Lobby) Voluptuous, *despite* the name, is a Python data validation library. It is primarily intended for validating data coming into Python as JSON, YAML, etc. It has three goals: - 1. Simplicity. - 2. Support for complex data structures. - 3. Provide useful error messages. + 1. Simplicity. + 2. Support for complex data structures. + 3. Provide useful error messages. - Contact - ------- + ## Contact Voluptuous now has a mailing list! Send a mail to - ` `__ to - subscribe. Instructions will follow. - - You can also contact me directly via `email `__ - or `Twitter `__. + [](mailto:voluptuous@librelist.com) to subscribe. Instructions + will follow. - To file a bug, create a `new - issue `__ on GitHub - with a short example of how to replicate the issue. + You can also contact me directly via [email](mailto:alec@swapoff.org) or + [Twitter](https://twitter.com/alecthomas). - Documentation - ------------- + To file a bug, create a [new issue](https://github.com/alecthomas/voluptuous/issues/new) on GitHub with a short example of how to replicate the issue. + + ## Documentation - The documentation is provided [here] - (http://alecthomas.github.io/voluptuous/). + The documentation is provided [here](http://alecthomas.github.io/voluptuous/). - Changelog - --------- + ## Changelog - See `CHANGELOG.md `__. + See [CHANGELOG.md](https://github.com/alecthomas/voluptuous/blob/master/CHANGELOG.md). - Show me an example - ------------------ + ## Show me an example - Twitter's `user search - API `__ accepts + Twitter's [user search API](https://dev.twitter.com/rest/reference/get/users/search) accepts query URLs like: - :: - - $ curl 'http://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' + ``` + $ curl 'https://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' + ``` To validate this we might use a schema like: - .. code:: pycon + ```pycon + >>> from voluptuous import Schema + >>> schema = Schema({ + ... 'q': str, + ... 'per_page': int, + ... 'page': int, + ... }) - >>> from voluptuous import Schema - >>> schema = Schema({ - ... 'q': str, - ... 'per_page': int, - ... 'page': int, - ... }) + ``` This schema very succinctly and roughly describes the data required by the API, and will work fine. But it has a few problems. Firstly, it doesn't fully express the constraints of the API. According to the API, - ``per_page`` should be restricted to at most 20, defaulting to 5, for + `per_page` should be restricted to at most 20, defaulting to 5, for example. To describe the semantics of the API more accurately, our schema will need to be more thoroughly defined: - .. code:: pycon + ```pycon + >>> from voluptuous import Required, All, Length, Range + >>> schema = Schema({ + ... Required('q'): All(str, Length(min=1)), + ... Required('per_page', default=5): All(int, Range(min=1, max=20)), + ... 'page': All(int, Range(min=0)), + ... }) - >>> from voluptuous import Required, All, Length, Range - >>> schema = Schema({ - ... Required('q'): All(str, Length(min=1)), - ... Required('per_page', default=5): All(int, Range(min=1, max=20)), - ... 'page': All(int, Range(min=0)), - ... }) + ``` This schema fully enforces the interface defined in Twitter's documentation, and goes a little further for completeness. "q" is required: - .. code:: pycon + ```pycon + >>> from voluptuous import MultipleInvalid, Invalid + >>> try: + ... schema({}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data['q']" + True - >>> from voluptuous import MultipleInvalid, Invalid - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data['q']" - True + ``` ...must be a string: - .. code:: pycon + ```pycon + >>> try: + ... schema({'q': 123}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "expected str for dictionary value @ data['q']" + True - >>> try: - ... schema({'q': 123}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected str for dictionary value @ data['q']" - True + ``` ...and must be at least one character in length: - .. code:: pycon + ```pycon + >>> try: + ... schema({'q': ''}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" + True + >>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} + True - >>> try: - ... schema({'q': ''}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" - True - >>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} - True + ``` "per\_page" is a positive integer no greater than 20: - .. code:: pycon - - >>> try: - ... schema({'q': '#topic', 'per_page': 900}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" - True - >>> try: - ... schema({'q': '#topic', 'per_page': -10}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" - True + ```pycon + >>> try: + ... schema({'q': '#topic', 'per_page': 900}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" + True + >>> try: + ... schema({'q': '#topic', 'per_page': -10}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" + True - "page" is an integer >= 0: + ``` - .. code:: pycon + "page" is an integer \>= 0: - >>> try: - ... schema({'q': '#topic', 'per_page': 'one'}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "expected int for dictionary value @ data['per_page']" - >>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} - True + ```pycon + >>> try: + ... schema({'q': '#topic', 'per_page': 'one'}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) + "expected int for dictionary value @ data['per_page']" + >>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} + True - Defining schemas - ---------------- + ``` + + ## Defining schemas Schemas are nested data structures consisting of dictionaries, lists, scalars and *validators*. Each node in the input schema is pattern matched against corresponding nodes in the input data. - Literals - ~~~~~~~~ + ### Literals Literals in the schema are matched using normal equality checks: - .. code:: pycon + ```pycon + >>> schema = Schema(1) + >>> schema(1) + 1 + >>> schema = Schema('a string') + >>> schema('a string') + 'a string' - >>> schema = Schema(1) - >>> schema(1) - 1 - >>> schema = Schema('a string') - >>> schema('a string') - 'a string' + ``` - Types - ~~~~~ + ### Types Types in the schema are matched by checking if the corresponding value is an instance of the type: - .. code:: pycon + ```pycon + >>> schema = Schema(int) + >>> schema(1) + 1 + >>> try: + ... schema('one') + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "expected int" + True - >>> schema = Schema(int) - >>> schema(1) - 1 - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected int" - True + ``` - URL's - ~~~~~ + ### URL's - URL's in the schema are matched by using ``urlparse`` library. - - .. code:: pycon + URL's in the schema are matched by using `urlparse` library. - >>> from voluptuous import Url - >>> schema = Schema(Url()) - >>> schema('http://w3.org') - 'http://w3.org' - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected a URL" - True + ```pycon + >>> from voluptuous import Url + >>> schema = Schema(Url()) + >>> schema('http://w3.org') + 'http://w3.org' + >>> try: + ... schema('one') + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "expected a URL" + True - Lists - ~~~~~ + ``` + + ### Lists Lists in the schema are treated as a set of valid values. Each element in the schema list is compared to each value in the input data: - .. code:: pycon + ```pycon + >>> schema = Schema([1, 'a', 'string']) + >>> schema([1]) + [1] + >>> schema([1, 1, 1]) + [1, 1, 1] + >>> schema(['a', 1, 'string', 1, 'string']) + ['a', 1, 'string', 1, 'string'] + + ``` + + However, an empty list (`[]`) is treated as is. If you want to specify a list that can + contain anything, specify it as `list`: - >>> schema = Schema([1, 'a', 'string']) - >>> schema([1]) - [1] - >>> schema([1, 1, 1]) - [1, 1, 1] - >>> schema(['a', 1, 'string', 1, 'string']) - ['a', 1, 'string', 1, 'string'] + ```pycon + >>> schema = Schema([]) + >>> try: + ... schema([1]) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "not a valid value @ data[1]" + True + >>> schema([]) + [] + >>> schema = Schema(list) + >>> schema([]) + [] + >>> schema([1, 2]) + [1, 2] - However, an empty list (``[]``) is treated as is. If you want to specify - a list that can contain anything, specify it as ``list``: + ``` - .. code:: pycon + ### Sets and frozensets + + Sets and frozensets are treated as a set of valid values. Each element + in the schema set is compared to each value in the input data: - >>> schema = Schema([]) - >>> try: - ... schema([1]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema([]) - [] - >>> schema = Schema(list) - >>> schema([]) - [] - >>> schema([1, 2]) - [1, 2] + ```pycon + >>> schema = Schema({42}) + >>> schema({42}) == {42} + True + >>> try: + ... schema({43}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "invalid value in set" + True + >>> schema = Schema({int}) + >>> schema({1, 2, 3}) == {1, 2, 3} + True + >>> schema = Schema({int, str}) + >>> schema({1, 2, 'abc'}) == {1, 2, 'abc'} + True + >>> schema = Schema(frozenset([int])) + >>> try: + ... schema({3}) + ... raise AssertionError('Invalid not raised') + ... except Invalid as e: + ... exc = e + >>> str(exc) == 'expected a frozenset' + True - Validation functions - ~~~~~~~~~~~~~~~~~~~~ + ``` + + However, an empty set (`set()`) is treated as is. If you want to specify a set + that can contain anything, specify it as `set`: - Validators are simple callables that raise an ``Invalid`` exception when + ```pycon + >>> schema = Schema(set()) + >>> try: + ... schema({1}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "invalid value in set" + True + >>> schema(set()) == set() + True + >>> schema = Schema(set) + >>> schema({1, 2}) == {1, 2} + True + + ``` + + ### Validation functions + + Validators are simple callables that raise an `Invalid` exception when they encounter invalid data. The criteria for determining validity is entirely up to the implementation; it may check that a value is a valid - username with ``pwd.getpwnam()``, it may check that a value is of a + username with `pwd.getpwnam()`, it may check that a value is of a specific type, and so on. The simplest kind of validator is a Python function that raises ValueError when its argument is invalid. Conveniently, many builtin Python functions have this property. Here's an example of a date validator: - .. code:: pycon + ```pycon + >>> from datetime import datetime + >>> def Date(fmt='%Y-%m-%d'): + ... return lambda v: datetime.strptime(v, fmt) - >>> from datetime import datetime - >>> def Date(fmt='%Y-%m-%d'): - ... return lambda v: datetime.strptime(v, fmt) - - .. code:: pycon + ``` - >>> schema = Schema(Date()) - >>> schema('2013-03-03') - datetime.datetime(2013, 3, 3, 0, 0) - >>> try: - ... schema('2013-03') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True + ```pycon + >>> schema = Schema(Date()) + >>> schema('2013-03-03') + datetime.datetime(2013, 3, 3, 0, 0) + >>> try: + ... schema('2013-03') + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "not a valid value" + True + + ``` In addition to simply determining if a value is valid, validators may mutate the value into a valid form. An example of this is the - ``Coerce(type)`` function, which returns a function that coerces its + `Coerce(type)` function, which returns a function that coerces its argument to the given type: - .. code:: python - - def Coerce(type, msg=None): - """Coerce a value to a type. + ```python + def Coerce(type, msg=None): + """Coerce a value to a type. - If the type constructor throws a ValueError, the value will be marked as - Invalid. - """ - def f(v): - try: - return type(v) - except ValueError: - raise Invalid(msg or ('expected %s' % type.__name__)) - return f + If the type constructor throws a ValueError, the value will be marked as + Invalid. + """ + def f(v): + try: + return type(v) + except ValueError: + raise Invalid(msg or ('expected %s' % type.__name__)) + return f + + ``` This example also shows a common idiom where an optional human-readable message can be provided. This can vastly improve the usefulness of the resulting error messages. - Dictionaries - ~~~~~~~~~~~~ + ### Dictionaries Each key-value pair in a schema dictionary is validated against each key-value pair in the corresponding data dictionary: - .. code:: pycon + ```pycon + >>> schema = Schema({1: 'one', 2: 'two'}) + >>> schema({1: 'one'}) + {1: 'one'} - >>> schema = Schema({1: 'one', 2: 'two'}) - >>> schema({1: 'one'}) - {1: 'one'} + ``` - Extra dictionary keys - ^^^^^^^^^^^^^^^^^^^^^ + #### Extra dictionary keys By default any additional keys in the data, not in the schema will trigger exceptions: - .. code:: pycon + ```pycon + >>> schema = Schema({2: 3}) + >>> try: + ... schema({1: 2, 2: 3}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "extra keys not allowed @ data[1]" + True - >>> schema = Schema({2: 3}) - >>> try: - ... schema({1: 2, 2: 3}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[1]" - True + ``` - This behaviour can be altered on a per-schema basis. To allow additional - keys use ``Schema(..., extra=ALLOW_EXTRA)``: + This behaviour can be altered on a per-schema basis. To allow + additional keys use + `Schema(..., extra=ALLOW_EXTRA)`: - .. code:: pycon + ```pycon + >>> from voluptuous import ALLOW_EXTRA + >>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) + >>> schema({1: 2, 2: 3}) + {1: 2, 2: 3} - >>> from voluptuous import ALLOW_EXTRA - >>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) - >>> schema({1: 2, 2: 3}) - {1: 2, 2: 3} + ``` - To remove additional keys use ``Schema(..., extra=REMOVE_EXTRA)``: - - .. code:: pycon + To remove additional keys use + `Schema(..., extra=REMOVE_EXTRA)`: - >>> from voluptuous import REMOVE_EXTRA - >>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) - >>> schema({1: 2, 2: 3}) - {2: 3} + ```pycon + >>> from voluptuous import REMOVE_EXTRA + >>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) + >>> schema({1: 2, 2: 3}) + {2: 3} + + ``` It can also be overridden per-dictionary by using the catch-all marker - token ``extra`` as a key: - - .. code:: pycon - - >>> from voluptuous import Extra - >>> schema = Schema({1: {Extra: object}}) - >>> schema({1: {'foo': 'bar'}}) - {1: {'foo': 'bar'}} - - However, an empty dict (``{}``) is treated as is. If you want to specify - a list that can contain anything, specify it as ``dict``: - - .. code:: pycon + token `extra` as a key: - >>> schema = Schema({}, extra=ALLOW_EXTRA) # don't do this - >>> try: - ... schema({'extra': 1}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema({}) - {} - >>> schema = Schema(dict) # do this instead - >>> schema({}) - {} - >>> schema({'extra': 1}) - {'extra': 1} + ```pycon + >>> from voluptuous import Extra + >>> schema = Schema({1: {Extra: object}}) + >>> schema({1: {'foo': 'bar'}}) + {1: {'foo': 'bar'}} - Required dictionary keys - ^^^^^^^^^^^^^^^^^^^^^^^^ + ``` + + #### Required dictionary keys By default, keys in the schema are not required to be in the data: - .. code:: pycon + ```pycon + >>> schema = Schema({1: 2, 3: 4}) + >>> schema({3: 4}) + {3: 4} - >>> schema = Schema({1: 2, 3: 4}) - >>> schema({3: 4}) - {3: 4} + ``` Similarly to how extra\_ keys work, this behaviour can be overridden per-schema: - .. code:: pycon + ```pycon + >>> schema = Schema({1: 2, 3: 4}, required=True) + >>> try: + ... schema({3: 4}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data[1]" + True - >>> schema = Schema({1: 2, 3: 4}, required=True) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True + ``` - And per-key, with the marker token ``Required(key)``: - - .. code:: pycon + And per-key, with the marker token `Required(key)`: - >>> schema = Schema({Required(1): 2, 3: 4}) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} + ```pycon + >>> schema = Schema({Required(1): 2, 3: 4}) + >>> try: + ... schema({3: 4}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data[1]" + True + >>> schema({1: 2}) + {1: 2} - Optional dictionary keys - ^^^^^^^^^^^^^^^^^^^^^^^^ + ``` - If a schema has ``required=True``, keys may be individually marked as - optional using the marker token ``Optional(key)``: + #### Optional dictionary keys - .. code:: pycon + If a schema has `required=True`, keys may be individually marked as + optional using the marker token `Optional(key)`: - >>> from voluptuous import Optional - >>> schema = Schema({1: 2, Optional(3): 4}, required=True) - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} - >>> try: - ... schema({1: 2, 4: 5}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[4]" - True + ```pycon + >>> from voluptuous import Optional + >>> schema = Schema({1: 2, Optional(3): 4}, required=True) + >>> try: + ... schema({}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data[1]" + True + >>> schema({1: 2}) + {1: 2} + >>> try: + ... schema({1: 2, 4: 5}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "extra keys not allowed @ data[4]" + True - .. code:: pycon - - >>> schema({1: 2, 3: 4}) - {1: 2, 3: 4} + ``` - Recursive schema - ~~~~~~~~~~~~~~~~ + ```pycon + >>> schema({1: 2, 3: 4}) + {1: 2, 3: 4} - There is no syntax to have a recursive schema. The best way to do it is - to have a wrapper like this: + ``` - .. code:: pycon + ### Recursive / nested schema + + You can use `voluptuous.Self` to define a nested schema: - >>> from voluptuous import Schema, Any - >>> def s2(v): - ... return s1(v) - ... - >>> s1 = Schema({"key": Any(s2, "value")}) - >>> s1({"key": {"key": "value"}}) - {'key': {'key': 'value'}} + ```pycon + >>> from voluptuous import Schema, Self + >>> recursive = Schema({"more": Self, "value": int}) + >>> recursive({"more": {"value": 42}, "value": 41}) == {'more': {'value': 42}, 'value': 41} + True - Extending an existing Schema - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ``` + + ### Extending an existing Schema - Often it comes handy to have a base ``Schema`` that is extended with - more requirements. In that case you can use ``Schema.extend`` to create - a new ``Schema``: - - .. code:: pycon + Often it comes handy to have a base `Schema` that is extended with more + requirements. In that case you can use `Schema.extend` to create a new + `Schema`: - >>> from voluptuous import Schema - >>> person = Schema({'name': str}) - >>> person_with_age = person.extend({'age': int}) - >>> sorted(list(person_with_age.schema.keys())) - ['age', 'name'] + ```pycon + >>> from voluptuous import Schema + >>> person = Schema({'name': str}) + >>> person_with_age = person.extend({'age': int}) + >>> sorted(list(person_with_age.schema.keys())) + ['age', 'name'] - The original ``Schema`` remains unchanged. + ``` - Objects - ~~~~~~~ + The original `Schema` remains unchanged. + + ### Objects Each key-value pair in a schema dictionary is validated against each attribute-value pair in the corresponding object: - .. code:: pycon + ```pycon + >>> from voluptuous import Object + >>> class Structure(object): + ... def __init__(self, q=None): + ... self.q = q + ... def __repr__(self): + ... return ''.format(self) + ... + >>> schema = Schema(Object({'q': 'one'}, cls=Structure)) + >>> schema(Structure(q='one')) + - >>> from voluptuous import Object - >>> class Structure(object): - ... def __init__(self, q=None): - ... self.q = q - ... def __repr__(self): - ... return ''.format(self) - ... - >>> schema = Schema(Object({'q': 'one'}, cls=Structure)) - >>> schema(Structure(q='one')) - + ``` - Allow None values - ~~~~~~~~~~~~~~~~~ + ### Allow None values To allow value to be None as well, use Any: - .. code:: pycon - - >>> from voluptuous import Any + ```pycon + >>> from voluptuous import Any - >>> schema = Schema(Any(None, int)) - >>> schema(None) - >>> schema(5) - 5 + >>> schema = Schema(Any(None, int)) + >>> schema(None) + >>> schema(5) + 5 - Error reporting - --------------- + ``` - Validators must throw an ``Invalid`` exception if invalid data is passed + ## Error reporting + + Validators must throw an `Invalid` exception if invalid data is passed to them. All other exceptions are treated as errors in the validator and will not be caught. - Each ``Invalid`` exception has an associated ``path`` attribute - representing the path in the data structure to our currently validating - value, as well as an ``error_message`` attribute that contains the - message of the original exception. This is especially useful when you - want to catch ``Invalid`` exceptions and give some feedback to the user, - for instance in the context of an HTTP API. + Each `Invalid` exception has an associated `path` attribute representing + the path in the data structure to our currently validating value, as well + as an `error_message` attribute that contains the message of the original + exception. This is especially useful when you want to catch `Invalid` + exceptions and give some feedback to the user, for instance in the context of + an HTTP API. - .. code:: pycon - >>> def validate_email(email): - ... """Validate email.""" - ... if not "@" in email: - ... raise Invalid("This email is invalid.") - ... return email - >>> schema = Schema({"email": validate_email}) - >>> exc = None - >>> try: - ... schema({"email": "whatever"}) - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "This email is invalid. for dictionary value @ data['email']" - >>> exc.path - ['email'] - >>> exc.msg - 'This email is invalid.' - >>> exc.error_message - 'This email is invalid.' + ```pycon + >>> def validate_email(email): + ... """Validate email.""" + ... if not "@" in email: + ... raise Invalid("This email is invalid.") + ... return email + >>> schema = Schema({"email": validate_email}) + >>> exc = None + >>> try: + ... schema({"email": "whatever"}) + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) + "This email is invalid. for dictionary value @ data['email']" + >>> exc.path + ['email'] + >>> exc.msg + 'This email is invalid.' + >>> exc.error_message + 'This email is invalid.' - The ``path`` attribute is used during error reporting, but also during - matching to determine whether an error should be reported to the user or - if the next match should be attempted. This is determined by comparing - the depth of the path where the check is, to the depth of the path where - the error occurred. If the error is more than one level deeper, it is - reported. + ``` + + The `path` attribute is used during error reporting, but also during matching + to determine whether an error should be reported to the user or if the next + match should be attempted. This is determined by comparing the depth of the + path where the check is, to the depth of the path where the error occurred. If + the error is more than one level deeper, it is reported. The upshot of this is that *matching is depth-first and fail-fast*. To illustrate this, here is an example schema: - .. code:: pycon + ```pycon + >>> schema = Schema([[2, 3], 6]) - >>> schema = Schema([[2, 3], 6]) + ``` Each value in the top-level list is matched depth-first in-order. Given - input data of ``[[6]]``, the inner list will match the first element of - the schema, but the literal ``6`` will not match any of the elements of + input data of `[[6]]`, the inner list will match the first element of + the schema, but the literal `6` will not match any of the elements of that list. This error will be reported back to the user immediately. No backtracking is attempted: - .. code:: pycon + ```pycon + >>> try: + ... schema([[6]]) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "not a valid value @ data[0][0]" + True + + ``` - >>> try: - ... schema([[6]]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value @ data[0][0]" - True + If we pass the data `[6]`, the `6` is not a list type and so will not + recurse into the first element of the schema. Matching will continue on + to the second element in the schema, and succeed: + + ```pycon + >>> schema([6]) + [6] + + ``` + + ## Multi-field validation + + Validation rules that involve multiple fields can be implemented as + custom validators. It's recommended to use `All()` to do a two-pass + validation - the first pass checking the basic structure of the data, + and only after that, the second pass applying your cross-field + validator: - If we pass the data ``[6]``, the ``6`` is not a list type and so will - not recurse into the first element of the schema. Matching will continue - on to the second element in the schema, and succeed: + ```python + def passwords_must_match(passwords): + if passwords['password'] != passwords['password_again']: + raise Invalid('passwords must match') + return passwords - .. code:: pycon + s=Schema(All( + # First "pass" for field types + {'password':str, 'password_again':str}, + # Follow up the first "pass" with your multi-field rules + passwords_must_match + )) + + # valid + s({'password':'123', 'password_again':'123'}) - >>> schema([6]) - [6] + # raises MultipleInvalid: passwords must match + s({'password':'123', 'password_again':'and now for something completely different'}) + + ``` + + With this structure, your multi-field validator will run with + pre-validated data from the first "pass" and so will not have to do + its own type checking on its inputs. - Running tests. - -------------- + The flipside is that if the first "pass" of validation fails, your + cross-field validator will not run: + + ``` + # raises Invalid because password_again is not a string + # passwords_must_match() will not run because first-pass validation already failed + s({'password':'123', 'password_again': 1337}) + ``` + + ## Running tests. Voluptuous is using nosetests: - :: - $ nosetests - Why use Voluptuous over another validation library? - --------------------------------------------------- + + ## Why use Voluptuous over another validation library? **Validators are simple callables** - No need to subclass anything, just use a function. + : No need to subclass anything, just use a function. + **Errors are simple exceptions.** - A validator can just ``raise Invalid(msg)`` and expect the user to - get useful messages. + : A validator can just `raise Invalid(msg)` and expect the user to get + useful messages. + **Schemas are basic Python data structures.** - Should your data be a dictionary of integer keys to strings? - ``{int: str}`` does what you expect. List of integers, floats or - strings? ``[int, float, str]``. + : Should your data be a dictionary of integer keys to strings? + `{int: str}` does what you expect. List of integers, floats or + strings? `[int, float, str]`. + **Designed from the ground up for validating more than just forms.** - Nested data structures are treated in the same way as any other - type. Need a list of dictionaries? ``[{}]`` + : Nested data structures are treated in the same way as any other + type. Need a list of dictionaries? `[{}]` + **Consistency.** - Types in the schema are checked as types. Values are compared as - values. Callables are called to validate. Simple. + : Types in the schema are checked as types. Values are compared as + values. Callables are called to validate. Simple. - Other libraries and inspirations - -------------------------------- + ## Other libraries and inspirations Voluptuous is heavily inspired by - `Validino `__, and to a lesser - extent, `jsonvalidator `__ and - `json\_schema `__. + [Validino](http://code.google.com/p/validino/), and to a lesser extent, + [jsonvalidator](http://code.google.com/p/jsonvalidator/) and + [json\_schema](http://blog.sendapatch.se/category/json_schema.html). + + [pytest-voluptuous](https://github.com/F-Secure/pytest-voluptuous) is a + [pytest](https://github.com/pytest-dev/pytest) plugin that helps in + using voluptuous validators in `assert`s. I greatly prefer the light-weight style promoted by these libraries to the complexity of libraries like FormEncode. - .. |Build Status| image:: https://travis-ci.org/alecthomas/voluptuous.png - :target: https://travis-ci.org/alecthomas/voluptuous - .. |Coverage Status| image:: https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master - :target: https://coveralls.io/github/alecthomas/voluptuous?branch=master - .. |Gitter chat| image:: https://badges.gitter.im/alecthomas.png - :target: https://gitter.im/alecthomas/Lobby - Platform: any Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.1 -Classifier: Programming Language :: Python :: 3.2 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Description-Content-Type: text/markdown diff --git a/third_party/python/voluptuous/README.md b/third_party/python/voluptuous/README.md --- a/third_party/python/voluptuous/README.md +++ b/third_party/python/voluptuous/README.md @@ -21,29 +21,29 @@ will follow. You can also contact me directly via [email](mailto:alec@swapoff.org) or [Twitter](https://twitter.com/alecthomas). To file a bug, create a [new issue](https://github.com/alecthomas/voluptuous/issues/new) on GitHub with a short example of how to replicate the issue. ## Documentation -The documentation is provided [here] (http://alecthomas.github.io/voluptuous/). +The documentation is provided [here](http://alecthomas.github.io/voluptuous/). ## Changelog -See [CHANGELOG.md](CHANGELOG.md). +See [CHANGELOG.md](https://github.com/alecthomas/voluptuous/blob/master/CHANGELOG.md). ## Show me an example Twitter's [user search API](https://dev.twitter.com/rest/reference/get/users/search) accepts query URLs like: ``` -$ curl 'http://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' +$ curl 'https://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' ``` To validate this we might use a schema like: ```pycon >>> from voluptuous import Schema >>> schema = Schema({ ... 'q': str, @@ -229,28 +229,81 @@ contain anything, specify it as `list`: ```pycon >>> schema = Schema([]) >>> try: ... schema([1]) ... raise AssertionError('MultipleInvalid not raised') ... except MultipleInvalid as e: ... exc = e ->>> str(exc) == "not a valid value" +>>> str(exc) == "not a valid value @ data[1]" True >>> schema([]) [] >>> schema = Schema(list) >>> schema([]) [] >>> schema([1, 2]) [1, 2] ``` +### Sets and frozensets + +Sets and frozensets are treated as a set of valid values. Each element +in the schema set is compared to each value in the input data: + +```pycon +>>> schema = Schema({42}) +>>> schema({42}) == {42} +True +>>> try: +... schema({43}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "invalid value in set" +True +>>> schema = Schema({int}) +>>> schema({1, 2, 3}) == {1, 2, 3} +True +>>> schema = Schema({int, str}) +>>> schema({1, 2, 'abc'}) == {1, 2, 'abc'} +True +>>> schema = Schema(frozenset([int])) +>>> try: +... schema({3}) +... raise AssertionError('Invalid not raised') +... except Invalid as e: +... exc = e +>>> str(exc) == 'expected a frozenset' +True + +``` + +However, an empty set (`set()`) is treated as is. If you want to specify a set +that can contain anything, specify it as `set`: + +```pycon +>>> schema = Schema(set()) +>>> try: +... schema({1}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "invalid value in set" +True +>>> schema(set()) == set() +True +>>> schema = Schema(set) +>>> schema({1, 2}) == {1, 2} +True + +``` + ### Validation functions Validators are simple callables that raise an `Invalid` exception when they encounter invalid data. The criteria for determining validity is entirely up to the implementation; it may check that a value is a valid username with `pwd.getpwnam()`, it may check that a value is of a specific type, and so on. @@ -363,38 +416,16 @@ token `extra` as a key: ```pycon >>> from voluptuous import Extra >>> schema = Schema({1: {Extra: object}}) >>> schema({1: {'foo': 'bar'}}) {1: {'foo': 'bar'}} ``` -However, an empty dict (`{}`) is treated as is. If you want to specify a list that can -contain anything, specify it as `dict`: - -```pycon ->>> schema = Schema({}, extra=ALLOW_EXTRA) # don't do this ->>> try: -... schema({'extra': 1}) -... raise AssertionError('MultipleInvalid not raised') -... except MultipleInvalid as e: -... exc = e ->>> str(exc) == "not a valid value" -True ->>> schema({}) -{} ->>> schema = Schema(dict) # do this instead ->>> schema({}) -{} ->>> schema({'extra': 1}) -{'extra': 1} - -``` - #### Required dictionary keys By default, keys in the schema are not required to be in the data: ```pycon >>> schema = Schema({1: 2, 3: 4}) >>> schema({3: 4}) {3: 4} @@ -460,28 +491,25 @@ True ``` ```pycon >>> schema({1: 2, 3: 4}) {1: 2, 3: 4} ``` -### Recursive schema +### Recursive / nested schema -There is no syntax to have a recursive schema. The best way to do it is to have a wrapper like this: +You can use `voluptuous.Self` to define a nested schema: ```pycon ->>> from voluptuous import Schema, Any ->>> def s2(v): -... return s1(v) -... ->>> s1 = Schema({"key": Any(s2, "value")}) ->>> s1({"key": {"key": "value"}}) -{'key': {'key': 'value'}} +>>> from voluptuous import Schema, Self +>>> recursive = Schema({"more": Self, "value": int}) +>>> recursive({"more": {"value": 42}, "value": 41}) == {'more': {'value': 42}, 'value': 41} +True ``` ### Extending an existing Schema Often it comes handy to have a base `Schema` that is extended with more requirements. In that case you can use `Schema.extend` to create a new `Schema`: @@ -604,16 +632,58 @@ recurse into the first element of the sc to the second element in the schema, and succeed: ```pycon >>> schema([6]) [6] ``` +## Multi-field validation + +Validation rules that involve multiple fields can be implemented as +custom validators. It's recommended to use `All()` to do a two-pass +validation - the first pass checking the basic structure of the data, +and only after that, the second pass applying your cross-field +validator: + +```python +def passwords_must_match(passwords): + if passwords['password'] != passwords['password_again']: + raise Invalid('passwords must match') + return passwords + +s=Schema(All( + # First "pass" for field types + {'password':str, 'password_again':str}, + # Follow up the first "pass" with your multi-field rules + passwords_must_match +)) + +# valid +s({'password':'123', 'password_again':'123'}) + +# raises MultipleInvalid: passwords must match +s({'password':'123', 'password_again':'and now for something completely different'}) + +``` + +With this structure, your multi-field validator will run with +pre-validated data from the first "pass" and so will not have to do +its own type checking on its inputs. + +The flipside is that if the first "pass" of validation fails, your +cross-field validator will not run: + +``` +# raises Invalid because password_again is not a string +# passwords_must_match() will not run because first-pass validation already failed +s({'password':'123', 'password_again': 1337}) +``` + ## Running tests. Voluptuous is using nosetests: $ nosetests ## Why use Voluptuous over another validation library? @@ -640,10 +710,14 @@ values. Callables are called to validate ## Other libraries and inspirations Voluptuous is heavily inspired by [Validino](http://code.google.com/p/validino/), and to a lesser extent, [jsonvalidator](http://code.google.com/p/jsonvalidator/) and [json\_schema](http://blog.sendapatch.se/category/json_schema.html). +[pytest-voluptuous](https://github.com/F-Secure/pytest-voluptuous) is a +[pytest](https://github.com/pytest-dev/pytest) plugin that helps in +using voluptuous validators in `assert`s. + I greatly prefer the light-weight style promoted by these libraries to the complexity of libraries like FormEncode. diff --git a/third_party/python/voluptuous/README.rst b/third_party/python/voluptuous/README.rst deleted file mode 100644 --- a/third_party/python/voluptuous/README.rst +++ /dev/null @@ -1,644 +0,0 @@ -Voluptuous is a Python data validation library -============================================== - -|Build Status| |Coverage Status| |Gitter chat| - -Voluptuous, *despite* the name, is a Python data validation library. It -is primarily intended for validating data coming into Python as JSON, -YAML, etc. - -It has three goals: - -1. Simplicity. -2. Support for complex data structures. -3. Provide useful error messages. - -Contact -------- - -Voluptuous now has a mailing list! Send a mail to -` `__ to -subscribe. Instructions will follow. - -You can also contact me directly via `email `__ -or `Twitter `__. - -To file a bug, create a `new -issue `__ on GitHub -with a short example of how to replicate the issue. - -Documentation -------------- - -The documentation is provided [here] -(http://alecthomas.github.io/voluptuous/). - -Changelog ---------- - -See `CHANGELOG.md `__. - -Show me an example ------------------- - -Twitter's `user search -API `__ accepts -query URLs like: - -:: - - $ curl 'http://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' - -To validate this we might use a schema like: - -.. code:: pycon - - >>> from voluptuous import Schema - >>> schema = Schema({ - ... 'q': str, - ... 'per_page': int, - ... 'page': int, - ... }) - -This schema very succinctly and roughly describes the data required by -the API, and will work fine. But it has a few problems. Firstly, it -doesn't fully express the constraints of the API. According to the API, -``per_page`` should be restricted to at most 20, defaulting to 5, for -example. To describe the semantics of the API more accurately, our -schema will need to be more thoroughly defined: - -.. code:: pycon - - >>> from voluptuous import Required, All, Length, Range - >>> schema = Schema({ - ... Required('q'): All(str, Length(min=1)), - ... Required('per_page', default=5): All(int, Range(min=1, max=20)), - ... 'page': All(int, Range(min=0)), - ... }) - -This schema fully enforces the interface defined in Twitter's -documentation, and goes a little further for completeness. - -"q" is required: - -.. code:: pycon - - >>> from voluptuous import MultipleInvalid, Invalid - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data['q']" - True - -...must be a string: - -.. code:: pycon - - >>> try: - ... schema({'q': 123}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected str for dictionary value @ data['q']" - True - -...and must be at least one character in length: - -.. code:: pycon - - >>> try: - ... schema({'q': ''}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" - True - >>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} - True - -"per\_page" is a positive integer no greater than 20: - -.. code:: pycon - - >>> try: - ... schema({'q': '#topic', 'per_page': 900}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" - True - >>> try: - ... schema({'q': '#topic', 'per_page': -10}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" - True - -"page" is an integer >= 0: - -.. code:: pycon - - >>> try: - ... schema({'q': '#topic', 'per_page': 'one'}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "expected int for dictionary value @ data['per_page']" - >>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} - True - -Defining schemas ----------------- - -Schemas are nested data structures consisting of dictionaries, lists, -scalars and *validators*. Each node in the input schema is pattern -matched against corresponding nodes in the input data. - -Literals -~~~~~~~~ - -Literals in the schema are matched using normal equality checks: - -.. code:: pycon - - >>> schema = Schema(1) - >>> schema(1) - 1 - >>> schema = Schema('a string') - >>> schema('a string') - 'a string' - -Types -~~~~~ - -Types in the schema are matched by checking if the corresponding value -is an instance of the type: - -.. code:: pycon - - >>> schema = Schema(int) - >>> schema(1) - 1 - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected int" - True - -URL's -~~~~~ - -URL's in the schema are matched by using ``urlparse`` library. - -.. code:: pycon - - >>> from voluptuous import Url - >>> schema = Schema(Url()) - >>> schema('http://w3.org') - 'http://w3.org' - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected a URL" - True - -Lists -~~~~~ - -Lists in the schema are treated as a set of valid values. Each element -in the schema list is compared to each value in the input data: - -.. code:: pycon - - >>> schema = Schema([1, 'a', 'string']) - >>> schema([1]) - [1] - >>> schema([1, 1, 1]) - [1, 1, 1] - >>> schema(['a', 1, 'string', 1, 'string']) - ['a', 1, 'string', 1, 'string'] - -However, an empty list (``[]``) is treated as is. If you want to specify -a list that can contain anything, specify it as ``list``: - -.. code:: pycon - - >>> schema = Schema([]) - >>> try: - ... schema([1]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema([]) - [] - >>> schema = Schema(list) - >>> schema([]) - [] - >>> schema([1, 2]) - [1, 2] - -Validation functions -~~~~~~~~~~~~~~~~~~~~ - -Validators are simple callables that raise an ``Invalid`` exception when -they encounter invalid data. The criteria for determining validity is -entirely up to the implementation; it may check that a value is a valid -username with ``pwd.getpwnam()``, it may check that a value is of a -specific type, and so on. - -The simplest kind of validator is a Python function that raises -ValueError when its argument is invalid. Conveniently, many builtin -Python functions have this property. Here's an example of a date -validator: - -.. code:: pycon - - >>> from datetime import datetime - >>> def Date(fmt='%Y-%m-%d'): - ... return lambda v: datetime.strptime(v, fmt) - -.. code:: pycon - - >>> schema = Schema(Date()) - >>> schema('2013-03-03') - datetime.datetime(2013, 3, 3, 0, 0) - >>> try: - ... schema('2013-03') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - -In addition to simply determining if a value is valid, validators may -mutate the value into a valid form. An example of this is the -``Coerce(type)`` function, which returns a function that coerces its -argument to the given type: - -.. code:: python - - def Coerce(type, msg=None): - """Coerce a value to a type. - - If the type constructor throws a ValueError, the value will be marked as - Invalid. - """ - def f(v): - try: - return type(v) - except ValueError: - raise Invalid(msg or ('expected %s' % type.__name__)) - return f - -This example also shows a common idiom where an optional human-readable -message can be provided. This can vastly improve the usefulness of the -resulting error messages. - -Dictionaries -~~~~~~~~~~~~ - -Each key-value pair in a schema dictionary is validated against each -key-value pair in the corresponding data dictionary: - -.. code:: pycon - - >>> schema = Schema({1: 'one', 2: 'two'}) - >>> schema({1: 'one'}) - {1: 'one'} - -Extra dictionary keys -^^^^^^^^^^^^^^^^^^^^^ - -By default any additional keys in the data, not in the schema will -trigger exceptions: - -.. code:: pycon - - >>> schema = Schema({2: 3}) - >>> try: - ... schema({1: 2, 2: 3}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[1]" - True - -This behaviour can be altered on a per-schema basis. To allow additional -keys use ``Schema(..., extra=ALLOW_EXTRA)``: - -.. code:: pycon - - >>> from voluptuous import ALLOW_EXTRA - >>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) - >>> schema({1: 2, 2: 3}) - {1: 2, 2: 3} - -To remove additional keys use ``Schema(..., extra=REMOVE_EXTRA)``: - -.. code:: pycon - - >>> from voluptuous import REMOVE_EXTRA - >>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) - >>> schema({1: 2, 2: 3}) - {2: 3} - -It can also be overridden per-dictionary by using the catch-all marker -token ``extra`` as a key: - -.. code:: pycon - - >>> from voluptuous import Extra - >>> schema = Schema({1: {Extra: object}}) - >>> schema({1: {'foo': 'bar'}}) - {1: {'foo': 'bar'}} - -However, an empty dict (``{}``) is treated as is. If you want to specify -a list that can contain anything, specify it as ``dict``: - -.. code:: pycon - - >>> schema = Schema({}, extra=ALLOW_EXTRA) # don't do this - >>> try: - ... schema({'extra': 1}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema({}) - {} - >>> schema = Schema(dict) # do this instead - >>> schema({}) - {} - >>> schema({'extra': 1}) - {'extra': 1} - -Required dictionary keys -^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, keys in the schema are not required to be in the data: - -.. code:: pycon - - >>> schema = Schema({1: 2, 3: 4}) - >>> schema({3: 4}) - {3: 4} - -Similarly to how extra\_ keys work, this behaviour can be overridden -per-schema: - -.. code:: pycon - - >>> schema = Schema({1: 2, 3: 4}, required=True) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - -And per-key, with the marker token ``Required(key)``: - -.. code:: pycon - - >>> schema = Schema({Required(1): 2, 3: 4}) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} - -Optional dictionary keys -^^^^^^^^^^^^^^^^^^^^^^^^ - -If a schema has ``required=True``, keys may be individually marked as -optional using the marker token ``Optional(key)``: - -.. code:: pycon - - >>> from voluptuous import Optional - >>> schema = Schema({1: 2, Optional(3): 4}, required=True) - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} - >>> try: - ... schema({1: 2, 4: 5}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[4]" - True - -.. code:: pycon - - >>> schema({1: 2, 3: 4}) - {1: 2, 3: 4} - -Recursive schema -~~~~~~~~~~~~~~~~ - -There is no syntax to have a recursive schema. The best way to do it is -to have a wrapper like this: - -.. code:: pycon - - >>> from voluptuous import Schema, Any - >>> def s2(v): - ... return s1(v) - ... - >>> s1 = Schema({"key": Any(s2, "value")}) - >>> s1({"key": {"key": "value"}}) - {'key': {'key': 'value'}} - -Extending an existing Schema -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Often it comes handy to have a base ``Schema`` that is extended with -more requirements. In that case you can use ``Schema.extend`` to create -a new ``Schema``: - -.. code:: pycon - - >>> from voluptuous import Schema - >>> person = Schema({'name': str}) - >>> person_with_age = person.extend({'age': int}) - >>> sorted(list(person_with_age.schema.keys())) - ['age', 'name'] - -The original ``Schema`` remains unchanged. - -Objects -~~~~~~~ - -Each key-value pair in a schema dictionary is validated against each -attribute-value pair in the corresponding object: - -.. code:: pycon - - >>> from voluptuous import Object - >>> class Structure(object): - ... def __init__(self, q=None): - ... self.q = q - ... def __repr__(self): - ... return ''.format(self) - ... - >>> schema = Schema(Object({'q': 'one'}, cls=Structure)) - >>> schema(Structure(q='one')) - - -Allow None values -~~~~~~~~~~~~~~~~~ - -To allow value to be None as well, use Any: - -.. code:: pycon - - >>> from voluptuous import Any - - >>> schema = Schema(Any(None, int)) - >>> schema(None) - >>> schema(5) - 5 - -Error reporting ---------------- - -Validators must throw an ``Invalid`` exception if invalid data is passed -to them. All other exceptions are treated as errors in the validator and -will not be caught. - -Each ``Invalid`` exception has an associated ``path`` attribute -representing the path in the data structure to our currently validating -value, as well as an ``error_message`` attribute that contains the -message of the original exception. This is especially useful when you -want to catch ``Invalid`` exceptions and give some feedback to the user, -for instance in the context of an HTTP API. - -.. code:: pycon - - >>> def validate_email(email): - ... """Validate email.""" - ... if not "@" in email: - ... raise Invalid("This email is invalid.") - ... return email - >>> schema = Schema({"email": validate_email}) - >>> exc = None - >>> try: - ... schema({"email": "whatever"}) - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "This email is invalid. for dictionary value @ data['email']" - >>> exc.path - ['email'] - >>> exc.msg - 'This email is invalid.' - >>> exc.error_message - 'This email is invalid.' - -The ``path`` attribute is used during error reporting, but also during -matching to determine whether an error should be reported to the user or -if the next match should be attempted. This is determined by comparing -the depth of the path where the check is, to the depth of the path where -the error occurred. If the error is more than one level deeper, it is -reported. - -The upshot of this is that *matching is depth-first and fail-fast*. - -To illustrate this, here is an example schema: - -.. code:: pycon - - >>> schema = Schema([[2, 3], 6]) - -Each value in the top-level list is matched depth-first in-order. Given -input data of ``[[6]]``, the inner list will match the first element of -the schema, but the literal ``6`` will not match any of the elements of -that list. This error will be reported back to the user immediately. No -backtracking is attempted: - -.. code:: pycon - - >>> try: - ... schema([[6]]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value @ data[0][0]" - True - -If we pass the data ``[6]``, the ``6`` is not a list type and so will -not recurse into the first element of the schema. Matching will continue -on to the second element in the schema, and succeed: - -.. code:: pycon - - >>> schema([6]) - [6] - -Running tests. --------------- - -Voluptuous is using nosetests: - -:: - - $ nosetests - -Why use Voluptuous over another validation library? ---------------------------------------------------- - -**Validators are simple callables** - No need to subclass anything, just use a function. -**Errors are simple exceptions.** - A validator can just ``raise Invalid(msg)`` and expect the user to - get useful messages. -**Schemas are basic Python data structures.** - Should your data be a dictionary of integer keys to strings? - ``{int: str}`` does what you expect. List of integers, floats or - strings? ``[int, float, str]``. -**Designed from the ground up for validating more than just forms.** - Nested data structures are treated in the same way as any other - type. Need a list of dictionaries? ``[{}]`` -**Consistency.** - Types in the schema are checked as types. Values are compared as - values. Callables are called to validate. Simple. - -Other libraries and inspirations --------------------------------- - -Voluptuous is heavily inspired by -`Validino `__, and to a lesser -extent, `jsonvalidator `__ and -`json\_schema `__. - -I greatly prefer the light-weight style promoted by these libraries to -the complexity of libraries like FormEncode. - -.. |Build Status| image:: https://travis-ci.org/alecthomas/voluptuous.png - :target: https://travis-ci.org/alecthomas/voluptuous -.. |Coverage Status| image:: https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master - :target: https://coveralls.io/github/alecthomas/voluptuous?branch=master -.. |Gitter chat| image:: https://badges.gitter.im/alecthomas.png - :target: https://gitter.im/alecthomas/Lobby diff --git a/third_party/python/voluptuous/setup.py b/third_party/python/voluptuous/setup.py --- a/third_party/python/voluptuous/setup.py +++ b/third_party/python/voluptuous/setup.py @@ -1,51 +1,40 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup +from setuptools import setup import sys +import io import os import atexit sys.path.insert(0, '.') version = __import__('voluptuous').__version__ -try: - import pypandoc - long_description = pypandoc.convert('README.md', 'rst') - with open('README.rst', 'w') as f: - f.write(long_description) - atexit.register(lambda: os.unlink('README.rst')) -except (ImportError, OSError): - print('WARNING: Could not locate pandoc, using Markdown long_description.') - with open('README.md') as f: - long_description = f.read() -description = long_description.splitlines()[0].strip() +with io.open('README.md', encoding='utf-8') as f: + long_description = f.read() + description = long_description.splitlines()[0].strip() setup( name='voluptuous', url='https://github.com/alecthomas/voluptuous', download_url='https://pypi.python.org/pypi/voluptuous', version=version, description=description, long_description=long_description, + long_description_content_type='text/markdown', license='BSD', platforms=['any'], packages=['voluptuous'], author='Alec Thomas', author_email='alec@swapoff.org', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ] ) diff --git a/third_party/python/voluptuous/voluptuous/__init__.py b/third_party/python/voluptuous/voluptuous/__init__.py --- a/third_party/python/voluptuous/voluptuous/__init__.py +++ b/third_party/python/voluptuous/voluptuous/__init__.py @@ -1,15 +1,9 @@ # flake8: noqa -try: - from schema_builder import * - from validators import * - from util import * - from error import * -except ImportError: - from .schema_builder import * - from .validators import * - from .util import * - from .error import * +from voluptuous.schema_builder import * +from voluptuous.validators import * +from voluptuous.util import * +from voluptuous.error import * -__version__ = '0.10.5' -__author__ = 'tusharmakkar08' +__version__ = '0.11.5' +__author__ = 'alecthomas' diff --git a/third_party/python/voluptuous/voluptuous/error.py b/third_party/python/voluptuous/voluptuous/error.py --- a/third_party/python/voluptuous/voluptuous/error.py +++ b/third_party/python/voluptuous/voluptuous/error.py @@ -182,8 +182,18 @@ class InInvalid(Invalid): class NotInInvalid(Invalid): pass class ExactSequenceInvalid(Invalid): pass + + +class NotEnoughValid(Invalid): + """The value did not pass enough validations.""" + pass + + +class TooManyValid(Invalid): + """The value passed more than expected validations.""" + pass diff --git a/third_party/python/voluptuous/voluptuous/schema_builder.py b/third_party/python/voluptuous/voluptuous/schema_builder.py --- a/third_party/python/voluptuous/voluptuous/schema_builder.py +++ b/third_party/python/voluptuous/voluptuous/schema_builder.py @@ -1,21 +1,17 @@ import collections import inspect import re from functools import wraps import sys from contextlib import contextmanager import itertools - -try: - import error as er -except ImportError: - from . import error as er +from voluptuous import error as er if sys.version_info >= (3,): long = int unicode = str basestring = str ifilter = filter def iteritems(d): @@ -121,16 +117,20 @@ class Undefined(object): def __repr__(self): return '...' UNDEFINED = Undefined() +def Self(): + raise er.SchemaError('"Self" should never be called') + + def default_factory(value): if value is UNDEFINED or callable(value): return value return lambda: value @contextmanager def raises(exc, msg=None, regex=None): @@ -196,21 +196,67 @@ class Schema(object): - Any value other than the above defaults to :const:`~voluptuous.PREVENT_EXTRA` """ self.schema = schema self.required = required self.extra = int(extra) # ensure the value is an integer self._compiled = self._compile(schema) + @classmethod + def infer(cls, data, **kwargs): + """Create a Schema from concrete data (e.g. an API response). + + For example, this will take a dict like: + + { + 'foo': 1, + 'bar': { + 'a': True, + 'b': False + }, + 'baz': ['purple', 'monkey', 'dishwasher'] + } + + And return a Schema: + + { + 'foo': int, + 'bar': { + 'a': bool, + 'b': bool + }, + 'baz': [str] + } + + Note: only very basic inference is supported. + """ + def value_to_schema_type(value): + if isinstance(value, dict): + if len(value) == 0: + return dict + return {k: value_to_schema_type(v) + for k, v in iteritems(value)} + if isinstance(value, list): + if len(value) == 0: + return list + else: + return [value_to_schema_type(v) + for v in value] + return type(value) + + return cls(value_to_schema_type(data), **kwargs) + def __eq__(self, other): - if str(other) == str(self.schema): - # Because repr is combination mixture of object and schema - return True - return False + if not isinstance(other, Schema): + return False + return other.schema == self.schema + + def __ne__(self, other): + return not (self == other) def __str__(self): return str(self.schema) def __repr__(self): return "" % ( self.schema, self._extra_to_name.get(self.extra, '??'), self.required, id(self)) @@ -223,26 +269,32 @@ class Schema(object): raise except er.Invalid as e: raise er.MultipleInvalid([e]) # return self.validate([], self.schema, data) def _compile(self, schema): if schema is Extra: return lambda _, v: v + if schema is Self: + return lambda p, v: self._compiled(p, v) + elif hasattr(schema, "__voluptuous_compile__"): + return schema.__voluptuous_compile__(self) if isinstance(schema, Object): return self._compile_object(schema) - if isinstance(schema, collections.Mapping) and len(schema): + if isinstance(schema, collections.Mapping): return self._compile_dict(schema) - elif isinstance(schema, list) and len(schema): + elif isinstance(schema, list): return self._compile_list(schema) elif isinstance(schema, tuple): return self._compile_tuple(schema) + elif isinstance(schema, (frozenset, set)): + return self._compile_set(schema) type_ = type(schema) - if type_ is type: + if inspect.isclass(schema): type_ = schema if type_ in (bool, bytes, int, long, str, unicode, float, complex, object, list, dict, type(None)) or callable(schema): return _compile_scalar(schema) raise er.SchemaError('unsupported schema data type %r' % type(schema).__name__) def _compile_mapping(self, schema, invalid_msg=None): @@ -279,21 +331,35 @@ class Schema(object): elif isinstance(skey, Marker) and type(skey.schema) in primitive_types: candidates_by_key.setdefault(skey.schema, []).append((skey, (ckey, cvalue))) else: # These are wildcards such as 'int', 'str', 'Remove' and others which should be applied to all keys additional_candidates.append((skey, (ckey, cvalue))) def validate_mapping(path, iterable, out): required_keys = all_required_keys.copy() - # keeps track of all default keys that haven't been filled - default_keys = all_default_keys.copy() + + # Build a map of all provided key-value pairs. + # The type(out) is used to retain ordering in case a ordered + # map type is provided as input. + key_value_map = type(out)() + for key, value in iterable: + key_value_map[key] = value + + # Insert default values for non-existing keys. + for key in all_default_keys: + if not isinstance(key.default, Undefined) and \ + key.schema not in key_value_map: + # A default value has been specified for this missing + # key, insert it. + key_value_map[key.schema] = key.default() + error = None errors = [] - for key, value in iterable: + for key, value in key_value_map.items(): key_path = path + [key] remove_key = False # Optimization. Validate against the matching key first, then fallback to the rest relevant_candidates = itertools.chain(candidates_by_key.get(key, []), additional_candidates) # compare each given key/value against all compiled key/values # schema key, (compiled key, compiled value) @@ -333,40 +399,31 @@ class Schema(object): errors.append(err) # If there is a validation error for a required # key, this means that the key was provided. # Discard the required key so it does not # create an additional, noisy exception. required_keys.discard(skey) break - # Key and value okay, mark any Required() fields as found. + # Key and value okay, mark as found in case it was + # a Required() field. required_keys.discard(skey) - # No need for a default if it was filled - default_keys.discard(skey) - break else: if remove_key: # remove key continue elif self.extra == ALLOW_EXTRA: out[key] = value elif self.extra != REMOVE_EXTRA: errors.append(er.Invalid('extra keys not allowed', key_path)) # else REMOVE_EXTRA: ignore the key so it's removed from output - # set defaults for any that can have defaults - for key in default_keys: - if not isinstance(key.default, Undefined): # if the user provides a default with the node - out[key.schema] = key.default() - if key in required_keys: - required_keys.discard(key) - # for any required keys left that weren't found and don't have defaults: for key in required_keys: msg = key.msg if hasattr(key, 'msg') and key.msg else 'required key not provided' errors.append(er.RequiredFieldInvalid(msg, path + [key])) if errors: raise er.MultipleInvalid(errors) return out @@ -407,51 +464,45 @@ class Schema(object): def _compile_dict(self, schema): """Validate a dictionary. A dictionary schema can contain a set of values, or at most one validator function/type. A dictionary schema will only validate a dictionary: - >>> validate = Schema({'prop': str}) + >>> validate = Schema({}) >>> with raises(er.MultipleInvalid, 'expected a dictionary'): ... validate([]) An invalid dictionary value: >>> validate = Schema({'one': 'two', 'three': 'four'}) >>> with raises(er.MultipleInvalid, "not a valid value for dictionary value @ data['one']"): ... validate({'one': 'three'}) An invalid key: >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data['two']"): ... validate({'two': 'three'}) + Validation function, in this case the "int" type: >>> validate = Schema({'one': 'two', 'three': 'four', int: str}) Valid integer input: >>> validate({10: 'twenty'}) {10: 'twenty'} - An empty dictionary is matched as value: - - >>> validate = Schema({}) - >>> with raises(er.MultipleInvalid, 'not a valid value'): - ... validate([]) - By default, a "type" in the schema (in this case "int") will be used purely to validate that the corresponding value is of that type. It will not Coerce the value: - >>> validate = Schema({'one': 'two', 'three': 'four', int: str}) >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data['10']"): ... validate({'10': 'twenty'}) Wrap them in the Coerce() function to achieve this: >>> from voluptuous import Coerce >>> validate = Schema({'one': 'two', 'three': 'four', ... Coerce(int): str}) >>> validate({'10': 'twenty'}) @@ -556,16 +607,20 @@ class Schema(object): seq_type_name = seq_type.__name__ def validate_sequence(path, data): if not isinstance(data, seq_type): raise er.SequenceTypeInvalid('expected a %s' % seq_type_name, path) # Empty seq schema, allow any data. if not schema: + if data: + raise er.MultipleInvalid([ + er.ValueInvalid('not a valid value', [value]) for value in data + ]) return data out = [] invalid = None errors = [] index_path = UNDEFINED for i, value in enumerate(data): index_path = path + [i] @@ -617,16 +672,56 @@ class Schema(object): ['one'] >>> with raises(er.MultipleInvalid, 'expected int @ data[0]'): ... validator([3.5]) >>> validator([1]) [1] """ return self._compile_sequence(schema, list) + def _compile_set(self, schema): + """Validate a set. + + A set is an unordered collection of unique elements. + + >>> validator = Schema({int}) + >>> validator(set([42])) == set([42]) + True + >>> with raises(er.Invalid, 'expected a set'): + ... validator(42) + >>> with raises(er.MultipleInvalid, 'invalid value in set'): + ... validator(set(['a'])) + """ + type_ = type(schema) + type_name = type_.__name__ + + def validate_set(path, data): + if not isinstance(data, type_): + raise er.Invalid('expected a %s' % type_name, path) + + _compiled = [self._compile(s) for s in schema] + errors = [] + for value in data: + for validate in _compiled: + try: + validate(path, value) + break + except er.Invalid: + pass + else: + invalid = er.Invalid('invalid value in %s' % type_name, path) + errors.append(invalid) + + if errors: + raise er.MultipleInvalid(errors) + + return data + + return validate_set + def extend(self, schema, required=None, extra=None): """Create a new `Schema` by merging this and the provided `schema`. Neither this `Schema` nor the provided `schema` are modified. The resulting `Schema` inherits the `required` and `extra` parameters of this, unless overridden. Both schemas must be dictionary-based. @@ -695,17 +790,17 @@ def _compile_scalar(schema): >>> _compile_scalar(lambda v: float(v))([], '1') 1.0 As a convenience, ValueError's are trapped: >>> with raises(er.Invalid, 'not a valid value'): ... _compile_scalar(lambda v: float(v))([], 'a') """ - if isinstance(schema, type): + if inspect.isclass(schema): def validate_instance(path, data): if isinstance(data, schema): return data else: msg = 'expected %s' % schema.__name__ raise er.TypeInvalid(msg, path) return validate_instance @@ -798,17 +893,16 @@ def _iterate_object(obj): try: slots = obj.__slots__ except AttributeError: pass else: for key in slots: if key != '__dict__': yield (key, getattr(obj, key)) - raise StopIteration() class Msg(object): """Report a user-friendly message if a schema fails to validate. >>> validate = Schema( ... Msg(['one', 'two', int], ... 'should be one of "one", "two" or an integer')) @@ -874,37 +968,40 @@ class VirtualPathComponent(str): # Markers.py class Marker(object): """Mark nodes for special treatment.""" - def __init__(self, schema_, msg=None): + def __init__(self, schema_, msg=None, description=None): self.schema = schema_ self._schema = Schema(schema_) self.msg = msg + self.description = description def __call__(self, v): try: return self._schema(v) except er.Invalid as e: if not self.msg or len(e.path) > 1: raise raise er.Invalid(self.msg) def __str__(self): return str(self.schema) def __repr__(self): return repr(self.schema) def __lt__(self, other): - return self.schema < other.schema + if isinstance(other, Marker): + return self.schema < other.schema + return self.schema < other def __hash__(self): return hash(self.schema) def __eq__(self, other): return self.schema == other def __ne__(self, other): @@ -929,18 +1026,19 @@ class Optional(Marker): >>> schema = Schema({ ... Optional('key'): str, ... 'key2': str ... }, required=True) >>> schema({'key2':'value'}) {'key2': 'value'} """ - def __init__(self, schema, msg=None, default=UNDEFINED): - super(Optional, self).__init__(schema, msg=msg) + def __init__(self, schema, msg=None, default=UNDEFINED, description=None): + super(Optional, self).__init__(schema, msg=msg, + description=description) self.default = default_factory(default) class Exclusive(Optional): """Mark a node in the schema as exclusive. Exclusive keys inherited from Optional: @@ -970,18 +1068,19 @@ class Exclusive(Optional): ... } ... }) >>> with raises(er.MultipleInvalid, "Please, use only one type of authentication at the same time. @ data[]"): ... schema({'classic': {'email': 'foo@example.com', 'password': 'bar'}, ... 'social': {'social_network': 'barfoo', 'token': 'tEMp'}}) """ - def __init__(self, schema, group_of_exclusion, msg=None): - super(Exclusive, self).__init__(schema, msg=msg) + def __init__(self, schema, group_of_exclusion, msg=None, description=None): + super(Exclusive, self).__init__(schema, msg=msg, + description=description) self.group_of_exclusion = group_of_exclusion class Inclusive(Optional): """ Mark a node in the schema as inclusive. Inclusive keys inherited from Optional: @@ -1037,18 +1136,19 @@ class Required(Marker): >>> schema = Schema({Required('key', default='value'): str}) >>> schema({}) {'key': 'value'} >>> schema = Schema({Required('key', default=list): list}) >>> schema({}) {'key': []} """ - def __init__(self, schema, msg=None, default=UNDEFINED): - super(Required, self).__init__(schema, msg=msg) + def __init__(self, schema, msg=None, default=UNDEFINED, description=None): + super(Required, self).__init__(schema, msg=msg, + description=description) self.default = default_factory(default) class Remove(Marker): """Mark a node in the schema to be removed and excluded from the validated output. Keys that fail validation will not raise ``Invalid``. Instead, these keys will be treated as extras. @@ -1067,16 +1167,17 @@ class Remove(Marker): return self.__class__ def __repr__(self): return "Remove(%r)" % (self.schema,) def __hash__(self): return object.__hash__(self) + def message(default=None, cls=None): """Convenience decorator to allow functions to provide a message. Set a default message: >>> @message('not an integer') ... def isint(v): ... return int(v) @@ -1169,17 +1270,18 @@ def validate(*a, **kw): schema_args_dict = _args_to_dict(func, a) schema_arguments = _merge_args_with_kwargs(schema_args_dict, kw) if RETURNS_KEY in schema_arguments: returns_defined = True returns = schema_arguments[RETURNS_KEY] del schema_arguments[RETURNS_KEY] - input_schema = Schema(schema_arguments) if len(schema_arguments) != 0 else lambda x: x + input_schema = (Schema(schema_arguments, extra=ALLOW_EXTRA) + if len(schema_arguments) != 0 else lambda x: x) output_schema = Schema(returns) if returns_defined else lambda x: x @wraps(func) def func_wrapper(*args, **kwargs): args_dict = _args_to_dict(func, args) arguments = _merge_args_with_kwargs(args_dict, kwargs) validated_arguments = input_schema(arguments) output = func(**validated_arguments) diff --git a/third_party/python/voluptuous/voluptuous/tests/tests.md b/third_party/python/voluptuous/voluptuous/tests/tests.md --- a/third_party/python/voluptuous/voluptuous/tests/tests.md +++ b/third_party/python/voluptuous/voluptuous/tests/tests.md @@ -261,8 +261,13 @@ Ensure that subclasses of Invalid of are ... >>> schema = Schema({'thing': custom_validator}) >>> try: ... schema({'thing': 'not an int'}) ... except MultipleInvalid as e: ... exc = e >>> exc.errors[0].__class__.__name__ 'SpecialInvalid' + +Ensure that Optional('Classification') < 'Name' will return True instead of throwing an AttributeError + + >>> Optional('Classification') < 'Name' + True diff --git a/third_party/python/voluptuous/voluptuous/tests/tests.py b/third_party/python/voluptuous/voluptuous/tests/tests.py --- a/third_party/python/voluptuous/voluptuous/tests/tests.py +++ b/third_party/python/voluptuous/voluptuous/tests/tests.py @@ -1,19 +1,22 @@ import copy import collections +import os import sys -from nose.tools import assert_equal, assert_raises, assert_true + +from nose.tools import assert_equal, assert_false, assert_raises, assert_true from voluptuous import ( - Schema, Required, Optional, Extra, Invalid, In, Remove, Literal, - Url, MultipleInvalid, LiteralInvalid, NotIn, Match, Email, + Schema, Required, Exclusive, Optional, Extra, Invalid, In, Remove, Literal, + Url, MultipleInvalid, LiteralInvalid, TypeInvalid, NotIn, Match, Email, Replace, Range, Coerce, All, Any, Length, FqdnUrl, ALLOW_EXTRA, PREVENT_EXTRA, validate, ExactSequence, Equal, Unordered, Number, Maybe, Datetime, Date, - Contains, Marker) + Contains, Marker, IsDir, IsFile, PathExists, SomeOf, TooManyValid, Self, + raises) from voluptuous.humanize import humanize_error from voluptuous.util import u def test_exact_sequence(): schema = Schema(ExactSequence([int, int])) try: schema([1, 2, 3]) @@ -148,16 +151,49 @@ def test_literal(): except MultipleInvalid as e: assert_equal(str(e), "{'b': 1} not match for {'a': 1}") assert_equal(len(e.errors), 1) assert_equal(type(e.errors[0]), LiteralInvalid) else: assert False, "Did not raise Invalid" +def test_class(): + class C1(object): + pass + + schema = Schema(C1) + schema(C1()) + + try: + schema(None) + except MultipleInvalid as e: + assert_equal(str(e), "expected C1") + assert_equal(len(e.errors), 1) + assert_equal(type(e.errors[0]), TypeInvalid) + else: + assert False, "Did not raise Invalid" + + # In Python 2, this will be an old-style class (classobj instance) + class C2: + pass + + schema = Schema(C2) + schema(C2()) + + try: + schema(None) + except MultipleInvalid as e: + assert_equal(str(e), "expected C2") + assert_equal(len(e.errors), 1) + assert_equal(type(e.errors[0]), TypeInvalid) + else: + assert False, "Did not raise Invalid" + + def test_email_validation(): """ test with valid email """ schema = Schema({"email": Email()}) out_ = schema({"email": "example@example.com"}) assert 'example@example.com"', out_.get("url") @@ -370,16 +406,79 @@ def test_subschema_extension(): extension = {'d': str, 'a': {'b': str, 'e': int}} extended = base.extend(extension) assert_equal(base.schema, {'a': {'b': int, 'c': float}}) assert_equal(extension, {'d': str, 'a': {'b': str, 'e': int}}) assert_equal(extended.schema, {'a': {'b': str, 'c': float, 'e': int}, 'd': str}) +def test_equality(): + assert_equal(Schema('foo'), Schema('foo')) + + assert_equal(Schema(['foo', 'bar', 'baz']), + Schema(['foo', 'bar', 'baz'])) + + # Ensure two Schemas w/ two equivalent dicts initialized in a different + # order are considered equal. + dict_a = {} + dict_a['foo'] = 1 + dict_a['bar'] = 2 + dict_a['baz'] = 3 + + dict_b = {} + dict_b['baz'] = 3 + dict_b['bar'] = 2 + dict_b['foo'] = 1 + + assert_equal(Schema(dict_a), Schema(dict_b)) + + +def test_equality_negative(): + """Verify that Schema objects are not equal to string representations""" + assert_false(Schema('foo') == 'foo') + + assert_false(Schema(['foo', 'bar']) == "['foo', 'bar']") + assert_false(Schema(['foo', 'bar']) == Schema("['foo', 'bar']")) + + assert_false(Schema({'foo': 1, 'bar': 2}) == "{'foo': 1, 'bar': 2}") + assert_false(Schema({'foo': 1, 'bar': 2}) == Schema("{'foo': 1, 'bar': 2}")) + + +def test_inequality(): + assert_true(Schema('foo') != 'foo') + + assert_true(Schema(['foo', 'bar']) != "['foo', 'bar']") + assert_true(Schema(['foo', 'bar']) != Schema("['foo', 'bar']")) + + assert_true(Schema({'foo': 1, 'bar': 2}) != "{'foo': 1, 'bar': 2}") + assert_true(Schema({'foo': 1, 'bar': 2}) != Schema("{'foo': 1, 'bar': 2}")) + + +def test_inequality_negative(): + assert_false(Schema('foo') != Schema('foo')) + + assert_false(Schema(['foo', 'bar', 'baz']) != + Schema(['foo', 'bar', 'baz'])) + + # Ensure two Schemas w/ two equivalent dicts initialized in a different + # order are considered equal. + dict_a = {} + dict_a['foo'] = 1 + dict_a['bar'] = 2 + dict_a['baz'] = 3 + + dict_b = {} + dict_b['baz'] = 3 + dict_b['bar'] = 2 + dict_b['foo'] = 1 + + assert_false(Schema(dict_a) != Schema(dict_b)) + + def test_repr(): """Verify that __repr__ returns valid Python expressions""" match = Match('a pattern', msg='message') replace = Replace('you', 'I', msg='you and I') range_ = Range(min=0, max=42, min_included=False, max_included=False, msg='number not in range') coerce_ = Coerce(int, msg="moo") all_ = All('10', Coerce(int), msg='all msg') @@ -388,17 +487,17 @@ def test_repr(): assert_equal(repr(match), "Match('a pattern', msg='message')") assert_equal(repr(replace), "Replace('you', 'I', msg='you and I')") assert_equal( repr(range_), "Range(min=0, max=42, min_included=False, max_included=False, msg='number not in range')" ) assert_equal(repr(coerce_), "Coerce(int, msg='moo')") assert_equal(repr(all_), "All('10', Coerce(int, msg=None), msg='all msg')") - assert_equal(repr(maybe_int), "Maybe(%s)" % str(int)) + assert_equal(repr(maybe_int), "Any(None, %s, msg=None)" % str(int)) def test_list_validation_messages(): """ Make sure useful error messages are available """ def is_even(value): if value % 2: raise Invalid('%i is not even' % value) @@ -509,65 +608,33 @@ def test_unordered(): # Most accurate validators must be positioned on left s = Schema(Unordered([int, 3])) assert_raises(Invalid, s, [3, 2]) s = Schema(Unordered([3, int])) s([3, 2]) def test_maybe(): - assert_raises(TypeError, Maybe, lambda x: x) - s = Schema(Maybe(int)) assert s(1) == 1 assert s(None) is None + assert_raises(Invalid, s, 'foo') - assert_raises(Invalid, s, 'foo') + s = Schema(Maybe({str: Coerce(int)})) + assert s({'foo': '100'}) == {'foo': 100} + assert s(None) is None + assert_raises(Invalid, s, {'foo': 'bar'}) def test_empty_list_as_exact(): s = Schema([]) assert_raises(Invalid, s, [1]) s([]) -def test_empty_dict_as_exact(): - # {} always evaluates as {} - s = Schema({}) - assert_raises(Invalid, s, {'extra': 1}) - s = Schema({}, extra=ALLOW_EXTRA) # this should not be used - assert_raises(Invalid, s, {'extra': 1}) - - # {...} evaluates as Schema({...}) - s = Schema({'foo': int}) - assert_raises(Invalid, s, {'foo': 1, 'extra': 1}) - s = Schema({'foo': int}, extra=ALLOW_EXTRA) - s({'foo': 1, 'extra': 1}) - - # dict matches {} or {...} - s = Schema(dict) - s({'extra': 1}) - s({}) - s = Schema(dict, extra=PREVENT_EXTRA) - s({'extra': 1}) - s({}) - - # nested {} evaluate as {} - s = Schema({ - 'inner': {} - }, extra=ALLOW_EXTRA) - assert_raises(Invalid, s, {'inner': {'extra': 1}}) - s({}) - s = Schema({ - 'inner': Schema({}, extra=ALLOW_EXTRA) - }) - assert_raises(Invalid, s, {'inner': {'extra': 1}}) - s({}) - - def test_schema_decorator_match_with_args(): @validate(int) def fn(arg): return arg fn(1) @@ -638,16 +705,48 @@ def test_schema_decorator_return_only_ma def test_schema_decorator_return_only_unmatch(): @validate(__return__=int) def fn(arg): return "hello" assert_raises(Invalid, fn, 1) +def test_schema_decorator_partial_match_called_with_args(): + @validate(arg1=int) + def fn(arg1, arg2): + return arg1 + + fn(1, "foo") + + +def test_schema_decorator_partial_unmatch_called_with_args(): + @validate(arg1=int) + def fn(arg1, arg2): + return arg1 + + assert_raises(Invalid, fn, "bar", "foo") + + +def test_schema_decorator_partial_match_called_with_kwargs(): + @validate(arg2=int) + def fn(arg1, arg2): + return arg1 + + fn(arg1="foo", arg2=1) + + +def test_schema_decorator_partial_unmatch_called_with_kwargs(): + @validate(arg2=int) + def fn(arg1, arg2): + return arg1 + + assert_raises(Invalid, fn, arg1=1, arg2="foo") + + def test_unicode_as_key(): if sys.version_info >= (3,): text_type = str else: text_type = unicode schema = Schema({text_type: int}) schema({u("foobar"): 1}) @@ -757,20 +856,25 @@ def test_datetime(): schema = Schema({"datetime": Datetime()}) schema({"datetime": "2016-10-24T14:01:57.102152Z"}) assert_raises(MultipleInvalid, schema, {"datetime": "2016-10-24T14:01:57"}) def test_date(): schema = Schema({"date": Date()}) schema({"date": "2016-10-24"}) - assert_raises(MultipleInvalid, schema, {"date": "2016-10-2"}) assert_raises(MultipleInvalid, schema, {"date": "2016-10-24Z"}) +def test_date_custom_format(): + schema = Schema({"date": Date("%Y%m%d")}) + schema({"date": "20161024"}) + assert_raises(MultipleInvalid, schema, {"date": "2016-10-24"}) + + def test_ordered_dict(): if not hasattr(collections, 'OrderedDict'): # collections.OrderedDict was added in Python2.7; only run if present return schema = Schema({Number(): Number()}) # x, y pairs (for interpolation or something) data = collections.OrderedDict([(5.0, 3.7), (24.0, 8.7), (43.0, 1.5), (62.0, 2.1), (71.5, 6.7), (90.5, 4.1), (109.0, 3.9)]) @@ -788,16 +892,89 @@ def test_marker_hashable(): assert_equal(definition.get('x'), int) assert_equal(definition.get('y'), float) assert_true(Required('x') == Required('x')) assert_true(Required('x') != Required('y')) # Remove markers are not hashable assert_equal(definition.get('j'), None) +def test_schema_infer(): + schema = Schema.infer({ + 'str': 'foo', + 'bool': True, + 'int': 42, + 'float': 3.14 + }) + assert_equal(schema, Schema({ + Required('str'): str, + Required('bool'): bool, + Required('int'): int, + Required('float'): float + })) + + +def test_schema_infer_dict(): + schema = Schema.infer({ + 'a': { + 'b': { + 'c': 'foo' + } + } + }) + + assert_equal(schema, Schema({ + Required('a'): { + Required('b'): { + Required('c'): str + } + } + })) + + +def test_schema_infer_list(): + schema = Schema.infer({ + 'list': ['foo', True, 42, 3.14] + }) + + assert_equal(schema, Schema({ + Required('list'): [str, bool, int, float] + })) + + +def test_schema_infer_scalar(): + assert_equal(Schema.infer('foo'), Schema(str)) + assert_equal(Schema.infer(True), Schema(bool)) + assert_equal(Schema.infer(42), Schema(int)) + assert_equal(Schema.infer(3.14), Schema(float)) + assert_equal(Schema.infer({}), Schema(dict)) + assert_equal(Schema.infer([]), Schema(list)) + + +def test_schema_infer_accepts_kwargs(): + schema = Schema.infer({ + 'str': 'foo', + 'bool': True + }, required=False, extra=True) + + # Subset of schema should be acceptable thanks to required=False. + schema({'bool': False}) + + # Keys that are in schema should still match required types. + try: + schema({'str': 42}) + except Invalid: + pass + else: + assert False, 'Did not raise Invalid for Number' + + # Extra fields should be acceptable thanks to extra=True. + schema({'str': 'bar', 'int': 42}) + + def test_validation_performance(): """ This test comes to make sure the validation complexity of dictionaries is done in a linear time. to achieve this a custom marker is used in the scheme that counts each time it is evaluated. By doing so we can determine if the validation is done in linear complexity. Prior to issue https://github.com/alecthomas/voluptuous/issues/259 this was exponential """ num_of_keys = 1000 @@ -811,20 +988,278 @@ def test_validation_performance(): class CounterMarker(Marker): def __call__(self, *args, **kwargs): counter[0] += 1 return super(CounterMarker, self).__call__(*args, **kwargs) for i in range(num_of_keys): schema_dict[CounterMarker(str(i))] = str data[str(i)] = str(i) - data_extra_keys[str(i*2)] = str(i) # half of the keys are present, and half aren't + data_extra_keys[str(i * 2)] = str(i) # half of the keys are present, and half aren't schema = Schema(schema_dict, extra=ALLOW_EXTRA) schema(data) assert counter[0] <= num_of_keys, "Validation complexity is not linear! %s > %s" % (counter[0], num_of_keys) counter[0] = 0 # reset counter schema(data_extra_keys) assert counter[0] <= num_of_keys, "Validation complexity is not linear! %s > %s" % (counter[0], num_of_keys) + + +def test_IsDir(): + schema = Schema(IsDir()) + assert_raises(MultipleInvalid, schema, 3) + schema(os.path.dirname(os.path.abspath(__file__))) + + +def test_IsFile(): + schema = Schema(IsFile()) + assert_raises(MultipleInvalid, schema, 3) + schema(os.path.abspath(__file__)) + + +def test_PathExists(): + schema = Schema(PathExists()) + assert_raises(MultipleInvalid, schema, 3) + schema(os.path.abspath(__file__)) + + +def test_description(): + marker = Marker(Schema(str), description='Hello') + assert marker.description == 'Hello' + + optional = Optional('key', description='Hello') + assert optional.description == 'Hello' + + exclusive = Exclusive('alpha', 'angles', description='Hello') + assert exclusive.description == 'Hello' + + required = Required('key', description='Hello') + assert required.description == 'Hello' + + +def test_SomeOf_min_validation(): + validator = All(Length(min=8), SomeOf( + min_valid=3, + validators=[Match(r'.*[A-Z]', 'no uppercase letters'), + Match(r'.*[a-z]', 'no lowercase letters'), + Match(r'.*[0-9]', 'no numbers'), + Match(r'.*[$@$!%*#?&^:;/<,>|{}()\-\'._+=]', 'no symbols')])) + + validator('ffe532A1!') + with raises(MultipleInvalid, 'length of value must be at least 8'): + validator('a') + + with raises(MultipleInvalid, 'no uppercase letters, no lowercase letters'): + validator('wqs2!#s111') + + with raises(MultipleInvalid, 'no lowercase letters, no symbols'): + validator('3A34SDEF5') + + +def test_SomeOf_max_validation(): + validator = SomeOf( + max_valid=2, + validators=[Match(r'.*[A-Z]', 'no uppercase letters'), + Match(r'.*[a-z]', 'no lowercase letters'), + Match(r'.*[0-9]', 'no numbers')], + msg='max validation test failed') + + validator('Aa') + with raises(TooManyValid, 'max validation test failed'): + validator('Aa1') + + +def test_self_validation(): + schema = Schema({"number": int, + "follow": Self}) + try: + schema({"number": "abc"}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + try: + schema({"follow": {"number": '123456.712'}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + schema({"follow": {"number": 123456}}) + schema({"follow": {"follow": {"number": 123456}}}) + + +def test_any_error_has_path(): + """https://github.com/alecthomas/voluptuous/issues/347""" + s = Schema({ + Optional('q'): int, + Required('q2'): Any(int, msg='toto') + }) + try: + s({'q': 'str', 'q2': 'tata'}) + except MultipleInvalid as exc: + assert ( + (exc.errors[0].path == ['q'] and exc.errors[1].path == ['q2']) or + (exc.errors[1].path == ['q'] and exc.errors[0].path == ['q2']) + ) + else: + assert False, "Did not raise AnyInvalid" + + +def test_all_error_has_path(): + """https://github.com/alecthomas/voluptuous/issues/347""" + s = Schema({ + Optional('q'): int, + Required('q2'): All([str, Length(min=10)], msg='toto'), + }) + try: + s({'q': 'str', 'q2': 12}) + except MultipleInvalid as exc: + assert ( + (exc.errors[0].path == ['q'] and exc.errors[1].path == ['q2']) or + (exc.errors[1].path == ['q'] and exc.errors[0].path == ['q2']) + ) + else: + assert False, "Did not raise AllInvalid" + + +def test_match_error_has_path(): + """https://github.com/alecthomas/voluptuous/issues/347""" + s = Schema({ + Required('q2'): Match("a"), + }) + try: + s({'q2': 12}) + except MultipleInvalid as exc: + assert exc.errors[0].path == ['q2'] + else: + assert False, "Did not raise MatchInvalid" + + +def test_self_any(): + schema = Schema({"number": int, + "follow": Any(Self, "stop")}) + try: + schema({"number": "abc"}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + try: + schema({"follow": {"number": '123456.712'}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + schema({"follow": {"number": 123456}}) + schema({"follow": {"follow": {"number": 123456}}}) + schema({"follow": {"follow": {"number": 123456, "follow": "stop"}}}) + + +def test_self_all(): + schema = Schema({"number": int, + "follow": All(Self, + Schema({"extra_number": int}, + extra=ALLOW_EXTRA))}, + extra=ALLOW_EXTRA) + try: + schema({"number": "abc"}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + try: + schema({"follow": {"number": '123456.712'}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + schema({"follow": {"number": 123456}}) + schema({"follow": {"follow": {"number": 123456}}}) + schema({"follow": {"number": 123456, "extra_number": 123}}) + try: + schema({"follow": {"number": 123456, "extra_number": "123"}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + + +def test_SomeOf_on_bounds_assertion(): + with raises(AssertionError, 'when using "SomeOf" you should specify at least one of min_valid and max_valid'): + SomeOf(validators=[]) + + +def test_comparing_voluptuous_object_to_str(): + assert_true(Optional('Classification') < 'Name') + + +def test_set_of_integers(): + schema = Schema({int}) + with raises(Invalid, 'expected a set'): + schema(42) + with raises(Invalid, 'expected a set'): + schema(frozenset([42])) + + schema(set()) + schema(set([42])) + schema(set([42, 43, 44])) + try: + schema(set(['abc'])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in set") + else: + assert False, "Did not raise Invalid" + + +def test_frozenset_of_integers(): + schema = Schema(frozenset([int])) + with raises(Invalid, 'expected a frozenset'): + schema(42) + with raises(Invalid, 'expected a frozenset'): + schema(set([42])) + + schema(frozenset()) + schema(frozenset([42])) + schema(frozenset([42, 43, 44])) + try: + schema(frozenset(['abc'])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in frozenset") + else: + assert False, "Did not raise Invalid" + + +def test_set_of_integers_and_strings(): + schema = Schema({int, str}) + with raises(Invalid, 'expected a set'): + schema(42) + + schema(set()) + schema(set([42])) + schema(set(['abc'])) + schema(set([42, 'abc'])) + try: + schema(set([None])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in set") + else: + assert False, "Did not raise Invalid" + + +def test_frozenset_of_integers_and_strings(): + schema = Schema(frozenset([int, str])) + with raises(Invalid, 'expected a frozenset'): + schema(42) + + schema(frozenset()) + schema(frozenset([42])) + schema(frozenset(['abc'])) + schema(frozenset([42, 'abc'])) + try: + schema(frozenset([None])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in frozenset") + else: + assert False, "Did not raise Invalid" diff --git a/third_party/python/voluptuous/voluptuous/util.py b/third_party/python/voluptuous/voluptuous/util.py --- a/third_party/python/voluptuous/voluptuous/util.py +++ b/third_party/python/voluptuous/voluptuous/util.py @@ -1,18 +1,13 @@ import sys -try: - from error import LiteralInvalid, TypeInvalid, Invalid - from schema_builder import Schema, default_factory, raises - import validators -except ImportError: - from .error import LiteralInvalid, TypeInvalid, Invalid - from .schema_builder import Schema, default_factory, raises - from . import validators +from voluptuous.error import LiteralInvalid, TypeInvalid, Invalid +from voluptuous.schema_builder import Schema, default_factory, raises +from voluptuous import validators __author__ = 'tusharmakkar08' def Lower(v): """Transform a string to lower case. >>> s = Schema(Lower) diff --git a/third_party/python/voluptuous/voluptuous/validators.py b/third_party/python/voluptuous/voluptuous/validators.py --- a/third_party/python/voluptuous/voluptuous/validators.py +++ b/third_party/python/voluptuous/voluptuous/validators.py @@ -1,31 +1,25 @@ import os import re import datetime import sys from functools import wraps from decimal import Decimal, InvalidOperation -try: - from schema_builder import Schema, raises, message - from error import (MultipleInvalid, CoerceInvalid, TrueInvalid, FalseInvalid, BooleanInvalid, Invalid, AnyInvalid, - AllInvalid, MatchInvalid, UrlInvalid, EmailInvalid, FileInvalid, DirInvalid, RangeInvalid, - PathInvalid, ExactSequenceInvalid, LengthInvalid, DatetimeInvalid, DateInvalid, InInvalid, - TypeInvalid, NotInInvalid, ContainsInvalid) -except ImportError: - from .schema_builder import Schema, raises, message - from .error import (MultipleInvalid, CoerceInvalid, TrueInvalid, FalseInvalid, BooleanInvalid, Invalid, AnyInvalid, - AllInvalid, MatchInvalid, UrlInvalid, EmailInvalid, FileInvalid, DirInvalid, RangeInvalid, - PathInvalid, ExactSequenceInvalid, LengthInvalid, DatetimeInvalid, DateInvalid, InInvalid, - TypeInvalid, NotInInvalid, ContainsInvalid) - +from voluptuous.schema_builder import Schema, raises, message +from voluptuous.error import (MultipleInvalid, CoerceInvalid, TrueInvalid, FalseInvalid, BooleanInvalid, Invalid, + AnyInvalid, AllInvalid, MatchInvalid, UrlInvalid, EmailInvalid, FileInvalid, DirInvalid, + RangeInvalid, PathInvalid, ExactSequenceInvalid, LengthInvalid, DatetimeInvalid, + DateInvalid, InInvalid, TypeInvalid, NotInInvalid, ContainsInvalid, NotEnoughValid, + TooManyValid) if sys.version_info >= (3,): import urllib.parse as urlparse + basestring = str else: import urlparse # Taken from https://github.com/kvesteri/validators/blob/master/validators/email.py USER_REGEX = re.compile( # dot-atom r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+" @@ -94,17 +88,17 @@ class Coerce(object): def __init__(self, type, msg=None): self.type = type self.msg = msg self.type_name = type.__name__ def __call__(self, v): try: return self.type(v) - except (ValueError, TypeError): + except (ValueError, TypeError, InvalidOperation): msg = self.msg or ('expected %s' % self.type_name) raise CoerceInvalid(msg) def __repr__(self): return 'Coerce(%s, msg=%r)' % (self.type_name, self.msg) @message('value was not true', cls=TrueInvalid) @@ -182,17 +176,50 @@ def Boolean(v): if v in ('1', 'true', 'yes', 'on', 'enable'): return True if v in ('0', 'false', 'no', 'off', 'disable'): return False raise ValueError return bool(v) -class Any(object): +class _WithSubValidators(object): + """Base class for validators that use sub-validators. + + Special class to use as a parent class for validators using sub-validators. + This class provides the `__voluptuous_compile__` method so the + sub-validators are compiled by the parent `Schema`. + """ + + def __init__(self, *validators, **kwargs): + self.validators = validators + self.msg = kwargs.pop('msg', None) + + def __voluptuous_compile__(self, schema): + self._compiled = [ + schema._compile(v) + for v in self.validators + ] + return self._run + + def _run(self, path, value): + return self._exec(self._compiled, value, path) + + def __call__(self, v): + return self._exec((Schema(val) for val in self.validators), v) + + def __repr__(self): + return '%s(%s, msg=%r)' % ( + self.__class__.__name__, + ", ".join(repr(v) for v in self.validators), + self.msg + ) + + +class Any(_WithSubValidators): """Use the first validated value. :param msg: Message to deliver to user if validation fails. :param kwargs: All other keyword arguments are passed to the sub-Schema constructors. :returns: Return value of the first validator that passes. >>> validate = Schema(Any('true', 'false', ... All(Any(int, bool), Coerce(bool)))) @@ -207,74 +234,63 @@ class Any(object): >>> validate = Schema(Any(1, 2, 3, msg="Expected 1 2 or 3")) >>> validate(1) 1 >>> with raises(MultipleInvalid, "Expected 1 2 or 3"): ... validate(4) """ - def __init__(self, *validators, **kwargs): - self.validators = validators - self.msg = kwargs.pop('msg', None) - self._schemas = [Schema(val, **kwargs) for val in validators] - - def __call__(self, v): + def _exec(self, funcs, v, path=None): error = None - for schema in self._schemas: + for func in funcs: try: - return schema(v) + if path is None: + return func(v) + else: + return func(path, v) except Invalid as e: if error is None or len(e.path) > len(error.path): error = e else: if error: - raise error if self.msg is None else AnyInvalid(self.msg) - raise AnyInvalid(self.msg or 'no valid value found') - - def __repr__(self): - return 'Any([%s])' % (", ".join(repr(v) for v in self.validators)) + raise error if self.msg is None else AnyInvalid( + self.msg, path=path) + raise AnyInvalid(self.msg or 'no valid value found', + path=path) # Convenience alias Or = Any -class All(object): +class All(_WithSubValidators): """Value must pass all validators. The output of each validator is passed as input to the next. :param msg: Message to deliver to user if validation fails. :param kwargs: All other keyword arguments are passed to the sub-Schema constructors. >>> validate = Schema(All('10', Coerce(int))) >>> validate('10') 10 """ - def __init__(self, *validators, **kwargs): - self.validators = validators - self.msg = kwargs.pop('msg', None) - self._schemas = [Schema(val, **kwargs) for val in validators] - - def __call__(self, v): + def _exec(self, funcs, v, path=None): try: - for schema in self._schemas: - v = schema(v) + for func in funcs: + if path is None: + v = func(v) + else: + v = func(path, v) except Invalid as e: - raise e if self.msg is None else AllInvalid(self.msg) + raise e if self.msg is None else AllInvalid(self.msg, path=path) return v - def __repr__(self): - return 'All(%s, msg=%r)' % ( - ", ".join(repr(v) for v in self.validators), - self.msg - ) - # Convenience alias And = All class Match(object): """Value must be a string that matches the regular expression. @@ -414,83 +430,82 @@ def IsFile(v): >>> os.path.basename(IsFile()(__file__)).startswith('validators.py') True >>> with raises(FileInvalid, 'not a file'): ... IsFile()("random_filename_goes_here.py") >>> with raises(FileInvalid, 'Not a file'): ... IsFile()(None) """ - if v: - return os.path.isfile(v) - else: + try: + if v: + v = str(v) + return os.path.isfile(v) + else: + raise FileInvalid('Not a file') + except TypeError: raise FileInvalid('Not a file') @message('not a directory', cls=DirInvalid) @truth def IsDir(v): """Verify the directory exists. >>> IsDir()('/') '/' >>> with raises(DirInvalid, 'Not a directory'): ... IsDir()(None) """ - if v: - return os.path.isdir(v) - else: + try: + if v: + v = str(v) + return os.path.isdir(v) + else: + raise DirInvalid("Not a directory") + except TypeError: raise DirInvalid("Not a directory") @message('path does not exist', cls=PathInvalid) @truth def PathExists(v): """Verify the path exists, regardless of its type. >>> os.path.basename(PathExists()(__file__)).startswith('validators.py') True >>> with raises(Invalid, 'path does not exist'): ... PathExists()("random_filename_goes_here.py") >>> with raises(PathInvalid, 'Not a Path'): ... PathExists()(None) """ - if v: - return os.path.exists(v) - else: + try: + if v: + v = str(v) + return os.path.exists(v) + else: + raise PathInvalid("Not a Path") + except TypeError: raise PathInvalid("Not a Path") -class Maybe(object): - """Validate that the object is of a given type or is None. +def Maybe(validator): + """Validate that the object matches given validator or is None. - :raises Invalid: if the value is not of the type declared and is not None + :raises Invalid: if the value does not match the given validator and is not + None >>> s = Schema(Maybe(int)) >>> s(10) 10 >>> with raises(Invalid): ... s("string") """ - def __init__(self, kind, msg=None): - if not isinstance(kind, type): - raise TypeError("kind has to be a type") - - self.kind = kind - self.msg = msg - - def __call__(self, v): - if v is not None and not isinstance(v, self.kind): - raise Invalid(self.msg or "%s must be None or of type %s" % (v, self.kind)) - - return v - - def __repr__(self): - return 'Maybe(%s)' % str(self.kind) + return Any(None, validator) class Range(object): """Limit a value to a range. Either min or max may be omitted. Either min or max can be excluded from the range of accepted values. @@ -616,25 +631,20 @@ class Datetime(object): def __repr__(self): return 'Datetime(format=%s)' % self.format class Date(Datetime): """Validate that the value matches the date format.""" DEFAULT_FORMAT = '%Y-%m-%d' - FORMAT_DESCRIPTION = 'yyyy-mm-dd' def __call__(self, v): try: datetime.datetime.strptime(v, self.format) - if len(v) != len(self.FORMAT_DESCRIPTION): - raise DateInvalid( - self.msg or 'value has invalid length' - ' expected length %d (%s)' % (len(self.FORMAT_DESCRIPTION), self.FORMAT_DESCRIPTION)) except (TypeError, ValueError): raise DateInvalid( self.msg or 'value does not match' ' expected format %s' % self.format) return v def __repr__(self): return 'Date(format=%s)' % self.format @@ -699,17 +709,17 @@ class Contains(object): check = self.item not in v except TypeError: check = True if check: raise ContainsInvalid(self.msg or 'value is not allowed') return v def __repr__(self): - return 'Contains(%s)' % (self.item, ) + return 'Contains(%s)' % (self.item,) class ExactSequence(object): """Matches each element in a sequence against the corresponding element in the validators. :param msg: Message to deliver to user if validation fails. :param kwargs: All other keyword arguments are passed to the sub-Schema @@ -861,20 +871,18 @@ class Unordered(object): break if not found: missing.append((index, value)) if len(missing) == 1: el = missing[0] raise Invalid(self.msg or 'Element #{} ({}) is not valid against any validator'.format(el[0], el[1])) elif missing: - raise MultipleInvalid([ - Invalid(self.msg or 'Element #{} ({}) is not valid against any validator'.format(el[0], el[1])) - for el in missing - ]) + raise MultipleInvalid([Invalid(self.msg or 'Element #{} ({}) is not valid against any validator'.format( + el[0], el[1])) for el in missing]) return v def __repr__(self): return 'Unordered([{}])'.format(", ".join(repr(v) for v in self.validators)) class Number(object): """ @@ -899,25 +907,26 @@ class Number(object): def __call__(self, v): """ :param v: is a number enclosed with string :return: Decimal number """ precision, scale, decimal_num = self._get_precision_scale(v) - if self.precision is not None and self.scale is not None and\ - precision != self.precision and scale != self.scale: - raise Invalid(self.msg or "Precision must be equal to %s, and Scale must be equal to %s" %(self.precision, self.scale)) + if self.precision is not None and self.scale is not None and precision != self.precision\ + and scale != self.scale: + raise Invalid(self.msg or "Precision must be equal to %s, and Scale must be equal to %s" % (self.precision, + self.scale)) else: if self.precision is not None and precision != self.precision: - raise Invalid(self.msg or "Precision must be equal to %s"%self.precision) + raise Invalid(self.msg or "Precision must be equal to %s" % self.precision) - if self.scale is not None and scale != self.scale : - raise Invalid(self.msg or "Scale must be equal to %s"%self.scale) + if self.scale is not None and scale != self.scale: + raise Invalid(self.msg or "Scale must be equal to %s" % self.scale) if self.yield_decimal: return decimal_num else: return v def __repr__(self): return ('Number(precision=%s, scale=%s, msg=%s)' % (self.precision, self.scale, self.msg)) @@ -928,8 +937,68 @@ class Number(object): :return: tuple(precision, scale, decimal_number) """ try: decimal_num = Decimal(number) except InvalidOperation: raise Invalid(self.msg or 'Value must be a number enclosed with string') return (len(decimal_num.as_tuple().digits), -(decimal_num.as_tuple().exponent), decimal_num) + + +class SomeOf(_WithSubValidators): + """Value must pass at least some validations, determined by the given parameter. + Optionally, number of passed validations can be capped. + + The output of each validator is passed as input to the next. + + :param min_valid: Minimum number of valid schemas. + :param validators: a list of schemas or validators to match input against + :param max_valid: Maximum number of valid schemas. + :param msg: Message to deliver to user if validation fails. + :param kwargs: All other keyword arguments are passed to the sub-Schema constructors. + + :raises NotEnoughValid: if the minimum number of validations isn't met + :raises TooManyValid: if the more validations than the given amount is met + + >>> validate = Schema(SomeOf(min_valid=2, validators=[Range(1, 5), Any(float, int), 6.6])) + >>> validate(6.6) + 6.6 + >>> validate(3) + 3 + >>> with raises(MultipleInvalid, 'value must be at most 5, not a valid value'): + ... validate(6.2) + """ + + def __init__(self, validators, min_valid=None, max_valid=None, **kwargs): + assert min_valid is not None or max_valid is not None, \ + 'when using "%s" you should specify at least one of min_valid and max_valid' % (type(self).__name__,) + self.min_valid = min_valid or 0 + self.max_valid = max_valid or len(validators) + super(SomeOf, self).__init__(*validators, **kwargs) + + def _exec(self, funcs, v, path=None): + errors = [] + funcs = list(funcs) + for func in funcs: + try: + if path is None: + v = func(v) + else: + v = func(path, v) + except Invalid as e: + errors.append(e) + + passed_count = len(funcs) - len(errors) + if self.min_valid <= passed_count <= self.max_valid: + return v + + msg = self.msg + if not msg: + msg = ', '.join(map(str, errors)) + + if passed_count > self.max_valid: + raise TooManyValid(msg) + raise NotEnoughValid(msg) + + def __repr__(self): + return 'SomeOf(min_valid=%s, validators=[%s], max_valid=%s, msg=%r)' % ( + self.min_valid, ", ".join(repr(v) for v in self.validators), self.max_valid, self.msg)