From 348bc173eb605d162492324611f1e0a6852e27f2 Mon Sep 17 00:00:00 2001 From: ringeringeraja Date: Fri, 25 Aug 2023 20:04:33 -0300 Subject: [PATCH] first commit --- .github/workflows/ci.yaml | 42 ++++++++++++++ .gitignore | 4 ++ LICENSE | 19 +++++++ Pipfile | 11 ++++ Pipfile.lock | 20 +++++++ README.md | 79 ++++++++++++++++++++++++++ pyproject.toml | 24 ++++++++ pyrightconfig.json | 6 ++ src/micromodel/__init__.py | 1 + src/micromodel/micromodel.py | 97 ++++++++++++++++++++++++++++++++ tests/__init__.py | 0 tests/validation.py | 106 +++++++++++++++++++++++++++++++++++ 12 files changed, 409 insertions(+) create mode 100644 .github/workflows/ci.yaml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 pyrightconfig.json create mode 100644 src/micromodel/__init__.py create mode 100644 src/micromodel/micromodel.py create mode 100644 tests/__init__.py create mode 100644 tests/validation.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..7feee8b --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,42 @@ +name: Python package + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - 3.11 + services: + redis: + image: redis + ports: + - 6379:6379 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pipenv + pipenv run pip install pyright + pipenv sync + - name: Perform typechecking + run: | + pipenv run python -m pyright + - name: Perform tests + run: | + pipenv run python -m unittest tests/*.py + env: + REDIS_HOST: localhost + REDIS_PORT: 6379 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..826f745 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__ +dist +build +*.egg-info diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..dd67b0f --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright 2023 João Santos (joaosan177@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..0757494 --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..54a7078 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,20 @@ +{ + "_meta": { + "hash": { + "sha256": "ed6d5d614626ae28e274e453164affb26694755170ccab3aa5866f093d51d3e4" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.11" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": {} +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..68065cc --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Micromodel + +Static and runtime dictionary validation. + +## Install + +```sh +$ pip install micromodel +``` + +## Why + +We had a HUGE Python code base which was using `pydantic` to provide a validation layer for MongoDB operations. The code was all messy but it worked fine, until we decided to switch the type checker config from "basic" to "strict", then over a thousand of dictionary-related errors popped up, not to mention the annoying conversions from `dict` to classes that were going on on every validation and that should be checked everytime. + +We then decided to make this validation in-loco using a more vanilla approach with only `TypedDict`s. Now our dictionaries containing MongoDB documents are consistently dicts that match with the static typing. + +## Usage + +```python +import typing +from micromodel import model + +Animal = typing.TypedDict('Animal', { + 'name': str, + 'specie': list[typing.Literal[ + 'dog', + 'cat', + 'bird' + ]] +}) + +# even another TypedDicts can be used! +Person = typing.TypedDict('Person', { + 'name': str, + 'age': int, + 'animal': Animal +}) + +m = model(Person, { + 'Animal': Animal +}) + +old_validate = m.validate +def new_validate(target: Person): + new = target.copy() + new['name'] = new['name'].capitalize() + return validate(Person, typing.cast(typing.Any, new)) + +# hooks can be implemented using monkeypatching +m.validate = new_validate + +result = m.validate({ + 'name': 'joao', + 'animal': { + 'name': 'thor', + 'specie': [ + 'dog', + # 'turtle' (this would produce both static and runtime errors) + ] + } +}) + +""" +{ + "name": "Joao", + "animal": { + "name": "thor", + "specie": [ + "dog" + ] + } +} +""" +print(result) +``` + +## License + +This library is [MIT licensed](https://github.com/capsulbrasil/normalize-json/tree/master/LICENSE). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5b45bf5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = 'micromodel' +version = '0.0.0' +authors = [ + { name = 'João Gabriel Santos', email = 'joaosan177@gmail.com' } +] +description = 'todo' +readme = 'README.md' +requires-python = '>=3.11' +classifiers = [ + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent' +] + +[project.urls] +'Homepage' = 'https://github.com/capsulbrasil/micromodel' + +[tool.setuptools] +packages = [ + 'micromodel' +] + +package-dir = { '' = 'src' } diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..78e02ed --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,6 @@ +{ + "include": [ + "src" + ], + "typeCheckingMode": "strict" +} diff --git a/src/micromodel/__init__.py b/src/micromodel/__init__.py new file mode 100644 index 0000000..d5af4ea --- /dev/null +++ b/src/micromodel/__init__.py @@ -0,0 +1 @@ +from .micromodel import * diff --git a/src/micromodel/micromodel.py b/src/micromodel/micromodel.py new file mode 100644 index 0000000..fe59539 --- /dev/null +++ b/src/micromodel/micromodel.py @@ -0,0 +1,97 @@ +import typing +import types +from abc import ABCMeta + +T = typing.TypeVar('T') + +ValidationOptions = typing.TypedDict('ValidationOptions', { + 'allow_extraneous': typing.NotRequired[bool] +}) + +class Model(typing.Generic[T]): + def __init__(self, model_type: typing.Callable[[typing.Any], T], ct: dict[str, typing.Any] = {}): + self.model_type = model_type + self.ct = ct + + def cast(self, target: T): return target + def validate(self, target: T, options: ValidationOptions = {}): return validate(self.model_type, typing.cast(typing.Any, target), options, self.ct) + +def raise_missing_key(k: int | str): + raise TypeError('missing key: %s' % k) + +def raise_extraneous_key(k: int | str): + raise TypeError('extraneous key: %s' % k) + +def raise_type_error(k: int | str, args: str, v: typing.Any): + raise TypeError('incorrect type for %s: expected %s, got %s' % (k, args, v)) + +def unwrap_type(obj: dict[int | str, typing.Any] | list[typing.Any], k: int | str, v: typing.Any, ct: dict[str, typing.Any] = {}): + origin = typing.get_origin(v) + args = typing.get_args(v) + + if (isinstance(obj, dict) and not k in obj) or (isinstance(obj, list) and int(k) > len(obj)): + if types.NoneType not in args: + raise_missing_key(k) + return + + value = obj[int(k)] \ + if isinstance(obj, list) \ + else obj[k] + + match origin: + case _ if origin == list: + for i in range(len(value)): + unwrap_type(value, i, args[0], ct) + + case _ if origin == tuple: + for i in range(len(value)): + unwrap_type(value, i, args[i], ct) + + case typing.Literal: + if value not in args: + raise_type_error(k, str(v), value) + + case types.UnionType: + for candidate in args: + if isinstance(candidate(), type(value)): + unwrap_type(obj, k, candidate, ct) + break + else: + raise_type_error(k, str(args), type(value)) + + case None: + if complex_type := ct.get(v.__name__): + value = validate(complex_type, value) + return value + + if not isinstance(value, v): + raise_type_error(k, str(v), type(value)) + case _: + if not isinstance(value, origin): + raise_type_error(k, str(args), type(value)) + + return value + +def validate(model_type: typing.Callable[[typing.Any], T], target: dict[str, typing.Any], options: ValidationOptions = {}, ct: dict[str, typing.Any] = {}) -> T: + obj: dict[int | str, typing.Any] = {} + hints = get_hints(typing.cast(ABCMeta, model_type)) + + for k, v in target.items(): + if k not in hints: + if options.get('allow_extraneous'): + continue + raise_extraneous_key(k) + obj[k] = v + + for k, v in hints.items(): + obj[k] = unwrap_type(obj, k, v, ct) + + return typing.cast(T, obj) + +def get_hints(model_type: ABCMeta): + hints = typing.get_type_hints(model_type) + return hints + +def model(model_type: typing.Callable[[typing.Any], T], ct: dict[str, typing.Any] = {}) -> Model[T]: + return Model(model_type, ct) + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/validation.py b/tests/validation.py new file mode 100644 index 0000000..3d5e46c --- /dev/null +++ b/tests/validation.py @@ -0,0 +1,106 @@ +import typing +from unittest import TestCase +from src.micromodel import model + +Animal = typing.TypedDict('Animal', { + 'name': str, + 'specie': typing.Literal[ + 'dog', + 'bird' + ] +}) + +Person = typing.TypedDict('Person', { + 'name': str, + 'age': int, + 'hobbies': typing.NotRequired[None | list[typing.Literal[ + 'programming', + 'reading', + 'swimming' + ]]], + 'hour': tuple[int, int] +}) + +PetOwner = typing.TypedDict('PetOwner', { + 'name': str, + 'pet': list[Animal] +}) + +class TestValidation(TestCase): + def test_object_equality(self): + m = model(Person) + target = Person({ + 'name': 'hello', + 'age': 5, + 'hobbies': [ + 'reading', + ], + 'hour': (12, 30) + }) + + result = m.validate(target) + self.assertEqual(target, result) + + def test_reports_missing(self): + m = model(Person) + target = Person({ # type: ignore + 'name': 'hello', + 'age': 5, + 'hobbies': [ + 'reading', + ] + }) + + with self.assertRaisesRegex(TypeError, 'missing key: hour'): + m.validate(target) + + def test_reports_extraneous(self): + m = model(Person) + target = Person({ + 'name': 'hello', + 'age': 5, + 'hobbies': [ + 'reading', + ], + 'hour': (12, 30), + 'hey': 'heyy' # type: ignore + }) + + with self.assertRaisesRegex(TypeError, 'extraneous key: hey'): + m.validate(target) + + def test_generics_validation(self): + m = model(Person) + target = Person({ + 'name': 'hello', + 'age': 5, + 'hobbies': [ + 'reading', + ], + 'hour': (12, 'thirty') # type: ignore + }) + + with self.assertRaisesRegex(TypeError, 'incorrect type for 1'): + m.validate(target) + + def test_indepth_validation(self): + m = model(PetOwner, ct={ + 'Animal': Animal + }) + + target = PetOwner({ + 'name': 'joao', + 'pet': [ + { + 'name': 'thor', + 'specie': 'dog' + }, + { + 'name': 'spike', + 'specie': 'dogx' # type: ignore + } + ] + }) + + with self.assertRaisesRegex(TypeError, 'got dogx'): + m.validate(target)