From ea34c3db40c151cd56738d79e38cc9bb478d05e0 Mon Sep 17 00:00:00 2001 From: markus-96 <73887906+markus-96@users.noreply.github.com> Date: Sun, 17 Nov 2024 20:12:26 +0100 Subject: [PATCH] pydantic_model_creator refactorization (#1745) * naming is not working * naming and description * naming is not working * naming and description * include improvements from #1741 * remove print statements * i should learn git better... * type hints, recursion protector, naming, .... should be ready for review * unused import statements... * test_early_init.py: naming of $defs changed * move dataclasses to a dedicated place * python 3.8 and 3.9: typing * test computed fields, remove own PydanticMeta class * remove print statements from tests * re-add pydantic_model_creator docstring it got lost during refactoring * remove _stack from pydantic_model_creator this is now handled by PydanticModelCreator * make some methods private * slim down dataclasses to only include necessary information * remove unused imports, satisfy mypy * add type annotation for ``from_pydantic_meta`` * better indexing of pydantic models * remove dataclasses for field descriptions pydantic_model_creator now accesses the fields directly * remove forgotten line of #1465 * include optional in hashed value * move dataclasses.py to descriptions.py * stupid unused import. * formatting --- tests/contrib/test_pydantic.py | 393 +++++++--- tests/test_early_init.py | 4 +- tortoise/contrib/pydantic/creator.py | 895 ++++++++++++---------- tortoise/contrib/pydantic/descriptions.py | 210 +++++ 4 files changed, 994 insertions(+), 508 deletions(-) create mode 100644 tortoise/contrib/pydantic/descriptions.py diff --git a/tests/contrib/test_pydantic.py b/tests/contrib/test_pydantic.py index 5b894f62a..feeec4f04 100644 --- a/tests/contrib/test_pydantic.py +++ b/tests/contrib/test_pydantic.py @@ -67,7 +67,7 @@ def test_event_schema(self): self.Event_Pydantic.model_json_schema(), { "$defs": { - "tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf": { + "Address_coqnj7_leaf": { "additionalProperties": False, "properties": { "city": {"maxLength": 64, "title": "City", "type": "string"}, @@ -83,7 +83,7 @@ def test_event_schema(self): "title": "Address", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf": { + "Reporter_fgnv33_leaf": { "additionalProperties": False, "description": "Whom is assigned as the reporter", "properties": { @@ -99,7 +99,7 @@ def test_event_schema(self): "title": "Reporter", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf": { + "Team_ip4pg6_leaf": { "additionalProperties": False, "description": "Team that is a playing", "properties": { @@ -119,15 +119,16 @@ def test_event_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, }, - "required": ["id", "name", "alias"], + "required": ["id", "name"], "title": "Team", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf": { + "Tournament_5y7e7j_leaf": { "additionalProperties": False, "properties": { "id": { @@ -139,6 +140,7 @@ def test_event_schema(self): "name": {"maxLength": 255, "title": "Name", "type": "string"}, "desc": { "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, "nullable": True, "title": "Desc", }, @@ -149,7 +151,7 @@ def test_event_schema(self): "type": "string", }, }, - "required": ["id", "name", "desc", "created"], + "required": ["id", "name", "created"], "title": "Tournament", "type": "object", }, @@ -165,13 +167,13 @@ def test_event_schema(self): }, "name": {"description": "The name", "title": "Name", "type": "string"}, "tournament": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf", + "$ref": "#/$defs/Tournament_5y7e7j_leaf", "description": "What tournaments is a happenin'", }, "reporter": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf" + "$ref": "#/$defs/Reporter_fgnv33_leaf" }, {"type": "null"}, ], @@ -180,7 +182,7 @@ def test_event_schema(self): }, "participants": { "items": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf" + "$ref": "#/$defs/Team_ip4pg6_leaf" }, "title": "Participants", "type": "array", @@ -197,13 +199,14 @@ def test_event_schema(self): {"maximum": 2147483647, "minimum": -2147483648, "type": "integer"}, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, "address": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf" + "$ref": "#/$defs/Address_coqnj7_leaf" }, {"type": "null"}, ], @@ -219,7 +222,6 @@ def test_event_schema(self): "participants", "modified", "token", - "alias", "address", ], "title": "Event", @@ -232,7 +234,7 @@ def test_eventlist_schema(self): self.Event_Pydantic_List.model_json_schema(), { "$defs": { - "Event_bliobj": { + "Event_padfez": { "additionalProperties": False, "description": "Events on the calendar", "properties": { @@ -244,13 +246,13 @@ def test_eventlist_schema(self): }, "name": {"description": "The name", "title": "Name", "type": "string"}, "tournament": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf", + "$ref": "#/$defs/Tournament_5y7e7j_leaf", "description": "What tournaments is a happenin'", }, "reporter": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf" + "$ref": "#/$defs/Reporter_fgnv33_leaf" }, {"type": "null"}, ], @@ -259,7 +261,7 @@ def test_eventlist_schema(self): }, "participants": { "items": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf" + "$ref": "#/$defs/Team_ip4pg6_leaf" }, "title": "Participants", "type": "array", @@ -283,13 +285,14 @@ def test_eventlist_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, "address": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf" + "$ref": "#/$defs/Address_coqnj7_leaf" }, {"type": "null"}, ], @@ -305,13 +308,12 @@ def test_eventlist_schema(self): "participants", "modified", "token", - "alias", "address", ], "title": "Event", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf": { + "Address_coqnj7_leaf": { "additionalProperties": False, "properties": { "city": {"maxLength": 64, "title": "City", "type": "string"}, @@ -327,7 +329,7 @@ def test_eventlist_schema(self): "title": "Address", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf": { + "Reporter_fgnv33_leaf": { "additionalProperties": False, "description": "Whom is assigned as the reporter", "properties": { @@ -343,7 +345,7 @@ def test_eventlist_schema(self): "title": "Reporter", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf": { + "Team_ip4pg6_leaf": { "additionalProperties": False, "description": "Team that is a playing", "properties": { @@ -363,15 +365,16 @@ def test_eventlist_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, }, - "required": ["id", "name", "alias"], + "required": ["id", "name"], "title": "Team", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf": { + "Tournament_5y7e7j_leaf": { "additionalProperties": False, "properties": { "id": { @@ -383,6 +386,7 @@ def test_eventlist_schema(self): "name": {"maxLength": 255, "title": "Name", "type": "string"}, "desc": { "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, "nullable": True, "title": "Desc", }, @@ -393,13 +397,13 @@ def test_eventlist_schema(self): "type": "string", }, }, - "required": ["id", "name", "desc", "created"], + "required": ["id", "name", "created"], "title": "Tournament", "type": "object", }, }, "description": "Events on the calendar", - "items": {"$ref": "#/$defs/Event_bliobj"}, + "items": {"$ref": "#/$defs/Event_padfez"}, "title": "Event_list", "type": "array", }, @@ -410,7 +414,7 @@ def test_address_schema(self): self.Address_Pydantic.model_json_schema(), { "$defs": { - "Event_jim4na": { + "Event_zvunzw_leaf": { "additionalProperties": False, "description": "Events on the calendar", "properties": { @@ -422,13 +426,13 @@ def test_address_schema(self): }, "name": {"description": "The name", "title": "Name", "type": "string"}, "tournament": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf", + "$ref": "#/$defs/Tournament_5y7e7j_leaf", "description": "What tournaments is a happenin'", }, "reporter": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf" + "$ref": "#/$defs/Reporter_fgnv33_leaf" }, {"type": "null"}, ], @@ -437,7 +441,7 @@ def test_address_schema(self): }, "participants": { "items": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf" + "$ref": "#/$defs/Team_ip4pg6_leaf" }, "title": "Participants", "type": "array", @@ -461,6 +465,7 @@ def test_address_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, @@ -473,12 +478,11 @@ def test_address_schema(self): "participants", "modified", "token", - "alias", ], "title": "Event", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf": { + "Reporter_fgnv33_leaf": { "additionalProperties": False, "description": "Whom is assigned as the reporter", "properties": { @@ -494,7 +498,7 @@ def test_address_schema(self): "title": "Reporter", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf": { + "Team_ip4pg6_leaf": { "additionalProperties": False, "description": "Team that is a playing", "properties": { @@ -514,15 +518,16 @@ def test_address_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, }, - "required": ["id", "name", "alias"], + "required": ["id", "name"], "title": "Team", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf": { + "Tournament_5y7e7j_leaf": { "additionalProperties": False, "properties": { "id": { @@ -534,6 +539,7 @@ def test_address_schema(self): "name": {"maxLength": 255, "title": "Name", "type": "string"}, "desc": { "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, "nullable": True, "title": "Desc", }, @@ -544,7 +550,7 @@ def test_address_schema(self): "type": "string", }, }, - "required": ["id", "name", "desc", "created"], + "required": ["id", "name", "created"], "title": "Tournament", "type": "object", }, @@ -553,7 +559,7 @@ def test_address_schema(self): "properties": { "city": {"maxLength": 64, "title": "City", "type": "string"}, "street": {"maxLength": 128, "title": "Street", "type": "string"}, - "event": {"$ref": "#/$defs/Event_jim4na"}, + "event": {"$ref": "#/$defs/Event_zvunzw_leaf"}, "event_id": { "maximum": 9223372036854775807, "minimum": -9223372036854775808, @@ -572,7 +578,7 @@ def test_tournament_schema(self): self.Tournament_Pydantic.model_json_schema(), { "$defs": { - "Event_ml4ytz": { + "Event_jgrv4c_leaf": { "additionalProperties": False, "description": "Events on the calendar", "properties": { @@ -586,7 +592,7 @@ def test_tournament_schema(self): "reporter": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf" + "$ref": "#/$defs/Reporter_fgnv33_leaf" }, {"type": "null"}, ], @@ -595,7 +601,7 @@ def test_tournament_schema(self): }, "participants": { "items": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf" + "$ref": "#/$defs/Team_ip4pg6_leaf" }, "title": "Participants", "type": "array", @@ -619,13 +625,14 @@ def test_tournament_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, "address": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf" + "$ref": "#/$defs/Address_coqnj7_leaf" }, {"type": "null"}, ], @@ -640,13 +647,12 @@ def test_tournament_schema(self): "participants", "modified", "token", - "alias", "address", ], "title": "Event", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf": { + "Address_coqnj7_leaf": { "additionalProperties": False, "properties": { "city": {"maxLength": 64, "title": "City", "type": "string"}, @@ -662,7 +668,7 @@ def test_tournament_schema(self): "title": "Address", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf": { + "Reporter_fgnv33_leaf": { "additionalProperties": False, "description": "Whom is assigned as the reporter", "properties": { @@ -678,7 +684,7 @@ def test_tournament_schema(self): "title": "Reporter", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Team__leaf": { + "Team_ip4pg6_leaf": { "additionalProperties": False, "description": "Team that is a playing", "properties": { @@ -698,11 +704,12 @@ def test_tournament_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, }, - "required": ["id", "name", "alias"], + "required": ["id", "name"], "title": "Team", "type": "object", }, @@ -713,6 +720,7 @@ def test_tournament_schema(self): "name": {"maxLength": 255, "title": "Name", "type": "string"}, "desc": { "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, "nullable": True, "title": "Desc", }, @@ -724,12 +732,12 @@ def test_tournament_schema(self): }, "events": { "description": "What tournaments is a happenin'", - "items": {"$ref": "#/$defs/Event_ml4ytz"}, + "items": {"$ref": "#/$defs/Event_jgrv4c_leaf"}, "title": "Events", "type": "array", }, }, - "required": ["id", "name", "desc", "created", "events"], + "required": ["id", "name", "created", "events"], "title": "Tournament", "type": "object", }, @@ -740,7 +748,7 @@ def test_team_schema(self): self.Team_Pydantic.model_json_schema(), { "$defs": { - "Event_vrm2bi": { + "Event_n2kadx_leaf": { "additionalProperties": False, "description": "Events on the calendar", "properties": { @@ -752,13 +760,13 @@ def test_team_schema(self): }, "name": {"description": "The name", "title": "Name", "type": "string"}, "tournament": { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf", + "$ref": "#/$defs/Tournament_5y7e7j_leaf", "description": "What tournaments is a happenin'", }, "reporter": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf" + "$ref": "#/$defs/Reporter_fgnv33_leaf" }, {"type": "null"}, ], @@ -784,13 +792,14 @@ def test_team_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, "address": { "anyOf": [ { - "$ref": "#/$defs/tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf" + "$ref": "#/$defs/Address_coqnj7_leaf" }, {"type": "null"}, ], @@ -805,13 +814,12 @@ def test_team_schema(self): "reporter", "modified", "token", - "alias", "address", ], "title": "Event", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Address__leaf": { + "Address_coqnj7_leaf": { "additionalProperties": False, "properties": { "city": {"maxLength": 64, "title": "City", "type": "string"}, @@ -827,7 +835,7 @@ def test_team_schema(self): "title": "Address", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Reporter__leaf": { + "Reporter_fgnv33_leaf": { "additionalProperties": False, "description": "Whom is assigned as the reporter", "properties": { @@ -843,7 +851,7 @@ def test_team_schema(self): "title": "Reporter", "type": "object", }, - "tortoise__contrib__pydantic__creator__tests__testmodels__Tournament__leaf": { + "Tournament_5y7e7j_leaf": { "additionalProperties": False, "properties": { "id": { @@ -855,6 +863,7 @@ def test_team_schema(self): "name": {"maxLength": 255, "title": "Name", "type": "string"}, "desc": { "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, "nullable": True, "title": "Desc", }, @@ -865,7 +874,7 @@ def test_team_schema(self): "type": "string", }, }, - "required": ["id", "name", "desc", "created"], + "required": ["id", "name", "created"], "title": "Tournament", "type": "object", }, @@ -885,16 +894,17 @@ def test_team_schema(self): {"maximum": 2147483647, "minimum": -2147483648, "type": "integer"}, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Alias", }, "events": { - "items": {"$ref": "#/$defs/Event_vrm2bi"}, + "items": {"$ref": "#/$defs/Event_n2kadx_leaf"}, "title": "Events", "type": "array", }, }, - "required": ["id", "name", "alias", "events"], + "required": ["id", "name", "events"], "title": "Team", "type": "object", }, @@ -902,7 +912,6 @@ def test_team_schema(self): async def test_eventlist(self): eventlp = await self.Event_Pydantic_List.from_queryset(Event.all()) - # print(eventlp.json(indent=4)) eventldict = eventlp.model_dump() # Remove timestamps @@ -961,7 +970,6 @@ async def test_eventlist(self): async def test_event(self): eventp = await self.Event_Pydantic.from_tortoise_orm(await Event.get(name="Test")) - # print(eventp.json(indent=4)) eventdict = eventp.model_dump() # Remove timestamps @@ -993,7 +1001,6 @@ async def test_event(self): async def test_address(self): addressp = await self.Address_Pydantic.from_tortoise_orm(await Address.get(street="Ocean")) - # print(addressp.json(indent=4)) addressdict = addressp.model_dump() # Remove timestamps @@ -1029,7 +1036,6 @@ async def test_tournament(self): tournamentp = await self.Tournament_Pydantic.from_tortoise_orm( await Tournament.all().first() ) - # print(tournamentp.json(indent=4)) tournamentdict = tournamentp.model_dump() # Remove timestamps @@ -1081,7 +1087,6 @@ async def test_tournament(self): async def test_team(self): teamp = await self.Team_Pydantic.from_tortoise_orm(await Team.get(id=self.team1.id)) - # print(teamp.json(indent=4)) teamdict = teamp.model_dump() # Remove timestamps @@ -1298,44 +1303,7 @@ def test_schema(self): self.Employee_Pydantic.model_json_schema(), { "$defs": { - "Employee_ibbaiu": { - "additionalProperties": False, - "properties": { - "id": { - "maximum": 2147483647, - "minimum": -2147483648, - "title": "Id", - "type": "integer", - }, - "name": {"maxLength": 50, "title": "Name", "type": "string"}, - "talks_to": { - "items": {"$ref": "#/$defs/leaf"}, - "title": "Talks To", - "type": "array", - }, - "manager_id": { - "anyOf": [ - { - "maximum": 2147483647, - "minimum": -2147483648, - "type": "integer", - }, - {"type": "null"}, - ], - "nullable": True, - "title": "Manager Id", - }, - "team_members": { - "items": {"$ref": "#/$defs/leaf"}, - "title": "Team Members", - "type": "array", - }, - }, - "required": ["id", "name", "talks_to", "manager_id", "team_members"], - "title": "Employee", - "type": "object", - }, - "Employee_obdn4z": { + "Employee_6tkbjb_leaf": { "additionalProperties": False, "properties": { "id": { @@ -1346,7 +1314,7 @@ def test_schema(self): }, "name": {"maxLength": 50, "title": "Name", "type": "string"}, "talks_to": { - "items": {"$ref": "#/$defs/leaf"}, + "items": {"$ref": "#/$defs/Employee_fj2ly4_leaf"}, "title": "Talks To", "type": "array", }, @@ -1359,20 +1327,21 @@ def test_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Manager Id", }, "team_members": { - "items": {"$ref": "#/$defs/leaf"}, + "items": {"$ref": "#/$defs/Employee_fj2ly4_leaf"}, "title": "Team Members", "type": "array", }, }, - "required": ["id", "name", "talks_to", "manager_id", "team_members"], + "required": ["id", "name", "talks_to", "team_members"], "title": "Employee", "type": "object", }, - "leaf": { + "Employee_fj2ly4_leaf": { "additionalProperties": False, "properties": { "id": { @@ -1391,11 +1360,12 @@ def test_schema(self): }, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Manager Id", }, }, - "required": ["id", "name", "manager_id"], + "required": ["id", "name"], "title": "Employee", "type": "object", }, @@ -1410,7 +1380,7 @@ def test_schema(self): }, "name": {"maxLength": 50, "title": "Name", "type": "string"}, "talks_to": { - "items": {"$ref": "#/$defs/Employee_obdn4z"}, + "items": {"$ref": "#/$defs/Employee_6tkbjb_leaf"}, "title": "Talks To", "type": "array", }, @@ -1419,16 +1389,17 @@ def test_schema(self): {"maximum": 2147483647, "minimum": -2147483648, "type": "integer"}, {"type": "null"}, ], + "default": None, "nullable": True, "title": "Manager Id", }, "team_members": { - "items": {"$ref": "#/$defs/Employee_ibbaiu"}, + "items": {"$ref": "#/$defs/Employee_6tkbjb_leaf"}, "title": "Team Members", "type": "array", }, }, - "required": ["id", "name", "talks_to", "manager_id", "team_members"], + "required": ["id", "name", "talks_to", "team_members"], "title": "Employee", "type": "object", }, @@ -1436,7 +1407,6 @@ def test_schema(self): async def test_serialisation(self): empp = await self.Employee_Pydantic.from_tortoise_orm(await Employee.get(name="Root")) - # print(empp.json(indent=4)) empdict = empp.model_dump() self.assertEqual( @@ -1517,6 +1487,211 @@ async def test_serialisation(self): ) +class TestPydanticComputed(test.TestCase): + async def asyncSetUp(self) -> None: + await super(TestPydanticComputed, self).asyncSetUp() + self.Employee_Pydantic = pydantic_model_creator(Employee) + self.employee = await Employee.create(name="Some Employee") + self.maxDiff = None + + async def test_computed_field(self): + employee_pyd = await self.Employee_Pydantic.from_tortoise_orm(await Employee.get(name="Some Employee")) + employee_serialised = employee_pyd.model_dump() + self.assertEqual(employee_serialised.get("name_length"), self.employee.name_length()) + + async def test_computed_field_schema(self): + self.assertEqual( + self.Employee_Pydantic.model_json_schema(mode="serialization"), + { + "$defs": { + "Employee_fj2ly4_leaf": { + "additionalProperties": False, + "properties": { + "id": { + "maximum": 2147483647, + "minimum": -2147483648, + "title": "Id", + "type": "integer" + }, + "name": { + "maxLength": 50, + "title": "Name", + "type": "string" + }, + "manager_id": { + "anyOf": [ + { + "maximum": 2147483647, + "minimum": -2147483648, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": None, + "nullable": True, + "title": "Manager Id" + }, + "name_length": { + "description": "", + "readOnly": True, + "title": "Name Length", + "type": "integer" + }, + "team_size": { + "description": "Computes team size.

Note that this function needs to be annotated with a return type so that pydantic can
generate a valid schema.

Note that the pydantic serializer can't call async methods, but the tortoise helpers
pre-fetch relational data, so that it is available before serialization. So we don't
need to await the relation. We do however have to protect against the case where no
prefetching was done, hence catching and handling the
``tortoise.exceptions.NoValuesFetched`` exception.", + "readOnly": True, + "title": "Team Size", + "type": "integer" + } + }, + "required": [ + "id", + "name", + "name_length", + "team_size" + ], + "title": "Employee", + "type": "object" + }, + "Employee_6tkbjb_leaf": { + "additionalProperties": False, + "properties": { + "id": { + "maximum": 2147483647, + "minimum": -2147483648, + "title": "Id", + "type": "integer" + }, + "name": { + "maxLength": 50, + "title": "Name", + "type": "string" + }, + "talks_to": { + "items": { + "$ref": "#/$defs/Employee_fj2ly4_leaf" + }, + "title": "Talks To", + "type": "array" + }, + "manager_id": { + "anyOf": [ + { + "maximum": 2147483647, + "minimum": -2147483648, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": None, + "nullable": True, + "title": "Manager Id" + }, + "team_members": { + "items": { + "$ref": "#/$defs/Employee_fj2ly4_leaf" + }, + "title": "Team Members", + "type": "array" + }, + "name_length": { + "description": "", + "readOnly": True, + "title": "Name Length", + "type": "integer" + }, + "team_size": { + "description": "Computes team size.

Note that this function needs to be annotated with a return type so that pydantic can
generate a valid schema.

Note that the pydantic serializer can't call async methods, but the tortoise helpers
pre-fetch relational data, so that it is available before serialization. So we don't
need to await the relation. We do however have to protect against the case where no
prefetching was done, hence catching and handling the
``tortoise.exceptions.NoValuesFetched`` exception.", + "readOnly": True, + "title": "Team Size", + "type": "integer" + } + }, + "required": [ + "id", + "name", + "talks_to", + "team_members", + "name_length", + "team_size" + ], + "title": "Employee", + "type": "object" + } + }, + "additionalProperties": False, + "properties": { + "id": { + "maximum": 2147483647, + "minimum": -2147483648, + "title": "Id", + "type": "integer" + }, + "name": { + "maxLength": 50, + "title": "Name", + "type": "string" + }, + "talks_to": { + "items": { + "$ref": "#/$defs/Employee_6tkbjb_leaf" + }, + "title": "Talks To", + "type": "array" + }, + "manager_id": { + "anyOf": [ + { + "maximum": 2147483647, + "minimum": -2147483648, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": None, + "nullable": True, + "title": "Manager Id" + }, + "team_members": { + "items": { + "$ref": "#/$defs/Employee_6tkbjb_leaf" + }, + "title": "Team Members", + "type": "array" + }, + "name_length": { + "description": "", + "readOnly": True, + "title": "Name Length", + "type": "integer" + }, + "team_size": { + "description": "Computes team size.

Note that this function needs to be annotated with a return type so that pydantic can
generate a valid schema.

Note that the pydantic serializer can't call async methods, but the tortoise helpers
pre-fetch relational data, so that it is available before serialization. So we don't
need to await the relation. We do however have to protect against the case where no
prefetching was done, hence catching and handling the
``tortoise.exceptions.NoValuesFetched`` exception.", + "readOnly": True, + "title": "Team Size", + "type": "integer" + } + }, + "required": [ + "id", + "name", + "talks_to", + "team_members", + "name_length", + "team_size" + ], + "title": "Employee", + "type": "object" + } + ) + + class TestPydanticUpdate(test.TestCase): def setUp(self) -> None: self.UserCreate_Pydantic = pydantic_model_creator( diff --git a/tests/test_early_init.py b/tests/test_early_init.py index 72d2e4314..24b454a8a 100644 --- a/tests/test_early_init.py +++ b/tests/test_early_init.py @@ -167,7 +167,7 @@ def test_early_init(self): Event_Pydantic.model_json_schema(), { "$defs": { - "leaf": { + "Tournament_aapnxb_leaf": { "additionalProperties": False, "properties": { "id": { @@ -211,7 +211,7 @@ def test_early_init(self): "type": "string", }, "tournament": { - "anyOf": [{"$ref": "#/$defs/leaf"}, {"type": "null"}], + "anyOf": [{"$ref": "#/$defs/Tournament_aapnxb_leaf"}, {"type": "null"}], "nullable": True, "title": "Tournament", }, diff --git a/tortoise/contrib/pydantic/creator.py b/tortoise/contrib/pydantic/creator.py index f73d5e4fc..8d0e3c828 100644 --- a/tortoise/contrib/pydantic/creator.py +++ b/tortoise/contrib/pydantic/creator.py @@ -1,66 +1,35 @@ import inspect from base64 import b32encode +from copy import copy +from typing import MutableMapping + from hashlib import sha3_224 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union -from pydantic import ConfigDict, Field, computed_field, create_model -from pydantic._internal._decorators import PydanticDescriptorProxy +from pydantic import ConfigDict, computed_field, create_model +from pydantic import Field as PydanticField +from tortoise import ForeignKeyFieldInstance, BackwardFKRelation, ManyToManyFieldInstance, OneToOneFieldInstance, \ + BackwardOneToOneRelation from tortoise.contrib.pydantic.base import PydanticListModel, PydanticModel from tortoise.contrib.pydantic.utils import get_annotations -from tortoise.fields import IntField, JSONField, TextField, relational +from tortoise.fields import JSONField, Field +from tortoise.contrib.pydantic.descriptions import ModelDescription, PydanticMetaData, ComputedFieldDescription if TYPE_CHECKING: # pragma: nocoverage from tortoise.models import Model _MODEL_INDEX: Dict[str, Type[PydanticModel]] = {} - - -class PydanticMeta: - """ - The ``PydanticMeta`` class is used to configure metadata for generating the pydantic Model. - - Usage: - - .. code-block:: python3 - - class Foo(Model): - ... - - class PydanticMeta: - exclude = ("foo", "baa") - computed = ("count_peanuts", ) - """ - - #: If not empty, only fields this property contains will be in the pydantic model - include: Tuple[str, ...] = () - - #: Fields listed in this property will be excluded from pydantic model - exclude: Tuple[str, ...] = ("Meta",) - - #: Computed fields can be listed here to use in pydantic model - computed: Tuple[str, ...] = () - - #: Use backward relations without annotations - not recommended, it can be huge data - #: without control - backward_relations: bool = True - - #: Maximum recursion level allowed - max_recursion: int = 3 - - #: Allow cycles in recursion - This can result in HUGE data - Be careful! - #: Please use this with ``exclude``/``include`` and sane ``max_recursion`` - allow_cycles: bool = False - - #: If we should exclude raw fields (the ones have _id suffixes) of relations - exclude_raw_fields: bool = True - - #: Sort fields alphabetically. - #: If not set (or ``False``) then leave fields in declaration order - sort_alphabetically: bool = False - - #: Allows user to specify custom config for generated model - model_config: Optional[ConfigDict] = None +""" +The index works as follows: +1. the hash is calculated from the following: + - the fully qualified name of the model + - the names of the contained fields + - the names of all relational fields and the corresponding names of the pydantic model. + This is because if the model is not yet fully initialized, the relational fields are not yet present. +2. the hash does not take into account the resulting name of the model; this must be checked separately. +3. the hash can only be calculated after a complete analysis of the given model. +""" def _br_it(val: str) -> str: @@ -72,15 +41,15 @@ def _cleandoc(obj: Any) -> str: def _pydantic_recursion_protector( - cls: "Type[Model]", - *, - stack: tuple, - exclude: Tuple[str, ...] = (), - include: Tuple[str, ...] = (), - computed: Tuple[str, ...] = (), - name=None, - allow_cycles: bool = False, - sort_alphabetically: Optional[bool] = None, + cls: "Type[Model]", + *, + stack: Tuple, + exclude: Tuple[str, ...] = (), + include: Tuple[str, ...] = (), + computed: Tuple[str, ...] = (), + name=None, + allow_cycles: bool = False, + sort_alphabetically: Optional[bool] = None, ) -> Optional[Type[PydanticModel]]: """ It is an inner function to protect pydantic model creator against cyclic recursion @@ -104,8 +73,7 @@ def _pydantic_recursion_protector( return None level += 1 - - return pydantic_model_creator( + pmc = PydanticModelCreator( cls, exclude=exclude, include=include, @@ -114,36 +82,90 @@ def _pydantic_recursion_protector( _stack=stack, allow_cycles=allow_cycles, sort_alphabetically=sort_alphabetically, + _as_submodel=True, ) + return pmc.create_pydantic_model() -def pydantic_model_creator( - cls: "Type[Model]", - *, - name=None, - exclude: Tuple[str, ...] = (), - include: Tuple[str, ...] = (), - computed: Tuple[str, ...] = (), - optional: Tuple[str, ...] = (), - allow_cycles: Optional[bool] = None, - sort_alphabetically: Optional[bool] = None, - _stack: tuple = (), - exclude_readonly: bool = False, - meta_override: Optional[Type] = None, - model_config: Optional[ConfigDict] = None, - validators: Optional[Dict[str, Any]] = None, - module: str = __name__, -) -> Type[PydanticModel]: +class FieldMap(MutableMapping[str, Union[Field, ComputedFieldDescription]]): + def __init__(self, meta: PydanticMetaData, pk_field: Optional[Field] = None): + self._field_map: Dict[str, Union[Field, ComputedFieldDescription]] = {} + self.pk_raw_field = pk_field.model_field_name if pk_field is not None else "" + if pk_field: + self.pk_raw_field = pk_field.model_field_name + self.field_map_update([pk_field], meta) + self.computed_fields: Dict[str, ComputedFieldDescription] = {} + + def __delitem__(self, __key): + self._field_map.__delitem__(__key) + + def __getitem__(self, __key): + return self._field_map.__getitem__(__key) + + def __len__(self): # pragma: no-coverage + return self._field_map.__len__() + + def __iter__(self): + return self._field_map.__iter__() + + def __setitem__(self, __key, __value): + self._field_map.__setitem__(__key, __value) + + def sort_alphabetically(self) -> None: + self._field_map = {k: self._field_map[k] for k in sorted(self._field_map)} + + def sort_definition_order(self, cls: "Type[Model]", computed: Tuple[str, ...]) -> None: + self._field_map = { + k: self._field_map[k] for k in tuple(cls._meta.fields_map.keys()) + computed if k in self._field_map + } + + def field_map_update(self, fields: List[Field], meta: PydanticMetaData) -> None: + for field in fields: + name = field.model_field_name + # Include or exclude field + if (meta.include and name not in meta.include) or name in meta.exclude: + continue + # Remove raw fields + if isinstance(field, ForeignKeyFieldInstance): + raw_field = field.source_field + if raw_field is not None and meta.exclude_raw_fields and raw_field != self.pk_raw_field: + self.pop(raw_field, None) + self[name] = field + + def computed_field_map_update(self, computed: Tuple[str, ...], cls: "Type[Model]"): + self._field_map.update( + { + k: ComputedFieldDescription( + field_type=callable, + function=getattr(cls, k), + description=None, + ) + for k in computed + } + ) + + +def pydantic_queryset_creator( + cls: "Type[Model]", + *, + name=None, + exclude: Tuple[str, ...] = (), + include: Tuple[str, ...] = (), + computed: Tuple[str, ...] = (), + allow_cycles: Optional[bool] = None, + sort_alphabetically: Optional[bool] = None, +) -> Type[PydanticListModel]: """ - Function to build `Pydantic Model `__ off Tortoise Model. + Function to build a `Pydantic Model `__ list off Tortoise Model. - :param _stack: Internal parameter to track recursion - :param cls: The Tortoise Model + :param cls: The Tortoise Model to put in a list. :param name: Specify a custom name explicitly, instead of a generated name. + + The list generated name is currently naive and merely adds a "s" to the end + of the singular name. :param exclude: Extra fields to exclude from the provided model. :param include: Extra fields to include from the provided model. :param computed: Extra computed fields to include from the provided model. - :param optional: Extra optional fields for the provided model. :param allow_cycles: Do we allow any cycles in the generated model? This is only useful for recursive/self-referential models. @@ -155,332 +177,409 @@ def pydantic_model_creator( * Field definition order + * order of reverse relations (as discovered) + * order of computed functions (as provided). - :param exclude_readonly: Build a subset model that excludes any readonly fields - :param meta_override: A PydanticMeta class to override model's values. - :param model_config: A custom config to use as pydantic config. - :param validators: A dictionary of methods that validate fields. - :param module: The name of the module that the model belongs to. - - Note: Created pydantic model uses config_class parameter and PydanticMeta's - config_class as its Config class's bases(Only if provided!), but it - ignores ``fields`` config. pydantic_model_creator will generate fields by - include/exclude/computed parameters automatically. """ - # Fully qualified class name - fqname = cls.__module__ + "." + cls.__qualname__ - postfix = "" + submodel = pydantic_model_creator( + cls, + exclude=exclude, + include=include, + computed=computed, + allow_cycles=allow_cycles, + sort_alphabetically=sort_alphabetically, + name=name, + ) + lname = name or f"{submodel.__name__}_list" - def get_name() -> str: - # If arguments are specified (different from the defaults), we append a hash to the - # class name, to make it unique - # We don't check by stack, as cycles get explicitly renamed. - # When called later, include is explicitly set, so fence passes. - nonlocal postfix - is_default = ( - exclude == () - and include == () - and computed == () - and sort_alphabetically is None - and allow_cycles is None - and not exclude_readonly + # Creating Pydantic class for the properties generated before + model = create_model( + lname, + __base__=PydanticListModel, + root=(List[submodel], PydanticField(default_factory=list)), # type: ignore + ) + # Copy the Model docstring over + model.__doc__ = _cleandoc(cls) + # The title of the model to hide the hash postfix + model.model_config["title"] = name or f"{submodel.model_config['title']}_list" + model.model_config["submodel"] = submodel # type: ignore + return model + + +class PydanticModelCreator: + def __init__( + self, + cls: "Type[Model]", + name: Optional[str] = None, + exclude: Optional[Tuple[str, ...]] = None, + include: Optional[Tuple[str, ...]] = None, + computed: Optional[Tuple[str, ...]] = None, + optional: Optional[Tuple[str, ...]] = None, + allow_cycles: Optional[bool] = None, + sort_alphabetically: Optional[bool] = None, + exclude_readonly: bool = False, + meta_override: Optional[Type] = None, + model_config: Optional[ConfigDict] = None, + validators: Optional[Dict[str, Any]] = None, + module: str = __name__, + _stack: tuple = (), + _as_submodel: bool = False + ) -> None: + self._cls: "Type[Model]" = cls + self._stack: Tuple[Tuple["Type[Model]", str, int], ...] = _stack # ((Type[Model], field_name, max_recursion),) + self._is_default: bool = ( + exclude is None + and include is None + and computed is None + and optional is None + and sort_alphabetically is None + and allow_cycles is None + and meta_override is None + and not exclude_readonly ) - hashval = f"{fqname};{exclude};{include};{computed};{_stack}:{sort_alphabetically}:{allow_cycles}:{exclude_readonly}" - postfix = ( - ":" + b32encode(sha3_224(hashval.encode("utf-8")).digest()).decode("utf-8").lower()[:6] - if not is_default - else "" + if exclude is None: + exclude = () + if include is None: + include = () + if computed is None: + computed = () + if optional is None: + optional = () + + if meta := getattr(cls, "PydanticMeta", None): + meta_from_class = PydanticMetaData.from_pydantic_meta(meta) + else: # default + meta_from_class = PydanticMetaData() + if meta_override: + meta_from_class = meta_from_class.construct_pydantic_meta(meta_override) + self.meta = meta_from_class.finalize_meta( + exclude=exclude, + include=include, + computed=computed, + allow_cycles=allow_cycles, + sort_alphabetically=sort_alphabetically, + model_config=model_config, ) - return fqname + postfix - # We need separate model class for different exclude, include and computed parameters - _name = name or get_name() - has_submodel = False + self._exclude_read_only: bool = exclude_readonly - # Get settings and defaults - meta = getattr(cls, "PydanticMeta", PydanticMeta) + self._fqname = cls.__module__ + "." + cls.__qualname__ + self._name: str + self._title: str + self.given_name = name + self.__hash: str = "" - def get_param(attr: str) -> Any: - if meta_override: - return getattr(meta_override, attr, getattr(meta, attr, getattr(PydanticMeta, attr))) - return getattr(meta, attr, getattr(PydanticMeta, attr)) - - default_include: Tuple[str, ...] = tuple(get_param("include")) - default_exclude: Tuple[str, ...] = tuple(get_param("exclude")) - default_computed: Tuple[str, ...] = tuple(get_param("computed")) - default_config: Optional[ConfigDict] = get_param("model_config") - - backward_relations: bool = bool(get_param("backward_relations")) - - max_recursion: int = int(get_param("max_recursion")) - exclude_raw_fields: bool = bool(get_param("exclude_raw_fields")) - _sort_fields: bool = ( - bool(get_param("sort_alphabetically")) - if sort_alphabetically is None - else sort_alphabetically - ) - _allow_cycles: bool = bool(get_param("allow_cycles") if allow_cycles is None else allow_cycles) - - # Update parameters with defaults - include = tuple(include) + default_include - exclude = tuple(exclude) + default_exclude - computed = tuple(computed) + default_computed - - annotations = get_annotations(cls) - - pconfig = PydanticModel.model_config.copy() - if default_config: - pconfig.update(default_config) - if model_config: - pconfig.update(model_config) - if "title" not in pconfig: - pconfig["title"] = name or cls.__name__ - if "extra" not in pconfig: - pconfig["extra"] = "forbid" - - properties: Dict[str, Any] = {} - - # Get model description - model_description = cls.describe(serializable=False) - - # Field map we use - field_map: Dict[str, dict] = {} - pk_raw_field: str = "" - - def field_map_update(keys: tuple, is_relation=True) -> None: - nonlocal pk_raw_field - - for key in keys: - fds = model_description[key] - if isinstance(fds, dict): - fds = [fds] - for fd in fds: - n = fd["name"] - if key == "pk_field": - pk_raw_field = n - # Include or exclude field - if (include and n not in include) or n in exclude: - continue - # Remove raw fields - raw_field = fd.get("raw_field", None) - if raw_field is not None and exclude_raw_fields and raw_field != pk_raw_field: - field_map.pop(raw_field, None) - field_map[n] = fd - - # Update field definitions from description - if not exclude_readonly: - field_map_update(("pk_field",), is_relation=False) - field_map_update(("data_fields",), is_relation=False) - if not exclude_readonly: - included_fields: tuple = ( - "fk_fields", - "o2o_fields", - "m2m_fields", - ) - if backward_relations: - included_fields = ( - *included_fields, - "backward_fk_fields", - "backward_o2o_fields", + self._as_submodel = _as_submodel + + self._annotations = get_annotations(cls) + + self._pconfig: ConfigDict + + self._properties: Dict[str, Any] = dict() + self._relational_fields_index: List[Tuple[str, str]] = list() + + self._model_description: ModelDescription = ModelDescription.from_model(cls) + + self._field_map: FieldMap = self._initialize_field_map() + self._construct_field_map() + + self._optional = optional + + self._validators = validators + self._module = module + + self._stack = _stack + + @property + def _hash(self): + if self.__hash == "": + hashval = ( + f"{self._fqname};{self._properties.keys()};{self._relational_fields_index};{self._optional};" + f"{self.meta.allow_cycles}" ) + self.__hash = b32encode(sha3_224(hashval.encode("utf-8")).digest()).decode("utf-8").lower()[:6] + return self.__hash - field_map_update(included_fields) - # Add possible computed fields - field_map.update( - { - k: { - "field_type": callable, - "function": getattr(cls, k), - "description": None, - } - for k in computed - } + def get_name(self) -> Tuple[str, str]: + # If arguments are specified (different from the defaults), we append a hash to the + # class name, to make it unique + # We don't check by stack, as cycles get explicitly renamed. + # When called later, include is explicitly set, so fence passes. + if self.given_name is not None: + return self.given_name, self.given_name + name = ( + f"{self._fqname}:{self._hash}" + if not self._is_default + else self._fqname + ) + name = ( + f"{name}:leaf" + if self._as_submodel + else name + ) + return name, self._cls.__name__ + + def _initialize_pconfig(self) -> ConfigDict: + pconfig: ConfigDict = PydanticModel.model_config.copy() + if self.meta.model_config: + pconfig.update(self.meta.model_config) + if "title" not in pconfig: + pconfig["title"] = self._title + if "extra" not in pconfig: + pconfig["extra"] = 'forbid' + return pconfig + + def _initialize_field_map(self) -> FieldMap: + return ( + FieldMap(self.meta) + if self._exclude_read_only + else FieldMap(self.meta, pk_field=self._model_description.pk_field) ) - # Sort field map (Python 3.7+ has guaranteed ordered dictionary keys) - if _sort_fields: - # Sort Alphabetically - field_map = {k: field_map[k] for k in sorted(field_map)} - else: - # Sort to definition order - field_map = { - k: field_map[k] for k in tuple(cls._meta.fields_map.keys()) + computed if k in field_map - } - # Process fields - for fname, fdesc in field_map.items(): - comment = "" + def _construct_field_map(self) -> None: + self._field_map.field_map_update(fields=self._model_description.data_fields, meta=self.meta) + if not self._exclude_read_only: + for fields in ( + self._model_description.fk_fields, + self._model_description.o2o_fields, + self._model_description.m2m_fields + ): + self._field_map.field_map_update(fields, self.meta) + if self.meta.backward_relations: + for fields in ( + self._model_description.backward_fk_fields, + self._model_description.backward_o2o_fields + ): + self._field_map.field_map_update(fields, self.meta) + self._field_map.computed_field_map_update(self.meta.computed, self._cls) + if self.meta.sort_alphabetically: + self._field_map.sort_alphabetically() + else: + self._field_map.sort_definition_order(self._cls, self.meta.computed) + + def create_pydantic_model(self) -> Type[PydanticModel]: + for field_name, field in self._field_map.items(): + self._process_field(field_name, field) + + self._name, self._title = self.get_name() + + if self._hash in _MODEL_INDEX: + # there is a model exactly the same, but the name could be different + hashed_model = _MODEL_INDEX[self._hash] + if hashed_model.__name__ == self._name: + # also the same name + return _MODEL_INDEX[self._hash] + + self._pconfig = self._initialize_pconfig() + self._properties["model_config"] = self._pconfig + model = create_model( + self._name, + __base__=PydanticModel, + __module__=self._module, + __validators__=self._validators, + **self._properties, + ) + # Copy the Model docstring over + model.__doc__ = _cleandoc(self._cls) + # Store the base class + model.model_config["orig_model"] = self._cls # type: ignore + # Store model reference so we can de-dup it later on if needed. + _MODEL_INDEX[self._hash] = model + return model + + def _process_field( + self, + field_name: str, + field: Union[Field, ComputedFieldDescription], + ) -> None: json_schema_extra: Dict[str, Any] = {} fconfig: Dict[str, Any] = { "json_schema_extra": json_schema_extra, } - field_type = fdesc["field_type"] - field_default = fdesc.get("default") - is_optional_field = fname in optional - - def get_submodel(_model: "Type[Model]") -> Optional[Type[PydanticModel]]: - """Get Pydantic model for the submodel""" - nonlocal exclude, _name, has_submodel - - if _model: - new_stack = _stack + ((cls, fname, max_recursion),) - - # Get pydantic schema for the submodel - prefix_len = len(fname) + 1 - pmodel = _pydantic_recursion_protector( - _model, - exclude=tuple( - str(v[prefix_len:]) for v in exclude if v.startswith(fname + ".") - ), - include=tuple( - str(v[prefix_len:]) for v in include if v.startswith(fname + ".") - ), - computed=tuple( - str(v[prefix_len:]) for v in computed if v.startswith(fname + ".") - ), - stack=new_stack, - allow_cycles=_allow_cycles, - sort_alphabetically=sort_alphabetically, + field_property: Optional[Any] = None + is_to_one_relation: bool = False + if isinstance(field, Field): + field_property, is_to_one_relation = self._process_normal_field( + field_name, field, json_schema_extra, fconfig + ) + if field_property: + fconfig["title"] = field_name.replace("_", " ").title() + description = _br_it(field.docstring or field.description or "") + if description: + fconfig["description"] = description + if ( + field_name in self._optional + or (field.default is not None and not callable(field.default)) + ): + self._properties[field_name] = (field_property, PydanticField(default=field.default, **fconfig)) + else: + if ( + ( + json_schema_extra.get("nullable") + and not is_to_one_relation + ) + or (self._exclude_read_only and json_schema_extra.get("readOnly")) + ): + # see: https://docs.pydantic.dev/latest/migration/#required-optional-and-nullable-fields + fconfig["default"] = None + self._properties[field_name] = (field_property, PydanticField(**fconfig)) + elif isinstance(field, ComputedFieldDescription): + field_property, is_to_one_relation = self._process_computed_field(field), False + if field_property: + comment = _cleandoc(field.function) + fconfig["title"] = field_name.replace("_", " ").title() + description = comment or _br_it(field.description or "") + if description: + fconfig["description"] = description + self._properties[field_name] = field_property + + def _process_normal_field( + self, + field_name: str, + field: Field, + json_schema_extra: Dict[str, Any], + fconfig: Dict[str, Any], + ) -> Tuple[Optional[Any], bool]: + if isinstance( + field, + ( + ForeignKeyFieldInstance, + OneToOneFieldInstance, + BackwardOneToOneRelation ) - else: - pmodel = None - - # If the result is None it has been excluded and we need to exclude the field - if pmodel is None: - exclude += (fname,) - else: - has_submodel = True - # We need to rename if there are duplicate instances of this model - if cls in (c[0] for c in _stack): - _name = name or get_name() - - return pmodel - - # Foreign keys and OneToOne fields are embedded schemas - is_to_one_relation = False - if ( - field_type is relational.ForeignKeyFieldInstance - or field_type is relational.OneToOneFieldInstance - or field_type is relational.BackwardOneToOneRelation ): - is_to_one_relation = True - model = get_submodel(fdesc["python_type"]) - if model: - if fdesc.get("nullable"): - json_schema_extra["nullable"] = True - if fdesc.get("nullable") or field_default is not None: - model = Optional[model] # type: ignore - - properties[fname] = model - - # Backward FK and ManyToMany fields are list of embedded schemas - elif ( - field_type is relational.BackwardFKRelation - or field_type is relational.ManyToManyFieldInstance - ): - model = get_submodel(fdesc["python_type"]) - if model: - properties[fname] = List[model] # type: ignore - - # Computed fields as methods - elif field_type is callable: - func = fdesc["function"] - annotation = get_annotations(cls, func).get("return", None) - comment = _cleandoc(func) - if annotation is not None: - properties[fname] = computed_field(return_type=annotation, description=comment)( - func - ) - - # Json fields - elif field_type is JSONField: - properties[fname] = Any - # Any other tortoise fields - else: - annotation = annotations.get(fname, None) - if "readOnly" in fdesc["constraints"]: - json_schema_extra["readOnly"] = fdesc["constraints"]["readOnly"] - del fdesc["constraints"]["readOnly"] - fconfig.update(fdesc["constraints"]) - ptype = fdesc["python_type"] - if fdesc.get("nullable"): + return self._process_single_field_relation(field_name, field, json_schema_extra), True + elif isinstance(field, (BackwardFKRelation, ManyToManyFieldInstance)): + return self._process_many_field_relation(field_name, field), False + elif field.field_type is JSONField: + return Any, False + return self._process_data_field(field_name, field, json_schema_extra, fconfig), False + + def _process_single_field_relation( + self, + field_name: str, + field: Union[ + ForeignKeyFieldInstance, + OneToOneFieldInstance, + BackwardOneToOneRelation + ], + json_schema_extra: Dict[str, Any], + ) -> Optional[Type[PydanticModel]]: + python_type = getattr(field, "related_model", field.field_type) + model: Optional[Type[PydanticModel]] = self._get_submodel(python_type, field_name) + if model: + self._relational_fields_index.append((field_name, model.__name__)) + if field.null: json_schema_extra["nullable"] = True - if is_optional_field or field_default is not None or fdesc.get("nullable"): - ptype = Optional[ptype] - if not (exclude_readonly and json_schema_extra.get("readOnly") is True): - properties[fname] = annotation or ptype - - if fname in properties and not isinstance(properties[fname], tuple): - fconfig["title"] = fname.replace("_", " ").title() - description = comment or _br_it(fdesc.get("docstring") or fdesc["description"] or "") - if description: - fconfig["description"] = description - ftype = properties[fname] - if isinstance(ftype, PydanticDescriptorProxy): - continue - if is_optional_field or (field_default is not None and not callable(field_default)): - properties[fname] = (ftype, Field(default=field_default, **fconfig)) - else: - if (j := fconfig.get("json_schema_extra")) and ( - ( - j.get("nullable") - and not is_to_one_relation - and field_type not in (IntField, TextField) - ) - or (exclude_readonly and j.get("readOnly")) - ): - fconfig["default_factory"] = lambda: None - properties[fname] = (ftype, Field(**fconfig)) + if field.null or field.default is not None: + model = Optional[model] # type: ignore - # Here we endure that the name is unique, but complete objects are still labeled verbatim - if not has_submodel and _stack: - _name = name or f"{fqname}.leaf" - else: - _name = name or get_name() + return model + return None - # Here we de-dup to ensure that a uniquely named object is a unique object - # This fixes some Pydantic constraints. - if _name in _MODEL_INDEX: - return _MODEL_INDEX[_name] + def _process_many_field_relation( + self, + field_name: str, + field: Union[BackwardFKRelation, ManyToManyFieldInstance], + ) -> Optional[Type[List[Type[PydanticModel]]]]: + python_type = field.related_model + model = self._get_submodel(python_type, field_name) + if model: + self._relational_fields_index.append((field_name, model.__name__)) + return List[model] # type: ignore + return None - # Creating Pydantic class for the properties generated before - properties["model_config"] = pconfig - model = create_model( - _name, - __base__=PydanticModel, - __module__=module, - __validators__=validators, - **properties, - ) - # Copy the Model docstring over - model.__doc__ = _cleandoc(cls) - # Store the base class - model.model_config["orig_model"] = cls # type: ignore - # Store model reference so we can de-dup it later on if needed. - _MODEL_INDEX[_name] = model - return model + def _process_data_field( + self, + field_name: str, + field: Field, + json_schema_extra: Dict[str, Any], + fconfig: Dict[str, Any], + ) -> Optional[Any]: + annotation = self._annotations.get(field_name, None) + constraints = copy(field.constraints) + if "readOnly" in constraints: + json_schema_extra["readOnly"] = constraints["readOnly"] + del constraints["readOnly"] + fconfig.update(constraints) + python_type = getattr(field, "related_model", field.field_type) + ptype = python_type + if field.null: + json_schema_extra["nullable"] = True + if field_name in self._optional or field.default is not None or field.null: + ptype = Optional[ptype] + if not (self._exclude_read_only and json_schema_extra.get("readOnly") is True): + return annotation or ptype + return None + def _process_computed_field( + self, + field: ComputedFieldDescription, + ) -> Optional[Any]: + func = field.function + annotation = get_annotations(self._cls, func).get("return", None) + comment = _cleandoc(func) + if annotation is not None: + c_f = computed_field(return_type=annotation, description=comment) + ret = c_f(func) + return ret + return None -def pydantic_queryset_creator( - cls: "Type[Model]", - *, - name=None, - exclude: Tuple[str, ...] = (), - include: Tuple[str, ...] = (), - computed: Tuple[str, ...] = (), - allow_cycles: Optional[bool] = None, - sort_alphabetically: Optional[bool] = None, -) -> Type[PydanticListModel]: + def _get_submodel(self, _model: Optional["Type[Model]"], field_name: str) -> Optional[Type[PydanticModel]]: + """Get Pydantic model for the submodel""" + + if _model: + new_stack = self._stack + ((self._cls, field_name, self.meta.max_recursion),) + + # Get pydantic schema for the submodel + prefix_len = len(field_name) + 1 + + def get_fields_to_carry_on(field_tuple: Tuple[str, ...]) -> Tuple[str, ...]: + return tuple( + str(v[prefix_len:]) for v in field_tuple if v.startswith(field_name + ".") + ) + pmodel = _pydantic_recursion_protector( + _model, + exclude=get_fields_to_carry_on(self.meta.exclude), + include=get_fields_to_carry_on(self.meta.include), + computed=get_fields_to_carry_on(self.meta.computed), + stack=new_stack, + allow_cycles=self.meta.allow_cycles, + sort_alphabetically=self.meta.sort_alphabetically, + ) + else: + pmodel = None + + # If the result is None it has been excluded and we need to exclude the field + if pmodel is None: + self.meta.exclude += (field_name,) + + return pmodel + + +def pydantic_model_creator( + cls: "Type[Model]", + *, + name=None, + exclude: Optional[Tuple[str, ...]] = None, + include: Optional[Tuple[str, ...]] = None, + computed: Optional[Tuple[str, ...]] = None, + optional: Optional[Tuple[str, ...]] = None, + allow_cycles: Optional[bool] = None, + sort_alphabetically: Optional[bool] = None, + exclude_readonly: bool = False, + meta_override: Optional[Type] = None, + model_config: Optional[ConfigDict] = None, + validators: Optional[Dict[str, Any]] = None, + module: str = __name__, +) -> Type[PydanticModel]: """ - Function to build a `Pydantic Model `__ list off Tortoise Model. + Function to build `Pydantic Model `__ off Tortoise Model. - :param cls: The Tortoise Model to put in a list. + :param cls: The Tortoise Model :param name: Specify a custom name explicitly, instead of a generated name. - - The list generated name is currently naive and merely adds a "s" to the end - of the singular name. :param exclude: Extra fields to exclude from the provided model. :param include: Extra fields to include from the provided model. :param computed: Extra computed fields to include from the provided model. + :param optional: Extra optional fields for the provided model. :param allow_cycles: Do we allow any cycles in the generated model? This is only useful for recursive/self-referential models. @@ -492,28 +591,30 @@ def pydantic_queryset_creator( * Field definition order + * order of reverse relations (as discovered) + * order of computed functions (as provided). - """ + :param exclude_readonly: Build a subset model that excludes any readonly fields + :param meta_override: A PydanticMeta class to override model's values. + :param model_config: A custom config to use as pydantic config. + :param validators: A dictionary of methods that validate fields. + :param module: The name of the module that the model belongs to. - submodel = pydantic_model_creator( - cls, + Note: Created pydantic model uses config_class parameter and PydanticMeta's + config_class as its Config class's bases(Only if provided!), but it + ignores ``fields`` config. pydantic_model_creator will generate fields by + include/exclude/computed parameters automatically. + """ + pmc = PydanticModelCreator( + cls=cls, + name=name, exclude=exclude, include=include, computed=computed, + optional=optional, allow_cycles=allow_cycles, sort_alphabetically=sort_alphabetically, - name=name, - ) - lname = name or f"{submodel.__name__}_list" - - # Creating Pydantic class for the properties generated before - model = create_model( - lname, - __base__=PydanticListModel, - root=(List[submodel], Field(default_factory=list)), # type: ignore + exclude_readonly=exclude_readonly, + meta_override=meta_override, + model_config=model_config, + validators=validators, + module=module ) - # Copy the Model docstring over - model.__doc__ = _cleandoc(cls) - # The title of the model to hide the hash postfix - model.model_config["title"] = name or f"{submodel.model_config['title']}_list" - model.model_config["submodel"] = submodel # type: ignore - return model + return pmc.create_pydantic_model() diff --git a/tortoise/contrib/pydantic/descriptions.py b/tortoise/contrib/pydantic/descriptions.py new file mode 100644 index 000000000..8d3770b8a --- /dev/null +++ b/tortoise/contrib/pydantic/descriptions.py @@ -0,0 +1,210 @@ +import dataclasses +import sys +from typing import Type, Optional, Any, TYPE_CHECKING, List, Tuple, Callable + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + +from pydantic import ConfigDict + +from tortoise.fields import Field + +if TYPE_CHECKING: # pragma: nocoverage + from tortoise.models import Model + + +@dataclasses.dataclass +class ModelDescription: + pk_field: Field + data_fields: List[Field] = dataclasses.field(default_factory=list) + fk_fields: List[Field] = dataclasses.field(default_factory=list) + backward_fk_fields: List[Field] = dataclasses.field(default_factory=list) + o2o_fields: List[Field] = dataclasses.field(default_factory=list) + backward_o2o_fields: List[Field] = dataclasses.field(default_factory=list) + m2m_fields: List[Field] = dataclasses.field(default_factory=list) + + @classmethod + def from_model(cls, model: Type["Model"]) -> Self: + return cls( + pk_field=model._meta.fields_map[model._meta.pk_attr], + data_fields=[ + field + for name, field in model._meta.fields_map.items() + if name != model._meta.pk_attr and name in (model._meta.fields - model._meta.fetch_fields) + ], + fk_fields=[ + field + for name, field in model._meta.fields_map.items() + if name in model._meta.fk_fields + ], + backward_fk_fields=[ + field + for name, field in model._meta.fields_map.items() + if name in model._meta.backward_fk_fields + ], + o2o_fields=[ + field + for name, field in model._meta.fields_map.items() + if name in model._meta.o2o_fields + ], + backward_o2o_fields=[ + field + for name, field in model._meta.fields_map.items() + if name in model._meta.backward_o2o_fields + ], + m2m_fields=[ + field + for name, field in model._meta.fields_map.items() + if name in model._meta.m2m_fields + ], + ) + + +@dataclasses.dataclass +class ComputedFieldDescription: + field_type: Any + function: Callable[[], Any] + description: Optional[str] + + +@dataclasses.dataclass +class PydanticMetaData: + #: If not empty, only fields this property contains will be in the pydantic model + include: Tuple[str, ...] = () + + #: Fields listed in this property will be excluded from pydantic model + exclude: Tuple[str, ...] = dataclasses.field(default_factory=lambda: ("Meta",)) + + #: Computed fields can be listed here to use in pydantic model + computed: Tuple[str, ...] = dataclasses.field(default_factory=tuple) + + #: Use backward relations without annotations - not recommended, it can be huge data + #: without control + backward_relations: bool = True + + #: Maximum recursion level allowed + max_recursion: int = 3 + + #: Allow cycles in recursion - This can result in HUGE data - Be careful! + #: Please use this with ``exclude``/``include`` and sane ``max_recursion`` + allow_cycles: bool = False + + #: If we should exclude raw fields (the ones have _id suffixes) of relations + exclude_raw_fields: bool = True + + #: Sort fields alphabetically. + #: If not set (or ``False``) then leave fields in declaration order + sort_alphabetically: bool = False + + #: Allows user to specify custom config for generated model + model_config: Optional[ConfigDict] = None + + @classmethod + def from_pydantic_meta(cls, old_pydantic_meta: Any) -> Self: + default_meta = cls() + + def get_param_from_pydantic_meta(attr: str, default: Any) -> Any: + return getattr(old_pydantic_meta, attr, default) + include = tuple(get_param_from_pydantic_meta("include", default_meta.include)) + exclude = tuple(get_param_from_pydantic_meta("exclude", default_meta.exclude)) + computed = tuple(get_param_from_pydantic_meta("computed", default_meta.computed)) + backward_relations = bool( + get_param_from_pydantic_meta("backward_relations_raw", default_meta.backward_relations) + ) + max_recursion = int(get_param_from_pydantic_meta("max_recursion", default_meta.max_recursion)) + allow_cycles = bool(get_param_from_pydantic_meta("allow_cycles", default_meta.allow_cycles)) + exclude_raw_fields = bool( + get_param_from_pydantic_meta("exclude_raw_fields", default_meta.exclude_raw_fields) + ) + sort_alphabetically = bool( + get_param_from_pydantic_meta("sort_alphabetically", default_meta.sort_alphabetically) + ) + model_config = get_param_from_pydantic_meta("model_config", default_meta.model_config) + pmd = cls( + include=include, + exclude=exclude, + computed=computed, + backward_relations=backward_relations, + max_recursion=max_recursion, + allow_cycles=allow_cycles, + exclude_raw_fields=exclude_raw_fields, + sort_alphabetically=sort_alphabetically, + model_config=model_config + ) + return pmd + + def construct_pydantic_meta( + self, + meta_override: Type + ) -> "PydanticMetaData": + def get_param_from_meta_override(attr: str) -> Any: + return getattr(meta_override, attr, getattr(self, attr)) + + default_include: Tuple[str, ...] = tuple(get_param_from_meta_override("include")) + default_exclude: Tuple[str, ...] = tuple(get_param_from_meta_override("exclude")) + default_computed: Tuple[str, ...] = tuple(get_param_from_meta_override("computed")) + default_config: Optional[ConfigDict] = self.model_config + + backward_relations: bool = bool(get_param_from_meta_override("backward_relations")) + + max_recursion: int = int(get_param_from_meta_override("max_recursion")) + exclude_raw_fields: bool = bool(get_param_from_meta_override("exclude_raw_fields")) + sort_alphabetically: bool = bool(get_param_from_meta_override("sort_alphabetically")) + allow_cycles: bool = bool(get_param_from_meta_override("allow_cycles")) + + pmd = PydanticMetaData( + include=default_include, + exclude=default_exclude, + computed=default_computed, + model_config=default_config, + backward_relations=backward_relations, + max_recursion=max_recursion, + exclude_raw_fields=exclude_raw_fields, + sort_alphabetically=sort_alphabetically, + allow_cycles=allow_cycles + ) + return pmd + + def finalize_meta( + self, + exclude: Tuple[str, ...] = (), + include: Tuple[str, ...] = (), + computed: Tuple[str, ...] = (), + allow_cycles: Optional[bool] = None, + sort_alphabetically: Optional[bool] = None, + model_config: Optional[ConfigDict] = None, + ) -> "PydanticMetaData": + _sort_fields: bool = ( + self.sort_alphabetically + if sort_alphabetically is None + else sort_alphabetically + ) + _allow_cycles: bool = ( + self.allow_cycles + if allow_cycles is None + else allow_cycles + ) + + include = tuple(include) + self.include + exclude = tuple(exclude) + self.exclude + computed = tuple(computed) + self.computed + + _model_config = ConfigDict() + if self.model_config: + _model_config.update(self.model_config) + if model_config: + _model_config.update(model_config) + + return PydanticMetaData( + include=include, + exclude=exclude, + computed=computed, + backward_relations=self.backward_relations, + max_recursion=self.max_recursion, + exclude_raw_fields=self.exclude_raw_fields, + sort_alphabetically=_sort_fields, + allow_cycles=_allow_cycles, + model_config=_model_config + )