Add JsonOrPython validator and serializer (#598)

Co-authored-by: Samuel Colvin <[email protected]>

Author: adriangb

pydantic_core/_pydantic_core.pyi

@@ -53,14 +53,6 @@ class SchemaValidator:
         context: Any = None,
         self_instance: 'Any | None' = None,
     ) -> Any: ...
-    def isinstance_json(
-        self,
-        input: 'str | bytes | bytearray',
-        *,
-        strict: 'bool | None' = None,
-        context: Any = None,
-        self_instance: 'Any | None' = None,
-    ) -> bool: ...
     def validate_assignment(
         self, obj: Any, field: str, input: Any, *, strict: 'bool | None' = None, context: Any = None
     ) -> 'dict[str, Any]': ...

pydantic_core/core_schema.py

@@ -124,6 +124,11 @@ def config(self) -> CoreConfig | None:
         """The CoreConfig that applies to this validation."""
         ...
 
+    @property
+    def mode(self) -> Literal['python', 'json']:
+        """The type of input data we are currently validating"""
+        ...
+
 
 class FieldValidationInfo(ValidationInfo, Protocol):
     """
@@ -1050,8 +1055,6 @@ class IsInstanceSchema(TypedDict, total=False):
     type: Required[Literal['is-instance']]
     cls: Required[Any]
     cls_repr: str
-    json_types: Set[JsonType]
-    json_function: Callable[[Any], Any]
     ref: str
     metadata: Any
     serialization: SerSchema
@@ -1060,8 +1063,6 @@ class IsInstanceSchema(TypedDict, total=False):
 def is_instance_schema(
     cls: Any,
     *,
-    json_types: Set[JsonType] | None = None,
-    json_function: Callable[[Any], Any] | None = None,
     cls_repr: str | None = None,
     ref: str | None = None,
     metadata: Any = None,
@@ -1083,23 +1084,13 @@ class A:
 
     Args:
         cls: The value must be an instance of this class
-        json_types: When parsing JSON directly, the value must be one of these json types
-        json_function: When parsing JSON directly, If provided, the JSON value is passed to this
-            function and the return value used as the output value
         cls_repr: If provided this string is used in the validator name instead of `repr(cls)`
         ref: optional unique identifier of the schema, used to reference the schema in other places
         metadata: Any other information you want to include with the schema, not used by pydantic-core
         serialization: Custom serialization schema
     """
     return dict_not_none(
-        type='is-instance',
-        cls=cls,
-        json_types=json_types,
-        json_function=json_function,
-        cls_repr=cls_repr,
-        ref=ref,
-        metadata=metadata,
-        serialization=serialization,
+        type='is-instance', cls=cls, cls_repr=cls_repr, ref=ref, metadata=metadata, serialization=serialization
     )
 
 
@@ -2604,6 +2595,63 @@ def fn(v: str, info: core_schema.ValidationInfo) -> str:
     )
 
 
+class JsonOrPythonSchema(TypedDict, total=False):
+    type: Required[Literal['json-or-python']]
+    json_schema: Required[CoreSchema]
+    python_schema: Required[CoreSchema]
+    ref: str
+    metadata: Any
+    serialization: SerSchema
+
+
+def json_or_python_schema(
+    json_schema: CoreSchema,
+    python_schema: CoreSchema,
+    *,
+    ref: str | None = None,
+    metadata: Any = None,
+    serialization: SerSchema | None = None,
+) -> JsonOrPythonSchema:
+    """
+    Returns a schema that uses the Json or Python schema depending on the input:
+
+    ```py
+    from pydantic_core import SchemaValidator, ValidationError, core_schema
+
+    v = SchemaValidator(
+        core_schema.json_or_python_schema(
+            json_schema=core_schema.int_schema(),
+            python_schema=core_schema.int_schema(strict=True),
+        )
+    )
+
+    assert v.validate_json('"123"') == 123
+
+    try:
+        v.validate_python('123')
+    except ValidationError:
+        pass
+    else:
+        raise AssertionError('Validation should have failed')
+    ```
+
+    Args:
+        json_schema: The schema to use for Json inputs
+        python_schema: The schema to use for Python inputs
+        ref: optional unique identifier of the schema, used to reference the schema in other places
+        metadata: Any other information you want to include with the schema, not used by pydantic-core
+        serialization: Custom serialization schema
+    """
+    return dict_not_none(
+        type='json-or-python',
+        json_schema=json_schema,
+        python_schema=python_schema,
+        ref=ref,
+        metadata=metadata,
+        serialization=serialization,
+    )
+
+
 class TypedDictField(TypedDict, total=False):
     type: Required[Literal['typed-dict-field']]
     schema: Required[CoreSchema]
@@ -3612,6 +3660,7 @@ def definition_reference_schema(
         TaggedUnionSchema,
         ChainSchema,
         LaxOrStrictSchema,
+        JsonOrPythonSchema,
         TypedDictSchema,
         ModelFieldsSchema,
         ModelSchema,
@@ -3664,6 +3713,7 @@ def definition_reference_schema(
     'tagged-union',
     'chain',
     'lax-or-strict',
+    'json-or-python',
     'typed-dict',
     'model-fields',
     'model',

src/input/input_abstract.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 
-use pyo3::prelude::*;
 use pyo3::types::{PyDict, PyString, PyType};
+use pyo3::{intern, prelude::*};
 
 use crate::errors::{InputValue, LocItem, ValResult};
 use crate::{PyMultiHostUrl, PyUrl};
@@ -10,15 +10,18 @@ use super::datetime::{EitherDate, EitherDateTime, EitherTime, EitherTimedelta};
 use super::return_enums::{EitherBytes, EitherString};
 use super::{GenericArguments, GenericCollection, GenericIterator, GenericMapping, JsonInput};
 
+#[derive(Debug, Clone, Copy)]
 pub enum InputType {
     Python,
     Json,
-    String,
 }
 
-impl InputType {
-    pub fn is_json(&self) -> bool {
-        matches!(self, Self::Json)
+impl IntoPy<PyObject> for InputType {
+    fn into_py(self, py: Python<'_>) -> PyObject {
+        match self {
+            Self::Json => intern!(py, "json").into(),
+            Self::Python => intern!(py, "python").into(),
+        }
     }
 }
 
@@ -27,8 +30,6 @@ impl InputType {
 /// * `strict_*` & `lax_*` if they have different behavior
 /// * or, `validate_*` and `strict_*` to just call `validate_*` if the behavior for strict and lax is the same
 pub trait Input<'a>: fmt::Debug + ToPyObject {
-    fn get_type(&self) -> &'static InputType;
-
     fn as_loc_item(&self) -> LocItem;
 
     fn as_error_value(&'a self) -> InputValue<'a>;
@@ -44,9 +45,6 @@ pub trait Input<'a>: fmt::Debug + ToPyObject {
         None
     }
 
-    // input_ prefix to differentiate from the function on PyAny
-    fn input_is_instance(&self, class: &PyAny, json_mask: u8) -> PyResult<bool>;
-
     fn is_exact_instance(&self, _class: &PyType) -> bool {
         false
     }

src/input/input_json.rs

@@ -7,19 +7,14 @@ use super::datetime::{
     bytes_as_date, bytes_as_datetime, bytes_as_time, bytes_as_timedelta, float_as_datetime, float_as_duration,
     float_as_time, int_as_datetime, int_as_duration, int_as_time, EitherDate, EitherDateTime, EitherTime,
 };
-use super::input_abstract::InputType;
 use super::parse_json::JsonArray;
 use super::shared::{float_as_int, int_as_bool, map_json_err, str_as_bool, str_as_int};
 use super::{
     EitherBytes, EitherString, EitherTimedelta, GenericArguments, GenericCollection, GenericIterator, GenericMapping,
-    Input, JsonArgs, JsonInput, JsonType,
+    Input, JsonArgs, JsonInput,
 };
 
 impl<'a> Input<'a> for JsonInput {
-    fn get_type(&self) -> &'static InputType {
-        &InputType::Json
-    }
-
     /// This is required by since JSON object keys are always strings, I don't think it can be called
     #[cfg_attr(has_no_coverage, no_coverage)]
     fn as_loc_item(&self) -> LocItem {
@@ -38,23 +33,6 @@ impl<'a> Input<'a> for JsonInput {
         matches!(self, JsonInput::Null)
     }
 
-    fn input_is_instance(&self, _class: &PyAny, json_mask: u8) -> PyResult<bool> {
-        if json_mask == 0 {
-            Ok(false)
-        } else {
-            let json_type: JsonType = match self {
-                JsonInput::Null => JsonType::Null,
-                JsonInput::Bool(_) => JsonType::Bool,
-                JsonInput::Int(_) => JsonType::Int,
-                JsonInput::Float(_) => JsonType::Float,
-                JsonInput::String(_) => JsonType::String,
-                JsonInput::Array(_) => JsonType::Array,
-                JsonInput::Object(_) => JsonType::Object,
-            };
-            Ok(json_type.matches(json_mask))
-        }
-    }
-
     fn as_kwargs(&'a self, py: Python<'a>) -> Option<&'a PyDict> {
         match self {
             JsonInput::Object(object) => {
@@ -330,10 +308,6 @@ impl<'a> Input<'a> for JsonInput {
 
 /// Required for Dict keys so the string can behave like an Input
 impl<'a> Input<'a> for String {
-    fn get_type(&self) -> &'static InputType {
-        &InputType::String
-    }
-
     fn as_loc_item(&self) -> LocItem {
         self.to_string().into()
     }
@@ -347,14 +321,6 @@ impl<'a> Input<'a> for String {
         false
     }
 
-    fn input_is_instance(&self, _class: &PyAny, json_mask: u8) -> PyResult<bool> {
-        if json_mask == 0 {
-            Ok(false)
-        } else {
-            Ok(JsonType::String.matches(json_mask))
-        }
-    }
-
     fn as_kwargs(&'a self, _py: Python<'a>) -> Option<&'a PyDict> {
         None
     }

src/input/input_python.rs

@@ -20,11 +20,10 @@ use super::datetime::{
     float_as_duration, float_as_time, int_as_datetime, int_as_duration, int_as_time, EitherDate, EitherDateTime,
     EitherTime,
 };
-use super::input_abstract::InputType;
 use super::shared::{float_as_int, int_as_bool, map_json_err, str_as_bool, str_as_int};
 use super::{
-    py_error_on_minusone, py_string_str, EitherBytes, EitherString, EitherTimedelta, GenericArguments,
-    GenericCollection, GenericIterator, GenericMapping, Input, JsonInput, PyArgs,
+    py_string_str, EitherBytes, EitherString, EitherTimedelta, GenericArguments, GenericCollection, GenericIterator,
+    GenericMapping, Input, JsonInput, PyArgs,
 };
 
 /// Extract generators and deques into a `GenericCollection`
@@ -72,10 +71,6 @@ macro_rules! extract_dict_iter {
 }
 
 impl<'a> Input<'a> for PyAny {
-    fn get_type(&self) -> &'static InputType {
-        &InputType::Python
-    }
-
     fn as_loc_item(&self) -> LocItem {
         if let Ok(py_str) = self.downcast::<PyString>() {
             py_str.to_string_lossy().as_ref().into()
@@ -102,14 +97,6 @@ impl<'a> Input<'a> for PyAny {
         Some(self.getattr(name))
     }
 
-    fn input_is_instance(&self, class: &PyAny, _json_mask: u8) -> PyResult<bool> {
-        // See PyO3/pyo3#2694 - we can't use `is_instance` here since it requires PyType,
-        // and some check objects are not types, this logic is lifted from `is_instance` in PyO3
-        let result = unsafe { ffi::PyObject_IsInstance(self.as_ptr(), class.as_ptr()) };
-        py_error_on_minusone(self.py(), result)?;
-        Ok(result == 1)
-    }
-
     fn is_exact_instance(&self, class: &PyType) -> bool {
         self.get_type().is(class)
     }

src/input/mod.rs

@@ -14,8 +14,8 @@ pub(crate) use datetime::{
     pydate_as_date, pydatetime_as_datetime, pytime_as_time, pytimedelta_as_duration, EitherDate, EitherDateTime,
     EitherTime, EitherTimedelta,
 };
-pub(crate) use input_abstract::Input;
-pub(crate) use parse_json::{JsonInput, JsonObject, JsonType};
+pub(crate) use input_abstract::{Input, InputType};
+pub(crate) use parse_json::{JsonInput, JsonObject};
 pub(crate) use return_enums::{
     py_string_str, AttributesGenericIterator, DictGenericIterator, EitherBytes, EitherString, GenericArguments,
     GenericCollection, GenericIterator, GenericMapping, JsonArgs, JsonObjectGenericIterator, MappingGenericIterator,

src/input/parse_json.rs

@@ -1,52 +1,11 @@
 use std::fmt;
 
 use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyList, PySet};
+use pyo3::types::{PyDict, PyList};
 use serde::de::{Deserialize, DeserializeSeed, Error as SerdeError, MapAccess, SeqAccess, Visitor};
 
 use crate::lazy_index_map::LazyIndexMap;
 
-use crate::build_tools::py_err;
-
-#[derive(Copy, Clone, Debug)]
-pub enum JsonType {
-    Null = 0b10000000,
-    Bool = 0b01000000,
-    Int = 0b00100000,
-    Float = 0b00010000,
-    String = 0b00001000,
-    Array = 0b00000100,
-    Object = 0b00000010,
-}
-
-impl JsonType {
-    pub fn combine(set: &PySet) -> PyResult<u8> {
-        set.iter().map(Self::try_from).try_fold(0u8, |a, b| Ok(a | b? as u8))
-    }
-
-    pub fn matches(&self, mask: u8) -> bool {
-        *self as u8 & mask > 0
-    }
-}
-
-impl TryFrom<&PyAny> for JsonType {
-    type Error = PyErr;
-
-    fn try_from(value: &PyAny) -> PyResult<Self> {
-        let s: &str = value.extract()?;
-        match s {
-            "null" => Ok(Self::Null),
-            "bool" => Ok(Self::Bool),
-            "int" => Ok(Self::Int),
-            "float" => Ok(Self::Float),
-            "str" => Ok(Self::String),
-            "list" => Ok(Self::Array),
-            "dict" => Ok(Self::Object),
-            _ => py_err!("Invalid json type: {}", s),
-        }
-    }
-}
-
 /// similar to serde `Value` but with int and float split
 #[derive(Clone, Debug)]
 pub enum JsonInput {

src/serializers/shared.rs

@@ -130,6 +130,7 @@ combined_serializer! {
         ToString: super::type_serializers::format::ToStringSerializer;
         WithDefault: super::type_serializers::with_default::WithDefaultSerializer;
         Json: super::type_serializers::json::JsonSerializer;
+        JsonOrPython: super::type_serializers::json_or_python::JsonOrPythonSerializer;
         Union: super::type_serializers::union::UnionSerializer;
         Literal: super::type_serializers::literal::LiteralSerializer;
         Recursive: super::type_serializers::definitions::DefinitionRefSerializer;