general to_json method and more inferred types (#372)

* general "to_json" method and more inferred types

* support or filtering generators

* increase CI width

* allow windows tests to fail

* another xfail

* remove incorrect xfail

Author: samuelcolvin

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
   pull_request: {}
 
 env:
-  COLUMNS: 120
+  COLUMNS: 150
 
 jobs:
   test-cpython:

pydantic_core/__init__.py

@@ -11,6 +11,7 @@
     Url,
     ValidationError,
     __version__,
+    to_json,
 )
 from .core_schema import CoreConfig, CoreSchema
 
@@ -29,4 +30,5 @@
     'PydanticOmit',
     'PydanticSerializationError',
     'PydanticSerializationUnexpectedValue',
+    'to_json',
 )

pydantic_core/_pydantic_core.pyi

@@ -5,9 +5,9 @@ from typing import Any, TypedDict
 from pydantic_core.core_schema import CoreConfig, CoreSchema, ErrorType
 
 if sys.version_info < (3, 11):
-    from typing_extensions import NotRequired, TypeAlias
+    from typing_extensions import Literal, NotRequired, TypeAlias
 else:
-    from typing import NotRequired, TypeAlias
+    from typing import Literal, NotRequired, TypeAlias
 
 __all__ = (
     '__version__',
@@ -75,6 +75,17 @@ class SchemaSerializer:
         warnings: bool = True,
     ) -> bytes: ...
 
+def to_json(
+    value: Any,
+    indent: int | None = None,
+    include: IncEx = None,
+    exclude: IncEx = None,
+    exclude_none: bool = False,
+    round_trip: bool = False,
+    timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
+    bytes_mode: Literal['utf8', 'base64'] = 'utf8',
+) -> bytes: ...
+
 class Url:
     scheme: str
     username: 'str | None'

src/lib.rs

@@ -24,7 +24,7 @@ mod validators;
 pub use self::url::{PyMultiHostUrl, PyUrl};
 pub use build_tools::SchemaError;
 pub use errors::{list_all_errors, PydanticCustomError, PydanticKnownError, PydanticOmit, ValidationError};
-pub use serializers::{PydanticSerializationError, PydanticSerializationUnexpectedValue, SchemaSerializer};
+pub use serializers::{to_json, PydanticSerializationError, PydanticSerializationUnexpectedValue, SchemaSerializer};
 pub use validators::SchemaValidator;
 
 pub fn get_version() -> String {
@@ -52,6 +52,7 @@ fn _pydantic_core(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<PyUrl>()?;
     m.add_class::<PyMultiHostUrl>()?;
     m.add_class::<SchemaSerializer>()?;
+    m.add_function(wrap_pyfunction!(to_json, m)?)?;
     m.add_function(wrap_pyfunction!(list_all_errors, m)?)?;
     Ok(())
 }

src/serializers/config.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::str::{from_utf8, Utf8Error};
 
 use pyo3::prelude::*;
-use pyo3::types::{PyBytes, PyDelta, PyDict};
+use pyo3::types::{PyDelta, PyDict};
 use pyo3::{intern, PyNativeType};
 
 use serde::ser::Error;
@@ -27,6 +27,15 @@ impl SerializationConfig {
             bytes_mode,
         })
     }
+
+    pub fn from_args(timedelta_mode: Option<&str>, bytes_mode: Option<&str>) -> PyResult<Self> {
+        let timedelta_mode = TimedeltaMode::from_str(timedelta_mode)?;
+        let bytes_mode = BytesMode::from_str(bytes_mode)?;
+        Ok(Self {
+            timedelta_mode,
+            bytes_mode,
+        })
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -41,7 +50,11 @@ impl TimedeltaMode {
             Some(c) => c.get_as::<&str>(intern!(c.py(), "ser_json_timedelta"))?,
             None => None,
         };
-        match raw_mode {
+        Self::from_str(raw_mode)
+    }
+
+    pub fn from_str(s: Option<&str>) -> PyResult<Self> {
+        match s {
             Some("iso8601") => Ok(Self::Iso8601),
             Some("float") => Ok(Self::Float),
             Some(s) => py_err!(
@@ -113,7 +126,11 @@ impl BytesMode {
             Some(c) => c.get_as::<&str>(intern!(c.py(), "ser_json_bytes"))?,
             None => None,
         };
-        let base64_config = match raw_mode {
+        Self::from_str(raw_mode)
+    }
+
+    pub fn from_str(s: Option<&str>) -> PyResult<Self> {
+        let base64_config = match s {
             Some("utf8") => None,
             Some("base64") => Some(base64::Config::new(base64::CharacterSet::UrlSafe, true)),
             Some(s) => return py_err!("Invalid bytes serialization mode: `{}`, expected `utf8` or `base64`", s),
@@ -122,23 +139,21 @@ impl BytesMode {
         Ok(Self { base64_config })
     }
 
-    pub fn bytes_to_string<'py>(&self, py_bytes: &'py PyBytes) -> PyResult<Cow<'py, str>> {
+    pub fn bytes_to_string<'py>(&self, py: Python, bytes: &'py [u8]) -> PyResult<Cow<'py, str>> {
         if let Some(config) = self.base64_config {
-            Ok(Cow::Owned(base64::encode_config(py_bytes.as_bytes(), config)))
+            Ok(Cow::Owned(base64::encode_config(bytes, config)))
         } else {
-            py_bytes_to_str(py_bytes).map(Cow::Borrowed)
+            from_utf8(bytes)
+                .map_err(|err| utf8_py_error(py, err, bytes))
+                .map(Cow::Borrowed)
         }
     }
 
-    pub fn serialize_bytes<S: serde::ser::Serializer>(
-        &self,
-        py_bytes: &PyBytes,
-        serializer: S,
-    ) -> Result<S::Ok, S::Error> {
+    pub fn serialize_bytes<S: serde::ser::Serializer>(&self, bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> {
         if let Some(config) = self.base64_config {
-            serializer.serialize_str(&base64::encode_config(py_bytes.as_bytes(), config))
+            serializer.serialize_str(&base64::encode_config(bytes, config))
         } else {
-            match from_utf8(py_bytes.as_bytes()) {
+            match from_utf8(bytes) {
                 Ok(s) => serializer.serialize_str(s),
                 Err(e) => Err(Error::custom(e.to_string())),
             }
@@ -152,9 +167,3 @@ pub fn utf8_py_error(py: Python, err: Utf8Error, data: &[u8]) -> PyErr {
         Err(err) => err,
     }
 }
-
-fn py_bytes_to_str(py_bytes: &PyBytes) -> PyResult<&str> {
-    let py = py_bytes.py();
-    let data = py_bytes.as_bytes();
-    from_utf8(data).map_err(|err| utf8_py_error(py, err, data))
-}

src/serializers/filter.rs

@@ -9,7 +9,7 @@ use pyo3::types::{PyBool, PyDict, PySet, PyString};
 use crate::build_tools::SchemaDict;
 
 #[derive(Debug, Clone, Default)]
-pub(super) struct SchemaFilter<T> {
+pub(crate) struct SchemaFilter<T> {
     include: Option<AHashSet<T>>,
     exclude: Option<AHashSet<T>>,
 }

src/serializers/infer.rs

@@ -1,19 +1,19 @@
 use std::borrow::Cow;
-use std::str::from_utf8;
 
 use pyo3::exceptions::PyTypeError;
 use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::types::{
-    PyByteArray, PyBytes, PyDate, PyDateTime, PyDelta, PyDict, PyFrozenSet, PyList, PySet, PyString, PyTime, PyTuple,
+    PyByteArray, PyBytes, PyDate, PyDateTime, PyDelta, PyDict, PyFrozenSet, PyIterator, PyList, PySet, PyString,
+    PyTime, PyTuple,
 };
 
 use serde::ser::{Serialize, SerializeMap, SerializeSeq, Serializer};
 
 use crate::build_tools::{py_err, safe_repr};
+use crate::serializers::filter::SchemaFilter;
 use crate::url::{PyMultiHostUrl, PyUrl};
 
-use super::config::utf8_py_error;
 use super::errors::{py_err_se_err, PydanticSerializationError};
 use super::extra::{Extra, SerMode};
 use super::filter::AnyFilter;
@@ -97,21 +97,23 @@ pub(crate) fn infer_to_python_known(
             // have to do this to make sure subclasses of for example str are upcast to `str`
             ObType::IntSubclass => value.extract::<i64>()?.into_py(py),
             ObType::FloatSubclass => value.extract::<f64>()?.into_py(py),
+            ObType::Decimal => value.to_string().into_py(py),
             ObType::StrSubclass => value.extract::<&str>()?.into_py(py),
             ObType::Bytes => extra
                 .config
                 .bytes_mode
-                .bytes_to_string(value.downcast()?)
+                .bytes_to_string(py, value.downcast::<PyBytes>()?.as_bytes())
                 .map(|s| s.into_py(py))?,
             ObType::Bytearray => {
                 let py_byte_array: &PyByteArray = value.downcast()?;
                 // see https://docs.rs/pyo3/latest/pyo3/types/struct.PyByteArray.html#method.as_bytes
                 // for why this is marked unsafe
                 let bytes = unsafe { py_byte_array.as_bytes() };
-                match from_utf8(bytes) {
-                    Ok(s) => s.into_py(py),
-                    Err(err) => return Err(utf8_py_error(py, err, bytes)),
-                }
+                extra
+                    .config
+                    .bytes_mode
+                    .bytes_to_string(py, bytes)
+                    .map(|s| s.into_py(py))?
             }
             ObType::Tuple => {
                 let elements = serialize_seq_filter!(PyTuple);
@@ -163,6 +165,20 @@ pub(crate) fn infer_to_python_known(
                 let v = value.getattr(intern!(py, "value"))?;
                 infer_to_python(v, include, exclude, extra)?.into_py(py)
             }
+            ObType::Generator => {
+                let py_seq: &PyIterator = value.downcast()?;
+                let mut items = Vec::new();
+                let filter = AnyFilter::new();
+
+                for (index, r) in py_seq.iter()?.enumerate() {
+                    let element = r?;
+                    let op_next = filter.value_filter(index, include, exclude)?;
+                    if let Some((next_include, next_exclude)) = op_next {
+                        items.push(infer_to_python(element, next_include, next_exclude, extra)?);
+                    }
+                }
+                PyList::new(py, items).into_py(py)
+            }
             ObType::Unknown => return Err(unknown_type_error(value)),
         },
         _ => match ob_type {
@@ -199,6 +215,17 @@ pub(crate) fn infer_to_python_known(
             }
             ObType::Dataclass => serialize_dict(object_to_dict(value, false, extra)?)?,
             ObType::PydanticModel => serialize_dict(object_to_dict(value, true, extra)?)?,
+            ObType::Generator => {
+                let iter = super::type_serializers::generator::SerializationIterator::new(
+                    value.downcast()?,
+                    super::type_serializers::any::AnySerializer::default().into(),
+                    SchemaFilter::default(),
+                    include,
+                    exclude,
+                    extra,
+                );
+                iter.into_py(py)
+            }
             _ => value.into_py(py),
         },
     };
@@ -321,21 +348,19 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
         ObType::Int | ObType::IntSubclass => serialize!(i64),
         ObType::Bool => serialize!(bool),
         ObType::Float | ObType::FloatSubclass => serialize!(f64),
+        ObType::Decimal => value.to_string().serialize(serializer),
         ObType::Str | ObType::StrSubclass => {
             let py_str: &PyString = value.downcast().map_err(py_err_se_err)?;
             super::type_serializers::string::serialize_py_str(py_str, serializer)
         }
         ObType::Bytes => {
             let py_bytes: &PyBytes = value.downcast().map_err(py_err_se_err)?;
-            extra.config.bytes_mode.serialize_bytes(py_bytes, serializer)
+            extra.config.bytes_mode.serialize_bytes(py_bytes.as_bytes(), serializer)
         }
         ObType::Bytearray => {
             let py_byte_array: &PyByteArray = value.downcast().map_err(py_err_se_err)?;
             let bytes = unsafe { py_byte_array.as_bytes() };
-            match from_utf8(bytes) {
-                Ok(s) => serializer.serialize_str(s),
-                Err(e) => Err(py_err_se_err(e)),
-            }
+            extra.config.bytes_mode.serialize_bytes(bytes, serializer)
         }
         ObType::Dict => serialize_dict!(value.downcast::<PyDict>().map_err(py_err_se_err)?),
         ObType::List => serialize_seq_filter!(PyList),
@@ -378,6 +403,20 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
             let v = value.getattr(intern!(value.py(), "value")).map_err(py_err_se_err)?;
             infer_serialize(v, serializer, include, exclude, extra)
         }
+        ObType::Generator => {
+            let py_seq: &PyIterator = value.downcast().map_err(py_err_se_err)?;
+            let mut seq = serializer.serialize_seq(None)?;
+            let filter = AnyFilter::new();
+            for (index, r) in py_seq.iter().map_err(py_err_se_err)?.enumerate() {
+                let element = r.map_err(py_err_se_err)?;
+                let op_next = filter.value_filter(index, include, exclude).map_err(py_err_se_err)?;
+                if let Some((next_include, next_exclude)) = op_next {
+                    let item_serializer = SerializeInfer::new(element, next_include, next_exclude, extra);
+                    seq.serialize_element(&item_serializer)?
+                }
+            }
+            seq.end()
+        }
         ObType::Unknown => return Err(py_err_se_err(unknown_type_error(value))),
     };
     extra.rec_guard.pop(value_id);
@@ -399,19 +438,20 @@ pub(crate) fn infer_json_key_known<'py>(ob_type: &ObType, key: &'py PyAny, extra
         ObType::Int | ObType::IntSubclass | ObType::Float | ObType::FloatSubclass => {
             super::type_serializers::simple::to_str_json_key(key)
         }
+        ObType::Decimal => Ok(Cow::Owned(key.to_string())),
         ObType::Bool => super::type_serializers::simple::bool_json_key(key),
         ObType::Str | ObType::StrSubclass => {
             let py_str: &PyString = key.downcast()?;
             Ok(Cow::Borrowed(py_str.to_str()?))
         }
-        ObType::Bytes => extra.config.bytes_mode.bytes_to_string(key.downcast()?),
+        ObType::Bytes => extra
+            .config
+            .bytes_mode
+            .bytes_to_string(key.py(), key.downcast::<PyBytes>()?.as_bytes()),
         ObType::Bytearray => {
             let py_byte_array: &PyByteArray = key.downcast()?;
             let bytes = unsafe { py_byte_array.as_bytes() };
-            match from_utf8(bytes) {
-                Ok(s) => Ok(Cow::Borrowed(s)),
-                Err(err) => Err(utf8_py_error(key.py(), err, bytes)),
-            }
+            extra.config.bytes_mode.bytes_to_string(key.py(), bytes)
         }
         ObType::Datetime => {
             let py_dt: &PyDateTime = key.downcast()?;
@@ -447,7 +487,7 @@ pub(crate) fn infer_json_key_known<'py>(ob_type: &ObType, key: &'py PyAny, extra
             }
             Ok(Cow::Owned(key_build.finish()))
         }
-        ObType::List | ObType::Set | ObType::Frozenset | ObType::Dict => {
+        ObType::List | ObType::Set | ObType::Frozenset | ObType::Dict | ObType::Generator => {
             py_err!(PyTypeError; "`{}` not valid as object key", ob_type)
         }
         ObType::Dataclass | ObType::PydanticModel => {

src/serializers/mod.rs

@@ -135,3 +135,39 @@ impl SchemaSerializer {
         )
     }
 }
+
+#[allow(clippy::too_many_arguments)]
+#[pyfunction]
+pub fn to_json(
+    py: Python,
+    value: &PyAny,
+    indent: Option<usize>,
+    include: Option<&PyAny>,
+    exclude: Option<&PyAny>,
+    exclude_none: Option<bool>,
+    round_trip: Option<bool>,
+    timedelta_mode: Option<&str>,
+    bytes_mode: Option<&str>,
+) -> PyResult<PyObject> {
+    let warnings = CollectWarnings::new(None);
+    let rec_guard = SerRecursionGuard::default();
+    let config = SerializationConfig::from_args(timedelta_mode, bytes_mode)?;
+    let extra = Extra::new(
+        py,
+        &SerMode::Json,
+        &[],
+        None,
+        &warnings,
+        None,
+        None,
+        exclude_none,
+        round_trip,
+        &config,
+        &rec_guard,
+    );
+    let serializer = type_serializers::any::AnySerializer::default().into();
+    let bytes = to_json_bytes(value, &serializer, include, exclude, &extra, indent, 1024)?;
+    warnings.final_check(py)?;
+    let py_bytes = PyBytes::new(py, &bytes);
+    Ok(py_bytes.into())
+}