ffi: update object.rs for 3.13 (#4447)

* ffi: update `object.rs` for 3.13

* add newsfragment

* fixup `_Py_IsImmortal` cfgs

* also update `cpython/object.rs`

* fix `Py_Is` on PyPy 3.10

* `Py_GetConstant` & `Py_GetConstantBorrowed` are pub

* [review] mejrs feedback

* correct typo

Co-authored-by: Alex Gaynor <[email protected]>

* fix freethreaded build

---------

Co-authored-by: Alex Gaynor <[email protected]>

Author: davidhewitt

Architecture.md

@@ -37,12 +37,9 @@ automated tooling because:
   - it gives us best control about how to adapt C conventions to Rust, and
   - there are many Python interpreter versions we support in a single set of files.
 
-We aim to provide straight-forward Rust wrappers resembling the file structure of
-[`cpython/Include`](https://github.com/python/cpython/tree/v3.9.2/Include).
+We aim to provide straight-forward Rust wrappers resembling the file structure of [`cpython/Include`](https://github.com/python/cpython/tree/3.13/Include).
 
-However, we still lack some APIs and are continuously updating the module to match
-the file contents upstream in CPython.
-The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome.
+We are continuously updating the module to match the latest CPython version which PyO3 supports (i.e. as of time of writing Python 3.13). The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome.
 
 In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`,
 `#[cfg(Py_3_7)]`, and `#[cfg(PyPy)]`.

newsfragments/4447.added.md

@@ -0,0 +1 @@
+Add Python 3.13 FFI definitions `PyObject_GetOptionalAttr`, `PyObject_GetOptionalAttrString`, `PyObject_HasAttrWithError`, `PyObject_HasAttrStringWithError`, `Py_CONSTANT_*` constants, `Py_GetConstant`, `Py_GetConstantBorrowed`, and `PyType_GetModuleByDef`.

newsfragments/4447.fixed.md

@@ -0,0 +1 @@
+Fix FFI definition `Py_Is` for PyPy on 3.10 to call the function defined by PyPy.

newsfragments/4447.removed.md

@@ -0,0 +1 @@
+Remove private FFI definitions `_Py_IMMORTAL_REFCNT`, `_Py_IsImmortal`, `_Py_TPFLAGS_STATIC_BUILTIN`, `_Py_Dealloc`, `_Py_IncRef`, `_Py_DecRef`.

pyo3-ffi/src/abstract_.rs

@@ -1,23 +1,17 @@
 use crate::object::*;
 use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
-use std::ptr;
-
-extern "C" {
-    #[cfg(PyPy)]
-    #[link_name = "PyPyObject_DelAttrString"]
-    pub fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int;
-}
 
 #[inline]
-#[cfg(not(PyPy))]
+#[cfg(all(not(Py_3_13), not(PyPy)))] // CPython exposed as a function in 3.13, in object.h
 pub unsafe fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int {
-    PyObject_SetAttrString(o, attr_name, ptr::null_mut())
+    PyObject_SetAttrString(o, attr_name, std::ptr::null_mut())
 }
 
 #[inline]
+#[cfg(all(not(Py_3_13), not(PyPy)))] // CPython exposed as a function in 3.13, in object.h
 pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_int {
-    PyObject_SetAttr(o, attr_name, ptr::null_mut())
+    PyObject_SetAttr(o, attr_name, std::ptr::null_mut())
 }
 
 extern "C" {

pyo3-ffi/src/boolobject.rs

@@ -4,12 +4,6 @@ use crate::object::*;
 use std::os::raw::{c_int, c_long};
 use std::ptr::addr_of_mut;
 
-#[cfg_attr(windows, link(name = "pythonXY"))]
-extern "C" {
-    #[cfg_attr(PyPy, link_name = "PyPyBool_Type")]
-    pub static mut PyBool_Type: PyTypeObject;
-}
-
 #[inline]
 pub unsafe fn PyBool_Check(op: *mut PyObject) -> c_int {
     (Py_TYPE(op) == addr_of_mut!(PyBool_Type)) as c_int

pyo3-ffi/src/cpython/object.rs

@@ -1,20 +1,23 @@
 #[cfg(Py_3_8)]
 use crate::vectorcallfunc;
-#[cfg(Py_3_11)]
-use crate::PyModuleDef;
 use crate::{object, PyGetSetDef, PyMemberDef, PyMethodDef, PyObject, Py_ssize_t};
 use std::mem;
-use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void};
+use std::os::raw::{c_char, c_int, c_uint, c_void};
+
+// skipped private _Py_NewReference
+// skipped private _Py_NewReferenceNoTotal
+// skipped private _Py_ResurrectReference
 
-// skipped _Py_NewReference
-// skipped _Py_ForgetReference
-// skipped _Py_GetRefTotal
+// skipped private _Py_GetGlobalRefTotal
+// skipped private _Py_GetRefTotal
+// skipped private _Py_GetLegacyRefTotal
+// skipped private _PyInterpreterState_GetRefTotal
 
-// skipped _Py_Identifier
+// skipped private _Py_Identifier
 
-// skipped _Py_static_string_init
-// skipped _Py_static_string
-// skipped _Py_IDENTIFIER
+// skipped private _Py_static_string_init
+// skipped private _Py_static_string
+// skipped private _Py_IDENTIFIER
 
 #[cfg(not(Py_3_11))] // moved to src/buffer.rs from Python
 mod bufferinfo {
@@ -240,7 +243,10 @@ pub struct PyTypeObject {
     pub tp_getattro: Option<object::getattrofunc>,
     pub tp_setattro: Option<object::setattrofunc>,
     pub tp_as_buffer: *mut PyBufferProcs,
-    pub tp_flags: c_ulong,
+    #[cfg(not(Py_GIL_DISABLED))]
+    pub tp_flags: std::os::raw::c_ulong,
+    #[cfg(Py_GIL_DISABLED)]
+    pub tp_flags: crate::impl_::AtomicCULong,
     pub tp_doc: *const c_char,
     pub tp_traverse: Option<object::traverseproc>,
     pub tp_clear: Option<object::inquiry>,
@@ -292,12 +298,12 @@ pub struct PyTypeObject {
 #[cfg(Py_3_11)]
 #[repr(C)]
 #[derive(Clone)]
-pub struct _specialization_cache {
-    pub getitem: *mut PyObject,
+struct _specialization_cache {
+    getitem: *mut PyObject,
     #[cfg(Py_3_12)]
-    pub getitem_version: u32,
+    getitem_version: u32,
     #[cfg(Py_3_13)]
-    pub init: *mut PyObject,
+    init: *mut PyObject,
 }
 
 #[repr(C)]
@@ -316,9 +322,9 @@ pub struct PyHeapTypeObject {
     #[cfg(Py_3_9)]
     pub ht_module: *mut object::PyObject,
     #[cfg(Py_3_11)]
-    pub _ht_tpname: *mut c_char,
+    _ht_tpname: *mut c_char,
     #[cfg(Py_3_11)]
-    pub _spec_cache: _specialization_cache,
+    _spec_cache: _specialization_cache,
 }
 
 impl Default for PyHeapTypeObject {
@@ -329,82 +335,75 @@ impl Default for PyHeapTypeObject {
 }
 
 #[inline]
+#[cfg(not(Py_3_11))]
 pub unsafe fn PyHeapType_GET_MEMBERS(etype: *mut PyHeapTypeObject) -> *mut PyMemberDef {
     let py_type = object::Py_TYPE(etype as *mut object::PyObject);
     let ptr = etype.offset((*py_type).tp_basicsize);
     ptr as *mut PyMemberDef
 }
 
-// skipped _PyType_Name
-// skipped _PyType_Lookup
-// skipped _PyType_LookupId
-// skipped _PyObject_LookupSpecial
-// skipped _PyType_CalculateMetaclass
-// skipped _PyType_GetDocFromInternalDoc
-// skipped _PyType_GetTextSignatureFromInternalDoc
+// skipped private _PyType_Name
+// skipped private _PyType_Lookup
+// skipped private _PyType_LookupRef
 
 extern "C" {
-    #[cfg(Py_3_11)]
-    #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleByDef")]
-    pub fn PyType_GetModuleByDef(ty: *mut PyTypeObject, def: *mut PyModuleDef) -> *mut PyObject;
-
     #[cfg(Py_3_12)]
     pub fn PyType_GetDict(o: *mut PyTypeObject) -> *mut PyObject;
 
     #[cfg_attr(PyPy, link_name = "PyPyObject_Print")]
     pub fn PyObject_Print(o: *mut PyObject, fp: *mut ::libc::FILE, flags: c_int) -> c_int;
 
-    // skipped _Py_BreakPoint
-    // skipped _PyObject_Dump
-    // skipped _PyObject_IsFreed
-    // skipped _PyObject_IsAbstract
+    // skipped private _Py_BreakPoint
+    // skipped private _PyObject_Dump
+
     // skipped _PyObject_GetAttrId
-    // skipped _PyObject_SetAttrId
-    // skipped _PyObject_LookupAttr
-    // skipped _PyObject_LookupAttrId
-    // skipped _PyObject_GetMethod
 
-    #[cfg(not(PyPy))]
-    pub fn _PyObject_GetDictPtr(obj: *mut PyObject) -> *mut *mut PyObject;
-    #[cfg(not(PyPy))]
-    pub fn _PyObject_NextNotImplemented(arg1: *mut PyObject) -> *mut PyObject;
+    // skipped private _PyObject_GetDictPtr
     pub fn PyObject_CallFinalizer(arg1: *mut PyObject);
     #[cfg_attr(PyPy, link_name = "PyPyObject_CallFinalizerFromDealloc")]
     pub fn PyObject_CallFinalizerFromDealloc(arg1: *mut PyObject) -> c_int;
 
-    // skipped _PyObject_GenericGetAttrWithDict
-    // skipped _PyObject_GenericSetAttrWithDict
-    // skipped _PyObject_FunctionStr
+    // skipped private _PyObject_GenericGetAttrWithDict
+    // skipped private _PyObject_GenericSetAttrWithDict
+    // skipped private _PyObject_FunctionStr
 }
 
 // skipped Py_SETREF
 // skipped Py_XSETREF
 
-#[cfg_attr(windows, link(name = "pythonXY"))]
-extern "C" {
-    pub static mut _PyNone_Type: PyTypeObject;
-    pub static mut _PyNotImplemented_Type: PyTypeObject;
-}
-
-// skipped _Py_SwappedOp
+// skipped private _PyObject_ASSERT_FROM
+// skipped private _PyObject_ASSERT_WITH_MSG
+// skipped private _PyObject_ASSERT
+// skipped private _PyObject_ASSERT_FAILED_MSG
+// skipped private _PyObject_AssertFailed
 
-// skipped _PyDebugAllocatorStats
-// skipped _PyObject_DebugTypeStats
-// skipped _PyObject_ASSERT_FROM
-// skipped _PyObject_ASSERT_WITH_MSG
-// skipped _PyObject_ASSERT
-// skipped _PyObject_ASSERT_FAILED_MSG
-// skipped _PyObject_AssertFailed
-// skipped _PyObject_CheckConsistency
+// skipped private _PyTrash_begin
+// skipped private _PyTrash_end
 
 // skipped _PyTrash_thread_deposit_object
 // skipped _PyTrash_thread_destroy_chain
-// skipped _PyTrash_begin
-// skipped _PyTrash_end
-// skipped _PyTrash_cond
-// skipped PyTrash_UNWIND_LEVEL
-// skipped Py_TRASHCAN_BEGIN_CONDITION
-// skipped Py_TRASHCAN_END
+
 // skipped Py_TRASHCAN_BEGIN
-// skipped Py_TRASHCAN_SAFE_BEGIN
-// skipped Py_TRASHCAN_SAFE_END
+// skipped Py_TRASHCAN_END
+
+// skipped PyObject_GetItemData
+
+// skipped PyObject_VisitManagedDict
+// skipped _PyObject_SetManagedDict
+// skipped PyObject_ClearManagedDict
+
+// skipped TYPE_MAX_WATCHERS
+
+// skipped PyType_WatchCallback
+// skipped PyType_AddWatcher
+// skipped PyType_ClearWatcher
+// skipped PyType_Watch
+// skipped PyType_Unwatch
+
+// skipped PyUnstable_Type_AssignVersionTag
+
+// skipped PyRefTracerEvent
+
+// skipped PyRefTracer
+// skipped PyRefTracer_SetTracer
+// skipped PyRefTracer_GetTracer

pyo3-ffi/src/impl_/mod.rs

@@ -0,0 +1,22 @@
+#[cfg(Py_GIL_DISABLED)]
+mod atomic_c_ulong {
+    pub struct GetAtomicCULong<const WIDTH: usize>();
+
+    pub trait AtomicCULongType {
+        type Type;
+    }
+    impl AtomicCULongType for GetAtomicCULong<32> {
+        type Type = std::sync::atomic::AtomicU32;
+    }
+    impl AtomicCULongType for GetAtomicCULong<64> {
+        type Type = std::sync::atomic::AtomicU64;
+    }
+
+    pub type TYPE =
+        <GetAtomicCULong<{ std::mem::size_of::<std::os::raw::c_ulong>() * 8 }> as AtomicCULongType>::Type;
+}
+
+/// Typedef for an atomic integer to match the platform-dependent c_ulong type.
+#[cfg(Py_GIL_DISABLED)]
+#[doc(hidden)]
+pub type AtomicCULong = atomic_c_ulong::TYPE;

pyo3-ffi/src/lib.rs

@@ -292,6 +292,7 @@ pub const fn _cstr_from_utf8_with_nul_checked(s: &str) -> &CStr {
 use std::ffi::CStr;
 
 pub mod compat;
+mod impl_;
 
 pub use self::abstract_::*;
 pub use self::bltinmodule::*;

pyo3-ffi/src/longobject.rs

@@ -6,12 +6,6 @@ use std::ptr::addr_of_mut;
 
 opaque_struct!(PyLongObject);
 
-#[cfg_attr(windows, link(name = "pythonXY"))]
-extern "C" {
-    #[cfg_attr(PyPy, link_name = "PyPyLong_Type")]
-    pub static mut PyLong_Type: PyTypeObject;
-}
-
 #[inline]
 pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int {
     PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)