Merge branch 'master' into sh_merge_master

.github/workflows/ci.yml

@@ -197,6 +197,35 @@ jobs:
         pytest tests/extra_setuptools
       if: "!(matrix.runs-on == 'windows-2022')"
 
+  manylinux:
+    name: Manylinux on 🐍 3.13t • GIL
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    container: quay.io/pypa/musllinux_1_2_x86_64:latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare venv
+        run: python3.13t -m venv .venv
+
+      - name: Install Python deps
+        run: .venv/bin/pip install -r tests/requirements.txt
+
+      - name: Configure C++11
+        run: >
+          cmake -S. -Bbuild
+          -DPYBIND11_WERROR=ON
+          -DDOWNLOAD_CATCH=ON
+          -DDOWNLOAD_EIGEN=ON
+          -DPython_ROOT_DIR=.venv
+
+      - name: Build C++11
+        run: cmake --build build -j2
+
+      - name: Python tests C++11
+        run: cmake --build build --target pytest -j2
 
   deadsnakes:
     strategy:

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
 
 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.5
+  rev: v0.4.7
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]

CMakeLists.txt

@@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
+option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
 
 if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
   add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
@@ -306,6 +307,7 @@ if(PYBIND11_INSTALL)
           tools/pybind11Common.cmake
           tools/pybind11Tools.cmake
           tools/pybind11NewTools.cmake
+          tools/pybind11GuessPythonExtSuffix.cmake
     DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
   if(NOT PYBIND11_EXPORT_NAME)

include/pybind11/detail/class.h

@@ -388,7 +388,11 @@ inline void clear_patients(PyObject *self) {
     auto *instance = reinterpret_cast<detail::instance *>(self);
     auto &internals = get_internals();
     auto pos = internals.patients.find(self);
-    assert(pos != internals.patients.end());
+
+    if (pos == internals.patients.end()) {
+        pybind11_fail("FATAL: Internal consistency check failed: Invalid clear_patients() call.");
+    }
+
     // Clearing the patients can cause more Python code to run, which
     // can invalidate the iterator. Extract the vector of patients
     // from the unordered_map first.

include/pybind11/pybind11.h

@@ -3040,7 +3040,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
             PyObject *locals = PyEval_GetFrameLocals();
 #        else
             PyObject *locals = PyEval_GetLocals();
-            Py_INCREF(locals);
+            Py_XINCREF(locals);
 #        endif
             if (locals != nullptr) {
 #        if PY_VERSION_HEX >= 0x030b0000

include/pybind11/pytypes.h

@@ -183,7 +183,15 @@ class object_api : public pyobject_tag {
     str_attr_accessor doc() const;
 
     /// Return the object's current reference count
-    int ref_count() const { return static_cast<int>(Py_REFCNT(derived().ptr())); }
+    ssize_t ref_count() const {
+#ifdef PYPY_VERSION
+        // PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT
+        // Following pybind11 2.12.1 and older behavior and removing this part
+        return static_cast<ssize_t>(static_cast<int>(Py_REFCNT(derived().ptr())));
+#else
+        return Py_REFCNT(derived().ptr());
+#endif
+    }
 
     // TODO PYBIND11_DEPRECATED(
     //     "Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()")
@@ -2175,6 +2183,11 @@ class list : public object {
             throw error_already_set();
         }
     }
+    void clear() /* py-non-const */ {
+        if (PyList_SetSlice(m_ptr, 0, PyList_Size(m_ptr), nullptr) == -1) {
+            throw error_already_set();
+        }
+    }
 };
 
 class args : public tuple {

tests/CMakeLists.txt

@@ -358,7 +358,7 @@ if(Boost_FOUND)
     add_library(Boost::headers IMPORTED INTERFACE)
     if(TARGET Boost::boost)
       # Classic FindBoost
-      set_property(TARGET Boost::boost PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost)
+      set_property(TARGET Boost::headers PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost)
     else()
       # Very old FindBoost, or newer Boost than CMake in older CMakes
       set_property(TARGET Boost::headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES

tests/extra_python_package/test_files.py

@@ -80,6 +80,7 @@
     "share/cmake/pybind11/pybind11Common.cmake",
     "share/cmake/pybind11/pybind11Config.cmake",
     "share/cmake/pybind11/pybind11ConfigVersion.cmake",
+    "share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
     "share/cmake/pybind11/pybind11NewTools.cmake",
     "share/cmake/pybind11/pybind11Targets.cmake",
     "share/cmake/pybind11/pybind11Tools.cmake",

tests/pybind11_tests.cpp

@@ -89,6 +89,8 @@ PYBIND11_MODULE(pybind11_tests, m) {
 #endif
     m.attr("cpp_std") = cpp_std();
     m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID;
+    // Free threaded Python uses UINT32_MAX for immortal objects.
+    m.attr("PYBIND11_REFCNT_IMMORTAL") = UINT32_MAX;
     m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") =
 #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
         true;

tests/test_class.py

@@ -3,7 +3,7 @@
 import pytest
 
 import env
-from pybind11_tests import ConstructorStats, UserType
+from pybind11_tests import PYBIND11_REFCNT_IMMORTAL, ConstructorStats, UserType
 from pybind11_tests import class_ as m
 
 
@@ -377,7 +377,9 @@ class PyDog(m.Dog):
         refcount_3 = getrefcount(cls)
 
         assert refcount_1 == refcount_3
-        assert refcount_2 > refcount_1
+        assert (refcount_2 > refcount_1) or (
+            refcount_2 == refcount_1 == PYBIND11_REFCNT_IMMORTAL
+        )
 
 
 def test_reentrant_implicit_conversion_failure(msg):

tests/test_embed/CMakeLists.txt

@@ -7,6 +7,13 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STR
   return()
 endif()
 
+if(TARGET Python::Module AND NOT TARGET Python::Python)
+  message(STATUS "Skipping embed test since no embed libs found")
+  add_custom_target(cpptest) # Dummy target since embedding is not supported.
+  set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}")
+  return()
+endif()
+
 find_package(Catch 2.13.9)
 
 if(CATCH_FOUND)

tests/test_kwargs_and_defaults.cpp

@@ -150,10 +150,13 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
 
 // test_args_refcount
 // PyPy needs a garbage collection to get the reference count values to match CPython's behaviour
+// PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT, so truncate
 #ifdef PYPY_VERSION
 #    define GC_IF_NEEDED ConstructorStats::gc()
+#    define REFCNT(x) (int) Py_REFCNT(x)
 #else
 #    define GC_IF_NEEDED
+#    define REFCNT(x) Py_REFCNT(x)
 #endif
     m.def("arg_refcount_h", [](py::handle h) {
         GC_IF_NEEDED;
@@ -172,7 +175,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
         py::tuple t(a.size());
         for (size_t i = 0; i < a.size(); i++) {
             // Use raw Python API here to avoid an extra, intermediate incref on the tuple item:
-            t[i] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
+            t[i] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
         }
         return t;
     });
@@ -182,7 +185,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
         t[0] = o.ref_count();
         for (size_t i = 0; i < a.size(); i++) {
             // Use raw Python API here to avoid an extra, intermediate incref on the tuple item:
-            t[i + 1] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
+            t[i + 1] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
         }
         return t;
     });

tests/test_kwargs_and_defaults.py

@@ -381,7 +381,7 @@ def test_args_refcount():
     arguments"""
     refcount = m.arg_refcount_h
 
-    myval = 54321
+    myval = object()
     expected = refcount(myval)
     assert m.arg_refcount_h(myval) == expected
     assert m.arg_refcount_o(myval) == expected + 1
@@ -420,6 +420,7 @@ def test_args_refcount():
     # for the `py::args`; in the previous case, we could simply inc_ref and pass on Python's input
     # tuple without having to inc_ref the individual elements, but here we can't, hence the extra
     # refs.
-    assert m.mixed_args_refcount(myval, myval, myval) == (exp3 + 3, exp3 + 3, exp3 + 3)
+    exp3_3 = exp3 + 3
+    assert m.mixed_args_refcount(myval, myval, myval) == (exp3_3, exp3_3, exp3_3)
 
     assert m.class_default_argument() == "<class 'decimal.Decimal'>"

tests/test_pytypes.cpp

@@ -135,6 +135,7 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("list_size_t", []() { return py::list{(py::size_t) 0}; });
     m.def("list_insert_ssize_t", [](py::list *l) { return l->insert((py::ssize_t) 1, 83); });
     m.def("list_insert_size_t", [](py::list *l) { return l->insert((py::size_t) 3, 57); });
+    m.def("list_clear", [](py::list *l) { l->clear(); });
     m.def("get_list", []() {
         py::list list;
         list.append("value");

tests/test_pytypes.py

@@ -65,6 +65,8 @@ def test_list(capture, doc):
     assert lins == [1, 83, 2]
     m.list_insert_size_t(lins)
     assert lins == [1, 83, 2, 57]
+    m.list_clear(lins)
+    assert lins == []
 
     with capture:
         lst = m.get_list()
@@ -631,7 +633,8 @@ def test_memoryview(method, args, fmt, expected_view):
     ],
 )
 def test_memoryview_refcount(method):
-    buf = b"\x0a\x0b\x0c\x0d"
+    # Avoiding a literal to avoid an immortal object in free-threaded builds
+    buf = "\x0a\x0b\x0c\x0d".encode("ascii")
     ref_before = sys.getrefcount(buf)
     view = method(buf)
     ref_after = sys.getrefcount(buf)

tools/FindPythonLibsNew.cmake

@@ -205,7 +205,7 @@ endif()
 # Make sure the Python has the same pointer-size as the chosen compiler
 # Skip if CMAKE_SIZEOF_VOID_P is not defined
 # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)
-if(NOT CMAKE_CROSSCOMPILING
+if(NOT _PYBIND11_CROSSCOMPILING
    AND CMAKE_SIZEOF_VOID_P
    AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
   if(PythonLibsNew_FIND_REQUIRED)

tools/pybind11Common.cmake

@@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS
     "${pybind11_INCLUDE_DIR}"
     CACHE INTERNAL "Include directory for pybind11 (Python not requested)")
 
+if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING)
+  set(_PYBIND11_CROSSCOMPILING
+      ON
+      CACHE INTERNAL "")
+else()
+  set(_PYBIND11_CROSSCOMPILING
+      OFF
+      CACHE INTERNAL "")
+endif()
+
 # --------------------- Shared targets ----------------------------
 
 # Build an interface library target:
@@ -195,7 +205,7 @@ endif()
 
 # --------------------- pybind11_find_import -------------------------------
 
-if(NOT _pybind11_nopython)
+if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING)
   # Check to see if modules are importable. Use REQUIRED to force an error if
   # one of the modules is not found. <package_name>_FOUND will be set if the
   # package was found (underscores replace dashes if present). QUIET will hide

tools/pybind11GuessPythonExtSuffix.cmake

@@ -0,0 +1,86 @@
+cmake_minimum_required(VERSION 3.5)
+
+function(pybind11_guess_python_module_extension python)
+
+  # The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX})
+    message(
+      STATUS
+        "Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}"
+    )
+    set(PYTHON_MODULE_EXT_SUFFIX
+        "$ENV{SETUPTOOLS_EXT_SUFFIX}"
+        CACHE
+          STRING
+          "Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)")
+  endif()
+  # If that didn't work, use the Python_SOABI variable:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI)
+    message(
+      STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}")
+    # The final extension depends on the system
+    set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+      set(_PY_BUILD_EXTENSION ".pyd")
+    endif()
+    # If the SOABI already has an extension, use it as the full suffix
+    # (used for debug versions of Python on Windows)
+    if(${python}_SOABI MATCHES "\\.")
+      set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}")
+      # If the SOABI is empty, this is usually a bug, but we generate a
+      # correct extension anyway, which is the best we can do
+    elseif("${${python}_SOABI}" STREQUAL "")
+      message(
+        WARNING
+          "${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly."
+      )
+      set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}")
+      # Otherwise, add the system-dependent extension to it
+    else()
+      set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}")
+    endif()
+  endif()
+
+  # If we could not deduce the extension suffix, unset the results:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX)
+    unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE)
+    unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE)
+    unset(PYTHON_IS_DEBUG PARENT_SCOPE)
+    return()
+  endif()
+
+  # Sanity checks:
+  if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI
+                              OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\."))
+    message(
+      WARNING
+        "Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})."
+    )
+  endif()
+
+  # Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699)
+  get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+  get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT)
+
+  # Try to deduce the debug ABI from the extension suffix:
+  if(NOT DEFINED _PYTHON_IS_DEBUG)
+    if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-"
+       OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d")
+      set(_PYTHON_IS_DEBUG On)
+    else()
+      set(_PYTHON_IS_DEBUG Off)
+    endif()
+  endif()
+
+  # Return results
+  set(PYTHON_MODULE_DEBUG_POSTFIX
+      "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+      PARENT_SCOPE)
+  set(PYTHON_MODULE_EXTENSION
+      "${_PYTHON_MODULE_EXTENSION}"
+      PARENT_SCOPE)
+  set(PYTHON_IS_DEBUG
+      "${_PYTHON_IS_DEBUG}"
+      PARENT_SCOPE)
+
+endfunction()