Merge branch 'master' into sh_merge_master

.github/CONTRIBUTING.md

@@ -135,7 +135,7 @@ The valid options are:
 * Use `-G` and the name of a generator to use something different. `cmake
   --help` lists the generators available.
       - On Unix, setting `CMAKE_GENERATER=Ninja` in your environment will give
-        you automatic mulithreading on all your CMake projects!
+        you automatic multithreading on all your CMake projects!
 * Open the `CMakeLists.txt` with QtCreator to generate for that IDE.
 * You can use `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to generate the `.json` file
   that some tools expect.

.pre-commit-config.yaml

@@ -32,13 +32,13 @@ repos:
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: "23.7.0" # Keep in sync with blacken-docs
+  rev: "23.9.1" # Keep in sync with blacken-docs
   hooks:
   - id: black
 
 # Ruff, the Python auto-correcting linter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.287
+  rev: v0.0.292
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -126,15 +126,15 @@ repos:
 # Use tools/codespell_ignore_lines_from_errors.py
 # to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.2.5"
+  rev: "v2.2.6"
   hooks:
   - id: codespell
     exclude: ".supp$"
     args: ["-x.codespell-ignore-lines", "-Lccompiler"]
 
 # Check for common shell mistakes
 - repo: https://github.com/shellcheck-py/shellcheck-py
-  rev: "v0.9.0.5"
+  rev: "v0.9.0.6"
   hooks:
   - id: shellcheck
 
@@ -144,12 +144,12 @@ repos:
   - id: disallow-caps
     name: Disallow improper capitalization
     language: pygrep
-    entry: PyBind|Numpy|Cmake|CCache|PyTest
+    entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest
     exclude: ^\.pre-commit-config.yaml$
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v3.0.0a7"
+  rev: "v3.0.0"
   hooks:
   - id: pylint
     files: ^pybind11

CMakeLists.txt

@@ -5,6 +5,11 @@
 # All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+# Propagate this policy (FindPythonInterp removal) so it can be detected later
+if(NOT CMAKE_VERSION VERSION_LESS "3.27")
+  cmake_policy(GET CMP0148 _pybind11_cmp0148)
+endif()
+
 cmake_minimum_required(VERSION 3.5)
 
 # The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with
@@ -16,6 +21,11 @@ else()
   cmake_policy(VERSION 3.26)
 endif()
 
+if(_pybind11_cmp0148)
+  cmake_policy(SET CMP0148 ${_pybind11_cmp0148})
+  unset(_pybind11_cmp0148)
+endif()
+
 # Avoid infinite recursion if tests include this as a subdirectory
 if(DEFINED PYBIND11_MASTER_PROJECT)
   return()

include/pybind11/cast.h

@@ -695,7 +695,7 @@ class tuple_caster {
     }
 
     static constexpr auto name
-        = const_name("Tuple[") + concat(make_caster<Ts>::name...) + const_name("]");
+        = const_name("tuple[") + concat(make_caster<Ts>::name...) + const_name("]");
 
     template <typename T>
     using cast_op_type = type;
@@ -920,6 +920,10 @@ struct handle_type_name<bytes> {
     static constexpr auto name = const_name(PYBIND11_BYTES_NAME);
 };
 template <>
+struct handle_type_name<buffer> {
+    static constexpr auto name = const_name("Buffer");
+};
+template <>
 struct handle_type_name<int_> {
     static constexpr auto name = const_name("int");
 };
@@ -940,10 +944,18 @@ struct handle_type_name<function> {
     static constexpr auto name = const_name("Callable");
 };
 template <>
+struct handle_type_name<handle> {
+    static constexpr auto name = handle_type_name<object>::name;
+};
+template <>
 struct handle_type_name<none> {
     static constexpr auto name = const_name("None");
 };
 template <>
+struct handle_type_name<sequence> {
+    static constexpr auto name = const_name("Sequence");
+};
+template <>
 struct handle_type_name<args> {
     static constexpr auto name = const_name("*args");
 };

include/pybind11/detail/init.h

@@ -66,7 +66,7 @@ constexpr bool is_alias(void *) {
 }
 
 // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall
-// back to brace aggregate initiailization so that for aggregate initialization can be used with
+// back to brace aggregate initialization so that for aggregate initialization can be used with
 // py::init, e.g.  `py::init<int, int>` to initialize a `struct T { int a; int b; }`.  For
 // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually
 // works, but will not do the expected thing when `T` has an `initializer_list<T>` constructor).

include/pybind11/detail/internals.h

@@ -462,6 +462,7 @@ inline object get_python_state_dict() {
 #endif
     if (!state_dict) {
         raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED");
+        throw error_already_set();
     }
     return state_dict;
 }
@@ -474,6 +475,7 @@ inline internals **get_internals_pp_from_capsule(handle obj) {
     void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr);
     if (raw_ptr == nullptr) {
         raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED");
+        throw error_already_set();
     }
     return static_cast<internals **>(raw_ptr);
 }

include/pybind11/numpy.h

@@ -120,6 +120,20 @@ inline numpy_internals &get_numpy_internals() {
     return *ptr;
 }
 
+PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) {
+    module_ numpy = module_::import("numpy");
+    str version_string = numpy.attr("__version__");
+
+    module_ numpy_lib = module_::import("numpy.lib");
+    object numpy_version = numpy_lib.attr("NumpyVersion")(version_string);
+    int major_version = numpy_version.attr("major").cast<int>();
+
+    /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially
+        became a private module. */
+    std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core";
+    return module_::import((numpy_core_path + "." + submodule_name).c_str());
+}
+
 template <typename T>
 struct same_size {
     template <typename U>
@@ -263,9 +277,13 @@ struct npy_api {
     };
 
     static npy_api lookup() {
-        module_ m = module_::import("numpy.core.multiarray");
+        module_ m = detail::import_numpy_core_submodule("multiarray");
         auto c = m.attr("_ARRAY_API");
         void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr);
+        if (api_ptr == nullptr) {
+            raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer.");
+            throw error_already_set();
+        }
         npy_api api;
 #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func];
         DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion);
@@ -626,11 +644,8 @@ class dtype : public object {
 
 private:
     static object _dtype_from_pep3118() {
-        static PyObject *obj = module_::import("numpy.core._internal")
-                                   .attr("_dtype_from_pep3118")
-                                   .cast<object>()
-                                   .release()
-                                   .ptr();
+        module_ m = detail::import_numpy_core_submodule("_internal");
+        static PyObject *obj = m.attr("_dtype_from_pep3118").cast<object>().release().ptr();
         return reinterpret_borrow<object>(obj);
     }
 

include/pybind11/pybind11.h

@@ -58,11 +58,13 @@ inline std::string replace_newlines_and_squash(const char *text) {
     std::string result(text);
     bool previous_is_whitespace = false;
 
-    // Do not modify string representations
-    char first_char = result[0];
-    char last_char = result[result.size() - 1];
-    if (first_char == last_char && first_char == '\'') {
-        return result;
+    if (result.size() >= 2) {
+        // Do not modify string representations
+        char first_char = result[0];
+        char last_char = result[result.size() - 1];
+        if (first_char == last_char && first_char == '\'') {
+            return result;
+        }
     }
     result.clear();
 
@@ -1141,7 +1143,7 @@ class cpp_function : public function {
                     }
                     msg += "kwargs: ";
                     bool first = true;
-                    for (auto kwarg : kwargs) {
+                    for (const auto &kwarg : kwargs) {
                         if (first) {
                             first = false;
                         } else {

include/pybind11/stl.h

@@ -100,7 +100,7 @@ struct set_caster {
         return s.release();
     }
 
-    PYBIND11_TYPE_CASTER(type, const_name("Set[") + key_conv::name + const_name("]"));
+    PYBIND11_TYPE_CASTER(type, const_name("set[") + key_conv::name + const_name("]"));
 };
 
 template <typename Type, typename Key, typename Value>
@@ -157,7 +157,7 @@ struct map_caster {
     }
 
     PYBIND11_TYPE_CASTER(Type,
-                         const_name("Dict[") + key_conv::name + const_name(", ") + value_conv::name
+                         const_name("dict[") + key_conv::name + const_name(", ") + value_conv::name
                              + const_name("]"));
 };
 
@@ -208,7 +208,7 @@ struct list_caster {
         return l.release();
     }
 
-    PYBIND11_TYPE_CASTER(Type, const_name("List[") + value_conv::name + const_name("]"));
+    PYBIND11_TYPE_CASTER(Type, const_name("list[") + value_conv::name + const_name("]"));
 };
 
 template <typename Type, typename Alloc>
@@ -274,7 +274,7 @@ struct array_caster {
 
     PYBIND11_TYPE_CASTER(ArrayType,
                          const_name<Resizable>(const_name(""), const_name("Annotated["))
-                             + const_name("List[") + value_conv::name + const_name("]")
+                             + const_name("list[") + value_conv::name + const_name("]")
                              + const_name<Resizable>(const_name(""),
                                                      const_name(", FixedSize(")
                                                          + const_name<Size>() + const_name(")]")));

include/pybind11/typing.h

@@ -45,6 +45,16 @@ class Set : public set {
     using set::set;
 };
 
+template <typename T>
+class Iterable : public iterable {
+    using iterable::iterable;
+};
+
+template <typename T>
+class Iterator : public iterator {
+    using iterator::iterator;
+};
+
 template <typename Signature>
 class Callable;
 
@@ -85,6 +95,16 @@ struct handle_type_name<typing::Set<T>> {
     static constexpr auto name = const_name("set[") + make_caster<T>::name + const_name("]");
 };
 
+template <typename T>
+struct handle_type_name<typing::Iterable<T>> {
+    static constexpr auto name = const_name("Iterable[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::Iterator<T>> {
+    static constexpr auto name = const_name("Iterator[") + make_caster<T>::name + const_name("]");
+};
+
 template <typename Return, typename... Args>
 struct handle_type_name<typing::Callable<Return(Args...)>> {
     using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;

tests/test_buffers.py

@@ -219,3 +219,10 @@ def test_ctypes_from_buffer():
         assert cinfo.shape == pyinfo.shape
         assert cinfo.strides == pyinfo.strides
         assert not cinfo.readonly
+
+
+def test_buffer_docstring():
+    assert (
+        m.get_buffer_info.__doc__.strip()
+        == "get_buffer_info(arg0: Buffer) -> pybind11_tests.buffers.buffer_info"
+    )

tests/test_builtin_casters.py

@@ -352,15 +352,15 @@ def test_tuple(doc):
     assert (
         doc(m.pair_passthrough)
         == """
-        pair_passthrough(arg0: Tuple[bool, str]) -> Tuple[str, bool]
+        pair_passthrough(arg0: tuple[bool, str]) -> tuple[str, bool]
 
         Return a pair in reversed order
     """
     )
     assert (
         doc(m.tuple_passthrough)
         == """
-        tuple_passthrough(arg0: Tuple[bool, str, int]) -> Tuple[int, str, bool]
+        tuple_passthrough(arg0: tuple[bool, str, int]) -> tuple[int, str, bool]
 
         Return a triple in reversed order
     """

tests/test_factory_constructors.py

@@ -77,7 +77,7 @@ def test_init_factory_signature(msg):
             1. m.factory_constructors.TestFactory1(arg0: m.factory_constructors.tag.unique_ptr_tag, arg1: int)
             2. m.factory_constructors.TestFactory1(arg0: str)
             3. m.factory_constructors.TestFactory1(arg0: m.factory_constructors.tag.pointer_tag)
-            4. m.factory_constructors.TestFactory1(arg0: handle, arg1: int, arg2: handle)
+            4. m.factory_constructors.TestFactory1(arg0: object, arg1: int, arg2: object)
 
         Invoked with: 'invalid', 'constructor', 'arguments'
     """
@@ -95,7 +95,7 @@ def test_init_factory_signature(msg):
 
         3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None
 
-        4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None
+        4. __init__(self: m.factory_constructors.TestFactory1, arg0: object, arg1: int, arg2: object) -> None
     """
     )
 

tests/test_kwargs_and_defaults.cpp

@@ -85,6 +85,8 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
         "kw_lb_func7",
         [](const std::string &) {},
         py::arg("str_arg") = "First line.\n  Second line.");
+    m.def(
+        "kw_lb_func8", [](const CustomRepr &) {}, py::arg("custom") = CustomRepr(""));
 
     // test_args_and_kwargs
     m.def("args_function", [](py::args args) -> py::tuple {

tests/test_kwargs_and_defaults.py

@@ -8,7 +8,7 @@ def test_function_signatures(doc):
     assert doc(m.kw_func1) == "kw_func1(x: int, y: int) -> str"
     assert doc(m.kw_func2) == "kw_func2(x: int = 100, y: int = 200) -> str"
     assert doc(m.kw_func3) == "kw_func3(data: str = 'Hello world!') -> None"
-    assert doc(m.kw_func4) == "kw_func4(myList: List[int] = [13, 17]) -> str"
+    assert doc(m.kw_func4) == "kw_func4(myList: list[int] = [13, 17]) -> str"
     assert doc(m.kw_func_udl) == "kw_func_udl(x: int, y: int = 300) -> str"
     assert doc(m.kw_func_udl_z) == "kw_func_udl_z(x: int, y: int = 0) -> str"
     assert doc(m.args_function) == "args_function(*args) -> tuple"
@@ -55,6 +55,10 @@ def test_function_signatures(doc):
         doc(m.kw_lb_func7)
         == "kw_lb_func7(str_arg: str = 'First line.\\n  Second line.') -> None"
     )
+    assert (
+        doc(m.kw_lb_func8)
+        == "kw_lb_func8(custom: m.kwargs_and_defaults.CustomRepr = ) -> None"
+    )
 
 
 def test_named_arguments():

tests/test_numpy_array.py

@@ -298,7 +298,7 @@ def test_constructors():
     results = m.converting_constructors([1, 2, 3])
     for a in results.values():
         np.testing.assert_array_equal(a, [1, 2, 3])
-    assert results["array"].dtype == np.int_
+    assert results["array"].dtype == np.dtype(int)
     assert results["array_t<int32>"].dtype == np.int32
     assert results["array_t<double>"].dtype == np.float64
 

tests/test_pytypes.cpp

@@ -828,6 +828,8 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("annotate_dict_str_int", [](const py::typing::Dict<py::str, int> &) {});
     m.def("annotate_list_int", [](const py::typing::List<int> &) {});
     m.def("annotate_set_str", [](const py::typing::Set<std::string> &) {});
+    m.def("annotate_iterable_str", [](const py::typing::Iterable<std::string> &) {});
+    m.def("annotate_iterator_int", [](const py::typing::Iterator<int> &) {});
     m.def("annotate_fn",
           [](const py::typing::Callable<int(py::typing::List<py::str>, py::str)> &) {});
 }