From 17c68091658f036eb6aed05cebabfc9265407a5b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 20 Oct 2022 05:49:52 -0700 Subject: [PATCH 001/137] ci: update PGI build (old one no longer signed) (#4260) * Simply replace "22.3" with "22.9" to see what happens. * Remove PYBIND11_TEST_FILTER to see what happens. * Revert "Remove PYBIND11_TEST_FILTER to see what happens." This reverts commit 0cba2cef0c5c5bcbf57cad219c7d5dfb0cda241c. * Remove only test_smart_ptr.cpp to see what happens. * Revert "Remove only test_smart_ptr.cpp to see what happens." This reverts commit 8e9df22c85292003abcab50260393e547567fc18. * Remove only test_virtual_functions.cpp to see what happens. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc7175d829..2951a0ad19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,7 +391,7 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: runs-on: ubuntu-latest - name: "🐍 3 β€’ CentOS7 / PGI 22.3 β€’ x64" + name: "🐍 3 β€’ CentOS7 / PGI 22.9 β€’ x64" container: centos:7 steps: @@ -401,7 +401,7 @@ jobs: run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils - name: Install NVidia HPC SDK - run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.3 + run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 # On CentOS 7, we have to filter a few tests (compiler internal error) # and allow deeper template recursion (not needed on CentOS 8 with a newer @@ -411,12 +411,12 @@ jobs: shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.3 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ -DCMAKE_CXX_STANDARD=11 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ - -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" # Building before installing Pip should produce a warning but not an error - name: Build From c3854682e6644c60ff4f99f81595636f582b04f5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 09:39:19 -0400 Subject: [PATCH 002/137] ci(fix): don't label weekly dep updates & ci fixes (#4264) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- .github/workflows/labeler.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index d2b5979681..5152cebca2 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -10,7 +10,10 @@ jobs: steps: - uses: actions/labeler@main - if: github.event.pull_request.merged == true + if: > + github.event.pull_request.merged == true && + !startsWith(github.event.pull_request.title, 'chore(deps):') && + !startsWith(github.event.plul_request.title, 'ci(fix):' with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/labeler_merged.yml From 1d4a65e2f145a25b66e44e9d5edae9ec11abbc73 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:35:18 -0400 Subject: [PATCH 003/137] feat: add entrypoint for cmake modules dir (#4258) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- setup.py | 2 ++ tests/extra_python_package/test_files.py | 2 ++ tools/setup_main.py.in | 3 +++ 3 files changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 68573519c1..9097440bfe 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,8 @@ def remove_output(*sources: str) -> Iterator[None]: stdout=sys.stdout, stderr=sys.stderr, ) + if not global_sdist: + Path("pybind11/share/cmake/pybind11/__init__.py").touch() txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 9a9bb1556a..ae69e56d3a 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -166,6 +166,7 @@ def test_build_sdist(monkeypatch, tmpdir): files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") + files.add("pybind11/share/cmake/pybind11/__init__.py") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: @@ -252,6 +253,7 @@ def tests_build_wheel(monkeypatch, tmpdir): "dist-info/entry_points.txt", "dist-info/top_level.txt", } + files.add("pybind11/share/cmake/pybind11/__init__.py") with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 6358cc7b9b..b3681bba8c 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -38,6 +38,9 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", + ], + "cmake.modules": [ + "pybind11 = pybind11.share.cmake.pybind11", ] }, cmdclass=cmdclass From 128d988ef183cfb6334714f0bcf8b8e7b7e2e95b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:37:12 -0400 Subject: [PATCH 004/137] ci: fix labeler --- .github/workflows/labeler.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 5152cebca2..165a2fd87b 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -13,7 +13,8 @@ jobs: if: > github.event.pull_request.merged == true && !startsWith(github.event.pull_request.title, 'chore(deps):') && - !startsWith(github.event.plul_request.title, 'ci(fix):' + !startsWith(github.event.pull_request.title, 'ci(fix):') && + !startsWith(github.event.pull_request.title, 'docs(changelog):') with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/labeler_merged.yml From 36ccb08b0d8f007f674ce373095b4b278689fc10 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:58:04 -0400 Subject: [PATCH 005/137] docs: update changelog (#4265) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9c8ff423f4..1199ecdedf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,107 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Version 2.10.1 (Oct 2?, 2022) +----------------------------- + + +Changes: + + +* Allow ``pybind11::capsule`` constructor to take null destructor pointers. + `#4221 `_ + +* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11 + (established behavior). + `#4119 `_ + +Bug fixes: + +* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. + `#4188 `_ + +* Make augmented assignment operators non-const for the object-api. Behavior + was previously broken for augmented assignment operators. + `#4065 `_ + +* Add proper error checking to C++ bindings for Python list append and insert. + `#4208 `_ + +* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0. + `#4220 `_ + +* A workaround for PyPy was added in the ``py::error_already_set`` + implementation, related to PR `#1895 `_ + released with v2.10.0. + `#4079 `_ + +* Fixed compiler errors when C++23 ``std::forward_like`` is available. + `#4136 `_ + +* Properly raise exceptions in contains methods (like when an object in unhashable). + `#4209 `_ + +* Further improve another error in exception handling. + `#4232 `_ + +* ``get_local_internals()`` was made compatible with + ``finalize_interpreter()``, fixing potential freezes during interpreter + finalization. + `#4192 `_ + + +Performance and style: + +* Reserve space in set and STL map casters if possible. This will prevent + unnecessary rehashing / resizing by knowing the number of keys ahead of time + for Python to C++ casting. This improvement will greatly speed up the casting + of large unordered maps and sets. + `#4194 `_ + +* GIL RAII scopes are non-copyable to avoid potential bugs. + `#4183 `_ + +* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT`` + macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in + macros as well. + `#4017 `_ + +* Optimize iterator advancement in C++ bindings. + `#4237 `_ + +* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict`` + for handling dynamic attribute dictionaries. + `#4106 `_ + +* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when + opening namespaces. Using namespace declarations and namespace qualification + remain the same as ``pybind11``. This is done to ensure consistent symbol + visibility. + `#4098 `_ + +* Mark ``detail::forward_like`` as constexpr. + `#4147 `_ + +* Optimize unpacking_collector when processing ``arg_v`` arguments. + `#4219 `_ + + +Build system improvements: + +* Experimental support for ``cmake.modules`` entrypoint. + `#4258 `_ + +* Include a pkg-config file when installing pybind11, such as in the Python + package. + `#4077 `_ + +* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG`` + instead of ``Debug``. + `#4078 `_ + +* Followup to `#3948 `_, fixing vcpkg again. + `#4123 `_ + Version 2.10.0 (Jul 15, 2022) ----------------------------- From 2ce76f783373b1dd34e94c8283598af3051ef3d7 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 21 Oct 2022 12:51:26 -0400 Subject: [PATCH 006/137] Cleanup casters to release none() to avoid ref counting (#4269) --- include/pybind11/cast.h | 6 +++--- include/pybind11/functional.h | 2 +- include/pybind11/stl.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index b89e4946ad..5699d4fb00 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -248,7 +248,7 @@ struct void_caster { return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } PYBIND11_TYPE_CASTER(T, const_name("None")); }; @@ -291,7 +291,7 @@ class type_caster : public type_caster { if (ptr) { return capsule(ptr).release(); } - return none().inc_ref(); + return none().release(); } template @@ -537,7 +537,7 @@ struct type_caster::value>> { static handle cast(const CharT *src, return_value_policy policy, handle parent) { if (src == nullptr) { - return pybind11::none().inc_ref(); + return pybind11::none().release(); } return StringCaster::cast(StringType(src), policy, parent); } diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 4034990d89..102d1a938b 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -110,7 +110,7 @@ struct type_caster> { template static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) { if (!f_) { - return none().inc_ref(); + return none().release(); } auto result = f_.template target(); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 8f243502ef..2d144b598b 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -311,7 +311,7 @@ struct optional_caster { template static handle cast(T &&src, return_value_policy policy, handle parent) { if (!src) { - return none().inc_ref(); + return none().release(); } if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); From 91cfb7702280d3176e56a6667d8a091d9731a779 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 21 Oct 2022 17:25:53 -0400 Subject: [PATCH 007/137] Revert "feat: add entrypoint for cmake modules dir" (#4270) * Revert "feat: add entrypoint for cmake modules dir (#4258)" This reverts commit 1d4a65e2f145a25b66e44e9d5edae9ec11abbc73. * docs: revert changelog mention too --- docs/changelog.rst | 3 --- setup.py | 2 -- tests/extra_python_package/test_files.py | 2 -- tools/setup_main.py.in | 3 --- 4 files changed, 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1199ecdedf..df3181c524 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -103,9 +103,6 @@ Performance and style: Build system improvements: -* Experimental support for ``cmake.modules`` entrypoint. - `#4258 `_ - * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ diff --git a/setup.py b/setup.py index 9097440bfe..68573519c1 100644 --- a/setup.py +++ b/setup.py @@ -144,8 +144,6 @@ def remove_output(*sources: str) -> Iterator[None]: stdout=sys.stdout, stderr=sys.stderr, ) - if not global_sdist: - Path("pybind11/share/cmake/pybind11/__init__.py").touch() txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index ae69e56d3a..9a9bb1556a 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -166,7 +166,6 @@ def test_build_sdist(monkeypatch, tmpdir): files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") - files.add("pybind11/share/cmake/pybind11/__init__.py") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: @@ -253,7 +252,6 @@ def tests_build_wheel(monkeypatch, tmpdir): "dist-info/entry_points.txt", "dist-info/top_level.txt", } - files.add("pybind11/share/cmake/pybind11/__init__.py") with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index b3681bba8c..6358cc7b9b 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -38,9 +38,6 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", - ], - "cmake.modules": [ - "pybind11 = pybind11.share.cmake.pybind11", ] }, cmdclass=cmdclass From 17c1e27b3d202aaab6d5443b1800124bbf24d9ca Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 21 Oct 2022 18:04:01 -0400 Subject: [PATCH 008/137] fix: Revert pfect args make iterator (#4234) * Revert "chore: perfectly forward all make_iterator args (#3980)" This reverts commit 8da58da53996e9aa40d051b3b84cb3220fdbbb58. * Redo unrelated optimization in commit * Add tests for ambiguous overloads --- include/pybind11/pybind11.h | 22 ++++++++-------------- tests/test_sequences_and_iterators.cpp | 19 +++++++++++++++++++ tests/test_sequences_and_iterators.py | 8 ++++++++ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index fb4b7578d9..e662236d01 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2333,7 +2333,7 @@ template -iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values @@ -2359,7 +2359,7 @@ iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) Policy); } - return cast(state{std::forward(first), std::forward(last), true}); + return cast(state{first, last, true}); } PYBIND11_NAMESPACE_END(detail) @@ -2370,15 +2370,13 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2388,15 +2386,13 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, KeyType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2406,15 +2402,13 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index b867f49a25..1de65edbf2 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -559,4 +559,23 @@ TEST_SUBMODULE(sequences_and_iterators, m) { []() { return py::make_iterator(list); }); m.def("make_iterator_2", []() { return py::make_iterator(list); }); + + // test_iterator on c arrays + // #4100: ensure lvalue required as increment operand + class CArrayHolder { + public: + CArrayHolder(double x, double y, double z) { + values[0] = x; + values[1] = y; + values[2] = z; + }; + double values[3]; + }; + + py::class_(m, "CArrayHolder") + .def(py::init()) + .def( + "__iter__", + [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); }, + py::keep_alive<0, 1>()); } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index 062e3b3d30..de486e3e8e 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -241,3 +241,11 @@ def test_iterator_rvp(): assert list(m.make_iterator_1()) == [1, 2, 3] assert list(m.make_iterator_2()) == [1, 2, 3] assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2())) + + +def test_carray_iterator(): + """#4100: Check for proper iterator overload with C-Arrays""" + args_gt = list(float(i) for i in range(3)) + arr_h = m.CArrayHolder(*args_gt) + args = list(arr_h) + assert args_gt == args From 8ea75ab4d754305676d1bff5e324df2d9470dec7 Mon Sep 17 00:00:00 2001 From: Lalaland Date: Sat, 22 Oct 2022 16:52:35 -0700 Subject: [PATCH 009/137] Fix casts to void* (#4275) * Fix casts to void* * Improve tests * style: pre-commit fixes * remove c style cast Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Aaron Gokaslan --- include/pybind11/cast.h | 6 ++---- tests/test_class.cpp | 18 +++++++++++++++++- tests/test_class.py | 2 ++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 5699d4fb00..430c62f357 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1179,11 +1179,9 @@ enable_if_t::value, T> cast_safe(object &&) pybind11_fail("Internal error: cast_safe fallback invoked"); } template -enable_if_t>::value, void> cast_safe(object &&) {} +enable_if_t::value, void> cast_safe(object &&) {} template -enable_if_t, - std::is_same>>::value, - T> +enable_if_t, std::is_void>::value, T> cast_safe(object &&o) { return pybind11::cast(std::move(o)); } diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 18c8d358bb..fc1c17b179 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -384,6 +384,8 @@ TEST_SUBMODULE(class_, m) { protected: virtual int foo() const { return value; } + virtual void *void_foo() { return static_cast(&value); } + virtual void *get_self() { return static_cast(this); } private: int value = 42; @@ -392,6 +394,8 @@ TEST_SUBMODULE(class_, m) { class TrampolineB : public ProtectedB { public: int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); } + void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); } + void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); } }; class PublicistB : public ProtectedB { @@ -401,11 +405,23 @@ TEST_SUBMODULE(class_, m) { // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) ~PublicistB() override{}; // NOLINT(modernize-use-equals-default) using ProtectedB::foo; + using ProtectedB::get_self; + using ProtectedB::void_foo; }; + m.def("read_foo", [](const void *original) { + const int *ptr = reinterpret_cast(original); + return *ptr; + }); + + m.def("pointers_equal", + [](const void *original, const void *comparison) { return original == comparison; }); + py::class_(m, "ProtectedB") .def(py::init<>()) - .def("foo", &PublicistB::foo); + .def("foo", &PublicistB::foo) + .def("void_foo", &PublicistB::void_foo) + .def("get_self", &PublicistB::get_self); // test_brace_initialization struct BraceInitialization { diff --git a/tests/test_class.py b/tests/test_class.py index 47ba450fc2..7c1ed20608 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -313,6 +313,8 @@ def test_bind_protected_functions(): b = m.ProtectedB() assert b.foo() == 42 + assert m.read_foo(b.void_foo()) == 42 + assert m.pointers_equal(b.get_self(), b) class C(m.ProtectedB): def __init__(self): From 4fe905d4f01ce47bdea5fdcbcb6037acc9e2596c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 23 Oct 2022 00:32:17 -0400 Subject: [PATCH 010/137] fix: add flag for overriding classic Python search values (#4195) * fix: PyPy needs to overrite broken FindPythonInterp values Signed-off-by: Henry Schreiner * fix: add flag to opt-in to new (cross-compile) behavior Signed-off-by: Henry Schreiner * Apply suggestions from code review Signed-off-by: Henry Schreiner --- tools/FindPythonLibsNew.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index a5a628fd7a..bbbd9f9ec1 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -151,9 +151,13 @@ if(NOT _PYTHON_SUCCESS MATCHES 0) return() endif() +option( + PYBIND11_PYTHONLIBS_OVERWRITE + "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values." + ON) # Can manually set values when cross-compiling macro(_PYBIND11_GET_IF_UNDEF lst index name) - if(NOT DEFINED "${name}") + if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}") list(GET "${lst}" "${index}" "${name}") endif() endmacro() From d1c31e9aa0b8f7594b4c816b8745493272791c27 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 23 Oct 2022 08:08:00 -0400 Subject: [PATCH 011/137] chore: improve issue template (#4276) --- .github/ISSUE_TEMPLATE/bug-report.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index bd6a9a8e22..e6494ba6a0 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -6,7 +6,8 @@ body: - type: markdown attributes: value: | - Maintainers will only make a best effort to triage PRs. Please do your best to make the issue as easy to act on as possible, and only open if clearly a problem with pybind11 (ask first if unsure). + Please do your best to make the issue as easy to act on as possible, and only submit here if there is clearly a problem with pybind11 (ask first if unsure). **Note that a reproducer in a PR is much more likely to get immediate attention.** + - type: checkboxes id: steps attributes: @@ -20,6 +21,12 @@ body: - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new). required: false + - type: Input + id: version + attributes: + label: What version (or hash if on master) of pybind11 are you using? + required: true + - type: textarea id: description attributes: @@ -40,6 +47,14 @@ body: The code should be minimal, have no external dependencies, isolate the function(s) that cause breakage. Submit matched and complete C++ and Python snippets that can be easily compiled and run to diagnose the - issue. If possible, make a PR with a new, failing test to give us a - starting point to work on! + issue. β€” Note that a reproducer in a PR is much more likely to get + immediate attention: failing tests in the pybind11 CI are the best + starting point for working out fixes. render: text + + - type: Input + id: regression + attributes: + label: Is this a regression? Put the last known working version here if it is. + description: Put the last known working version here if this is a regression. + value: Not a regression From 07a61aa1c0a331431165103d49e1848e69373d62 Mon Sep 17 00:00:00 2001 From: Vemund Handeland Date: Sun, 23 Oct 2022 20:57:45 +0200 Subject: [PATCH 012/137] Fix char8_t support (#4278) Standard library macro __cpp_lib_char8_t is only available after including standard header --- include/pybind11/detail/common.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 6da7ef859c..b43100b95a 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -205,10 +205,6 @@ # endif #endif -#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING -#endif - #include #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." @@ -259,6 +255,11 @@ # endif #endif +// Must be after including or one of the other headers specified by the standard +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the From fcb5554d9fdd4ad3150bf12cb8be04099cb4976e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 26 Oct 2022 10:41:51 -0400 Subject: [PATCH 013/137] ci: move to final release of 3.11 (#4286) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2951a0ad19..b56e8f447f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - '3.6' - '3.9' - '3.10' - - '3.11-dev' + - '3.11' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -119,7 +119,7 @@ jobs: - name: C++11 tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 @@ -146,7 +146,7 @@ jobs: - name: C++ tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build build2 --target cpptest # Third build - C++17 mode with unstable ABI @@ -186,7 +186,7 @@ jobs: - python-version: "3.9" python-debug: true valgrind: true - - python-version: "3.11-dev" + - python-version: "3.11" python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' β€’ Valgrind' || '' }} β€’ x64" From b07223fa693f65da81e17c12858d636569947f98 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 29 Oct 2022 11:12:24 -0400 Subject: [PATCH 014/137] fix: improve bytes to str decoding error handling (#4294) * (bugfix): Improve bytes to str decoding error handling * regroup test * Further broaden tests * Add another decode error test * Fix bug in tests * Reviewer suggestions --- include/pybind11/pytypes.h | 9 +++++++++ tests/test_pytypes.cpp | 5 +++++ tests/test_pytypes.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 37fc49a0e1..80a2e2228e 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1432,6 +1432,9 @@ class str : public object { str(const char *c, const SzType &n) : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1441,6 +1444,9 @@ class str : public object { // NOLINTNEXTLINE(google-explicit-constructor) str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1598,6 +1604,9 @@ inline str::str(const bytes &b) { } auto obj = reinterpret_steal(PyUnicode_FromStringAndSize(buffer, length)); if (!obj) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } m_ptr = obj.release().ptr(); diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 99237ccef7..ea8d03958b 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -206,7 +206,12 @@ TEST_SUBMODULE(pytypes, m) { m.def("str_from_char_ssize_t", []() { return py::str{"red", (py::ssize_t) 3}; }); m.def("str_from_char_size_t", []() { return py::str{"blue", (py::size_t) 4}; }); m.def("str_from_string", []() { return py::str(std::string("baz")); }); + m.def("str_from_std_string_input", [](const std::string &stri) { return py::str(stri); }); + m.def("str_from_cstr_input", [](const char *c_str) { return py::str(c_str); }); m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); + m.def("str_from_bytes_input", + [](const py::bytes &encoded_str) { return py::str(encoded_str); }); + m.def("str_from_object", [](const py::object &obj) { return py::str(obj); }); m.def("repr_from_object", [](const py::object &obj) { return py::repr(obj); }); m.def("str_from_handle", [](py::handle h) { return py::str(h); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 3ed7b9c946..079ee7ca50 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -244,6 +244,20 @@ def __repr__(self): m.str_from_string_from_str(ucs_surrogates_str) +@pytest.mark.parametrize( + "func", + [ + m.str_from_bytes_input, + m.str_from_cstr_input, + m.str_from_std_string_input, + ], +) +def test_surrogate_pairs_unicode_error(func): + input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") + with pytest.raises(UnicodeDecodeError): + func(input_str) + + def test_bytes(doc): assert m.bytes_from_char_ssize_t().decode() == "green" assert m.bytes_from_char_size_t().decode() == "purple" From b07d08f6009a52050bb588a3e6fb0489c98af85e Mon Sep 17 00:00:00 2001 From: Chekov2k Date: Sun, 30 Oct 2022 15:57:23 +0000 Subject: [PATCH 015/137] Add `PYBIND11_SIMPLE_GIL_MANAGEMENT` option (cmake, C++ define) (#4216) * Add option to force the use of the PYPY GIL scoped acquire/release logic to support nested gil access, see https://github.com/pybind/pybind11/issues/1276 and https://github.com/pytorch/pytorch/issues/83101 * Apply suggestions from code review * Update CMakeLists.txt * docs: update upgrade guide * Update docs/upgrade.rst * All bells & whistles. * Add Reminder to common.h, so that we will not forget to purge `!WITH_THREAD` branches when dropping Python 3.6 * New sentence instead of semicolon. * Temporarily pull in snapshot of PR #4246 * Add `test_release_acquire` * Add more unit tests for nested gil locking * Add test_report_builtins_internals_keys * Very minor enhancement: sort list only after filtering. * Revert change in docs/upgrade.rst * Add test_multi_acquire_release_cross_module, while also forcing unique PYBIND11_INTERNALS_VERSION for cross_module_gil_utils.cpp * Hopefully fix apparently new ICC error. ``` 2022-10-28T07:57:54.5187728Z -- The CXX compiler identification is Intel 2021.7.0.20220726 ... 2022-10-28T07:58:53.6758994Z icpc: remark #10441: The Intel(R) C++ Compiler Classic (ICC) is deprecated and will be removed from product release in the second half of 2023. The Intel(R) oneAPI DPC++/C++ Compiler (ICX) is the recommended compiler moving forward. Please transition to use this compiler. Use '-diag-disable=10441' to disable this message. 2022-10-28T07:58:54.5801597Z In file included from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../detail/type_caster_base.h(15), 2022-10-28T07:58:54.5803794Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../cast.h(15), 2022-10-28T07:58:54.5805740Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../attr.h(14), 2022-10-28T07:58:54.5809556Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/class.h(12), 2022-10-28T07:58:54.5812154Z from /home/runner/work/pybind11/pybind11/include/pybind11/pybind11.h(13), 2022-10-28T07:58:54.5948523Z from /home/runner/work/pybind11/pybind11/tests/cross_module_gil_utils.cpp(13): 2022-10-28T07:58:54.5949009Z /home/runner/work/pybind11/pybind11/include/pybind11/detail/../detail/internals.h(177): error #2282: unrecognized GCC pragma 2022-10-28T07:58:54.5949374Z PYBIND11_TLS_KEY_INIT(tstate) 2022-10-28T07:58:54.5949579Z ^ 2022-10-28T07:58:54.5949695Z ``` * clang-tidy fixes * Workaround for PYPY WIN exitcode None * Revert "Temporarily pull in snapshot of PR #4246" This reverts commit 23ac16e859150f27fda25ca865cabcb4444e0770. * Another workaround for PYPY WIN exitcode None * Clean up how the tests are run "run in process" Part 1: uniformity * Clean up how the tests are run "run in process" Part 2: use `@pytest.mark.parametrize` and clean up the naming. * Skip some tests `#if defined(THREAD_SANITIZER)` (tested with TSAN using the Google-internal toolchain). * Run all tests again but ignore ThreadSanitizer exitcode 66 (this is less likely to mask unrelated ThreadSanitizer issues in the future). * bug fix: missing common.h include before using `PYBIND11_SIMPLE_GIL_MANAGEMENT` For the tests in the github CI this does not matter, because `PYBIND11_SIMPLE_GIL_MANAGEMENT` is always defined from the command line, but when monkey-patching common.h locally, it matters. * if process.exitcode is None: assert t_delta > 9.9 * More sophisiticated `_run_in_process()` implementation, clearly reporting `DEADLOCK`, additionally exercised via added `intentional_deadlock()` * Wrap m.intentional_deadlock in a Python function, for `ForkingPickler` compatibility. ``` > ForkingPickler(file, protocol).dump(obj) E TypeError: cannot pickle 'PyCapsule' object ``` Observed with all Windows builds including mingw but not PyPy, and macos-latest with Python 3.9, 3.10, 3.11 but not 3.6. * Add link to potential solution for WOULD-BE-NICE-TO-HAVE feature. * Add `SKIP_IF_DEADLOCK = True` option, to not pollute the CI results with expected `DEADLOCK` failures while we figure out what to do about them. * Add COPY-PASTE-THIS: gdb ... command (to be used for debugging the detected deadlock) * style: pre-commit fixes * Do better than automatic pre-commit fixes. * Add `PYBIND11_SIMPLE_GIL_MANAGEMENT` to `pytest_report_header()` (so that we can easily know when harvesting deadlock information from the CI logs). Co-authored-by: Arnim Balzer Co-authored-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 + CMakeLists.txt | 6 + include/pybind11/detail/common.h | 5 + include/pybind11/detail/internals.h | 16 +- include/pybind11/gil.h | 53 ++++-- tests/conftest.py | 1 + tests/cross_module_gil_utils.cpp | 67 +++++++- tests/pybind11_tests.cpp | 6 + tests/test_embed/test_interpreter.cpp | 1 - tests/test_gil_scoped.cpp | 107 +++++++++++- tests/test_gil_scoped.py | 227 +++++++++++++++++++++----- 11 files changed, 433 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56e8f447f..a11cae1ab0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,10 +102,12 @@ jobs: run: python -m pip install pytest-github-actions-annotate-failures # First build - C++11 mode and inplace + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here. - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 @@ -129,10 +131,12 @@ jobs: run: git clean -fdx # Second build - C++17 mode and in a build directory + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. - name: Configure C++17 run: > cmake -S . -B build2 -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 diff --git a/CMakeLists.txt b/CMakeLists.txt index 3284e21eb4..0d93203881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,10 +91,16 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() + cmake_dependent_option( USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index b43100b95a..a3e0bc9b37 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -206,6 +206,7 @@ #endif #include +// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." #endif @@ -229,6 +230,10 @@ # undef copysign #endif +#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d47084e263..7de7794344 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -9,6 +9,12 @@ #pragma once +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + #include "../pytypes.h" #include @@ -49,7 +55,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // `Py_LIMITED_API` anyway. # if PYBIND11_INTERNALS_VERSION > 4 # define PYBIND11_TLS_KEY_REF Py_tss_t & -# ifdef __GNUC__ +# if defined(__GNUC__) && !defined(__INTEL_COMPILER) // Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer // for every field. # define PYBIND11_TLS_KEY_INIT(var) \ @@ -169,10 +175,12 @@ struct internals { PyTypeObject *default_metaclass; PyObject *instance_base; #if defined(WITH_THREAD) + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PYBIND11_TLS_KEY_INIT(tstate) # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) # endif // PYBIND11_INTERNALS_VERSION > 4 + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 @@ -408,6 +416,10 @@ PYBIND11_NOINLINE internals &get_internals() { return **internals_pp; } +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + gil_scoped_acquire gil; +# else // Ensure that the GIL is held since we will need to make Python calls. // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { @@ -417,6 +429,8 @@ PYBIND11_NOINLINE internals &get_internals() { ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; +# endif +#endif error_scope err_scope; PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 1ef5f0a8c8..cb0028d505 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -10,7 +10,10 @@ #pragma once #include "detail/common.h" -#include "detail/internals.h" + +#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/internals.h" +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -21,7 +24,9 @@ PyThreadState *get_thread_state_unchecked(); PYBIND11_NAMESPACE_END(detail) -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) +#if defined(WITH_THREAD) + +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -62,11 +67,11 @@ class gil_scoped_acquire { if (!tstate) { tstate = PyThreadState_New(internals.istate); -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!tstate) { pybind11_fail("scoped_acquire: could not create thread state!"); } -# endif +# endif tstate->gilstate_counter = 0; PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); } else { @@ -87,20 +92,20 @@ class gil_scoped_acquire { PYBIND11_NOINLINE void dec_ref() { --tstate->gilstate_counter; -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate) { pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); } if (tstate->gilstate_counter < 0) { pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); } -# endif +# endif if (tstate->gilstate_counter == 0) { -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!release) { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); } -# endif +# endif PyThreadState_Clear(tstate); if (active) { PyThreadState_DeleteCurrent(); @@ -178,12 +183,14 @@ class gil_scoped_release { bool disassoc; bool active = true; }; -#elif defined(PYPY_VERSION) + +# else // PYBIND11_SIMPLE_GIL_MANAGEMENT + class gil_scoped_acquire { PyGILState_STATE state; public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } + gil_scoped_acquire() : state{PyGILState_Ensure()} {} gil_scoped_acquire(const gil_scoped_acquire &) = delete; gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_acquire() { PyGILState_Release(state); } @@ -194,19 +201,39 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() { state = PyEval_SaveThread(); } + gil_scoped_release() : state{PyEval_SaveThread()} {} gil_scoped_release(const gil_scoped_release &) = delete; gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; -#else + +# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT + +#else // WITH_THREAD + class gil_scoped_acquire { +public: + gil_scoped_acquire() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; + class gil_scoped_release { +public: + gil_scoped_release() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; -#endif + +#endif // WITH_THREAD PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/conftest.py b/tests/conftest.py index 02ce263afc..f5ddb9f129 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -210,4 +210,5 @@ def pytest_report_header(config): f" {pybind11_tests.compiler_info}" f" {pybind11_tests.cpp_std}" f" {pybind11_tests.PYBIND11_INTERNALS_ID}" + f" PYBIND11_SIMPLE_GIL_MANAGEMENT={pybind11_tests.PYBIND11_SIMPLE_GIL_MANAGEMENT}" ) diff --git a/tests/cross_module_gil_utils.cpp b/tests/cross_module_gil_utils.cpp index 1436c35d6e..7c20849dd9 100644 --- a/tests/cross_module_gil_utils.cpp +++ b/tests/cross_module_gil_utils.cpp @@ -6,9 +6,15 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance. #include #include +#include +#include // This file mimics a DSO that makes pybind11 calls but does not define a // PYBIND11_MODULE. The purpose is to test that such a DSO can create a @@ -21,8 +27,54 @@ namespace { namespace py = pybind11; + void gil_acquire() { py::gil_scoped_acquire gil; } +std::string gil_multi_acquire_release(unsigned bits) { + if ((bits & 0x1u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x2u) != 0u) { + py::gil_scoped_release gil; + } + if ((bits & 0x4u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x8u) != 0u) { + py::gil_scoped_release gil; + } + return PYBIND11_INTERNALS_ID; +} + +struct CustomAutoGIL { + CustomAutoGIL() : gstate(PyGILState_Ensure()) {} + ~CustomAutoGIL() { PyGILState_Release(gstate); } + + PyGILState_STATE gstate; +}; +struct CustomAutoNoGIL { + CustomAutoNoGIL() : save(PyEval_SaveThread()) {} + ~CustomAutoNoGIL() { PyEval_RestoreThread(save); } + + PyThreadState *save; +}; + +template +void gil_acquire_inner() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; +} + +template +void gil_acquire_nested() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; + auto thread = std::thread(&gil_acquire_inner); + thread.join(); +} + constexpr char kModuleName[] = "cross_module_gil_utils"; struct PyModuleDef moduledef = { @@ -30,6 +82,9 @@ struct PyModuleDef moduledef = { } // namespace +#define ADD_FUNCTION(Name, ...) \ + PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast(&__VA_ARGS__))); + extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { PyObject *m = PyModule_Create(&moduledef); @@ -37,8 +92,16 @@ extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { if (m != nullptr) { static_assert(sizeof(&gil_acquire) == sizeof(void *), "Function pointer must have the same size as void*"); - PyModule_AddObject( - m, "gil_acquire_funcaddr", PyLong_FromVoidPtr(reinterpret_cast(&gil_acquire))); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested) } return m; diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index aa3095594b..6240346487 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = +#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + true; +#else + false; +#endif bind_ConstructorStats(m); diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 6299293b91..44dcd1fdb0 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -293,7 +293,6 @@ TEST_CASE("Threads") { { py::gil_scoped_release gil_release{}; - REQUIRE(has_pybind11_internals_static()); auto threads = std::vector(); for (auto i = 0; i < num_threads; ++i) { diff --git a/tests/test_gil_scoped.cpp b/tests/test_gil_scoped.cpp index 97efdc1616..f136086e84 100644 --- a/tests/test_gil_scoped.cpp +++ b/tests/test_gil_scoped.cpp @@ -11,6 +11,13 @@ #include "pybind11_tests.h" +#include +#include + +#define CROSS_MODULE(Function) \ + auto cm = py::module_::import("cross_module_gil_utils"); \ + auto target = reinterpret_cast(PyLong_AsVoidPtr(cm.attr(Function).ptr())); + class VirtClass { public: virtual ~VirtClass() = default; @@ -28,6 +35,16 @@ class PyVirtClass : public VirtClass { }; TEST_SUBMODULE(gil_scoped, m) { + m.attr("defined_THREAD_SANITIZER") = +#if defined(THREAD_SANITIZER) + true; +#else + false; +#endif + + m.def("intentional_deadlock", + []() { std::thread([]() { py::gil_scoped_acquire gil_acquired; }).join(); }); + py::class_(m, "VirtClass") .def(py::init<>()) .def("virtual_func", &VirtClass::virtual_func) @@ -37,11 +54,91 @@ TEST_SUBMODULE(gil_scoped, m) { m.def("test_callback_std_func", [](const std::function &func) { func(); }); m.def("test_callback_virtual_func", [](VirtClass &virt) { virt.virtual_func(); }); m.def("test_callback_pure_virtual_func", [](VirtClass &virt) { virt.pure_virtual_func(); }); - m.def("test_cross_module_gil", []() { - auto cm = py::module_::import("cross_module_gil_utils"); - auto gil_acquire = reinterpret_cast( - PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); + m.def("test_cross_module_gil_released", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_acquired", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_custom_released", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_custom_acquired", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_released", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") py::gil_scoped_release gil_release; - gil_acquire(); + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_custom_released", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_custom_acquired", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_released", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_release_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired; + return py::str(obj); + }); + m.def("test_nested_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired_outer; + py::gil_scoped_acquire gil_acquired_inner; + return py::str(obj); + }); + m.def("test_multi_acquire_release_cross_module", [](unsigned bits) { + py::set internals_ids; + internals_ids.add(PYBIND11_INTERNALS_ID); + { + py::gil_scoped_release gil_released; + auto thread_f = [bits, &internals_ids]() { + py::gil_scoped_acquire gil_acquired; + auto cm = py::module_::import("cross_module_gil_utils"); + auto target = reinterpret_cast( + PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr())); + std::string cm_internals_id = target(bits >> 3); + internals_ids.add(cm_internals_id); + }; + if ((bits & 0x1u) != 0u) { + thread_f(); + } + if ((bits & 0x2u) != 0u) { + std::thread non_python_thread(thread_f); + non_python_thread.join(); + } + if ((bits & 0x4u) != 0u) { + thread_f(); + } + } + return internals_ids; }); } diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 52374b0cce..e890a7b0c8 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -1,45 +1,199 @@ import multiprocessing +import sys import threading +import time + +import pytest from pybind11_tests import gil_scoped as m +class ExtendedVirtClass(m.VirtClass): + def virtual_func(self): + pass + + def pure_virtual_func(self): + pass + + +def test_callback_py_obj(): + m.test_callback_py_obj(lambda: None) + + +def test_callback_std_func(): + m.test_callback_std_func(lambda: None) + + +def test_callback_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_virtual_func(extended) + + +def test_callback_pure_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_pure_virtual_func(extended) + + +def test_cross_module_gil_released(): + """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" + m.test_cross_module_gil_released() # Should not raise a SIGSEGV + + +def test_cross_module_gil_acquired(): + """Makes sure that the GIL can be acquired by another module from a GIL-acquired state.""" + m.test_cross_module_gil_acquired() # Should not raise a SIGSEGV + + +def test_cross_module_gil_inner_custom_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_released() + + +def test_cross_module_gil_inner_custom_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_acquired() + + +def test_cross_module_gil_inner_pybind11_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_released() + + +def test_cross_module_gil_inner_pybind11_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_acquired() + + +def test_cross_module_gil_nested_custom_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_released() + + +def test_cross_module_gil_nested_custom_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_acquired() + + +def test_cross_module_gil_nested_pybind11_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_released() + + +def test_cross_module_gil_nested_pybind11_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_acquired() + + +def test_release_acquire(): + assert m.test_release_acquire(0xAB) == "171" + + +def test_nested_acquire(): + assert m.test_nested_acquire(0xAB) == "171" + + +def test_multi_acquire_release_cross_module(): + for bits in range(16 * 8): + internals_ids = m.test_multi_acquire_release_cross_module(bits) + assert len(internals_ids) == 2 if bits % 8 else 1 + + +# Intentionally putting human review in the loop here, to guard against accidents. +VARS_BEFORE_ALL_BASIC_TESTS = dict(vars()) # Make a copy of the dict (critical). +ALL_BASIC_TESTS = ( + test_callback_py_obj, + test_callback_std_func, + test_callback_virtual_func, + test_callback_pure_virtual_func, + test_cross_module_gil_released, + test_cross_module_gil_acquired, + test_cross_module_gil_inner_custom_released, + test_cross_module_gil_inner_custom_acquired, + test_cross_module_gil_inner_pybind11_released, + test_cross_module_gil_inner_pybind11_acquired, + test_cross_module_gil_nested_custom_released, + test_cross_module_gil_nested_custom_acquired, + test_cross_module_gil_nested_pybind11_released, + test_cross_module_gil_nested_pybind11_acquired, + test_release_acquire, + test_nested_acquire, + test_multi_acquire_release_cross_module, +) + + +def test_all_basic_tests_completeness(): + num_found = 0 + for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items(): + if not key.startswith("test_"): + continue + assert value in ALL_BASIC_TESTS + num_found += 1 + assert len(ALL_BASIC_TESTS) == num_found + + +def _intentional_deadlock(): + m.intentional_deadlock() + + +ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) +SKIP_IF_DEADLOCK = True # See PR #4216 + + def _run_in_process(target, *args, **kwargs): - """Runs target in process and returns its exitcode after 10s (None if still alive).""" + if len(args) == 0: + test_fn = target + else: + test_fn = args[0] + # Do not need to wait much, 10s should be more than enough. + timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: + t_start = time.time() process.start() - # Do not need to wait much, 10s should be more than enough. - process.join(timeout=10) + if timeout >= 100: # For debugging. + print( + "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs) + ) + print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True) + process.join(timeout=timeout) + if timeout >= 100: + print("\nprocess.pid JOINED", process.pid, flush=True) + t_delta = time.time() - t_start + if process.exitcode == 66 and m.defined_THREAD_SANITIZER: # Issue #2754 + # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output. + # Maybe this could work: + # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e + pytest.skip( + "ThreadSanitizer: starting new threads after multi-threaded fork is not supported." + ) + elif test_fn is _intentional_deadlock: + assert process.exitcode is None + return 0 + elif process.exitcode is None: + assert t_delta > 0.9 * timeout + msg = "DEADLOCK, most likely, exactly what this test is meant to detect." + if SKIP_IF_DEADLOCK: + pytest.skip(msg) + raise RuntimeError(msg) return process.exitcode finally: if process.is_alive(): process.terminate() -def _python_to_cpp_to_python(): - """Calls different C++ functions that come back to Python.""" - - class ExtendedVirtClass(m.VirtClass): - def virtual_func(self): - pass - - def pure_virtual_func(self): - pass - - extended = ExtendedVirtClass() - m.test_callback_py_obj(lambda: None) - m.test_callback_std_func(lambda: None) - m.test_callback_virtual_func(extended) - m.test_callback_pure_virtual_func(extended) - - -def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): - """Calls different C++ functions that come back to Python, from Python threads.""" +def _run_in_threads(test_fn, num_threads, parallel): threads = [] for _ in range(num_threads): - thread = threading.Thread(target=_python_to_cpp_to_python) + thread = threading.Thread(target=test_fn) thread.daemon = True thread.start() if parallel: @@ -51,43 +205,40 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_parallel(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_sequential(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert ( - _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 - ) + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_process(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. This test is for completion, but it was never an issue. """ - assert _run_in_process(_python_to_cpp_to_python) == 0 - - -def test_cross_module_gil(): - """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" - m.test_cross_module_gil() # Should not raise a SIGSEGV + assert _run_in_process(test_fn) == 0 From 5bc0943ed96836f46489f53961f6c438d2935357 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 30 Oct 2022 13:24:41 -0700 Subject: [PATCH 016/137] Ensure config, build, toolchain, spelling, etc. issues are not masked. (#4255) --- include/pybind11/eigen/matrix.h | 1 - tests/test_eigen_tensor.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index 5f5ad3867b..c30dac2411 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -11,7 +11,6 @@ #include "../numpy.h" -// Similar to comments & pragma block in eigen_tensor.h. PLEASE KEEP IN SYNC. /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 5ee5fa01b2..653c9f288d 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -9,8 +9,16 @@ from pybind11_tests import eigen_tensor_avoid_stl_array as avoid submodules += [avoid.c_style, avoid.f_style] -except ImportError: - pass +except ImportError as e: + # Ensure config, build, toolchain, etc. issues are not masked here: + raise RuntimeError( + "import pybind11_tests.eigen_tensor_avoid_stl_array FAILED, while " + "import pybind11_tests.eigen_tensor succeeded. " + "Please ensure that " + "test_eigen_tensor.cpp & " + "test_eigen_tensor_avoid_stl_array.cpp " + "are built together (or both are not built if Eigen is not available)." + ) from e tensor_ref = np.empty((3, 5, 2), dtype=np.int64) From 3a2c96bd6f95f6b183c342380558238583415ab1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 09:18:05 -0700 Subject: [PATCH 017/137] fix: unicode surrogate character in Python exception message. (#4297) * Fix & test for issue #4288 (unicode surrogate character in Python exception message). * DRY `message_unavailable_exc` * fix: add a constexpr Co-authored-by: Aaron Gokaslan * style: pre-commit fixes Co-authored-by: Henry Schreiner Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pytypes.h | 22 ++++++++++++++++++++-- tests/test_exceptions.cpp | 5 ----- tests/test_exceptions.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80a2e2228e..91cbaaba28 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,11 +501,29 @@ struct error_fetch_and_normalize { std::string message_error_string; if (m_value) { auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; if (!value_str) { message_error_string = detail::error_string(); - result = ""; + result = message_unavailable_exc; } else { - result = value_str.cast(); + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } } } else { result = ""; diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 3583f22a50..f57e095068 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor { py::str s; }; -std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); - return py::error_already_set().what(); -} - TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 70b6ffea95..7eb1a9d62c 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -275,6 +275,20 @@ def test_local_translator(msg): assert msg(excinfo.value) == "this mod" +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + class FlakyException(Exception): def __init__(self, failure_point): if failure_point == "failure_point_init": From b1bd7f2600d650bf59c88800c637703dd89317f3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 10:36:26 -0700 Subject: [PATCH 018/137] fix: define (non-empty) `PYBIND11_EXPORT_EXCEPTION` only under macOS. (#4298) Background: #2999, #4105, #4283, #4284 In a nutshell: * Only macOS actually needs `PYBIND11_EXPORT_EXCEPTION` (#4284). * Evidently (#4283), under macOS `PYBIND11_EXPORT_EXCEPTION` does not run the risk of introducing ODR violations, * but evidently (#4283) under Linux it does, in the presumably rare/unusual situation that `RTLD_GLOBAL` is used. * Windows does no have the equivalent of `RTLD_GLOBAL`, therefore `PYBIND11_EXPORT_EXCEPTION` has no practical benefit, on the contrary, noisy warning suppression pragmas are needed, therefore it is best left empty. --- docs/advanced/exceptions.rst | 9 ++++++--- include/pybind11/detail/common.h | 18 +++--------------- include/pybind11/pytypes.h | 9 --------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 2211caf5d3..53981dc08f 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -177,9 +177,12 @@ section. may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. - Note that ``libc++`` and ``libstdc++`` `behave differently `_ - with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI boundaries need to be explicitly exported, as exercised in ``tests/test_exceptions.h``. - See also: "Problems with C++ exceptions" under `GCC Wiki `_. + Note that ``libc++`` and ``libstdc++`` `behave differently under macOS + `_ + with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI + boundaries need to be explicitly exported, as exercised in + ``tests/test_exceptions.h``. See also: + "Problems with C++ exceptions" under `GCC Wiki `_. Local vs Global Exception Translators diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a3e0bc9b37..9b74323f67 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -96,13 +96,10 @@ #endif #if !defined(PYBIND11_EXPORT_EXCEPTION) -# ifdef __MINGW32__ -// workaround for: -// error: 'dllexport' implies default visibility, but xxx has already been declared with a -// different visibility -# define PYBIND11_EXPORT_EXCEPTION -# else +# if defined(__apple_build_version__) # define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION # endif #endif @@ -904,12 +901,6 @@ using expand_side_effects = bool[]; PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// C++ bindings of builtin Python exceptions class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { public: @@ -917,9 +908,6 @@ class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { /// Set the error using the Python C API virtual void set_error() const = 0; }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif #define PYBIND11_RUNTIME_EXCEPTION(name, type) \ class PYBIND11_EXPORT_EXCEPTION name : public builtin_exception { \ diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 91cbaaba28..80b49ec397 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -623,12 +623,6 @@ inline std::string error_string() { PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275 4251) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// Fetch and hold an error which was already set in Python. An instance of this is typically /// thrown to propagate python-side errors back through C++ which can either be caught manually or /// else falls back to the function dispatcher (which then raises the captured error back to @@ -688,9 +682,6 @@ class PYBIND11_EXPORT_EXCEPTION error_already_set : public std::exception { /// crashes (undefined behavior) if the Python interpreter is finalizing. static void m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr); }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif /// Replaces the current Python error indicator with the chosen error, performing a /// 'raise from' to indicate that the chosen error was caused by the original error. From 252ed8fb52e46eb3ec3e3b8c621ca9f79b53b26a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:11:23 -0400 Subject: [PATCH 019/137] docs: prepare for 2.10.1 release (#4279) * docs: prepare for 2.10.1 release Signed-off-by: Henry Schreiner * Update changelog.rst * docs: update changelog with final list of PRs Signed-off-by: Henry Schreiner * Update docs/changelog.rst * chore: one more changelog bump Signed-off-by: Henry Schreiner --- docs/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df3181c524..7d6d0c0f56 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,19 +10,18 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. - IN DEVELOPMENT -------------- Changes will be summarized here periodically. -Version 2.10.1 (Oct 2?, 2022) +Version 2.10.1 (Oct 31, 2022) ----------------------------- +This is the first version to fully support embedding the newly released Python 3.11. Changes: - * Allow ``pybind11::capsule`` constructor to take null destructor pointers. `#4221 `_ @@ -30,8 +29,40 @@ Changes: (established behavior). `#4119 `_ +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + Bug fixes: +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + * Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. `#4188 `_ @@ -100,9 +131,15 @@ Performance and style: * Optimize unpacking_collector when processing ``arg_v`` arguments. `#4219 `_ +* Optimize casting C++ object to ``None``. + `#4269 `_ + Build system improvements: +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ From 2441d25b26af3e7b89d521a218694b604ec716e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 00:10:13 -0400 Subject: [PATCH 020/137] chore(deps): update pre-commit hooks (#4302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.38.2 β†’ v3.2.0](https://github.com/asottile/pyupgrade/compare/v2.38.2...v3.2.0) - [github.com/psf/black: 22.8.0 β†’ 22.10.0](https://github.com/psf/black/compare/22.8.0...22.10.0) - [github.com/PyCQA/pylint: v2.15.3 β†’ v2.15.5](https://github.com/PyCQA/pylint/compare/v2.15.3...v2.15.5) - [github.com/pre-commit/mirrors-mypy: v0.981 β†’ v0.982](https://github.com/pre-commit/mirrors-mypy/compare/v0.981...v0.982) - [github.com/codespell-project/codespell: v2.2.1 β†’ v2.2.2](https://github.com/codespell-project/codespell/compare/v2.2.1...v2.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 742e7a30aa..a60d84f22f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v2.38.2" + rev: "v3.2.0" hooks: - id: pyupgrade args: [--py36-plus] @@ -54,7 +54,7 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.8.0" # Keep in sync with blacken-docs + rev: "22.10.0" # Keep in sync with blacken-docs hooks: - id: black @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.3" + rev: "v2.15.5" hooks: - id: pylint files: ^pybind11 @@ -132,7 +132,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.981" + rev: "v0.982" hooks: - id: mypy args: [] @@ -152,7 +152,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.1" + rev: "v2.2.2" hooks: - id: codespell exclude: ".supp$" From 0176632e8cef72a4223f2df54d6dfb9e552ec71f Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 1 Nov 2022 12:19:34 -0400 Subject: [PATCH 021/137] chore: sync blacken-docs hook with black (#4304) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a60d84f22f..fd079dd037 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==22.8.0 # keep in sync with black hook + - black==22.10.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks From ee2b5226295d67b690faddd446a329bb2840a1a8 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 2 Nov 2022 11:32:53 -0700 Subject: [PATCH 022/137] Fix functional.h bug + introduce test to verify that it is fixed (#4254) * Illustrate bug in functional.h * style: pre-commit fixes * Make functional casting more robust / add workaround * Make function_record* casting even more robust * See if this fixes PyPy issue * It still fails on PyPy sadly * Do not make new CTOR just yet * Fix test * Add name to ensure correctness * style: pre-commit fixes * Clean up tests + remove ifdef guards * Add comments * Improve comments, error handling, and safety * Fix compile error * Fix magic logic * Extract helper function * Fix func signature * move to local internals * style: pre-commit fixes * Switch to simpler design * style: pre-commit fixes * Move to function_record * style: pre-commit fixes * Switch to internals, update tests and docs * Fix lint * Oops, forgot to resolve last comment * Fix typo * Update in response to comments * Implement suggestion to improve test * Update comment * Simple fixes Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Aaron Gokaslan --- include/pybind11/detail/internals.h | 31 ++++++++++++++++++++ include/pybind11/functional.h | 11 ++++++-- include/pybind11/pybind11.h | 44 ++++++++++++++++++++++------- tests/test_callbacks.cpp | 37 ++++++++++++++++++++++++ tests/test_callbacks.py | 13 +++++++++ 5 files changed, 124 insertions(+), 12 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 7de7794344..6fd61098c4 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -43,6 +43,8 @@ using ExceptionTranslator = void (*)(std::exception_ptr); PYBIND11_NAMESPACE_BEGIN(detail) +constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule"; + // Forward declarations inline PyTypeObject *make_static_property_type(); inline PyTypeObject *make_default_metaclass(); @@ -182,6 +184,16 @@ struct internals { # endif // PYBIND11_INTERNALS_VERSION > 4 // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; + +# if PYBIND11_INTERNALS_VERSION > 4 + // Note that we have to use a std::string to allocate memory to ensure a unique address + // We want unique addresses since we use pointer equality to compare function records + std::string function_record_capsule_name = internals_function_record_capsule_name; +# endif + + internals() = default; + internals(const internals &other) = delete; + internals &operator=(const internals &other) = delete; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_FREE(loader_life_support_tls_key); @@ -548,6 +560,25 @@ const char *c_str(Args &&...args) { return strings.front().c_str(); } +inline const char *get_function_record_capsule_name() { +#if PYBIND11_INTERNALS_VERSION > 4 + return get_internals().function_record_capsule_name.c_str(); +#else + return nullptr; +#endif +} + +// Determine whether or not the following capsule contains a pybind11 function record. +// Note that we use `internals` to make sure that only ABI compatible records are touched. +// +// This check is currently used in two places: +// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++ +// - The sibling feature of cpp_function to allow overloads +inline bool is_function_record_capsule(const capsule &cap) { + // Pointer equality as we rely on internals() to ensure unique pointers + return cap.name() == get_function_record_capsule_name(); +} + PYBIND11_NAMESPACE_END(detail) /// Returns a named pointer that is shared among all extension modules (using the same diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 102d1a938b..87ec4d10cb 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -48,9 +48,16 @@ struct type_caster> { */ if (auto cfunc = func.cpp_function()) { auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); - if (isinstance(cfunc_self)) { + if (cfunc_self == nullptr) { + PyErr_Clear(); + } else if (isinstance(cfunc_self)) { auto c = reinterpret_borrow(cfunc_self); - auto *rec = (function_record *) c; + + function_record *rec = nullptr; + // Check that we can safely reinterpret the capsule into a function_record + if (detail::is_function_record_capsule(c)) { + rec = c.get_pointer(); + } while (rec != nullptr) { if (rec->is_stateless diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index e662236d01..76d6eadcae 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -468,13 +468,20 @@ class cpp_function : public function { if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); - capsule rec_capsule = isinstance(self) ? reinterpret_borrow(self) - : capsule(self); - chain = (detail::function_record *) rec_capsule; - /* Never append a method to an overload chain of a parent class; - instead, hide the parent's overloads in this case */ - if (!chain->scope.is(rec->scope)) { + if (!isinstance(self)) { chain = nullptr; + } else { + auto rec_capsule = reinterpret_borrow(self); + if (detail::is_function_record_capsule(rec_capsule)) { + chain = rec_capsule.get_pointer(); + /* Never append a method to an overload chain of a parent class; + instead, hide the parent's overloads in this case */ + if (!chain->scope.is(rec->scope)) { + chain = nullptr; + } + } else { + chain = nullptr; + } } } // Don't trigger for things like the default __init__, which are wrapper_descriptors @@ -496,6 +503,7 @@ class cpp_function : public function { capsule rec_capsule(unique_rec.release(), [](void *ptr) { destruct((detail::function_record *) ptr); }); + rec_capsule.set_name(detail::get_function_record_capsule_name()); guarded_strdup.release(); object scope_module; @@ -661,10 +669,13 @@ class cpp_function : public function { /// Main dispatch logic for calls to functions bound using pybind11 static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { using namespace detail; + assert(isinstance(self)); /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + const function_record *overloads = reinterpret_cast( + PyCapsule_GetPointer(self, get_function_record_capsule_name())), *it = overloads; + assert(overloads != nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right overload */ @@ -1871,9 +1882,22 @@ class class_ : public detail::generic_type { static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow( - PyCFunction_GET_SELF(h.ptr())) - : nullptr; + if (!h) { + return nullptr; + } + + handle func_self = PyCFunction_GET_SELF(h.ptr()); + if (!func_self) { + throw error_already_set(); + } + if (!isinstance(func_self)) { + return nullptr; + } + auto cap = reinterpret_borrow(func_self); + if (!detail::is_function_record_capsule(cap)) { + return nullptr; + } + return cap.get_pointer(); } }; diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp index 92b8053de4..2fd05dec72 100644 --- a/tests/test_callbacks.cpp +++ b/tests/test_callbacks.cpp @@ -240,4 +240,41 @@ TEST_SUBMODULE(callbacks, m) { f(); } }); + + auto *custom_def = []() { + static PyMethodDef def; + def.ml_name = "example_name"; + def.ml_doc = "Example doc"; + def.ml_meth = [](PyObject *, PyObject *args) -> PyObject * { + if (PyTuple_Size(args) != 1) { + throw std::runtime_error("Invalid number of arguments for example_name"); + } + PyObject *first = PyTuple_GetItem(args, 0); + if (!PyLong_Check(first)) { + throw std::runtime_error("Invalid argument to example_name"); + } + auto result = py::cast(PyLong_AsLong(first) * 9); + return result.release().ptr(); + }; + def.ml_flags = METH_VARARGS; + return &def; + }(); + + // rec_capsule with name that has the same value (but not pointer) as our internal one + // This capsule should be detected by our code as foreign and not inspected as the pointers + // shouldn't match + constexpr const char *rec_capsule_name + = pybind11::detail::internals_function_record_capsule_name; + py::capsule rec_capsule(std::malloc(1), [](void *data) { std::free(data); }); + rec_capsule.set_name(rec_capsule_name); + m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr())); + + // This test requires a new ABI version to pass +#if PYBIND11_INTERNALS_VERSION > 4 + // rec_capsule with nullptr name + py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); }); + m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr())); +#else + m.add_object("custom_function2", py::none()); +#endif } diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 0b1047bbf2..57b6599880 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -193,3 +193,16 @@ def test_callback_num_times(): if len(rates) > 1: print("Min Mean Max") print(f"{min(rates):6.3f} {sum(rates) / len(rates):6.3f} {max(rates):6.3f}") + + +def test_custom_func(): + assert m.custom_function(4) == 36 + assert m.roundtrip(m.custom_function)(4) == 36 + + +@pytest.mark.skipif( + m.custom_function2 is None, reason="Current PYBIND11_INTERNALS_VERSION too low" +) +def test_custom_func2(): + assert m.custom_function2(3) == 27 + assert m.roundtrip(m.custom_function2)(3) == 27 From 1f04cc7062e33481c62c78231e9561b318bca67b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 10 Nov 2022 08:33:26 -0800 Subject: [PATCH 023/137] Add windows_clang to ci.yml (#4323) * Add windows_clang to ci.yml (previously tested under PRs #4321, #4319) * Add `pip install --upgrade pip`, Show env, cosmetic changes Already tested under PR #4321 --- .github/workflows/ci.yml | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a11cae1ab0..adbbf626f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -967,3 +967,73 @@ jobs: - name: Interface test C++17 run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target test_cmake_build + + windows_clang: + + strategy: + matrix: + os: [windows-latest] + python: ['3.10'] + + runs-on: "${{ matrix.os }}" + + name: "🐍 ${{ matrix.python }} β€’ ${{ matrix.os }} β€’ clang-latest" + + steps: + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.13 + + - name: Install ninja-build tool + uses: seanmiddleditch/gha-setup-ninja@v3 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show Clang++ version + run: clang++ --version + + - name: Show CMake version + run: cmake --version + + # TODO: WERROR=ON + - name: Configure Clang + run: > + cmake -G Ninja -S . -B . + -DCMAKE_VERBOSE_MAKEFILE=ON + -DPYBIND11_WERROR=OFF + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx From 88b019a8a5e116d7e4a4ffae6399a426364d4bcd Mon Sep 17 00:00:00 2001 From: gitartpiano <51239761+gitartpiano@users.noreply.github.com> Date: Fri, 11 Nov 2022 19:52:57 -0600 Subject: [PATCH 024/137] fix pybind11Tools.cmake typo causing Unknown arguments (#4327) * fix pybind11Tools.cmake typo causing Unknown arguments CMake Error at pybind11/tools/pybind11Tools.cmake:217 (if): if given arguments: "NOT" "MSVC" "AND" "NOT" "TEST" "MATCHES" "DEBUG|RELWITHDEBINFO" Unknown arguments specified https://github.com/pybind/pybind11/issues/4325 * Apply the same fix in tools/pybind11NewTools.cmake Co-authored-by: Ralf W. Grosse-Kunstleve --- tools/pybind11NewTools.cmake | 2 +- tools/pybind11Tools.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 9e13daf1a1..5a6a0cb8be 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -235,7 +235,7 @@ function(pybind11_add_module target_name) # Use case-insensitive comparison to match the result of $ string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 0dc61d3996..66ad00a478 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -214,7 +214,7 @@ function(pybind11_add_module target_name) # Use case-insensitive comparison to match the result of $ string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) pybind11_strip(${target_name}) endif() From 296615ad34f9d8f680efbab22553881ad9843063 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 12 Nov 2022 12:24:19 -0800 Subject: [PATCH 025/137] Add macos_brew_install_llvm to ci.yml (#4326) * Add macos_brew_install_llvm to ci.yml Added block transferred from PR #4324 * `test_cross_module_exception_translator` xfail 'Homebrew Clang' * Add `pip install numpy scipy` (tested already under PR #4324). --- .github/workflows/ci.yml | 67 ++++++++++++++++++++++++++++++++++++++++ tests/test_exceptions.py | 5 +-- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adbbf626f1..bd99ddd338 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1037,3 +1037,70 @@ jobs: - name: Clean directory run: git clean -fdx + + macos_brew_install_llvm: + name: "macos-latest β€’ brew install llvm" + runs-on: macos-latest + + env: + # https://apple.stackexchange.com/questions/227026/how-to-install-recent-clang-with-homebrew + LDFLAGS: '-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib' + + steps: + - name: Update PATH + run: echo "/usr/local/opt/llvm/bin" >> $GITHUB_PATH + + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Show Clang++ version before brew install llvm + run: clang++ --version + + - name: brew install llvm + run: brew install llvm + + - name: Show Clang++ version after brew install llvm + run: clang++ --version + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.13 + + - name: Run pip installs + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r tests/requirements.txt + python3 -m pip install numpy + python3 -m pip install scipy + + - name: Show CMake version + run: cmake --version + + - name: CMake Configure + run: > + cmake -S . -B . + -DCMAKE_VERBOSE_MAKEFILE=ON + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 7eb1a9d62c..0d2c808143 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -4,6 +4,7 @@ import env import pybind11_cross_module_tests as cm +import pybind11_tests # noqa: F401 from pybind11_tests import exceptions as m @@ -72,9 +73,9 @@ def test_cross_module_exceptions(msg): # TODO: FIXME @pytest.mark.xfail( - "env.PYPY and env.MACOS", + "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang'))", raises=RuntimeError, - reason="Expected failure with PyPy and libc++ (Issue #2847 & PR #2999)", + reason="See Issue #2847, PR #2999, PR #4324", ) def test_cross_module_exception_translator(): with pytest.raises(KeyError): From 48949222c637da9fc72b0ed6526ae1b40bb55237 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 22 Nov 2022 15:14:49 -0800 Subject: [PATCH 026/137] Use `PyEval_InitThreads()` as intended (#4350) * Use `PyEval_InitThreads()` as intended (actually matters only for Python 3.6). * Add `if defined(WITH_THREAD)` condition. https://docs.python.org/3.6/c-api/init.html#c.PyEval_InitThreads > This function is not available when thread support is disabled at compile time. * Fix oversight pointed out by @EricCousineau-TRI: Remove condition that is always false. --- include/pybind11/detail/internals.h | 3 --- include/pybind11/embed.h | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 6fd61098c4..ef1849fbea 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -468,9 +468,6 @@ PYBIND11_NOINLINE internals &get_internals() { internals_ptr = new internals(); #if defined(WITH_THREAD) -# if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); -# endif PyThreadState *tstate = PyThreadState_Get(); if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 0ac609e0f1..5b77594b35 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -118,6 +118,9 @@ inline void initialize_interpreter(bool init_signal_handlers = true, #if PY_VERSION_HEX < 0x030B0000 Py_InitializeEx(init_signal_handlers ? 1 : 0); +# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +# endif // Before it was special-cased in python 3.8, passing an empty or null argv // caused a segfault, so we have to reimplement the special case ourselves. From 9c18a74e377dece2f2acd22c2c6e63ecb2a59c77 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 22 Nov 2022 17:17:02 -0800 Subject: [PATCH 027/137] Use `multiprocessing` `start_method` `"forkserver"` (#4306) * Use `multiprocessing` `start_method` `"forkserver"` Alternative to PR #4305 * Add link to comment under PR #4105 * Unconditionally `pytest.skip("DEADLOCK")` for PyPy Windows * Remove `SKIP_IF_DEADLOCK` entirely, for simplicity. Hopefully this PR will resolve the deadlocks for good. * Add "In a nutshell" comment, in response to request by @EricCousineau-TRI --- tests/conftest.py | 12 ++++++++++++ tests/test_gil_scoped.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f5ddb9f129..96dffc81cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,8 @@ import contextlib import difflib import gc +import multiprocessing +import os import re import textwrap @@ -15,6 +17,16 @@ # Early diagnostic for failed imports import pybind11_tests +if os.name != "nt": + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 + # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. + # It is actually a well-known pitfall, unfortunately without guard rails. + # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, + # visit the issuecomment link above for details). + # Windows does not have fork() and the associated pitfall, therefore it is best left + # running with defaults. + multiprocessing.set_start_method("forkserver") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index e890a7b0c8..6af6a472d5 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -5,6 +5,7 @@ import pytest +import env from pybind11_tests import gil_scoped as m @@ -144,7 +145,6 @@ def _intentional_deadlock(): ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) -SKIP_IF_DEADLOCK = True # See PR #4216 def _run_in_process(target, *args, **kwargs): @@ -181,7 +181,7 @@ def _run_in_process(target, *args, **kwargs): elif process.exitcode is None: assert t_delta > 0.9 * timeout msg = "DEADLOCK, most likely, exactly what this test is meant to detect." - if SKIP_IF_DEADLOCK: + if env.PYPY and env.WIN: pytest.skip(msg) raise RuntimeError(msg) return process.exitcode From 9907bedce517ea291c70c6a7b5dfec9e594f41df Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 26 Nov 2022 07:15:54 +0800 Subject: [PATCH 028/137] fix(.github): fix bug-report issue template (#4363) --- .github/ISSUE_TEMPLATE/bug-report.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index e6494ba6a0..4f1e78f33c 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -21,10 +21,11 @@ body: - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new). required: false - - type: Input + - type: input id: version attributes: label: What version (or hash if on master) of pybind11 are you using? + validations: required: true - type: textarea @@ -52,7 +53,7 @@ body: starting point for working out fixes. render: text - - type: Input + - type: input id: regression attributes: label: Is this a regression? Put the last known working version here if it is. From 06003e82b3ff48337b71b310b46c3d8b15ca6d5a Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 28 Nov 2022 07:39:38 -0800 Subject: [PATCH 029/137] Introduce a new style of warning suppression based on push/pop (#4285) * Introduce a new warning suppression system * Switch to better name * Nits --- include/pybind11/cast.h | 11 ++-- include/pybind11/detail/common.h | 82 +++++++++++++++++++++----- include/pybind11/detail/init.h | 9 ++- include/pybind11/eigen/matrix.h | 31 +++------- include/pybind11/eigen/tensor.h | 35 ++++------- include/pybind11/numpy.h | 15 ++--- include/pybind11/pybind11.h | 38 +++++------- include/pybind11/pytypes.h | 16 +++-- tests/test_builtin_casters.cpp | 9 ++- tests/test_class.cpp | 4 +- tests/test_constants_and_functions.cpp | 34 ++++------- tests/test_eigen_matrix.cpp | 4 +- tests/test_eigen_tensor.inl | 8 ++- tests/test_embed/catch.cpp | 4 +- tests/test_embed/test_interpreter.cpp | 4 +- tests/test_kwargs_and_defaults.cpp | 9 ++- tests/test_operator_overloading.cpp | 23 +++----- 17 files changed, 173 insertions(+), 163 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 430c62f357..8f92df8ab3 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -29,6 +29,9 @@ #include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template @@ -389,7 +392,7 @@ struct string_caster { // For UTF-8 we avoid the need for a temporary `bytes` object by using // `PyUnicode_AsUTF8AndSize`. - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N == 8)) { + if (UTF_N == 8) { Py_ssize_t size = -1; const auto *buffer = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); @@ -416,7 +419,7 @@ struct string_caster { = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); // Skip BOM for UTF-16/32 - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N > 8)) { + if (UTF_N > 8) { buffer++; length--; } @@ -572,7 +575,7 @@ struct type_caster::value>> { // figure out how long the first encoded character is in bytes to distinguish between these // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as // those can fit into a single char value. - if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 8) && str_len > 1 && str_len <= 4) { + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { auto v0 = static_cast(value[0]); // low bits only: 0-127 // 0b110xxxxx - start of 2-byte sequence @@ -598,7 +601,7 @@ struct type_caster::value>> { // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a // surrogate pair with total length 2 instantly indicates a range error (but not a "your // string was too long" error). - else if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 16) && str_len == 2) { + else if (StringCaster::UTF_N == 16 && str_len == 2) { one_char = static_cast(value[0]); if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9b74323f67..b58dc3afa7 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -17,8 +17,69 @@ // Additional convention: 0xD = dev #define PYBIND11_VERSION_HEX 0x020B00D1 -#define PYBIND11_NAMESPACE_BEGIN(name) namespace name { -#define PYBIND11_NAMESPACE_END(name) } +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif + +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH + +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } // Robust support for some features and loading modules compiled against different pybind versions // requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute @@ -151,9 +212,9 @@ /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# pragma warning(push) +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4505) // C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# pragma warning(disable : 4505) # if defined(_DEBUG) && !defined(Py_DEBUG) // Workaround for a VS 2022 issue. // NOTE: This workaround knowingly violates the Python.h include order requirement: @@ -236,7 +297,7 @@ # define _DEBUG # undef PYBIND11_DEBUG_MARKER # endif -# pragma warning(pop) +PYBIND11_WARNING_POP #endif #include @@ -1136,17 +1197,6 @@ constexpr # define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) #endif -#if defined(_MSC_VER) // All versions (as of July 2021). - -// warning C4127: Conditional expression is constant -constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } - -# define PYBIND11_SILENCE_MSVC_C4127(...) ::pybind11::detail::silence_msvc_c4127(__VA_ARGS__) - -#else -# define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__ -#endif - #if defined(__clang__) \ && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ available. */ \ diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 05f4fe54aa..0938c9bdeb 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -12,6 +12,9 @@ #include "class.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <> @@ -115,7 +118,7 @@ template void construct(value_and_holder &v_h, Cpp *ptr, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); no_nullptr(ptr); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { // We're going to try to construct an alias by moving the cpp type. Whether or not // that succeeds, we still need to destroy the original cpp pointer (either the // moved away leftover, if the alias construction works, or the value itself if we @@ -156,7 +159,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { auto *ptr = holder_helper>::get(holder); no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " "is not an alias instance"); } @@ -174,7 +177,7 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); static_assert(std::is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias) { + if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); } else { v_h.value_ptr() = new Cpp(std::move(result)); diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index c30dac2411..34fe329a82 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -16,29 +16,15 @@ https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler */ -// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could -// make it version specific, or even remove it later, but considering that -// 1. C4127 is generally far more distracting than useful for modern template code, and -// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, -// it is probably best to keep this around indefinitely. -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4127) // C4127: conditional expression is constant -# pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") #include #include -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit // move constructors that break things. We could detect this an explicitly copy, but an extra copy @@ -48,6 +34,8 @@ static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + // Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: using EigenDStride = Eigen::Stride; template @@ -189,8 +177,7 @@ struct EigenProps { EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); - if ((PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && np_rows != rows) - || (PYBIND11_SILENCE_MSVC_C4127(fixed_cols) && np_cols != cols)) { + if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) { return false; } @@ -203,7 +190,7 @@ struct EigenProps { stride = a.strides(0) / static_cast(sizeof(Scalar)); if (vector) { // Eigen type is a compile-time vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed) && size != n) { + if (fixed && size != n) { return false; // Vector size mismatch } return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; @@ -220,7 +207,7 @@ struct EigenProps { } return {1, n, stride}; } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && rows != n) { + if (fixed_rows && rows != n) { return false; } return {n, 1, stride}; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index a823c0f397..a129f99292 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -13,28 +13,23 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); #endif -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4554) // Tensor.h warning -# pragma warning(disable : 4127) // Tensor.h warning -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif +// Disable warnings for Eigen +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4554) +PYBIND11_WARNING_DISABLE_MSVC(4127) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") #include -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) inline bool is_tensor_aligned(const void *data) { @@ -138,10 +133,8 @@ struct get_tensor_descriptor { // // We need to disable the type-limits warning for the inner loop when size = 0. -#if defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wtype-limits") template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { @@ -165,9 +158,7 @@ Eigen::DSizes get_shape_for_array(const array &arr) { return result; } -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP template struct type_caster::ValidType> { @@ -389,7 +380,7 @@ struct type_caster, constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; - if (PYBIND11_SILENCE_MSVC_C4127(is_aligned) && !is_tensor_aligned(arr.data())) { + if (is_aligned && !is_tensor_aligned(arr.data())) { return false; } @@ -399,7 +390,7 @@ struct type_caster, return false; } - if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable) && !arr.writeable()) { + if (needs_writeable && !arr.writeable()) { return false; } diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 369e18649b..ea64aa7e7e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -36,6 +36,8 @@ static_assert(std::is_signed::value, "Py_intptr_t must be signed"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + class array; // Forward declaration PYBIND11_NAMESPACE_BEGIN(detail) @@ -875,7 +877,7 @@ class array : public buffer { */ template detail::unchecked_mutable_reference mutable_unchecked() & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -893,7 +895,7 @@ class array : public buffer { */ template detail::unchecked_reference unchecked() const & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -1865,9 +1867,10 @@ struct vectorize_helper { } auto result = returned_array::create(trivial, shape); + + PYBIND11_WARNING_PUSH #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wreturn-std-move" + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") #endif if (size == 0) { @@ -1883,9 +1886,7 @@ struct vectorize_helper { } return result; -#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic pop -#endif + PYBIND11_WARNING_POP } template diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 76d6eadcae..10b27254eb 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -35,6 +35,8 @@ # include #endif +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning This warning is about ABI compatibility, not code health. It is only actually needed in a couple places, but apparently GCC 7 "generates this warning if @@ -43,11 +45,10 @@ No other GCC version generates this warning. */ #if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wnoexcept-type" +PYBIND11_WARNING_DISABLE_GCC("-Wnoexcept-type") #endif -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) @@ -177,22 +178,22 @@ class cpp_function : public function { auto *rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(capture) <= sizeof(rec->data))) { + if (sizeof(capture) <= sizeof(rec->data)) { /* Without these pragmas, GCC warns that there might not be enough space to use the placement new operator. However, the 'if' statement above ensures that this is the case. */ -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wplacement-new" + PYBIND11_WARNING_PUSH + +#if defined(__GNUG__) && __GNUC__ >= 6 + PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif + new ((capture *) &rec->data) capture{std::forward(f)}; -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-aliasing" + +#if !PYBIND11_HAS_STD_LAUNDER + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") #endif + // UB without std::launder, but without breaking ABI and/or // a significant refactoring it's "impossible" to solve. if (!std::is_trivially_destructible::value) { @@ -202,9 +203,7 @@ class cpp_function : public function { data->~capture(); }; } -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP } else { rec->data[0] = new capture{std::forward(f)}; rec->free_data = [](function_record *r) { delete ((capture *) r->data[0]); }; @@ -1841,8 +1840,7 @@ class class_ : public detail::generic_type { if (holder_ptr) { init_holder_from_existing(v_h, holder_ptr, std::is_copy_constructible()); v_h.set_holder_constructed(); - } else if (PYBIND11_SILENCE_MSVC_C4127(detail::always_construct_holder::value) - || inst->owned) { + } else if (detail::always_construct_holder::value || inst->owned) { new (std::addressof(v_h.holder())) holder_type(v_h.value_ptr()); v_h.set_holder_constructed(); } @@ -2876,7 +2874,3 @@ inline function get_overload(const T *this_ptr, const char *name) { PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic pop // -Wnoexcept-type -#endif diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80b49ec397..f913565d36 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -33,6 +33,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + /* A few forward declarations */ class handle; class object; @@ -884,10 +886,8 @@ object object_or_cast(T &&o); // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(push) -# pragma warning(disable : 4522) // warning C4522: multiple assignment operators specified -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4522) // warning C4522: multiple assignment operators specified template class accessor : public object_api> { using key_type = typename Policy::key_type; @@ -951,9 +951,7 @@ class accessor : public object_api> { key_type key; mutable object cache; }; -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(pop) -#endif +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(accessor_policies) struct obj_attr { @@ -1693,7 +1691,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // unsigned type: (A)-1 != (B)-1 when A and B are unsigned types of different sizes). template Unsigned as_unsigned(PyObject *o) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(Unsigned) <= sizeof(unsigned long))) { + if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; } @@ -1710,7 +1708,7 @@ class int_ : public object { template ::value, int> = 0> // NOLINTNEXTLINE(google-explicit-constructor) int_(T value) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(T) <= sizeof(long))) { + if (sizeof(T) <= sizeof(long)) { if (std::is_signed::value) { m_ptr = PyLong_FromLong((long) value); } else { diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index 6ff63edfc6..0623b85dc9 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -73,6 +73,9 @@ PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) TEST_SUBMODULE(builtin_casters, m) { + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_MSVC(4127) + // test_simple_string m.def("string_roundtrip", [](const char *s) { return s; }); @@ -86,7 +89,7 @@ TEST_SUBMODULE(builtin_casters, m) { std::wstring wstr; wstr.push_back(0x61); // a wstr.push_back(0x2e18); // ⸘ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(wchar_t) == 2)) { + if (sizeof(wchar_t) == 2) { wstr.push_back(mathbfA16_1); wstr.push_back(mathbfA16_2); } // 𝐀, utf16 @@ -113,7 +116,7 @@ TEST_SUBMODULE(builtin_casters, m) { // Under Python 2.7, invalid unicode UTF-32 characters didn't appear to trigger // UnicodeDecodeError m.def("bad_utf32_string", [=]() { return std::u32string({a32, char32_t(0xd800), z32}); }); - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(wchar_t) == 2)) { + if (sizeof(wchar_t) == 2) { m.def("bad_wchar_string", [=]() { return std::wstring({wchar_t(0x61), wchar_t(0xd800)}); }); @@ -384,4 +387,6 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("takes_const_ref", [](const ConstRefCasted &x) { return x.tag; }); m.def("takes_const_ref_wrap", [](std::reference_wrapper x) { return x.get().tag; }); + + PYBIND11_WARNING_POP } diff --git a/tests/test_class.cpp b/tests/test_class.cpp index fc1c17b179..9ea10fae1e 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -22,10 +22,8 @@ #include -#if defined(_MSC_VER) -# pragma warning(disable : 4324) +PYBIND11_WARNING_DISABLE_MSVC(4324) // warning C4324: structure was padded due to alignment specifier -#endif // test_brace_initialization struct NoBraceInitialization { diff --git a/tests/test_constants_and_functions.cpp b/tests/test_constants_and_functions.cpp index 1918a429c8..922375c5ea 100644 --- a/tests/test_constants_and_functions.cpp +++ b/tests/test_constants_and_functions.cpp @@ -52,15 +52,12 @@ int f1(int x) noexcept { return x + 1; } #endif int f2(int x) noexcept(true) { return x + 2; } int f3(int x) noexcept(false) { return x + 3; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") +PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") // NOLINTNEXTLINE(modernize-use-noexcept) int f4(int x) throw() { return x + 4; } // Deprecated equivalent to noexcept(true) -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP struct C { int m1(int x) noexcept { return x - 1; } int m2(int x) const noexcept { return x - 2; } @@ -68,17 +65,14 @@ struct C { int m4(int x) const noexcept(true) { return x - 4; } int m5(int x) noexcept(false) { return x - 5; } int m6(int x) const noexcept(false) { return x - 6; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") + PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") // NOLINTNEXTLINE(modernize-use-noexcept) int m7(int x) throw() { return x - 7; } // NOLINTNEXTLINE(modernize-use-noexcept) int m8(int x) const throw() { return x - 8; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP }; } // namespace test_exc_sp @@ -126,14 +120,12 @@ TEST_SUBMODULE(constants_and_functions, m) { .def("m8", &C::m8); m.def("f1", f1); m.def("f2", f2); -#if defined(__INTEL_COMPILER) -# pragma warning push -# pragma warning disable 878 // incompatible exception specifications -#endif + + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_INTEL(878) // incompatible exception specifications m.def("f3", f3); -#if defined(__INTEL_COMPILER) -# pragma warning pop -#endif + PYBIND11_WARNING_POP + m.def("f4", f4); // test_function_record_leaks diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index 71e41d198a..554cc4d7f8 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -13,9 +13,7 @@ #include "constructor_stats.h" #include "pybind11_tests.h" -#if defined(_MSC_VER) -# pragma warning(disable : 4996) // C4996: std::unary_negation is deprecated -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) #include diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl index 09b35fa134..f46eb1803a 100644 --- a/tests/test_eigen_tensor.inl +++ b/tests/test_eigen_tensor.inl @@ -9,7 +9,9 @@ #include "pybind11_tests.h" -namespace PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE { +PYBIND11_NAMESPACE_BEGIN(PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) template void reset_tensor(M &x) { @@ -90,7 +92,7 @@ struct CustomExample { template void init_tensor_module(pybind11::module &m) { const char *needed_options = ""; - if (PYBIND11_SILENCE_MSVC_C4127(Options == Eigen::ColMajor)) { + if (Options == Eigen::ColMajor) { needed_options = "F"; } else { needed_options = "C"; @@ -330,4 +332,4 @@ void test_module(py::module_ &m) { init_tensor_module(c_style); } -} // namespace PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE +PYBIND11_NAMESPACE_END(PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE) diff --git a/tests/test_embed/catch.cpp b/tests/test_embed/catch.cpp index a03a8b37c4..558a7a35e5 100644 --- a/tests/test_embed/catch.cpp +++ b/tests/test_embed/catch.cpp @@ -3,11 +3,9 @@ #include -#ifdef _MSC_VER // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable : 4996) -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) // Catch uses _ internally, which breaks gettext style defines #ifdef _ diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 44dcd1fdb0..8cfd13fe52 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -1,10 +1,8 @@ #include -#ifdef _MSC_VER // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable : 4996) -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) #include #include diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 2f3cabaf0f..77e72c0c70 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -44,14 +44,13 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // test_args_and_kwargs m.def("args_function", [](py::args args) -> py::tuple { + PYBIND11_WARNING_PUSH + #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wreturn-std-move" + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") #endif return args; -#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic pop -#endif + PYBIND11_WARNING_POP }); m.def("args_kwargs_function", [](const py::args &args, const py::kwargs &kwargs) { return py::make_tuple(args, kwargs); diff --git a/tests/test_operator_overloading.cpp b/tests/test_operator_overloading.cpp index a4b895a897..112a363b4b 100644 --- a/tests/test_operator_overloading.cpp +++ b/tests/test_operator_overloading.cpp @@ -132,22 +132,18 @@ struct hash { // Not a good abs function, but easy to test. std::string abs(const Vector2 &) { return "abs(Vector2)"; } -// MSVC & Intel warns about unknown pragmas, and warnings are errors. -#if !defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push // clang 7.0.0 and Apple LLVM 10.0.1 introduce `-Wself-assign-overloaded` to // `-Wall`, which is used here for overloading (e.g. `py::self += py::self `). -// Here, we suppress the warning using `#pragma diagnostic`. +// Here, we suppress the warning // Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 // TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). -# if defined(__APPLE__) && defined(__clang__) -# if (__clang_major__ >= 10) -# pragma GCC diagnostic ignored "-Wself-assign-overloaded" -# endif -# elif defined(__clang__) -# if (__clang_major__ >= 7) -# pragma GCC diagnostic ignored "-Wself-assign-overloaded" -# endif +#if defined(__APPLE__) && defined(__clang__) +# if (__clang_major__ >= 10) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") +# endif +#elif defined(__clang__) +# if (__clang_major__ >= 7) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") # endif #endif @@ -283,6 +279,3 @@ TEST_SUBMODULE(operators, m) { m.def("get_unhashable_HashMe_set", []() { return std::unordered_set{{"one"}}; }); } -#if !defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif From 88699849264150f513dafb98d9ec1bb31a7fa904 Mon Sep 17 00:00:00 2001 From: Arman Date: Thu, 1 Dec 2022 07:17:59 +0200 Subject: [PATCH 030/137] scoped_interpreter. overloaded constructor: PyConfig param (#4330) * scoped_interpreter overloaded ctor: PyConfig param * style: pre-commit fixes * refact: some logics extracted into funcs (precheck_interpreter, _initialize_interpreter); config_guard * style: pre-commit fixes * refact: scoped_config, some funcs hidden in detail ns * refact: macro PYBIND11_PYCONFIG_SUPPORT_PY_VERSION + undef * feat: PYBIND11_PYCONFIG_SUPPORT_PY_VERSION set to 3.8 * tests: Custom PyConfig * ci: python 3.6 -> 3.8 * ci: reverted py 38 back to 36; refact: initialize_interpreter overloads * style: pre-commit fixes * fix: readability-implicit-bool-conversion * refact: each initialize_interpreter overloads in pybind11 ns * Move `initialize_interpreter_pre_pyconfig()` into the `detail` namespace. Move the `PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX` define down to where it is used for the first time, and check if it is defined already, so that it is possible to customize from the compilation command line, just in case there is some unforeseen issue for Python 3.8, 3.9, 3.10. * tests: Add program dir to path, Custom PyConfig with argv * refact: clang-formatted * tests: Add-program-dir-to-path covers both scoped_interpreter overloads * tests: Add-program-dir-to-path fixed * tests: Add-program-dir-to-path py_version dependant validation Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/embed.h | 119 +++++++++++++++++--------- tests/test_embed/test_interpreter.cpp | 66 ++++++++++++++ 2 files changed, 143 insertions(+), 42 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 5b77594b35..749c75beb6 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -86,37 +86,22 @@ inline wchar_t *widen_chars(const char *safe_arg) { return widened_arg; } -PYBIND11_NAMESPACE_END(detail) - -/** \rst - Initialize the Python interpreter. No other pybind11 or CPython API functions can be - called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The - optional `init_signal_handlers` parameter can be used to skip the registration of - signal handlers (see the `Python documentation`_ for details). Calling this function - again after the interpreter has already been initialized is a fatal error. - - If initializing the Python interpreter fails, then the program is terminated. (This - is controlled by the CPython runtime and is an exception to pybind11's normal behavior - of throwing exceptions on errors.) - - The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are - used to populate ``sys.argv`` and ``sys.path``. - See the |PySys_SetArgvEx documentation|_ for details. - - .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx - .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation - .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx - \endrst */ -inline void initialize_interpreter(bool init_signal_handlers = true, - int argc = 0, - const char *const *argv = nullptr, - bool add_program_dir_to_path = true) { +inline void precheck_interpreter() { if (Py_IsInitialized() != 0) { pybind11_fail("The interpreter is already running"); } +} -#if PY_VERSION_HEX < 0x030B0000 +#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX) +# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000) +#endif +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, + int argc, + const char *const *argv, + bool add_program_dir_to_path) { + detail::precheck_interpreter(); Py_InitializeEx(init_signal_handlers ? 1 : 0); # if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 PyEval_InitThreads(); @@ -150,26 +135,30 @@ inline void initialize_interpreter(bool init_signal_handlers = true, auto *pysys_argv = widened_argv.get(); PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -#else - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.isolated = 0; - config.use_environment = 1; - config.install_signal_handlers = init_signal_handlers ? 1 : 0; +} +#endif - PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); - if (PyStatus_Exception(status)) { +PYBIND11_NAMESPACE_END(detail) + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + detail::precheck_interpreter(); + PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast(argv)); + if (PyStatus_Exception(status) != 0) { // A failure here indicates a character-encoding failure or the python // interpreter out of memory. Give up. - PyConfig_Clear(&config); - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to prepare CPython"); + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to prepare CPython"); } - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to init CPython"); + status = Py_InitializeFromConfig(config); + if (PyStatus_Exception(status) != 0) { + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to init CPython"); } if (add_program_dir_to_path) { PyRun_SimpleString("import sys, os.path; " @@ -177,6 +166,43 @@ inline void initialize_interpreter(bool init_signal_handlers = true, "os.path.abspath(os.path.dirname(sys.argv[0])) " "if sys.argv and os.path.exists(sys.argv[0]) else '')"); } + PyConfig_Clear(config); +} +#endif + +/** \rst + Initialize the Python interpreter. No other pybind11 or CPython API functions can be + called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The + optional `init_signal_handlers` parameter can be used to skip the registration of + signal handlers (see the `Python documentation`_ for details). Calling this function + again after the interpreter has already been initialized is a fatal error. + + If initializing the Python interpreter fails, then the program is terminated. (This + is controlled by the CPython runtime and is an exception to pybind11's normal behavior + of throwing exceptions on errors.) + + The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are + used to populate ``sys.argv`` and ``sys.path``. + See the |PySys_SetArgvEx documentation|_ for details. + + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx + .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation + .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx + \endrst */ +inline void initialize_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + detail::initialize_interpreter_pre_pyconfig( + init_signal_handlers, argc, argv, add_program_dir_to_path); +#else + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + config.isolated = 0; + config.use_environment = 1; + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + initialize_interpreter(&config, argc, argv, add_program_dir_to_path); #endif } @@ -264,6 +290,15 @@ class scoped_interpreter { initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + explicit scoped_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(config, argc, argv, add_program_dir_to_path); + } +#endif + scoped_interpreter(const scoped_interpreter &) = delete; scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } scoped_interpreter &operator=(const scoped_interpreter &) = delete; diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 8cfd13fe52..5c306d3637 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -166,6 +166,72 @@ TEST_CASE("There can be only one interpreter") { py::initialize_interpreter(); } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Custom PyConfig") { + py::finalize_interpreter(); + PyConfig config; + PyConfig_InitPythonConfig(&config); + REQUIRE_NOTHROW(py::scoped_interpreter{&config}); + { + py::scoped_interpreter p{&config}; + REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast() == 42); + } + py::initialize_interpreter(); +} + +TEST_CASE("Custom PyConfig with argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + char *argv[] = {strdup("a.out")}; + py::scoped_interpreter argv_scope{&config, 1, argv}; + std::free(argv[0]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "a.out"); + } + py::initialize_interpreter(); +} +#endif + +TEST_CASE("Add program dir to path") { + static auto get_sys_path_size = []() -> size_t { + auto sys_path = py::module::import("sys").attr("path"); + return py::len(sys_path); + }; + static auto validate_path_len = [](size_t default_len) { +#if PY_VERSION_HEX < 0x030A0000 + // It seems a value remains in sys.path + // left by the previous call of scoped_interpreter ctor. + REQUIRE(get_sys_path_size() > default_len); +#else + REQUIRE(get_sys_path_size() == default_len + 1); +#endif + }; + py::finalize_interpreter(); + + size_t sys_path_default_size = 0; + { + py::scoped_interpreter scoped_interp{true, 0, nullptr, false}; + sys_path_default_size = get_sys_path_size(); + } + { + py::scoped_interpreter scoped_interp{}; // expected to append some to sys.path + validate_path_len(sys_path_default_size); + } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config}; // expected to append some to sys.path + validate_path_len(sys_path_default_size); + } +#endif + py::initialize_interpreter(); +} + bool has_pybind11_internals_builtin() { auto builtins = py::handle(PyEval_GetBuiltins()); return builtins.contains(PYBIND11_INTERNALS_ID); From b14d58b6151b3cc7a9d9b2bf90d636940935db5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 23:30:36 -0800 Subject: [PATCH 031/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.5.1 to 1.5.2 (#4370) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.5.1 to 1.5.2. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.5.1...v1.5.2) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index f03a397019..b787187ee0 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.5.2 with: password: ${{ secrets.pypi_password }} packages_dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.5.2 with: password: ${{ secrets.pypi_password_global }} packages_dir: global/ From 358ba459d2f05321a15555c3b57c606fe2597ec7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 1 Dec 2022 09:25:30 -0800 Subject: [PATCH 032/137] Fix test added with PR #4330 (#4372) --- tests/test_embed/test_interpreter.cpp | 47 ++++++++++++++------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 5c306d3637..10b20f3715 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -14,6 +14,11 @@ PYBIND11_WARNING_DISABLE_MSVC(4996) namespace py = pybind11; using namespace py::literals; +size_t get_sys_path_size() { + auto sys_path = py::module::import("sys").attr("path"); + return py::len(sys_path); +} + class Widget { public: explicit Widget(std::string message) : message(std::move(message)) {} @@ -196,41 +201,39 @@ TEST_CASE("Custom PyConfig with argv") { } #endif -TEST_CASE("Add program dir to path") { - static auto get_sys_path_size = []() -> size_t { - auto sys_path = py::module::import("sys").attr("path"); - return py::len(sys_path); - }; - static auto validate_path_len = [](size_t default_len) { -#if PY_VERSION_HEX < 0x030A0000 - // It seems a value remains in sys.path - // left by the previous call of scoped_interpreter ctor. - REQUIRE(get_sys_path_size() > default_len); -#else - REQUIRE(get_sys_path_size() == default_len + 1); -#endif - }; +TEST_CASE("Add program dir to path pre-PyConfig") { py::finalize_interpreter(); - - size_t sys_path_default_size = 0; + size_t path_size_add_program_dir_to_path_false = 0; { py::scoped_interpreter scoped_interp{true, 0, nullptr, false}; - sys_path_default_size = get_sys_path_size(); + path_size_add_program_dir_to_path_false = get_sys_path_size(); } { - py::scoped_interpreter scoped_interp{}; // expected to append some to sys.path - validate_path_len(sys_path_default_size); + py::scoped_interpreter scoped_interp{}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); } + py::initialize_interpreter(); +} + #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Add program dir to path using PyConfig") { + py::finalize_interpreter(); + size_t path_size_add_program_dir_to_path_false = 0; { PyConfig config; PyConfig_InitPythonConfig(&config); - py::scoped_interpreter scoped_interp{&config}; // expected to append some to sys.path - validate_path_len(sys_path_default_size); + py::scoped_interpreter scoped_interp{&config, 0, nullptr, false}; + path_size_add_program_dir_to_path_false = get_sys_path_size(); + } + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); } -#endif py::initialize_interpreter(); } +#endif bool has_pybind11_internals_builtin() { auto builtins = py::handle(PyEval_GetBuiltins()); From e133c33d5c6b19acab55fb1d6c10331d918bd830 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Thu, 1 Dec 2022 15:15:47 -0500 Subject: [PATCH 033/137] chore: Convert direct multiprocessing.set_start_method("forkserver") call to a pytest fixture. (#4377) * chore: convert multiprocessing set_spawn to fixture in pytest * Switch to early return --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 96dffc81cc..402fd4b25b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,12 @@ # Early diagnostic for failed imports import pybind11_tests -if os.name != "nt": + +@pytest.fixture(scope="session", autouse=True) +def always_forkserver_on_unix(): + if os.name == "nt": + return + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. # It is actually a well-known pitfall, unfortunately without guard rails. @@ -27,6 +32,7 @@ # running with defaults. multiprocessing.set_start_method("forkserver") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") From 5b55f8fe8261dc50397e26b08435cd27a9e1f89c Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 4 Dec 2022 19:39:25 -0800 Subject: [PATCH 034/137] Replace `ubuntu-latest` with `ubuntu-22.04` (#4382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Replace 🐍 3.6 β€’ ubuntu-latest β€’ x64 -DPYBIND11_FINDPYTHON=ON -DCMAKE_CXX_FLAGS="-D_=1" with 3.9 * Revert "Replace 🐍 3.6 β€’ ubuntu-latest β€’ x64 -DPYBIND11_FINDPYTHON=ON -DCMAKE_CXX_FLAGS="-D_=1" with 3.9" This reverts commit 3ec984e1ed7265ac152e6760c68a0bad97d5a2bd. * Systematically change all active ubuntu-latest to ubuntu-20.04, except in upstream.yml * Revert "Systematically change all active ubuntu-latest to ubuntu-20.04, except in upstream.yml" This reverts commit cdfd99526a9d3c172691685ac17054bfe76590fc. * Change only some ubuntu-latest to ubuntu-20.04 --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/configure.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd99ddd338..a6f2624dba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - runs-on: [ubuntu-latest, windows-2022, macos-latest] + runs-on: [ubuntu-20.04, windows-2022, macos-latest] python: - '3.6' - '3.9' @@ -42,12 +42,12 @@ jobs: # We support an optional key: args, for cmake args include: # Just add a key - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 python: '3.6' args: > -DPYBIND11_FINDPYTHON=ON -DCMAKE_CXX_FLAGS="-D_=1" - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 python: 'pypy-3.8' args: > -DPYBIND11_FINDPYTHON=ON @@ -194,7 +194,7 @@ jobs: python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' β€’ Valgrind' || '' }} β€’ x64" - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 5ec0dd462f..05e972c754 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -16,12 +16,12 @@ jobs: strategy: fail-fast: false matrix: - runs-on: [ubuntu-latest, macos-latest, windows-latest] + runs-on: [ubuntu-20.04, macos-latest, windows-latest] arch: [x64] cmake: ["3.23"] include: - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 arch: x64 cmake: 3.4 From 6a1023e3e9ba32f7a9c4f59dfa8ea3f9e8dec341 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 01:01:45 -0800 Subject: [PATCH 035/137] chore(deps): bump deadsnakes/action from 2.1.1 to 3.0.0 (#4383) Bumps [deadsnakes/action](https://github.com/deadsnakes/action) from 2.1.1 to 3.0.0. - [Release notes](https://github.com/deadsnakes/action/releases) - [Commits](https://github.com/deadsnakes/action/compare/v2.1.1...v3.0.0) --- updated-dependencies: - dependency-name: deadsnakes/action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6f2624dba..1c6a63f409 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,7 +200,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v2.1.1 + uses: deadsnakes/action@v3.0.0 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} From a6e75e4d8af51151587c13236fdc8570023fb57c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 01:03:05 -0800 Subject: [PATCH 036/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.5.2 to 1.6.1 (#4384) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.5.2 to 1.6.1. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.5.2...v1.6.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index b787187ee0..1f4422d138 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.5.2 + uses: pypa/gh-action-pypi-publish@v1.6.1 with: password: ${{ secrets.pypi_password }} packages_dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.5.2 + uses: pypa/gh-action-pypi-publish@v1.6.1 with: password: ${{ secrets.pypi_password_global }} packages_dir: global/ From a672de7cc88c053b77da0ae48f0f1a9b3afb86bb Mon Sep 17 00:00:00 2001 From: luzpaz Date: Tue, 6 Dec 2022 12:54:15 -0500 Subject: [PATCH 037/137] Fix source comment typo (#4388) --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index a129f99292..568c641a2d 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -279,7 +279,7 @@ struct type_caster::ValidType> { case return_value_policy::take_ownership: if (std::is_const::value) { // This cast is ugly, and might be UB in some cases, but we don't have an - // alterantive here as we must free that memory + // alternative here as we must free that memory Helper::free(const_cast(src)); pybind11_fail("Cannot take ownership of a const reference"); } From 4768a6f8f5e1abe106b4e3c9899d5866d88d77f6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Dec 2022 10:10:48 -0800 Subject: [PATCH 038/137] chore(deps): update pre-commit hooks (#4386) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/pre-commit/pre-commit-hooks: v4.3.0 β†’ v4.4.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.3.0...v4.4.0) - [github.com/asottile/pyupgrade: v3.2.0 β†’ v3.3.0](https://github.com/asottile/pyupgrade/compare/v3.2.0...v3.3.0) - [github.com/hadialqattan/pycln: v2.1.1 β†’ v2.1.2](https://github.com/hadialqattan/pycln/compare/v2.1.1...v2.1.2) - [github.com/PyCQA/flake8: 5.0.4 β†’ 6.0.0](https://github.com/PyCQA/flake8/compare/5.0.4...6.0.0) - [github.com/PyCQA/pylint: v2.15.5 β†’ v2.15.8](https://github.com/PyCQA/pylint/compare/v2.15.5...v2.15.8) - [github.com/pre-commit/mirrors-mypy: v0.982 β†’ v0.991](https://github.com/pre-commit/mirrors-mypy/compare/v0.982...v0.991) - [github.com/mgedmin/check-manifest: 0.48 β†’ 0.49](https://github.com/mgedmin/check-manifest/compare/0.48...0.49) - [github.com/pre-commit/mirrors-clang-format: v14.0.6 β†’ v15.0.4](https://github.com/pre-commit/mirrors-clang-format/compare/v14.0.6...v15.0.4) * style: pre-commit fixes Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 16 ++++++++-------- include/pybind11/cast.h | 3 ++- include/pybind11/detail/common.h | 6 +++--- include/pybind11/detail/init.h | 27 +++++++++++++++------------ include/pybind11/numpy.h | 2 +- include/pybind11/pybind11.h | 6 +++--- include/pybind11/pytypes.h | 3 ++- tests/test_virtual_functions.cpp | 3 ++- 8 files changed, 36 insertions(+), 30 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fd079dd037..4e80fbad15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ exclude: ^tools/JoinPaths.cmake$ repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.3.0" + rev: "v4.4.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -41,7 +41,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v3.2.0" + rev: "v3.3.0" hooks: - id: pyupgrade args: [--py36-plus] @@ -80,7 +80,7 @@ repos: # Autoremoves unused imports - repo: https://github.com/hadialqattan/pycln - rev: "v2.1.1" + rev: "v2.1.2" hooks: - id: pycln stages: [manual] @@ -108,7 +108,7 @@ repos: # Flake8 also supports pre-commit natively (same author) - repo: https://github.com/PyCQA/flake8 - rev: "5.0.4" + rev: "6.0.0" hooks: - id: flake8 exclude: ^(docs/.*|tools/.*)$ @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.5" + rev: "v2.15.8" hooks: - id: pylint files: ^pybind11 @@ -132,7 +132,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.982" + rev: "v0.991" hooks: - id: mypy args: [] @@ -141,7 +141,7 @@ repos: # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.48" + rev: "0.49" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v14.0.6" + rev: "v15.0.4" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 8f92df8ab3..3a40460276 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -91,7 +91,8 @@ public: template >::value, \ - int> = 0> \ + int> \ + = 0> \ static ::pybind11::handle cast( \ T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ if (!src) \ diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index b58dc3afa7..6e295fd2f8 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -34,17 +34,17 @@ # define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) #elif defined(__INTEL_COMPILER) # define PYBIND11_COMPILER_INTEL -# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) # define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) # define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) #elif defined(__clang__) # define PYBIND11_COMPILER_CLANG -# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) # define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) # define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) #elif defined(__GNUC__) # define PYBIND11_COMPILER_GCC -# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) # define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) # define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) #endif diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 0938c9bdeb..9f71278c26 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -209,10 +209,11 @@ struct constructor { extra...); } - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -229,10 +230,11 @@ struct constructor { extra...); } - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -248,10 +250,11 @@ struct constructor { // Implementing class for py::init_alias<...>() template struct alias_constructor { - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index ea64aa7e7e..8f072af267 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1471,7 +1471,7 @@ struct npy_format_descriptor { } // Extract name, offset and format descriptor for a struct field -# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, # Field) +# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 10b27254eb..a7310dfd4d 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1426,9 +1426,9 @@ template ::value, int> = 0> void call_operator_delete(T *p, size_t, size_t) { T::operator delete(p); } -template < - typename T, - enable_if_t::value && has_operator_delete_size::value, int> = 0> +template ::value && has_operator_delete_size::value, int> + = 0> void call_operator_delete(T *p, size_t s, size_t) { T::operator delete(p, s); } diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f913565d36..3282879312 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -232,7 +232,8 @@ class handle : public detail::object_api { detail::enable_if_t, detail::is_pyobj_ptr_or_nullptr_t>, std::is_convertible>::value, - int> = 0> + int> + = 0> // NOLINTNEXTLINE(google-explicit-constructor) handle(T &obj) : m_ptr(obj) {} diff --git a/tests/test_virtual_functions.cpp b/tests/test_virtual_functions.cpp index 323aa0d22d..93b136ad3c 100644 --- a/tests/test_virtual_functions.cpp +++ b/tests/test_virtual_functions.cpp @@ -173,7 +173,8 @@ struct AdderBase { using DataVisitor = std::function; virtual void - operator()(const Data &first, const Data &second, const DataVisitor &visitor) const = 0; + operator()(const Data &first, const Data &second, const DataVisitor &visitor) const + = 0; virtual ~AdderBase() = default; AdderBase() = default; AdderBase(const AdderBase &) = delete; From 65cc9d2a291c983c289b35afe970bdda5710d706 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Dec 2022 23:36:11 -0500 Subject: [PATCH 039/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.6.1 to 1.6.4 (#4389) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.6.1 to 1.6.4. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.6.1...v1.6.4) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 1f4422d138..7c6fc67a3e 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.6.1 + uses: pypa/gh-action-pypi-publish@v1.6.4 with: password: ${{ secrets.pypi_password }} packages_dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.6.1 + uses: pypa/gh-action-pypi-publish@v1.6.4 with: password: ${{ secrets.pypi_password_global }} packages_dir: global/ From 65374c8e62488c557dccf70c9276e3f9fe04b007 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 8 Dec 2022 22:06:51 -0800 Subject: [PATCH 040/137] `pybind11::handle` `inc_ref()` & `dec_ref()` `PyGILState_Check()` **excluding** `nullptr` (#4246) * pybind11/pytypes.h `inc_ref()`, `dec_ref()` `PyGILState_Check()` **excluding** `nullptr` Guarded by `PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF` * Disable `PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF` for PyPy under Windows. * Add reference to PR #4268 (PyPy Windows) --- include/pybind11/detail/common.h | 9 +++++++++ include/pybind11/pytypes.h | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 6e295fd2f8..9acf104668 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -323,6 +323,15 @@ PYBIND11_WARNING_POP # define PYBIND11_HAS_U8STRING #endif +// See description of PR #4246: +#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) \ + && !(defined(PYPY_VERSION) \ + && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the \ + process (see PR #4268) */ \ + && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +# define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 3282879312..56e0423ac4 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -249,6 +249,11 @@ class handle : public detail::object_api { const handle &inc_ref() const & { #ifdef PYBIND11_HANDLE_REF_DEBUG inc_ref_counter(1); +#endif +#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) + if (m_ptr != nullptr && !PyGILState_Check()) { + throw std::runtime_error("pybind11::handle::inc_ref() PyGILState_Check() failure."); + } #endif Py_XINCREF(m_ptr); return *this; @@ -260,6 +265,11 @@ class handle : public detail::object_api { this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { +#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) + if (m_ptr != nullptr && !PyGILState_Check()) { + throw std::runtime_error("pybind11::handle::dec_ref() PyGILState_Check() failure."); + } +#endif Py_XDECREF(m_ptr); return *this; } From 00126859a550d10fb9f5bc0d4fdf86988c2012f4 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 9 Dec 2022 08:10:10 +0100 Subject: [PATCH 041/137] Add option for enable/disable enum members in docstring. (#2768) * Add option for enable/disable enum members in docstring * Add tests for disable enum members docstring option * Add docstring options to documentation * style: pre-commit fixes * Fix typos in documentation * Improve documentation wording * Apply suggestions by @Skylion007 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- docs/advanced/misc.rst | 9 ++++++ include/pybind11/options.h | 16 ++++++++++ include/pybind11/pybind11.h | 50 +++++++++++++++++------------- tests/test_docstring_options.cpp | 53 ++++++++++++++++++++++++++++++++ tests/test_docstring_options.py | 23 ++++++++++++++ 5 files changed, 129 insertions(+), 22 deletions(-) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 71960b803d..35a6ebcd60 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -324,6 +324,15 @@ The class ``options`` allows you to selectively suppress auto-generated signatur m.def("add", [](int a, int b) { return a + b; }, "A function which adds two numbers"); } +pybind11 also appends all members of an enum to the resulting enum docstring. +This default behavior can be disabled by using the ``disable_enum_members_docstring()`` +function of the ``options`` class. + +With ``disable_user_defined_docstrings()`` all user defined docstrings of +``module_::def()``, ``class_::def()`` and ``enum_()`` are disabled, but the +function signatures and enum members are included in the docstring, unless they +are disabled separately. + Note that changes to the settings affect only function bindings created during the lifetime of the ``options`` instance. When it goes out of scope at the end of the module's init function, the default settings are restored to prevent unwanted side effects. diff --git a/include/pybind11/options.h b/include/pybind11/options.h index 1e493bdcc4..1b2122522d 100644 --- a/include/pybind11/options.h +++ b/include/pybind11/options.h @@ -47,6 +47,16 @@ class options { return *this; } + options &disable_enum_members_docstring() & { + global_state().show_enum_members_docstring = false; + return *this; + } + + options &enable_enum_members_docstring() & { + global_state().show_enum_members_docstring = true; + return *this; + } + // Getter methods (return the global state): static bool show_user_defined_docstrings() { @@ -55,6 +65,10 @@ class options { static bool show_function_signatures() { return global_state().show_function_signatures; } + static bool show_enum_members_docstring() { + return global_state().show_enum_members_docstring; + } + // This type is not meant to be allocated on the heap. void *operator new(size_t) = delete; @@ -63,6 +77,8 @@ class options { bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings. bool show_function_signatures = true; //< Include auto-generated function signatures // in docstrings. + bool show_enum_members_docstring = true; //< Include auto-generated member list in enum + // docstrings. }; static state &global_state() { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index a7310dfd4d..6205effd61 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1972,29 +1972,35 @@ struct enum_base { name("name"), is_method(m_base)); - m_base.attr("__doc__") = static_property( - cpp_function( - [](handle arg) -> std::string { - std::string docstring; - dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) { - docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; - } - docstring += "Members:"; - for (auto kv : entries) { - auto key = std::string(pybind11::str(kv.first)); - auto comment = kv.second[int_(1)]; - docstring += "\n\n " + key; - if (!comment.is_none()) { - docstring += " : " + (std::string) pybind11::str(comment); + if (options::show_enum_members_docstring()) { + m_base.attr("__doc__") = static_property( + cpp_function( + [](handle arg) -> std::string { + std::string docstring; + dict entries = arg.attr("__entries"); + if (((PyTypeObject *) arg.ptr())->tp_doc) { + docstring += std::string( + reinterpret_cast(arg.ptr())->tp_doc); + docstring += "\n\n"; } - } - return docstring; - }, - name("__doc__")), - none(), - none(), - ""); + docstring += "Members:"; + for (auto kv : entries) { + auto key = std::string(pybind11::str(kv.first)); + auto comment = kv.second[int_(1)]; + docstring += "\n\n "; + docstring += key; + if (!comment.is_none()) { + docstring += " : "; + docstring += pybind11::str(comment).cast(); + } + } + return docstring; + }, + name("__doc__")), + none(), + none(), + ""); + } m_base.attr("__members__") = static_property(cpp_function( [](handle arg) -> dict { diff --git a/tests/test_docstring_options.cpp b/tests/test_docstring_options.cpp index 4d44f4e20d..dda1cf6e41 100644 --- a/tests/test_docstring_options.cpp +++ b/tests/test_docstring_options.cpp @@ -85,4 +85,57 @@ TEST_SUBMODULE(docstring_options, m) { &DocstringTestFoo::setValue, "This is a property docstring"); } + + { + enum class DocstringTestEnum1 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum1", "Enum docstring") + .value("Member1", DocstringTestEnum1::Member1) + .value("Member2", DocstringTestEnum1::Member2); + } + + { + py::options options; + options.enable_enum_members_docstring(); + + enum class DocstringTestEnum2 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum2", "Enum docstring") + .value("Member1", DocstringTestEnum2::Member1) + .value("Member2", DocstringTestEnum2::Member2); + } + + { + py::options options; + options.disable_enum_members_docstring(); + + enum class DocstringTestEnum3 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum3", "Enum docstring") + .value("Member1", DocstringTestEnum3::Member1) + .value("Member2", DocstringTestEnum3::Member2); + } + + { + py::options options; + options.disable_user_defined_docstrings(); + + enum class DocstringTestEnum4 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum4", "Enum docstring") + .value("Member1", DocstringTestEnum4::Member1) + .value("Member2", DocstringTestEnum4::Member2); + } + + { + py::options options; + options.disable_user_defined_docstrings(); + options.disable_enum_members_docstring(); + + enum class DocstringTestEnum5 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum5", "Enum docstring") + .value("Member1", DocstringTestEnum5::Member1) + .value("Member2", DocstringTestEnum5::Member2); + } } diff --git a/tests/test_docstring_options.py b/tests/test_docstring_options.py index fcd16b89fd..e6f5a9d987 100644 --- a/tests/test_docstring_options.py +++ b/tests/test_docstring_options.py @@ -39,3 +39,26 @@ def test_docstring_options(): # Suppression of user-defined docstrings for non-function objects assert not m.DocstringTestFoo.__doc__ assert not m.DocstringTestFoo.value_prop.__doc__ + + # Check existig behaviour of enum docstings + assert ( + m.DocstringTestEnum1.__doc__ + == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" + ) + + # options.enable_enum_members_docstring() + assert ( + m.DocstringTestEnum2.__doc__ + == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" + ) + + # options.disable_enum_members_docstring() + assert m.DocstringTestEnum3.__doc__ == "Enum docstring" + + # options.disable_user_defined_docstrings() + assert m.DocstringTestEnum4.__doc__ == "Members:\n\n Member1\n\n Member2" + + # options.disable_user_defined_docstrings() + # options.disable_enum_members_docstring() + # When all options are disabled, no docstring (instead of an empty one) should be generated + assert m.DocstringTestEnum5.__doc__ is None From 9db988013ceb54ab15fb775b229ac9180fd08fbe Mon Sep 17 00:00:00 2001 From: aimir <48388610+aimir@users.noreply.github.com> Date: Fri, 9 Dec 2022 09:15:11 +0200 Subject: [PATCH 042/137] Correct class names for KeysView, ValuesView and ItemsView in bind_map (#4353) * Create templated abstract classes KeysView, ValuesView and ItemsView, and implement them on-the-fly when wrapping any specific map type * We don't want to wrap different ValuesView objects for double values and const double, for example, as both wrappers will be named ValuesView[float] * Fallback to C++ names if key or values types are not wrapped * Added a test for .keys(), .values() and .items() returning the same types for similarly-typed maps * Fixed wrong use of auto in a declarator list: the two descriptions might have different types * Fixes for clang-tidy issues: explicit single-argument constructor, using the 'override' keyword when overriding functions * Bugfix for old versions of clang++, which seem to have trouble with the struct being defined inside a module, which was also needlessly ugly anyway * Bugfix for clang++, which doesn't have some of the names in runtime uness they are specified to be static * A fix for clang-tidy performance-inefficient-string-concatenation issues - I personally think this looks uglier, but it's probably worth it for clang-tidy to be happy * Possible fix for clang++ linking issues - make the descriptions static constexpr to make sure they are known before linking * Correct names for previously-wrapped types as keys/values of maps * Bugfix - typo in type info names which caused things to segfault * Apply suggestions from code review Co-authored-by: Aaron Gokaslan * Use detail::remove_cvref_t instead of doing remove_cv and remove_reference separately * Avoid names with double underscore, as they are reserved * Improved testing for KeysView, ValuesView and ItemsView: check type names + stricter asserts * Moved description logic to helper function in type_caster_base.h * style: pre-commit fixes * Fix a clang-tidy issue: do not use 'else' after 'return' * Apply suggestion by @Skylion007, with additional trivial simplification. Co-authored-by: Amir Co-authored-by: aimir Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/detail/type_caster_base.h | 9 ++ include/pybind11/stl_bind.h | 154 ++++++++++++++------- tests/test_stl_binders.py | 26 ++++ 3 files changed, 142 insertions(+), 47 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 21f69c289f..0b710d7e4c 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -1006,5 +1006,14 @@ class type_caster_base : public type_caster_generic { static Constructor make_move_constructor(...) { return nullptr; } }; +PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { + if (auto *type_data = get_type_info(ti)) { + handle th((PyObject *) type_data->type); + return th.attr("__module__").cast() + '.' + + th.attr("__qualname__").cast(); + } + return clean_type_id(ti.name()); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 22a29b476f..0c634597ec 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -10,10 +10,13 @@ #pragma once #include "detail/common.h" +#include "detail/type_caster_base.h" +#include "cast.h" #include "operators.h" #include #include +#include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -636,18 +639,52 @@ auto map_if_insertion_operator(Class_ &cl, std::string const &name) "Return the canonical string representation of this map."); } -template +template struct keys_view { - Map ↦ + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual bool contains(const KeyType &k) = 0; + virtual bool contains(const object &k) = 0; + virtual ~keys_view() = default; }; -template +template struct values_view { - Map ↦ + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~values_view() = default; }; -template +template struct items_view { + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~items_view() = default; +}; + +template +struct KeysViewImpl : public KeysView { + explicit KeysViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_key_iterator(map.begin(), map.end()); } + bool contains(const typename Map::key_type &k) override { return map.find(k) != map.end(); } + bool contains(const object &) override { return false; } + Map ↦ +}; + +template +struct ValuesViewImpl : public ValuesView { + explicit ValuesViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_value_iterator(map.begin(), map.end()); } + Map ↦ +}; + +template +struct ItemsViewImpl : public ItemsView { + explicit ItemsViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_iterator(map.begin(), map.end()); } Map ↦ }; @@ -657,9 +694,11 @@ template , typename... class_ bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - using KeysView = detail::keys_view; - using ValuesView = detail::values_view; - using ItemsView = detail::items_view; + using StrippedKeyType = detail::remove_cvref_t; + using StrippedMappedType = detail::remove_cvref_t; + using KeysView = detail::keys_view; + using ValuesView = detail::values_view; + using ItemsView = detail::items_view; using Class_ = class_; // If either type is a non-module-local bound type then make the map binding non-local as well; @@ -673,12 +712,57 @@ class_ bind_map(handle scope, const std::string &name, Args && } Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward(args)...); - class_ keys_view( - scope, ("KeysView[" + name + "]").c_str(), pybind11::module_local(local)); - class_ values_view( - scope, ("ValuesView[" + name + "]").c_str(), pybind11::module_local(local)); - class_ items_view( - scope, ("ItemsView[" + name + "]").c_str(), pybind11::module_local(local)); + static constexpr auto key_type_descr = detail::make_caster::name; + static constexpr auto mapped_type_descr = detail::make_caster::name; + std::string key_type_name(key_type_descr.text), mapped_type_name(mapped_type_descr.text); + + // If key type isn't properly wrapped, fall back to C++ names + if (key_type_name == "%") { + key_type_name = detail::type_info_description(typeid(KeyType)); + } + // Similarly for value type: + if (mapped_type_name == "%") { + mapped_type_name = detail::type_info_description(typeid(MappedType)); + } + + // Wrap KeysView[KeyType] if it wasn't already wrapped + if (!detail::get_type_info(typeid(KeysView))) { + class_ keys_view( + scope, ("KeysView[" + key_type_name + "]").c_str(), pybind11::module_local(local)); + keys_view.def("__len__", &KeysView::len); + keys_view.def("__iter__", + &KeysView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + keys_view.def("__contains__", + static_cast(&KeysView::contains)); + // Fallback for when the object is not of the key type + keys_view.def("__contains__", + static_cast(&KeysView::contains)); + } + // Similarly for ValuesView: + if (!detail::get_type_info(typeid(ValuesView))) { + class_ values_view(scope, + ("ValuesView[" + mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + values_view.def("__len__", &ValuesView::len); + values_view.def("__iter__", + &ValuesView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } + // Similarly for ItemsView: + if (!detail::get_type_info(typeid(ItemsView))) { + class_ items_view( + scope, + ("ItemsView[" + key_type_name + ", ").append(mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + items_view.def("__len__", &ItemsView::len); + items_view.def("__iter__", + &ItemsView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } cl.def(init<>()); @@ -698,19 +782,25 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def( "keys", - [](Map &m) { return KeysView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::KeysViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "values", - [](Map &m) { return ValuesView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::ValuesViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "items", - [](Map &m) { return ItemsView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::ItemsViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); @@ -749,36 +839,6 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def("__len__", &Map::size); - keys_view.def("__len__", [](KeysView &view) { return view.map.size(); }); - keys_view.def( - "__iter__", - [](KeysView &view) { return make_key_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - keys_view.def("__contains__", [](KeysView &view, const KeyType &k) -> bool { - auto it = view.map.find(k); - if (it == view.map.end()) { - return false; - } - return true; - }); - // Fallback for when the object is not of the key type - keys_view.def("__contains__", [](KeysView &, const object &) -> bool { return false; }); - - values_view.def("__len__", [](ValuesView &view) { return view.map.size(); }); - values_view.def( - "__iter__", - [](ValuesView &view) { return make_value_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - - items_view.def("__len__", [](ItemsView &view) { return view.map.size(); }); - items_view.def( - "__iter__", - [](ItemsView &view) { return make_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - return cl; } diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index d5e9ccced7..9eb906f065 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -309,3 +309,29 @@ def test_map_delitem(): del um["ua"] assert sorted(list(um)) == ["ub"] assert sorted(list(um.items())) == [("ub", 2.6)] + + +def test_map_view_types(): + map_string_double = m.MapStringDouble() + unordered_map_string_double = m.UnorderedMapStringDouble() + map_string_double_const = m.MapStringDoubleConst() + unordered_map_string_double_const = m.UnorderedMapStringDoubleConst() + + assert map_string_double.keys().__class__.__name__ == "KeysView[str]" + assert map_string_double.values().__class__.__name__ == "ValuesView[float]" + assert map_string_double.items().__class__.__name__ == "ItemsView[str, float]" + + keys_type = type(map_string_double.keys()) + assert type(unordered_map_string_double.keys()) is keys_type + assert type(map_string_double_const.keys()) is keys_type + assert type(unordered_map_string_double_const.keys()) is keys_type + + values_type = type(map_string_double.values()) + assert type(unordered_map_string_double.values()) is values_type + assert type(map_string_double_const.values()) is values_type + assert type(unordered_map_string_double_const.values()) is values_type + + items_type = type(map_string_double.items()) + assert type(unordered_map_string_double.items()) is items_type + assert type(map_string_double_const.items()) is items_type + assert type(unordered_map_string_double_const.items()) is items_type From 663b86c26ca07c51f2981681ba79e2595e17fbb9 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 9 Dec 2022 10:53:03 -0800 Subject: [PATCH 043/137] Add flake8 `B905` to `extend-ignore` in setup.cfg (#4391) * Add flake8 `--ignore=B905,N818,W503` * Add B905 to `extend-ignore` in setup.cfg (thanks @Skylion007), leave .pre-commit-config.yaml as-is on master. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9af50ea483..16479ab000 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,5 +46,5 @@ zip_safe = False max-line-length = 120 show_source = True exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722, B903, B950 +extend-ignore = E203, E722, B903, B905, B950 extend-select = B9 From 9727dcdae5b723a01ec04bec5eb1a5a681cbb0fd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 9 Dec 2022 23:18:36 -0500 Subject: [PATCH 044/137] chore: future safe bugbear opinionated warnings (#4393) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 16479ab000..8b3361981a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,5 +46,5 @@ zip_safe = False max-line-length = 120 show_source = True exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722, B903, B905, B950 -extend-select = B9 +extend-ignore = E203, E722 +extend-select = B902, B904 From ff42f5254a2f1315d3b56e2692590f1ce10b8193 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 13 Dec 2022 11:04:05 -0800 Subject: [PATCH 045/137] Systematically add `-DCMAKE_VERBOSE_MAKEFILE=ON` to obtain full command lines related to `-Wodr` (#4398) --- .github/workflows/ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c6a63f409..cd0f33d279 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,7 @@ jobs: - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON @@ -135,6 +136,7 @@ jobs: - name: Configure C++17 run: > cmake -S . -B build2 + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON @@ -157,6 +159,7 @@ jobs: - name: Configure (unstable ABI) run: > cmake -S . -B build3 + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -243,6 +246,7 @@ jobs: SETUPTOOLS_USE_DISTUTILS: stdlib run: > cmake -S . -B build + -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON @@ -307,6 +311,7 @@ jobs: shell: bash run: > cmake -S . -B build + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=${{ matrix.std }} @@ -339,7 +344,7 @@ jobs: run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy - name: Configure - run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + run: cmake -S . -B build -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON - name: Build run: cmake --build build -j2 --verbose @@ -377,7 +382,7 @@ jobs: # run: | # source /etc/profile.d/modules.sh # module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.11 -# cmake -S . -B build -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") +# cmake -S . -B build -DCMAKE_VERBOSE_MAKEFILE=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") # # - name: Build # run: cmake --build build -j 2 --verbose @@ -475,6 +480,7 @@ jobs: shell: bash run: > cmake -S . -B build + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=${{ matrix.std }} @@ -529,6 +535,7 @@ jobs: run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e cmake -S . -B build-11 \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ -DPYBIND11_WERROR=ON \ -DDOWNLOAD_CATCH=ON \ -DDOWNLOAD_EIGEN=OFF \ @@ -561,6 +568,7 @@ jobs: run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e cmake -S . -B build-17 \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ -DPYBIND11_WERROR=ON \ -DDOWNLOAD_CATCH=ON \ -DDOWNLOAD_EIGEN=OFF \ @@ -627,6 +635,7 @@ jobs: shell: bash run: > cmake -S . -B build + -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=MinSizeRel -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON @@ -774,6 +783,7 @@ jobs: run: > cmake -S . -B build -G "Visual Studio 16 2019" -A Win32 + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -827,6 +837,7 @@ jobs: run: > cmake -S . -B build -G "Visual Studio 16 2019" -A Win32 + -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON @@ -867,6 +878,7 @@ jobs: - name: Configure C++20 run: > cmake -S . -B build + -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON From 89c3561d9a217f494900e82cd35723402c7d3e7c Mon Sep 17 00:00:00 2001 From: Martin Blanchard <49144287+mablanchard@users.noreply.github.com> Date: Wed, 14 Dec 2022 20:03:37 +0100 Subject: [PATCH 046/137] Fix multi-context new Python linking mode (#4401) Allow CMake find_package() from multiple directories. https://github.com/pybind/pybind11/issues/4400 Co-authored-by: Martin Blanchard --- tools/pybind11NewTools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 5a6a0cb8be..7d7424a790 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -9,7 +9,7 @@ if(CMAKE_VERSION VERSION_LESS 3.12) message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") endif() -include_guard(GLOBAL) +include_guard(DIRECTORY) get_property( is_config From ece1206b8ae4d1fc350d6f8129f7a098d223519d Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 15 Dec 2022 13:19:13 -0800 Subject: [PATCH 047/137] ci: set `env: VERBOSE: 1` (#4405) * Revert "Systematically add `-DCMAKE_VERBOSE_MAKEFILE=ON` to obtain full command lines related to `-Wodr` (#4398)" This reverts commit ff42f5254a2f1315d3b56e2692590f1ce10b8193. * Set `env: VERBOSE: 1` as suggested by @henryiii * Set `env: VERBOSE: 1` also in all other .yml files using cmake --- .github/workflows/ci.yml | 26 +++++++------------------- .github/workflows/configure.yml | 4 ++++ .github/workflows/format.yml | 2 ++ .github/workflows/upstream.yml | 2 ++ 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd0f33d279..af5a29d627 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ env: PIP_ONLY_BINARY: numpy FORCE_COLOR: 3 PYTEST_TIMEOUT: 300 + # For cmake: + VERBOSE: 1 jobs: # This is the "main" test suite, which tests a large number of different @@ -106,7 +108,6 @@ jobs: - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON @@ -136,7 +137,6 @@ jobs: - name: Configure C++17 run: > cmake -S . -B build2 - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON @@ -159,7 +159,6 @@ jobs: - name: Configure (unstable ABI) run: > cmake -S . -B build3 - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -246,7 +245,6 @@ jobs: SETUPTOOLS_USE_DISTUTILS: stdlib run: > cmake -S . -B build - -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON @@ -311,7 +309,6 @@ jobs: shell: bash run: > cmake -S . -B build - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=${{ matrix.std }} @@ -344,7 +341,7 @@ jobs: run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy - name: Configure - run: cmake -S . -B build -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON - name: Build run: cmake --build build -j2 --verbose @@ -382,7 +379,7 @@ jobs: # run: | # source /etc/profile.d/modules.sh # module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.11 -# cmake -S . -B build -DCMAKE_VERBOSE_MAKEFILE=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") +# cmake -S . -B build -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") # # - name: Build # run: cmake --build build -j 2 --verbose @@ -480,7 +477,6 @@ jobs: shell: bash run: > cmake -S . -B build - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=${{ matrix.std }} @@ -535,7 +531,6 @@ jobs: run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e cmake -S . -B build-11 \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ -DPYBIND11_WERROR=ON \ -DDOWNLOAD_CATCH=ON \ -DDOWNLOAD_EIGEN=OFF \ @@ -568,7 +563,6 @@ jobs: run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e cmake -S . -B build-17 \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ -DPYBIND11_WERROR=ON \ -DDOWNLOAD_CATCH=ON \ -DDOWNLOAD_EIGEN=OFF \ @@ -635,7 +629,6 @@ jobs: shell: bash run: > cmake -S . -B build - -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=MinSizeRel -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON @@ -783,7 +776,6 @@ jobs: run: > cmake -S . -B build -G "Visual Studio 16 2019" -A Win32 - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -837,7 +829,6 @@ jobs: run: > cmake -S . -B build -G "Visual Studio 16 2019" -A Win32 - -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON @@ -878,7 +869,6 @@ jobs: - name: Configure C++20 run: > cmake -S . -B build - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -930,7 +920,7 @@ jobs: - name: Configure C++11 # LTO leads to many undefined reference like # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&) - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build - name: Build C++11 run: cmake --build build -j 2 @@ -948,7 +938,7 @@ jobs: run: git clean -fdx - name: Configure C++14 - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2 - name: Build C++14 run: cmake --build build2 -j 2 @@ -966,7 +956,7 @@ jobs: run: git clean -fdx - name: Configure C++17 - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3 - name: Build C++17 run: cmake --build build3 -j 2 @@ -1027,7 +1017,6 @@ jobs: - name: Configure Clang run: > cmake -G Ninja -S . -B . - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=OFF -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON @@ -1093,7 +1082,6 @@ jobs: - name: CMake Configure run: > cmake -S . -B . - -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 05e972c754..29b041168e 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -9,6 +9,10 @@ on: - stable - v* +env: + # For cmake: + VERBOSE: 1 + jobs: # This tests various versions of CMake in various combinations, to make sure # the configure step passes. diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 31d893c479..b18474bc3d 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -14,6 +14,8 @@ on: env: FORCE_COLOR: 3 + # For cmake: + VERBOSE: 1 jobs: pre-commit: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 366284acf4..a15861ee47 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -11,6 +11,8 @@ concurrency: env: PIP_ONLY_BINARY: numpy + # For cmake: + VERBOSE: 1 jobs: standard: From 09db6445d8da6e918c2d2be3aa4e7b0ddd8077c7 Mon Sep 17 00:00:00 2001 From: Martin Blanchard <49144287+mablanchard@users.noreply.github.com> Date: Fri, 16 Dec 2022 06:10:46 +0100 Subject: [PATCH 048/137] IPO/LTO support for ICX (IntelLLVM) compiler (#4402) * IPO/LTO support for ICX (IntelLLVM) compiler https://github.com/pybind/pybind11/issues/4080 * style: pre-commit fixes * Add WARNING/HELP WANTED comment. Co-authored-by: Martin Blanchard Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- tools/pybind11Common.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index e1fb601acc..0c985bc8e5 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -311,6 +311,16 @@ function(_pybind11_generate_lto target prefer_thin_lto) HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") + # IntelLLVM equivalent to LTO is called IPO; also IntelLLVM is WIN32/UNIX + # WARNING/HELP WANTED: This block of code is currently not covered by pybind11 GitHub Actions! + if(WIN32) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-Qipo" "-Qipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + else() + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") # Intel equivalent to LTO is called IPO _pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO "-ipo" "-ipo" From a97c4d220ed8f10b6f2a67b0382887699a5ecf07 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 19 Dec 2022 14:36:39 -0500 Subject: [PATCH 049/137] fix(cmake): support Windows ARM cross-compilation (#4406) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- tools/FindPythonLibsNew.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index bbbd9f9ec1..ce558d4ece 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -208,7 +208,9 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX "${PYTHON_PREFIX}") string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES "${PYTHON_SITE_PACKAGES}") -if(CMAKE_HOST_WIN32) +if(DEFINED PYTHON_LIBRARY) + # Don't write to PYTHON_LIBRARY if it's already set +elseif(CMAKE_HOST_WIN32) set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/python${PYTHON_LIBRARY_SUFFIX}.lib") # when run in a venv, PYTHON_PREFIX points to it. But the libraries remain in the @@ -274,7 +276,7 @@ if(NOT PYTHON_DEBUG_LIBRARY) endif() set(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}") -find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARY}" +find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARIES}" "${PYTHON_EXECUTABLE}${PYTHON_VERSION_STRING}") set(PYTHONLIBS_FOUND TRUE) From ee4b9f5d7b43383c1d4344dd11caf3dd5b066695 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 20 Dec 2022 05:34:00 -0800 Subject: [PATCH 050/137] Fix ODR violations in our Eigen Tensor tests (#4412) * First * Fix centos 7 again :( * Fix minor nits --- tests/CMakeLists.txt | 15 +++++---------- ...array.cpp => eigen_tensor_avoid_stl_array.cpp} | 6 ++---- tests/test_eigen_tensor.cpp | 6 ++++-- tests/test_eigen_tensor.inl | 10 ++++------ tests/test_eigen_tensor.py | 8 ++++---- 5 files changed, 19 insertions(+), 26 deletions(-) rename tests/{test_eigen_tensor_avoid_stl_array.cpp => eigen_tensor_avoid_stl_array.cpp} (65%) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 491f215cef..9beb268ed7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -130,7 +130,6 @@ set(PYBIND11_TEST_FILES test_docstring_options test_eigen_matrix test_eigen_tensor - test_eigen_tensor_avoid_stl_array.cpp test_enum test_eval test_exceptions @@ -293,6 +292,11 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) set(EIGEN3_VERSION ${EIGEN3_VERSION_STRING}) endif() message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") + + if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) + tests_extra_targets("test_eigen_tensor.py" "eigen_tensor_avoid_stl_array") + endif() + else() list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) @@ -303,11 +307,6 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - list(FIND PYBIND11_TEST_FILES test_eigen_tensor_avoid_stl_array.cpp - PYBIND11_TEST_FILES_EIGEN_I) - if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) - endif() message( STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") endif() @@ -319,10 +318,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_L if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - list(FIND PYBIND11_TEST_FILES test_eigen_tensor_avoid_stl_array.cpp PYBIND11_TEST_FILES_EIGEN_I) - if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) - endif() endif() # Optional dependency for some tests (boost::variant is only supported with version >= 1.56) diff --git a/tests/test_eigen_tensor_avoid_stl_array.cpp b/tests/eigen_tensor_avoid_stl_array.cpp similarity index 65% rename from tests/test_eigen_tensor_avoid_stl_array.cpp rename to tests/eigen_tensor_avoid_stl_array.cpp index 58bedf62dd..eacc9e9bd6 100644 --- a/tests/test_eigen_tensor_avoid_stl_array.cpp +++ b/tests/eigen_tensor_avoid_stl_array.cpp @@ -5,12 +5,10 @@ BSD-style license that can be found in the LICENSE file. */ -constexpr const char *test_eigen_tensor_module_name = "eigen_tensor_avoid_stl_array"; - #ifndef EIGEN_AVOID_STL_ARRAY # define EIGEN_AVOID_STL_ARRAY #endif -#define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor_avoid_stl_array - #include "test_eigen_tensor.inl" + +PYBIND11_MODULE(eigen_tensor_avoid_stl_array, m) { eigen_tensor_test::test_module(m); } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 40b4940053..503c69c7d3 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -5,8 +5,6 @@ BSD-style license that can be found in the LICENSE file. */ -constexpr const char *test_eigen_tensor_module_name = "eigen_tensor"; - #define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor #ifdef EIGEN_AVOID_STL_ARRAY @@ -14,3 +12,7 @@ constexpr const char *test_eigen_tensor_module_name = "eigen_tensor"; #endif #include "test_eigen_tensor.inl" + +#include "pybind11_tests.h" + +test_initializer egien_tensor("eigen_tensor", eigen_tensor_test::test_module); diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl index f46eb1803a..d864ce7379 100644 --- a/tests/test_eigen_tensor.inl +++ b/tests/test_eigen_tensor.inl @@ -7,9 +7,9 @@ #include -#include "pybind11_tests.h" +PYBIND11_NAMESPACE_BEGIN(eigen_tensor_test) -PYBIND11_NAMESPACE_BEGIN(PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE) +namespace py = pybind11; PYBIND11_WARNING_DISABLE_MSVC(4127) @@ -108,7 +108,7 @@ void init_tensor_module(pybind11::module &m) { return check_tensor(get_tensor()) && check_tensor(get_fixed_tensor()); }); - py::class_>(m, "CustomExample") + py::class_>(m, "CustomExample", py::module_local()) .def(py::init<>()) .def_readonly( "member", &CustomExample::member, py::return_value_policy::reference_internal) @@ -322,8 +322,6 @@ void init_tensor_module(pybind11::module &m) { py::return_value_policy::reference); } -void test_module(py::module_ &); -test_initializer name(test_eigen_tensor_module_name, test_module); void test_module(py::module_ &m) { auto f_style = m.def_submodule("f_style"); auto c_style = m.def_submodule("c_style"); @@ -332,4 +330,4 @@ void test_module(py::module_ &m) { init_tensor_module(c_style); } -PYBIND11_NAMESPACE_END(PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE) +PYBIND11_NAMESPACE_END(eigen_tensor_test) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 653c9f288d..dc8aa4643e 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -6,17 +6,17 @@ eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") submodules = [eigen_tensor.c_style, eigen_tensor.f_style] try: - from pybind11_tests import eigen_tensor_avoid_stl_array as avoid + import eigen_tensor_avoid_stl_array as avoid submodules += [avoid.c_style, avoid.f_style] except ImportError as e: # Ensure config, build, toolchain, etc. issues are not masked here: raise RuntimeError( - "import pybind11_tests.eigen_tensor_avoid_stl_array FAILED, while " + "import eigen_tensor_avoid_stl_array FAILED, while " "import pybind11_tests.eigen_tensor succeeded. " "Please ensure that " "test_eigen_tensor.cpp & " - "test_eigen_tensor_avoid_stl_array.cpp " + "eigen_tensor_avoid_stl_array.cpp " "are built together (or both are not built if Eigen is not available)." ) from e @@ -42,7 +42,7 @@ def cleanup(): def test_import_avoid_stl_array(): - pytest.importorskip("pybind11_tests.eigen_tensor_avoid_stl_array") + pytest.importorskip("eigen_tensor_avoid_stl_array") assert len(submodules) == 4 From 3fd1520de294035325469a7fcb3d60a2d818a85b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 20 Dec 2022 14:47:41 -0500 Subject: [PATCH 051/137] docs: changelog for next version (#4413) * docs: changelog for next version Signed-off-by: Henry Schreiner * docs: address feedback Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 55 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7d6d0c0f56..9ff7281bc6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,60 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Version 2.10.2 (Dec 20, 2022) +----------------------------- + +Changes: + +* ``scoped_interpreter`` constructor taking ``PyConfig``. + `#4330 `_ + +* ``pybind11/eigen/tensor.h`` adds converters to and from ``Eigen::Tensor`` and + ``Eigen::TensorMap``. + `#4201 `_ + +* ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` + ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if + ``NDEBUG`` is not defined. + `#4246 `_ + +* Add option for enable/disable enum members in docstring. + `#2768 `_ + +* Fixed typing of ``KeysView``, ``ValuesView`` and ``ItemsView`` in ``bind_map``. + `#4353 `_ + +Bug fixes: + +* Bug fix affecting only Python 3.6 under very specific, uncommon conditions: + move ``PyEval_InitThreads()`` call to the correct location. + `#4350 `_ + +* Fix segfault bug when passing foreign native functions to functional.h. + `#4254 `_ + +Build system improvements: + +* Support setting PYTHON_LIBRARIES manually for Windows ARM cross-compilation + (classic mode). + `#4406 `_ + +* Extend IPO/LTO detection for ICX (a.k.a IntelLLVM) compiler. + `#4402 `_ + +* Allow calling ``find_package(pybind11 CONFIG)`` multiple times from separate + directories in the same CMake project and properly link Python (new mode). + `#4401 `_ + +* ``multiprocessing_set_spawn`` in pytest fixture for added safety. + `#4377 `_ + +* Fixed a bug in two pybind11/tools cmake scripts causing "Unknown arguments specified" errors. + `#4327 `_ + + + Version 2.10.1 (Oct 31, 2022) ----------------------------- @@ -95,7 +149,6 @@ Bug fixes: finalization. `#4192 `_ - Performance and style: * Reserve space in set and STL map casters if possible. This will prevent From 0694ec6a15863bff2e0ea5efe07c78de39b9a33c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 20 Dec 2022 17:57:47 -0500 Subject: [PATCH 052/137] chore: preapre for 2.10.2 release (#4414) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9acf104668..068ae0137a 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_MINOR 10 +#define PYBIND11_VERSION_PATCH 2 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020A0200 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 1cb51fc5c2..4c814edfac 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.10.2" version_info = tuple(_to_int(s) for s in __version__.split(".")) From 7f23e9f3a47cb6b6373467c18962c16dfdd82fe7 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 27 Dec 2022 15:14:10 -0500 Subject: [PATCH 053/137] chore: update clang-tidy to 15 (#4387) * chore: update clang-tidy to 15 * Add git * Add NOLINTNEXTLINE for assignment in if * Update CONTRIBUTING.md * Add NOLINTNEXTLINE where needed * Add one more NOLINTNEXTLINE * stl_bind: make more readable * Another missing NOLINTNEXTLINE * Match style elsewhere * Apply reviewer suggestion. Mark false positive --- .github/CONTRIBUTING.md | 4 ++-- .github/workflows/format.yml | 4 ++-- include/pybind11/detail/internals.h | 3 +++ include/pybind11/stl.h | 1 + include/pybind11/stl_bind.h | 12 ++++++++---- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 00b1fea4cf..ad79743953 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -235,8 +235,8 @@ directory inside your pybind11 git clone. Files will be modified in place, so you can use git to monitor the changes. ```bash -docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:13 -apt-get update && apt-get install -y python3-dev python3-pytest +docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:15-bullseye +apt-get update && apt-get install -y git python3-dev python3-pytest cmake -S /mounted_pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color" -DDOWNLOAD_EIGEN=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=17 cmake --build build -j 2 ``` diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b18474bc3d..46489feb31 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -38,12 +38,12 @@ jobs: # in .github/CONTRIBUTING.md and update as needed. name: Clang-Tidy runs-on: ubuntu-latest - container: silkeh/clang:13 + container: silkeh/clang:15-bullseye steps: - uses: actions/checkout@v3 - name: Install requirements - run: apt-get update && apt-get install -y python3-dev python3-pytest + run: apt-get update && apt-get install -y git python3-dev python3-pytest - name: Configure run: > diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index ef1849fbea..05e36ad18b 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -469,12 +469,14 @@ PYBIND11_NOINLINE internals &get_internals() { #if defined(WITH_THREAD) PyThreadState *tstate = PyThreadState_Get(); + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); } PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); # if PYBIND11_INTERNALS_VERSION > 4 + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { pybind11_fail("get_internals: could not successfully initialize the " "loader_life_support TSS key!"); @@ -514,6 +516,7 @@ struct local_internals { struct shared_loader_life_support_data { PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) shared_loader_life_support_data() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { pybind11_fail("local_internals: could not successfully initialize the " "loader_life_support TLS key!"); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 2d144b598b..84c5171081 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -316,6 +316,7 @@ struct optional_caster { if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); } + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return value_conv::cast(*std::forward(src), policy, parent); } diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 0c634597ec..7cfd996dda 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -355,13 +355,17 @@ void vector_accessor(enable_if_t::value, Class_> &cl) using DiffType = typename Vector::difference_type; using ItType = typename Vector::iterator; cl.def("__getitem__", [](const Vector &v, DiffType i) -> T { - if (i < 0 && (i += v.size()) < 0) { - throw index_error(); + if (i < 0) { + i += v.size(); + if (i < 0) { + throw index_error(); + } } - if ((SizeType) i >= v.size()) { + auto i_st = static_cast(i); + if (i_st >= v.size()) { throw index_error(); } - return v[(SizeType) i]; + return v[i_st]; }); cl.def( From 70af9873c26ea46cf97d66444f4d80c4af79dd7a Mon Sep 17 00:00:00 2001 From: kajananchinniah Date: Wed, 28 Dec 2022 12:49:57 -0500 Subject: [PATCH 054/137] docs: fixed typo in spelling of first (#4428) --- include/pybind11/detail/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 068ae0137a..421cbe192f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -436,7 +436,7 @@ PYBIND11_WARNING_POP /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. From 60f02f5f663829c9fd2d7d52c03d4064129e38df Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Dec 2022 10:46:55 -0800 Subject: [PATCH 055/137] fix: improve the error reporting for inc_ref GIL failures (#4427) * First * Fixs * Improve * Additional assertions comment * Improve docs --- docs/advanced/misc.rst | 28 ++++++++++++++++++++++++++++ include/pybind11/pytypes.h | 30 +++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 35a6ebcd60..805ec838fc 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -118,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy m.def("call_go", &call_go, py::call_guard()); +Common Sources Of Global Interpreter Lock Errors +================================================================== + +Failing to properly hold the Global Interpreter Lock (GIL) is one of the +more common sources of bugs within code that uses pybind11. If you are +running into GIL related errors, we highly recommend you consult the +following checklist. + +- Do you have any global variables that are pybind11 objects or invoke + pybind11 functions in either their constructor or destructor? You are generally + not allowed to invoke any Python function in a global static context. We recommend + using lazy initialization and then intentionally leaking at the end of the program. + +- Do you have any pybind11 objects that are members of other C++ structures? One + commonly overlooked requirement is that pybind11 objects have to increase their reference count + whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke + the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very + tricky to track for complicated programs Think carefully when you make a pybind11 object + a member in another struct. + +- C++ destructors that invoke Python functions can be particularly troublesome as + destructors can sometimes get invoked in weird and unexpected circumstances as a result + of exceptions. + +- You should try running your code in a debug build. That will enable additional assertions + within pybind11 that will throw exceptions on certain GIL handling errors + (reference counting operations). + Binding sequence data types, iterators, the slicing protocol, etc. ================================================================== diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 56e0423ac4..3660c180d9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -250,9 +250,9 @@ class handle : public detail::object_api { #ifdef PYBIND11_HANDLE_REF_DEBUG inc_ref_counter(1); #endif -#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF if (m_ptr != nullptr && !PyGILState_Check()) { - throw std::runtime_error("pybind11::handle::inc_ref() PyGILState_Check() failure."); + throw_gilstate_error("pybind11::handle::inc_ref()"); } #endif Py_XINCREF(m_ptr); @@ -265,9 +265,9 @@ class handle : public detail::object_api { this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { -#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF if (m_ptr != nullptr && !PyGILState_Check()) { - throw std::runtime_error("pybind11::handle::dec_ref() PyGILState_Check() failure."); + throw_gilstate_error("pybind11::handle::dec_ref()"); } #endif Py_XDECREF(m_ptr); @@ -296,8 +296,28 @@ class handle : public detail::object_api { protected: PyObject *m_ptr = nullptr; -#ifdef PYBIND11_HANDLE_REF_DEBUG private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + function_name.c_str()); + fflush(stderr); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + "The failing %s call was triggered on a %s object.\n", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + fflush(stderr); + } + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG static std::size_t inc_ref_counter(std::size_t add) { thread_local std::size_t counter = 0; counter += add; From f12e098f1d68d9e7445f84d9bf94ce7d0908c531 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 2 Jan 2023 03:46:17 -0800 Subject: [PATCH 056/137] Fix `detail::obj_class_name()` to work correctly for meta classes. (#4436) * Fix `detail::obj_class_name()` to work correctly for meta classes. * Adjust expected name for PyPy --- include/pybind11/pytypes.h | 2 +- tests/test_class.cpp | 2 ++ tests/test_class.py | 11 ++++++++++- tests/test_pytypes.cpp | 2 ++ tests/test_pytypes.py | 6 ++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 3660c180d9..a2824a0e01 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -463,7 +463,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline const char *obj_class_name(PyObject *obj) { - if (Py_TYPE(obj) == &PyType_Type) { + if (PyType_Check(obj)) { return reinterpret_cast(obj)->tp_name; } return Py_TYPE(obj)->tp_name; diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 9ea10fae1e..ca925917e6 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -55,6 +55,8 @@ void bind_empty0(py::module_ &m) { } // namespace test_class TEST_SUBMODULE(class_, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + // test_instance struct NoConstructor { NoConstructor() = default; diff --git a/tests/test_class.py b/tests/test_class.py index 7c1ed20608..9c964e001b 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -1,10 +1,19 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats, UserType from pybind11_tests import class_ as m +def test_obj_class_name(): + if env.PYPY: + expected_name = "UserType" + else: + expected_name = "pybind11_tests.UserType" + assert m.obj_class_name(UserType(1)) == expected_name + assert m.obj_class_name(UserType) == expected_name + + def test_repr(): assert "pybind11_type" in repr(type(UserType)) assert "UserType" in repr(UserType) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index ea8d03958b..1028bb58e8 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -99,6 +99,8 @@ void m_defs(py::module_ &m) { } // namespace handle_from_move_only_type_with_operator_PyObject TEST_SUBMODULE(pytypes, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + handle_from_move_only_type_with_operator_PyObject::m_defs(m); // test_bool diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 079ee7ca50..8f9f2987e9 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -9,6 +9,12 @@ from pybind11_tests import pytypes as m +def test_obj_class_name(): + assert m.obj_class_name(None) == "NoneType" + assert m.obj_class_name(list) == "list" + assert m.obj_class_name([]) == "list" + + def test_handle_from_move_only_type_with_operator_PyObject(): # noqa: N802 assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() assert m.handle_from_move_only_type_with_operator_PyObject_const() From 6da268a5c58b76c0577d0d7b223a78edd6a34bea Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 3 Jan 2023 05:46:55 -0800 Subject: [PATCH 057/137] ci: remove clang 10 C++20 (it broke recently) (#4438) * Remove clang 10 C++20 (it broke recently), add clang 15 C++20 while we are at it. * No luck trying clang15: Error response from daemon: manifest for silkeh/clang:15 not found: manifest unknown: manifest unknown. Giving up, this needs to be a separate PR. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af5a29d627..b36bbfe1b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,8 +283,6 @@ jobs: include: - clang: 5 std: 14 - - clang: 10 - std: 20 - clang: 10 std: 17 - clang: 11 From e3e24f3f656841142ef5897e33a0aabc2b34fd07 Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 3 Jan 2023 23:20:39 +0800 Subject: [PATCH 058/137] fix: issuses detected by static analyzer (#4440) * fix incorrect variable check * remove duplicated check * remove unneeded const cast --- include/pybind11/attr.h | 2 +- include/pybind11/eigen/tensor.h | 2 +- include/pybind11/pytypes.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index db7cd8efff..b5e3b7b22c 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -399,7 +399,7 @@ struct process_attribute : process_attribute_default { template <> struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; template <> struct process_attribute : process_attribute {}; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 568c641a2d..0877da8953 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -176,7 +176,7 @@ struct type_caster::ValidType> { return false; } - if (!convert && !temp.dtype().is(dtype::of())) { + if (!temp.dtype().is(dtype::of())) { return false; } } diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index a2824a0e01..f11ed5da78 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,7 +501,7 @@ struct error_fetch_and_normalize { "active exception."); } const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { + if (exc_type_name_norm == nullptr) { pybind11_fail("Internal error: " + std::string(called) + " failed to obtain the name " "of the normalized active exception type."); From 769fd3b889fef6cddb060f2a0be26aee62b4da05 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:21:05 -0500 Subject: [PATCH 059/137] chore(deps): update pre-commit hooks (#4439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.3.0 β†’ v3.3.1](https://github.com/asottile/pyupgrade/compare/v3.3.0...v3.3.1) - [github.com/PyCQA/isort: 5.10.1 β†’ 5.11.4](https://github.com/PyCQA/isort/compare/5.10.1...5.11.4) - [github.com/psf/black: 22.10.0 β†’ 22.12.0](https://github.com/psf/black/compare/22.10.0...22.12.0) - [github.com/PyCQA/pylint: v2.15.8 β†’ v2.15.9](https://github.com/PyCQA/pylint/compare/v2.15.8...v2.15.9) - [github.com/shellcheck-py/shellcheck-py: v0.8.0.4 β†’ v0.9.0.2](https://github.com/shellcheck-py/shellcheck-py/compare/v0.8.0.4...v0.9.0.2) - [github.com/pre-commit/mirrors-clang-format: v15.0.4 β†’ v15.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v15.0.4...v15.0.6) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e80fbad15..d625d5726b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,20 +41,20 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v3.3.0" + rev: "v3.3.1" hooks: - id: pyupgrade args: [--py36-plus] # Nicely sort includes - repo: https://github.com/PyCQA/isort - rev: "5.10.1" + rev: "5.11.4" hooks: - id: isort # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.10.0" # Keep in sync with blacken-docs + rev: "22.12.0" # Keep in sync with blacken-docs hooks: - id: black @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.8" + rev: "v2.15.9" hooks: - id: pylint files: ^pybind11 @@ -160,7 +160,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.8.0.4" + rev: "v0.9.0.2" hooks: - id: shellcheck @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.4" + rev: "v15.0.6" hooks: - id: clang-format types_or: [c++, c, cuda] From b2d7ad72c24ee8d2acc22a4ce39fa38a1e9d77b8 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Jan 2023 11:34:22 -0500 Subject: [PATCH 060/137] chore: prepare for 2.10.3 (#4437) * docs: update changelog for v2.10.3 Signed-off-by: Henry Schreiner * chore: bump versions for 2.10.3 Signed-off-by: Henry Schreiner * chore: fix make changelog script with entry is empty Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 35 +++++++++++++++++++++++++++++++- include/pybind11/detail/common.h | 4 ++-- pybind11/_version.py | 2 +- tools/make_changelog.py | 9 ++++---- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ff7281bc6..bb111c5f26 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,39 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Changes: + +* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & + ``dec_ref()`` are now enabled by default again. + `#4246 `_ + +Build system improvements: + +* Update clang-tidy to 15 in CI. + `#4387 `_ + + +Version 2.10.3 (Jan 3, 2023) +---------------------------- + +Changes: + +* Temporarily made our GIL status assertions (added in 2.10.2) disabled by + default (re-enable manually by defining + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11). + `#4432 `_ + +* Improved error messages when ``inc_ref``/``dec_ref`` are called with an + invalid GIL state. + `#4427 `_ + `#4436 `_ + +Bug Fixes: + +* Some minor touchups found by static analyzers. + `#4440 `_ + + Version 2.10.2 (Dec 20, 2022) ----------------------------- @@ -30,7 +63,7 @@ Changes: * ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if - ``NDEBUG`` is not defined. + ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11) `#4246 `_ * Add option for enable/disable enum members in docstring. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 421cbe192f..fc7e50776c 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 2 +#define PYBIND11_VERSION_PATCH 3 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0200 +#define PYBIND11_VERSION_HEX 0x020A0300 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 4c814edfac..63078bbe67 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.2" +__version__ = "2.10.3" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/tools/make_changelog.py b/tools/make_changelog.py index 839040a938..b5bd832940 100755 --- a/tools/make_changelog.py +++ b/tools/make_changelog.py @@ -31,8 +31,10 @@ missing = [] for issue in issues: - changelog = ENTRY.findall(issue.body) - if changelog: + changelog = ENTRY.findall(issue.body or "") + if not changelog or not changelog[0]: + missing.append(issue) + else: (msg,) = changelog if not msg.startswith("* "): msg = "* " + msg @@ -44,9 +46,6 @@ print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) print() - else: - missing.append(issue) - if missing: print() print("[blue]" + "-" * 30) From a34596bfe1947b4a6b0bcc4218e1f72d0c2e9b4c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Jan 2023 14:23:36 -0500 Subject: [PATCH 061/137] chore: back to work Signed-off-by: Henry Schreiner --- include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index fc7e50776c..0fc81e0534 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 3 +#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_PATCH 0.dev1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0300 +#define PYBIND11_VERSION_HEX 0x020B00D1 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 63078bbe67..1cb51fc5c2 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.3" +__version__ = "2.11.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) From d821788bb62fa911bb9797550b8d06ca41462455 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 12 Jan 2023 17:50:28 -0800 Subject: [PATCH 062/137] Add clang15 C++20 job (#4443) --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b36bbfe1b9..7c5fcb43ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,6 +280,8 @@ jobs: - dev std: - 11 + container_suffix: + - "" include: - clang: 5 std: 14 @@ -293,9 +295,12 @@ jobs: std: 20 - clang: 14 std: 20 + - clang: 15 + std: 20 + container_suffix: "-bullseye" name: "🐍 3 β€’ Clang ${{ matrix.clang }} β€’ C++${{ matrix.std }} β€’ x64" - container: "silkeh/clang:${{ matrix.clang }}" + container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 From e53d58af6c0ade85fa6edce6a228ad2aeace55ee Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Jan 2023 13:47:56 -0800 Subject: [PATCH 063/137] Ensure `import pybind11_tests` traceback is shown. (#4455) --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 402fd4b25b..c24d51d4d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,11 +11,17 @@ import os import re import textwrap +import traceback import pytest # Early diagnostic for failed imports -import pybind11_tests +try: + import pybind11_tests +except Exception: + # pytest does not show the traceback without this. + traceback.print_exc() + raise @pytest.fixture(scope="session", autouse=True) From c709d2a83e06aaec06987aa9761d0ebbfefbecd7 Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 18 Jan 2023 21:11:26 +0100 Subject: [PATCH 064/137] Make sure to properly untrack gc objects before freeing them (#4461) * Make sure to properly untrack gc objects before freeing them * style: pre-commit fixes * Fix lint * Add comment about where the original track comes from Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/class.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 528e716f78..bc2b40c50a 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -445,9 +445,17 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 From a500f439d06d220ee2c680cdd2c8828eac8e7dfc Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 19 Jan 2023 10:48:46 -0800 Subject: [PATCH 065/137] Resolve new flake8 error (#4462) * Resolve flake8 error by replacing `pytest.raises(Exception)` with `SystemError` * Also remove the obsolete comment. * Tweak comment instead of removing it. --- tests/test_modules.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index e11d68e78e..c3e8cea6e1 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -107,11 +107,10 @@ def test_def_submodule_failures(): sm_name_orig = sm.__name__ sm.__name__ = malformed_utf8 try: - with pytest.raises(Exception): - # Seen with Python 3.9: SystemError: nameless module - # But we do not want to exercise the internals of PyModule_GetName(), which could - # change in future versions of Python, but a bad __name__ is very likely to cause - # some kind of failure indefinitely. + # We want to assert that a bad __name__ causes some kind of failure, although we do not want to exercise + # the internals of PyModule_GetName(). Currently all supported Python versions raise SystemError. If that + # changes in future Python versions, simply add the new expected exception types here. + with pytest.raises(SystemError): m.def_submodule(sm, b"SubSubModuleName") finally: # Clean up to ensure nothing gets upset by a module with an invalid __name__. From c71e3af73f80140023ee399a982678a8b968b5c5 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 31 Jan 2023 22:44:18 -0800 Subject: [PATCH 066/137] Bump isort version to 5.12.0 (#4480) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d625d5726b..9ec31ae997 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Nicely sort includes - repo: https://github.com/PyCQA/isort - rev: "5.11.4" + rev: "5.12.0" hooks: - id: isort From 44e936822252df9e6420b42dc2520d622f430bc3 Mon Sep 17 00:00:00 2001 From: Daniel Jacobs <44678615+danielcjacobs@users.noreply.github.com> Date: Wed, 1 Feb 2023 02:42:05 -0500 Subject: [PATCH 067/137] Use PyConfig_InitPythonConfig instead of PyConfig_InitIsolatedConfig (#4473) * Use PyConfig_InitPythonConfig instead of PyConfig_InitIsolatedConfig * add unit test for default python configuration --------- Co-authored-by: Daniel Jacobs --- include/pybind11/embed.h | 7 ++++--- tests/test_embed/test_interpreter.cpp | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 749c75beb6..5a175b1341 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -198,9 +198,10 @@ inline void initialize_interpreter(bool init_signal_handlers = true, init_signal_handlers, argc, argv, add_program_dir_to_path); #else PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.isolated = 0; - config.use_environment = 1; + PyConfig_InitPythonConfig(&config); + // See PR #4473 for background + config.parse_argv = 0; + config.install_signal_handlers = init_signal_handlers ? 1 : 0; initialize_interpreter(&config, argc, argv, add_program_dir_to_path); #endif diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 10b20f3715..e54e822708 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -184,7 +184,7 @@ TEST_CASE("Custom PyConfig") { py::initialize_interpreter(); } -TEST_CASE("Custom PyConfig with argv") { +TEST_CASE("scoped_interpreter with PyConfig_InitIsolatedConfig and argv") { py::finalize_interpreter(); { PyConfig config; @@ -199,6 +199,26 @@ TEST_CASE("Custom PyConfig with argv") { } py::initialize_interpreter(); } + +TEST_CASE("scoped_interpreter with PyConfig_InitPythonConfig and argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + + // `initialize_interpreter() overrides the default value for config.parse_argv (`1`) by + // changing it to `0`. This test exercises `scoped_interpreter` with the default config. + char *argv[] = {strdup("a.out"), strdup("arg1")}; + py::scoped_interpreter argv_scope(&config, 2, argv); + std::free(argv[0]); + std::free(argv[1]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "arg1"); + } + py::initialize_interpreter(); +} #endif TEST_CASE("Add program dir to path pre-PyConfig") { From 3efe9d4cb5d7314faee722205e560b8b932aed9e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 1 Feb 2023 14:23:37 -0500 Subject: [PATCH 068/137] chore: update to black 23 (#4482) Signed-off-by: Henry Schreiner --- .pre-commit-config.yaml | 18 +++++++++--------- docs/conf.py | 1 - pybind11/__main__.py | 1 - pybind11/setup_helpers.py | 3 --- tests/extra_python_package/test_files.py | 2 -- tests/test_chrono.py | 4 ---- tests/test_class.py | 1 - tests/test_custom_type_casters.py | 3 ++- tests/test_eigen_tensor.py | 6 ------ tests/test_local_bindings.py | 3 ++- tests/test_operator_overloading.py | 1 - 11 files changed, 13 insertions(+), 30 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ec31ae997..363ac71c6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,40 +54,40 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.12.0" # Keep in sync with blacken-docs + rev: "23.1.0" # Keep in sync with blacken-docs hooks: - id: black # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "v1.12.1" + rev: "1.13.0" hooks: - id: blacken-docs additional_dependencies: - - black==22.10.0 # keep in sync with black hook + - black==23.1.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.3.1" + rev: "v1.4.2" hooks: - id: remove-tabs - repo: https://github.com/sirosen/texthooks - rev: "0.4.0" + rev: "0.5.0" hooks: - id: fix-ligatures - id: fix-smartquotes # Autoremoves unused imports - repo: https://github.com/hadialqattan/pycln - rev: "v2.1.2" + rev: "v2.1.3" hooks: - id: pycln stages: [manual] # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks - rev: "v1.9.0" + rev: "v1.10.0" hooks: - id: python-check-blanket-noqa - id: python-check-blanket-type-ignore @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.9" + rev: "v2.16.0" hooks: - id: pylint files: ^pybind11 @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.6" + rev: "v15.0.7" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/docs/conf.py b/docs/conf.py index 2da6773f4f..cdb9f63069 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -358,7 +358,6 @@ def clean_up(app, exception): def setup(app): - # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 8c89533843..e87b16beb4 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -24,7 +24,6 @@ def print_includes() -> None: def main() -> None: - parser = argparse.ArgumentParser() parser.add_argument( "--includes", diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 1fd04b9154..1db4795327 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -118,7 +118,6 @@ def _add_ldflags(self, flags: List[str]) -> None: self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: - self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -174,7 +173,6 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: - if self._cxx_level: warnings.warn("You cannot safely change the cxx_level after setting it!") @@ -439,7 +437,6 @@ def compile_function( extra_postargs: Optional[List[str]] = None, depends: Optional[List[str]] = None, ) -> Any: - # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 9a9bb1556a..dd6393bf0b 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -135,7 +135,6 @@ def normalize_line_endings(value: bytes) -> bytes: def test_build_sdist(monkeypatch, tmpdir): - monkeypatch.chdir(MAIN_DIR) subprocess.run( @@ -186,7 +185,6 @@ def test_build_sdist(monkeypatch, tmpdir): def test_build_global_dist(monkeypatch, tmpdir): - monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") subprocess.run( diff --git a/tests/test_chrono.py b/tests/test_chrono.py index 7f47b37a25..a29316c38d 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -7,7 +7,6 @@ def test_chrono_system_clock(): - # Get the time from both c++ and datetime date0 = datetime.datetime.today() date1 = m.test_chrono1() @@ -122,7 +121,6 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): def test_chrono_duration_roundtrip(): - # Get the difference between two times (a timedelta) date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -143,7 +141,6 @@ def test_chrono_duration_roundtrip(): def test_chrono_duration_subtraction_equivalence(): - date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -154,7 +151,6 @@ def test_chrono_duration_subtraction_equivalence(): def test_chrono_duration_subtraction_equivalence_date(): - date1 = datetime.date.today() date2 = datetime.date.today() diff --git a/tests/test_class.py b/tests/test_class.py index 9c964e001b..64abac133c 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -185,7 +185,6 @@ def test_inheritance(msg): def test_inheritance_init(msg): - # Single base class Python(m.Pet): def __init__(self): diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index adfa6cf86e..731380e46a 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -94,7 +94,8 @@ def test_noconvert_args(msg): def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets - destroyed when the function has py::return_value_policy::take_ownership policy applied.""" + destroyed when the function has py::return_value_policy::take_ownership policy applied. + """ cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index dc8aa4643e..cd54efa069 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -59,7 +59,6 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): - if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() @@ -108,7 +107,6 @@ def test_convert_tensor_to_py(m, func_name): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - with pytest.raises( RuntimeError, match="Cannot use reference internal when there is no parent" ): @@ -131,7 +129,6 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises( TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments" ): @@ -194,7 +191,6 @@ def test_bad_python_to_cpp_casts(m): @pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): - a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -211,7 +207,6 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): - assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) with pytest.raises(TypeError, match="^Cannot cast array data from"): @@ -260,7 +255,6 @@ def test_round_trip(m): @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): - # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) diff --git a/tests/test_local_bindings.py b/tests/test_local_bindings.py index 654d96d490..d641877396 100644 --- a/tests/test_local_bindings.py +++ b/tests/test_local_bindings.py @@ -130,7 +130,8 @@ def test_stl_bind_global(): def test_mixed_local_global(): """Local types take precedence over globally registered types: a module with a `module_local` type can be registered even if the type is already registered globally. With the module, - casting will go to the local type; outside the module casting goes to the global type.""" + casting will go to the local type; outside the module casting goes to the global type. + """ import pybind11_cross_module_tests as cm m.register_mixed_global() diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index b228da3cc3..9fde305a03 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -130,7 +130,6 @@ def test_nested(): def test_overriding_eq_reset_hash(): - assert m.Comparable(15) is not m.Comparable(15) assert m.Comparable(15) == m.Comparable(15) From 08a89fac3a11c5cb5498d76cfde3ba3988368d3a Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 4 Feb 2023 13:40:13 -0500 Subject: [PATCH 069/137] bugfix: delete proper ctors in gil.h (#4490) --- include/pybind11/gil.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index cb0028d505..570a5581d5 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -152,8 +152,8 @@ class gil_scoped_release { } } - gil_scoped_release(const gil_scoped_acquire &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter @@ -203,7 +203,7 @@ class gil_scoped_release { public: gil_scoped_release() : state{PyEval_SaveThread()} {} gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; @@ -230,7 +230,7 @@ class gil_scoped_release { (void) (this != (this + 1)); } gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; void disarm() {} }; From 9ef65cee0e780e6443fa0813122712c9ae4bdcc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 00:16:52 -0500 Subject: [PATCH 070/137] chore(deps): bump ilammy/msvc-dev-cmd from 1.12.0 to 1.12.1 (#4493) Bumps [ilammy/msvc-dev-cmd](https://github.com/ilammy/msvc-dev-cmd) from 1.12.0 to 1.12.1. - [Release notes](https://github.com/ilammy/msvc-dev-cmd/releases) - [Commits](https://github.com/ilammy/msvc-dev-cmd/compare/v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: ilammy/msvc-dev-cmd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c5fcb43ac..8befccdbc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -766,7 +766,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.12.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 @@ -819,7 +819,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.12.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 From b2c1978caaa6627095e3326f7ecba84b43037869 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 6 Feb 2023 11:36:05 -0500 Subject: [PATCH 071/137] bugfix: Keep registered types until after Py_Finalize(). Fix #4459 (#4486) * Keep registered types until after Py_Finalize(). Fix #4459 * Address reviewer comments --- include/pybind11/embed.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 5a175b1341..8450d53589 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -254,13 +254,16 @@ inline void finalize_interpreter() { if (builtins.contains(id) && isinstance(builtins[id])) { internals_ptr_ptr = capsule(builtins[id]); } + + Py_Finalize(); + // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter + // Must be cleared only after Py_Finalize() so atexit and other hooks can still use + // registered_types detail::get_local_internals().registered_types_cpp.clear(); detail::get_local_internals().registered_exception_translators.clear(); - Py_Finalize(); - if (internals_ptr_ptr) { delete *internals_ptr_ptr; *internals_ptr_ptr = nullptr; From 8a90b36772f36da577c485bd450130c8318c8111 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Feb 2023 01:16:25 -0500 Subject: [PATCH 072/137] chore(deps): update pre-commit hooks (#4495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/pylint: v2.16.0 β†’ v2.16.1](https://github.com/PyCQA/pylint/compare/v2.16.0...v2.16.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 363ac71c6f..27474f2f73 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.0" + rev: "v2.16.1" hooks: - id: pylint files: ^pybind11 From f8713ec43e6833033b43126c8b55de97242ab4dd Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 7 Feb 2023 16:55:00 -0800 Subject: [PATCH 073/137] Revert "bugfix: Keep registered types until after Py_Finalize(). Fix #4459 (#4486)" (#4501) This reverts commit b2c1978caaa6627095e3326f7ecba84b43037869. See #4500 for background. --- include/pybind11/embed.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 8450d53589..5a175b1341 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -254,16 +254,13 @@ inline void finalize_interpreter() { if (builtins.contains(id) && isinstance(builtins[id])) { internals_ptr_ptr = capsule(builtins[id]); } - - Py_Finalize(); - // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter - // Must be cleared only after Py_Finalize() so atexit and other hooks can still use - // registered_types detail::get_local_internals().registered_types_cpp.clear(); detail::get_local_internals().registered_exception_translators.clear(); + Py_Finalize(); + if (internals_ptr_ptr) { delete *internals_ptr_ptr; *internals_ptr_ptr = nullptr; From b8f28551cc3a98ea9fbfc15c05b513c8f2d23e84 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 7 Feb 2023 20:19:33 -0800 Subject: [PATCH 074/137] Go back to CMake 3.25.2 (#4496) --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8befccdbc2..5d93d42b17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,9 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 + # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: + with: + cmake-version: '3.25.2' - name: Cache wheels if: runner.os == 'macOS' @@ -1071,6 +1074,9 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 + # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: + with: + cmake-version: '3.25.2' - name: Run pip installs run: | From 531144dddc8abf05612523a83ff575f6834fd611 Mon Sep 17 00:00:00 2001 From: Mike Essenmacher <112431871+mikeessen@users.noreply.github.com> Date: Thu, 9 Feb 2023 13:11:01 -0500 Subject: [PATCH 075/137] Replace "whitelist" with "allowlist" (#4506) --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9beb268ed7..b1cb222b4a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -169,7 +169,7 @@ if(PYBIND11_TEST_OVERRIDE) # This allows the override to be done with extensions, preserving backwards compatibility. foreach(test_name ${TEST_FILES_NO_EXT}) if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT - )# If not in the whitelist, add to be filtered out. + )# If not in the allowlist, add to be filtered out. list(APPEND PYBIND11_TEST_FILTER ${test_name}) endif() endforeach() From 8dcced29ae4c7654c25bc50742a72556ee3ae36c Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Fri, 10 Feb 2023 00:21:17 -0500 Subject: [PATCH 076/137] Always display python type information in cast errors (#4463) * Always display python type information in cast errors * Address comments * Update comment --- include/pybind11/cast.h | 33 ++++++++++++++++++-------------- include/pybind11/detail/common.h | 5 +++-- tests/test_callbacks.py | 14 ++++++++++++-- tests/test_exceptions.cpp | 5 +++++ tests/test_exceptions.py | 9 +++++++++ tests/test_pytypes.py | 2 +- 6 files changed, 49 insertions(+), 19 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 3a40460276..9b013bc39f 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1017,11 +1017,14 @@ type_caster &load_type(type_caster &conv, const handle &ha "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error("Unable to cast Python instance to C++ type (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + throw cast_error( + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " - + (std::string) str(type::handle_of(handle)) + " to C++ type '" + + str(type::handle_of(handle)).cast() + " to C++ type '" + type_id() + "'"); #endif } @@ -1085,12 +1088,13 @@ detail::enable_if_t::value, T> move(object &&obj) { if (obj.ref_count() > 1) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + "Unable to cast Python " + str(type::handle_of(obj)).cast() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) - + " instance to C++ " + type_id() - + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast() + " instance to C++ " + + type_id() + " instance: instance has multiple references"); #endif } @@ -1195,9 +1199,10 @@ PYBIND11_NAMESPACE_END(detail) // The overloads could coexist, i.e. the #if is not strictly speaking needed, // but it is an easy minor optimization. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) -inline cast_error cast_error_unable_to_convert_call_arg() { - return cast_error("Unable to convert call argument to Python object (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } #else inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, @@ -1220,7 +1225,7 @@ tuple make_tuple(Args &&...args_) { for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else std::array argtypes{{type_id()...}}; throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); @@ -1510,7 +1515,7 @@ class unpacking_collector { detail::make_caster::cast(std::forward(x), policy, {})); if (!o) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); #else throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), type_id()); @@ -1542,7 +1547,7 @@ class unpacking_collector { } if (!a.value) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(a.name); #else throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 0fc81e0534..1032293d21 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1225,8 +1225,9 @@ constexpr #endif // Pybind offers detailed error messages by default for all builts that are debug (through the -// negation of ndebug). This can also be manually enabled by users, for any builds, through -// defining PYBIND11_DETAILED_ERROR_MESSAGES. +// negation of NDEBUG). This can also be manually enabled by users, for any builds, through +// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) # define PYBIND11_DETAILED_ERROR_MESSAGES #endif diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 57b6599880..4a652f53e8 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -5,6 +5,7 @@ import env # noqa: F401 from pybind11_tests import callbacks as m +from pybind11_tests import detailed_error_messages_enabled def test_callbacks(): @@ -70,11 +71,20 @@ def f(*args, **kwargs): with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error1(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'1' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error2(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'expected_name' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'expected_name' to Python object " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) def test_lambda_closure_cleanup(): diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index f57e095068..854c7e6f76 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -339,4 +339,9 @@ TEST_SUBMODULE(exceptions, m) { } return py::str("UNEXPECTED"); }); + + m.def("test_fn_cast_int", [](const py::function &fn) { + // function returns None instead of int, should give a useful error message + fn().cast(); + }); } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 0d2c808143..8bcefe3de6 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -381,3 +381,12 @@ def test_pypy_oserror_normalization(): # https://github.com/pybind/pybind11/issues/4075 what = m.test_pypy_oserror_normalization() assert "this_filename_must_not_exist" in what + + +def test_fn_cast_int_exception(): + with pytest.raises(RuntimeError) as excinfo: + m.test_fn_cast_int(lambda: None) + + assert str(excinfo.value).startswith( + "Unable to cast Python instance of type to C++ type" + ) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 8f9f2987e9..a1a80a2933 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -536,7 +536,7 @@ def test_print(capture): assert str(excinfo.value) == "Unable to convert call argument " + ( "'1' of type 'UnregisteredType' to Python object" if detailed_error_messages_enabled - else "to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" ) From 535f81a1c0f60f39aaf0f7661f2a1ac2a3a0af16 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Feb 2023 06:54:27 -0800 Subject: [PATCH 077/137] fix: tests dir has started to show up in packaging (#4510) Signed-off-by: Henry Schreiner --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 033303a74a..31632acc33 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +prune tests recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed From 08a4a47a6c0a8e8ca99a86c23c9b6ac843837395 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 16 Feb 2023 07:54:57 -0800 Subject: [PATCH 078/137] Revert "Go back to CMake 3.25.2 (#4496)" (#4503) * Revert "Go back to CMake 3.25.2 (#4496)" This reverts commit b8f28551cc3a98ea9fbfc15c05b513c8f2d23e84. * Apply patch provided by @bradking (with pre-commit cmake-format auto fixes). https://github.com/pybind/pybind11/pull/4503#issuecomment-1424768172 --- .github/workflows/ci.yml | 6 ------ tools/FindCatch.cmake | 8 ++++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d93d42b17..8befccdbc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,9 +83,6 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 - # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: - with: - cmake-version: '3.25.2' - name: Cache wheels if: runner.os == 'macOS' @@ -1074,9 +1071,6 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 - # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: - with: - cmake-version: '3.25.2' - name: Run pip installs run: | diff --git a/tools/FindCatch.cmake b/tools/FindCatch.cmake index 57bba58b30..5d3fcbfb13 100644 --- a/tools/FindCatch.cmake +++ b/tools/FindCatch.cmake @@ -36,10 +36,14 @@ endfunction() function(_download_catch version destination_dir) message(STATUS "Downloading catch v${version}...") set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp) - file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status) + file( + DOWNLOAD ${url} "${destination_dir}/catch.hpp" + STATUS status + LOG log) list(GET status 0 error) if(error) - message(FATAL_ERROR "Could not download ${url}") + string(REPLACE "\n" "\n " log " ${log}") + message(FATAL_ERROR "Could not download URL:\n" " ${url}\n" "Log:\n" "${log}") endif() set(CATCH_INCLUDE_DIR "${destination_dir}" From d1956eabb50d93428097a8a564cef4fc8b0ded82 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 17 Feb 2023 12:31:08 -0800 Subject: [PATCH 079/137] Appease new flake8 B028 error: (#4513) ``` flake8...................................................................Failed - hook id: flake8 - exit code: 1 pybind11/setup_helpers.py:177:13: B028 No explicit stacklevel keyword argument found. The warn method from the warnings module uses a stacklevel of 1 by default. This will only show a stack trace for the line on which the warn method is called. It is therefore recommended to use a stacklevel of 2 or greater to provide more information to the user. warnings.warn("You cannot safely change the cxx_level after setting it!") ^ ``` --- pybind11/setup_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 1db4795327..4c14498f6f 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -174,7 +174,9 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=1 + ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. From 6a5e6007cdfa9441fc05a589d7b819c6c37eb489 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 17 Feb 2023 12:58:35 -0800 Subject: [PATCH 080/137] Make warning suppressions MINGW-specific again. (#4515) Background: https://github.com/pybind/pybind11/pull/4285#issuecomment-1435066554 --- include/pybind11/eigen/matrix.h | 2 ++ include/pybind11/eigen/tensor.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index 34fe329a82..bd533bed37 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -19,7 +19,9 @@ PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include #include diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 0877da8953..de7dcba89d 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -17,7 +17,9 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0 PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(4554) PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include From 68211d41c1ef94fa65ef27ea2a6cdd8677996635 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 17 Feb 2023 14:13:29 -0800 Subject: [PATCH 081/137] fix: nicer stack level for warning (#4516) Signed-off-by: Henry Schreiner --- pybind11/setup_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 4c14498f6f..b9b433b656 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -175,7 +175,7 @@ def cxx_std(self) -> int: def cxx_std(self, level: int) -> None: if self._cxx_level: warnings.warn( - "You cannot safely change the cxx_level after setting it!", stacklevel=1 + "You cannot safely change the cxx_level after setting it!", stacklevel=2 ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so From a19daeac16685877fd0b5ffcadce4b9f60d90f7b Mon Sep 17 00:00:00 2001 From: xkszltl Date: Tue, 21 Feb 2023 06:58:37 +0800 Subject: [PATCH 082/137] Inconsistent comments between 2 templates of `unchecked()`. (#4519) This comment is unrelated to having T in tparam or not. Probably a typo. Copied the correct one here. Resolve https://github.com/pybind/pybind11/issues/4518 --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8f072af267..854d6e87fa 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1121,10 +1121,10 @@ class array_t : public array { /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const & { From 438034c5b8a6a8bf82dda7a195e3c00cafe575e0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 22 Feb 2023 06:18:55 -0800 Subject: [PATCH 083/137] chore: move to Ruff and add rules (#4483) --- .pre-commit-config.yaml | 44 ++-------- docs/conf.py | 2 +- pybind11/__init__.py | 2 +- pybind11/commands.py | 2 +- pybind11/setup_helpers.py | 2 +- pyproject.toml | 46 ++++++++-- setup.cfg | 8 -- tests/conftest.py | 50 ++++------- tests/env.py | 3 +- tests/test_async.py | 4 +- tests/test_buffers.py | 3 +- tests/test_builtin_casters.py | 10 ++- tests/test_class.py | 11 +-- tests/test_const_name.py | 4 +- tests/test_custom_type_casters.py | 3 +- tests/test_custom_type_setup.py | 2 +- tests/test_eigen_matrix.py | 93 +++++++++++++-------- tests/test_eigen_tensor.py | 20 ++--- tests/test_enum.py | 2 + tests/test_exceptions.py | 18 ++-- tests/test_factory_constructors.py | 2 +- tests/test_gil_scoped.py | 8 +- tests/test_iostream.py | 20 ++--- tests/test_kwargs_and_defaults.py | 31 ++++--- tests/test_methods_and_attributes.py | 9 +- tests/test_modules.py | 9 +- tests/test_numpy_array.py | 18 ++-- tests/test_numpy_dtypes.py | 28 +++---- tests/test_numpy_vectorize.py | 2 +- tests/test_pytypes.py | 48 +++++------ tests/test_sequences_and_iterators.py | 7 +- tests/test_stl.py | 7 +- tests/test_stl_binders.py | 12 +-- tests/test_virtual_functions.py | 7 +- tools/codespell_ignore_lines_from_errors.py | 8 +- 35 files changed, 266 insertions(+), 279 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27474f2f73..2255f2e4d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,19 +39,6 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Upgrade old Python syntax -- repo: https://github.com/asottile/pyupgrade - rev: "v3.3.1" - hooks: - - id: pyupgrade - args: [--py36-plus] - -# Nicely sort includes -- repo: https://github.com/PyCQA/isort - rev: "5.12.0" - hooks: - - id: isort - # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black rev: "23.1.0" # Keep in sync with blacken-docs @@ -72,47 +59,28 @@ repos: hooks: - id: remove-tabs +# Avoid directional quotes - repo: https://github.com/sirosen/texthooks rev: "0.5.0" hooks: - id: fix-ligatures - id: fix-smartquotes -# Autoremoves unused imports -- repo: https://github.com/hadialqattan/pycln - rev: "v2.1.3" +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.251 hooks: - - id: pycln - stages: [manual] + - id: ruff + args: ["--fix", "--show-fixes"] # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks rev: "v1.10.0" hooks: - - id: python-check-blanket-noqa - - id: python-check-blanket-type-ignore - - id: python-no-log-warn - - id: python-use-type-annotations - id: rst-backticks - id: rst-directive-colons - id: rst-inline-touching-normal -# Automatically remove noqa that are not used -- repo: https://github.com/asottile/yesqa - rev: "v1.4.0" - hooks: - - id: yesqa - additional_dependencies: &flake8_dependencies - - flake8-bugbear - - pep8-naming - -# Flake8 also supports pre-commit natively (same author) -- repo: https://github.com/PyCQA/flake8 - rev: "6.0.0" - hooks: - - id: flake8 - exclude: ^(docs/.*|tools/.*)$ - additional_dependencies: *flake8_dependencies # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint diff --git a/docs/conf.py b/docs/conf.py index cdb9f63069..6e24751e9e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -353,7 +353,7 @@ def prepare(app): f.write(contents) -def clean_up(app, exception): +def clean_up(app, exception): # noqa: ARG001 (DIR / "readme.rst").unlink() diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 4fbb17079f..7c10b30578 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -1,6 +1,6 @@ import sys -if sys.version_info < (3, 6): +if sys.version_info < (3, 6): # noqa: UP036 msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5." raise ImportError(msg) diff --git a/pybind11/commands.py b/pybind11/commands.py index 152fa20ce7..b11690f46b 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -3,7 +3,7 @@ DIR = os.path.abspath(os.path.dirname(__file__)) -def get_include(user: bool = False) -> str: # pylint: disable=unused-argument +def get_include(user: bool = False) -> str: # noqa: ARG001 """ Return the path to the pybind11 include directory. The historical "user" argument is unused, and may be removed. diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index b9b433b656..0fb4679e49 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -341,7 +341,7 @@ def naive_recompile(obj: str, src: str) -> bool: return os.stat(obj).st_mtime < os.stat(src).st_mtime -def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument +def no_recompile(obg: str, src: str) -> bool: # noqa: ARG001 """ This is the safest but slowest choice (and is the default) - will always recompile sources. diff --git a/pyproject.toml b/pyproject.toml index 3ba1b4b22f..96cf842650 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,6 @@ ignore = [ "noxfile.py", ] -[tool.isort] -# Needs the compiled .so modules and env.py from tests -known_first_party = "env,pybind11_cross_module_tests,pybind11_tests," -# For black compatibility -profile = "black" - [tool.mypy] files = ["pybind11"] python_version = "3.6" @@ -58,4 +52,44 @@ messages_control.disable = [ "invalid-name", "protected-access", "missing-module-docstring", + "unused-argument", # covered by Ruff ARG +] + +[tool.ruff] +select = [ + "E", "F", "W", # flake8 + "B", "B904", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF100", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 ] +ignore = [ + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118",# iter(x) is not always the same as iter(x.keys()) +] +target-version = "py37" +typing-modules = ["scikit_build_core._compat.typing"] +src = ["src"] +unfixable = ["T20"] +exclude = [] +line-length = 120 +isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] + +[tool.ruff.per-file-ignores] +"tests/**" = ["EM", "N"] diff --git a/setup.cfg b/setup.cfg index 8b3361981a..c35dd07ae6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,11 +40,3 @@ project_urls = [options] python_requires = >=3.6 zip_safe = False - - -[flake8] -max-line-length = 120 -show_source = True -exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722 -extend-select = B902, B904 diff --git a/tests/conftest.py b/tests/conftest.py index c24d51d4d1..b60d3c956b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,9 +83,8 @@ def __eq__(self, other): b = _strip_and_dedent(other).splitlines() if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Unordered(Output): @@ -96,9 +95,8 @@ def __eq__(self, other): b = _split_and_sort(other) if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Capture: @@ -119,9 +117,8 @@ def __eq__(self, other): b = other if a == b: return True - else: - self.explanation = a.explanation - return False + self.explanation = a.explanation + return False def __str__(self): return self.out @@ -138,7 +135,7 @@ def stderr(self): return Output(self.err) -@pytest.fixture +@pytest.fixture() def capture(capsys): """Extended `capsys` with context manager and custom equality operators""" return Capture(capsys) @@ -159,25 +156,22 @@ def __eq__(self, other): b = _strip_and_dedent(other) if a == b: return True - else: - self.explanation = _make_explanation(a.splitlines(), b.splitlines()) - return False + self.explanation = _make_explanation(a.splitlines(), b.splitlines()) + return False def _sanitize_general(s): s = s.strip() s = s.replace("pybind11_tests.", "m.") - s = _long_marker.sub(r"\1", s) - return s + return _long_marker.sub(r"\1", s) def _sanitize_docstring(thing): s = thing.__doc__ - s = _sanitize_general(s) - return s + return _sanitize_general(s) -@pytest.fixture +@pytest.fixture() def doc(): """Sanitize docstrings and add custom failure explanation""" return SanitizedString(_sanitize_docstring) @@ -186,30 +180,20 @@ def doc(): def _sanitize_message(thing): s = str(thing) s = _sanitize_general(s) - s = _hexadecimal.sub("0", s) - return s + return _hexadecimal.sub("0", s) -@pytest.fixture +@pytest.fixture() def msg(): """Sanitize messages and add custom failure explanation""" return SanitizedString(_sanitize_message) -# noinspection PyUnusedLocal -def pytest_assertrepr_compare(op, left, right): +def pytest_assertrepr_compare(op, left, right): # noqa: ARG001 """Hook to insert custom failure explanation""" if hasattr(left, "explanation"): return left.explanation - - -@contextlib.contextmanager -def suppress(exception): - """Suppress the desired exception""" - try: - yield - except exception: - pass + return None def gc_collect(): @@ -220,7 +204,7 @@ def gc_collect(): def pytest_configure(): - pytest.suppress = suppress + pytest.suppress = contextlib.suppress pytest.gc_collect = gc_collect diff --git a/tests/env.py b/tests/env.py index 0345df65dc..7eea5a3b3b 100644 --- a/tests/env.py +++ b/tests/env.py @@ -24,5 +24,4 @@ def deprecated_call(): pytest_major_minor = (int(pieces[0]), int(pieces[1])) if pytest_major_minor < (3, 9): return pytest.warns((DeprecationWarning, PendingDeprecationWarning)) - else: - return pytest.deprecated_call() + return pytest.deprecated_call() diff --git a/tests/test_async.py b/tests/test_async.py index b9ff9514d2..83a4c5036a 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -4,7 +4,7 @@ m = pytest.importorskip("pybind11_tests.async_module") -@pytest.fixture +@pytest.fixture() def event_loop(): loop = asyncio.new_event_loop() yield loop @@ -16,7 +16,7 @@ async def get_await_result(x): def test_await(event_loop): - assert 5 == event_loop.run_until_complete(get_await_result(m.SupportsAsync())) + assert event_loop.run_until_complete(get_await_result(m.SupportsAsync())) == 5 def test_await_missing(event_loop): diff --git a/tests/test_buffers.py b/tests/test_buffers.py index 8354b68cda..eb58c4675e 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -54,7 +54,8 @@ def test_to_python(): mat2 = np.array(mat, copy=False) assert mat2.shape == (5, 4) assert abs(mat2).sum() == 11 - assert mat2[2, 3] == 4 and mat2[3, 2] == 7 + assert mat2[2, 3] == 4 + assert mat2[3, 2] == 7 mat2[2, 3] = 5 assert mat2[2, 3] == 5 diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index d38ae68028..b1f57bdd98 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -126,8 +126,8 @@ def test_bytes_to_string(): assert m.strlen(b"hi") == 2 assert m.string_length(b"world") == 5 - assert m.string_length("a\x00b".encode()) == 3 - assert m.strlen("a\x00b".encode()) == 1 # C-string limitation + assert m.string_length(b"a\x00b") == 3 + assert m.strlen(b"a\x00b") == 1 # C-string limitation # passing in a utf8 encoded string should work assert m.string_length("πŸ’©".encode()) == 4 @@ -421,13 +421,15 @@ def test_reference_wrapper(): a2 = m.refwrap_list(copy=True) assert [x.value for x in a1] == [2, 3] assert [x.value for x in a2] == [2, 3] - assert not a1[0] is a2[0] and not a1[1] is a2[1] + assert a1[0] is not a2[0] + assert a1[1] is not a2[1] b1 = m.refwrap_list(copy=False) b2 = m.refwrap_list(copy=False) assert [x.value for x in b1] == [1, 2] assert [x.value for x in b2] == [1, 2] - assert b1[0] is b2[0] and b1[1] is b2[1] + assert b1[0] is b2[0] + assert b1[1] is b2[1] assert m.refwrap_iiw(IncType(5)) == 5 assert m.refwrap_call_iiw(IncType(10), m.refwrap_iiw) == [10, 10, 10, 10] diff --git a/tests/test_class.py b/tests/test_class.py index 64abac133c..ee7467cf8b 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -6,10 +6,7 @@ def test_obj_class_name(): - if env.PYPY: - expected_name = "UserType" - else: - expected_name = "pybind11_tests.UserType" + expected_name = "UserType" if env.PYPY else "pybind11_tests.UserType" assert m.obj_class_name(UserType(1)) == expected_name assert m.obj_class_name(UserType) == expected_name @@ -32,7 +29,7 @@ def test_instance(msg): assert cstats.alive() == 0 -def test_instance_new(msg): +def test_instance_new(): instance = m.NoConstructorNew() # .__new__(m.NoConstructor.__class__) cstats = ConstructorStats.get(m.NoConstructorNew) assert cstats.alive() == 1 @@ -221,7 +218,7 @@ def test_automatic_upcasting(): def test_isinstance(): - objects = [tuple(), dict(), m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 + objects = [(), {}, m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 expected = (True, True, True, True, True, False, False) assert m.check_instances(objects) == expected @@ -427,7 +424,7 @@ def test_exception_rvalue_abort(): # https://github.com/pybind/pybind11/issues/1568 -def test_multiple_instances_with_same_pointer(capture): +def test_multiple_instances_with_same_pointer(): n = 100 instances = [m.SamePointer() for _ in range(n)] for i in range(n): diff --git a/tests/test_const_name.py b/tests/test_const_name.py index 10b0caee2e..a145f0bbb5 100644 --- a/tests/test_const_name.py +++ b/tests/test_const_name.py @@ -3,9 +3,9 @@ from pybind11_tests import const_name as m -@pytest.mark.parametrize("func", (m.const_name_tests, m.underscore_tests)) +@pytest.mark.parametrize("func", [m.const_name_tests, m.underscore_tests]) @pytest.mark.parametrize( - "selector, expected", + ("selector", "expected"), enumerate( ( "", diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index 731380e46a..3a00ea964b 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -100,7 +100,8 @@ def test_custom_caster_destruction(): cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: z = m.custom_caster_no_destroy() - assert cstats.alive() == 1 and cstats.default_constructions == 1 + assert cstats.alive() == 1 + assert cstats.default_constructions == 1 assert z # take_ownership applied: this constructs a new object, casts it, then destroys it: diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 19b44c9de2..e63ff5758e 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -7,7 +7,7 @@ from pybind11_tests import custom_type_setup as m -@pytest.fixture +@pytest.fixture() def gc_tester(): """Tests that an object is garbage collected. diff --git a/tests/test_eigen_matrix.py b/tests/test_eigen_matrix.py index 4407fa6aee..b2e76740b1 100644 --- a/tests/test_eigen_matrix.py +++ b/tests/test_eigen_matrix.py @@ -263,79 +263,96 @@ def test_eigen_return_references(): primary = np.ones((10, 10)) a = m.ReturnTester() a_get1 = a.get() - assert not a_get1.flags.owndata and a_get1.flags.writeable + assert not a_get1.flags.owndata + assert a_get1.flags.writeable assign_both(a_get1, primary, 3, 3, 5) a_get2 = a.get_ptr() - assert not a_get2.flags.owndata and a_get2.flags.writeable + assert not a_get2.flags.owndata + assert a_get2.flags.writeable assign_both(a_get1, primary, 2, 3, 6) a_view1 = a.view() - assert not a_view1.flags.owndata and not a_view1.flags.writeable + assert not a_view1.flags.owndata + assert not a_view1.flags.writeable with pytest.raises(ValueError): a_view1[2, 3] = 4 a_view2 = a.view_ptr() - assert not a_view2.flags.owndata and not a_view2.flags.writeable + assert not a_view2.flags.owndata + assert not a_view2.flags.writeable with pytest.raises(ValueError): a_view2[2, 3] = 4 a_copy1 = a.copy_get() - assert a_copy1.flags.owndata and a_copy1.flags.writeable + assert a_copy1.flags.owndata + assert a_copy1.flags.writeable np.testing.assert_array_equal(a_copy1, primary) a_copy1[7, 7] = -44 # Shouldn't affect anything else c1want = array_copy_but_one(primary, 7, 7, -44) a_copy2 = a.copy_view() - assert a_copy2.flags.owndata and a_copy2.flags.writeable + assert a_copy2.flags.owndata + assert a_copy2.flags.writeable np.testing.assert_array_equal(a_copy2, primary) a_copy2[4, 4] = -22 # Shouldn't affect anything else c2want = array_copy_but_one(primary, 4, 4, -22) a_ref1 = a.ref() - assert not a_ref1.flags.owndata and a_ref1.flags.writeable + assert not a_ref1.flags.owndata + assert a_ref1.flags.writeable assign_both(a_ref1, primary, 1, 1, 15) a_ref2 = a.ref_const() - assert not a_ref2.flags.owndata and not a_ref2.flags.writeable + assert not a_ref2.flags.owndata + assert not a_ref2.flags.writeable with pytest.raises(ValueError): a_ref2[5, 5] = 33 a_ref3 = a.ref_safe() - assert not a_ref3.flags.owndata and a_ref3.flags.writeable + assert not a_ref3.flags.owndata + assert a_ref3.flags.writeable assign_both(a_ref3, primary, 0, 7, 99) a_ref4 = a.ref_const_safe() - assert not a_ref4.flags.owndata and not a_ref4.flags.writeable + assert not a_ref4.flags.owndata + assert not a_ref4.flags.writeable with pytest.raises(ValueError): a_ref4[7, 0] = 987654321 a_copy3 = a.copy_ref() - assert a_copy3.flags.owndata and a_copy3.flags.writeable + assert a_copy3.flags.owndata + assert a_copy3.flags.writeable np.testing.assert_array_equal(a_copy3, primary) a_copy3[8, 1] = 11 c3want = array_copy_but_one(primary, 8, 1, 11) a_copy4 = a.copy_ref_const() - assert a_copy4.flags.owndata and a_copy4.flags.writeable + assert a_copy4.flags.owndata + assert a_copy4.flags.writeable np.testing.assert_array_equal(a_copy4, primary) a_copy4[8, 4] = 88 c4want = array_copy_but_one(primary, 8, 4, 88) a_block1 = a.block(3, 3, 2, 2) - assert not a_block1.flags.owndata and a_block1.flags.writeable + assert not a_block1.flags.owndata + assert a_block1.flags.writeable a_block1[0, 0] = 55 primary[3, 3] = 55 a_block2 = a.block_safe(2, 2, 3, 2) - assert not a_block2.flags.owndata and a_block2.flags.writeable + assert not a_block2.flags.owndata + assert a_block2.flags.writeable a_block2[2, 1] = -123 primary[4, 3] = -123 a_block3 = a.block_const(6, 7, 4, 3) - assert not a_block3.flags.owndata and not a_block3.flags.writeable + assert not a_block3.flags.owndata + assert not a_block3.flags.writeable with pytest.raises(ValueError): a_block3[2, 2] = -44444 a_copy5 = a.copy_block(2, 2, 2, 3) - assert a_copy5.flags.owndata and a_copy5.flags.writeable + assert a_copy5.flags.owndata + assert a_copy5.flags.writeable np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5]) a_copy5[1, 1] = 777 c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777) a_corn1 = a.corners() - assert not a_corn1.flags.owndata and a_corn1.flags.writeable + assert not a_corn1.flags.owndata + assert a_corn1.flags.writeable a_corn1 *= 50 a_corn1[1, 1] = 999 primary[0, 0] = 50 @@ -343,7 +360,8 @@ def test_eigen_return_references(): primary[9, 0] = 50 primary[9, 9] = 999 a_corn2 = a.corners_const() - assert not a_corn2.flags.owndata and not a_corn2.flags.writeable + assert not a_corn2.flags.owndata + assert not a_corn2.flags.writeable with pytest.raises(ValueError): a_corn2[1, 0] = 51 @@ -503,10 +521,14 @@ def test_numpy_ref_mutators(): assert [zc[1, 2], zcro[1, 2], zr[1, 2], zrro[1, 2]] == [23] * 4 - assert not zc.flags.owndata and zc.flags.writeable - assert not zr.flags.owndata and zr.flags.writeable - assert not zcro.flags.owndata and not zcro.flags.writeable - assert not zrro.flags.owndata and not zrro.flags.writeable + assert not zc.flags.owndata + assert zc.flags.writeable + assert not zr.flags.owndata + assert zr.flags.writeable + assert not zcro.flags.owndata + assert not zcro.flags.writeable + assert not zrro.flags.owndata + assert not zrro.flags.writeable zc[1, 2] = 99 expect = np.array([[11.0, 12, 13], [21, 22, 99], [31, 32, 33]]) @@ -530,7 +552,8 @@ def test_numpy_ref_mutators(): # the const should drop away) y1 = np.array(m.get_cm_const_ref()) - assert y1.flags.owndata and y1.flags.writeable + assert y1.flags.owndata + assert y1.flags.writeable # We should get copies of the eigen data, which was modified above: assert y1[1, 2] == 99 y1[1, 2] += 12 @@ -603,38 +626,38 @@ def test_nocopy_wrapper(): # All but the second should fail with m.get_elem_nocopy: with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_colmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_rowmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(dbl_matrix_rowmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) # For the row-major test, we take a long matrix in row-major, so only the third is allowed: with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(int_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_rowmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) def test_eigen_ref_life_support(): diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index cd54efa069..3e7ee6b7f2 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -11,14 +11,15 @@ submodules += [avoid.c_style, avoid.f_style] except ImportError as e: # Ensure config, build, toolchain, etc. issues are not masked here: - raise RuntimeError( + msg = ( "import eigen_tensor_avoid_stl_array FAILED, while " "import pybind11_tests.eigen_tensor succeeded. " "Please ensure that " "test_eigen_tensor.cpp & " "eigen_tensor_avoid_stl_array.cpp " "are built together (or both are not built if Eigen is not available)." - ) from e + ) + raise RuntimeError(msg) from e tensor_ref = np.empty((3, 5, 2), dtype=np.int64) @@ -147,10 +148,7 @@ def test_bad_python_to_cpp_casts(m): m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64)) ) - if m.needed_options == "F": - bad_options = "C" - else: - bad_options = "F" + bad_options = "C" if m.needed_options == "F" else "F" # Shape, dtype and the order need to be correct for a TensorMap cast with pytest.raises( TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" @@ -173,19 +171,19 @@ def test_bad_python_to_cpp_casts(m): np.zeros((3, 5), dtype=np.float64, order=m.needed_options) ) + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) with pytest.raises( TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" ): - temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) m.round_trip_view_tensor( temp[:, ::-1, :], ) + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + temp.setflags(write=False) with pytest.raises( TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" ): - temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) - temp.setflags(write=False) m.round_trip_view_tensor(temp) @@ -282,9 +280,9 @@ def test_doc_string(m, doc): order_flag = f"flags.{m.needed_options.lower()}_contiguous" assert doc(m.round_trip_view_tensor) == ( f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" - + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" ) assert doc(m.round_trip_const_view_tensor) == ( f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])" - + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) diff --git a/tests/test_enum.py b/tests/test_enum.py index f14a72398f..4e85d29c31 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -1,3 +1,5 @@ +# ruff: noqa: SIM201 SIM300 SIM202 + import pytest from pybind11_tests import enums as m diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 8bcefe3de6..7f80a5da56 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -94,8 +94,7 @@ def ignore_pytest_unraisable_warning(f): if hasattr(pytest, unraisable): # Python >= 3.8 and pytest >= 6 dec = pytest.mark.filterwarnings(f"ignore::pytest.{unraisable}") return dec(f) - else: - return f + return f # TODO: find out why this fails on PyPy, https://foss.heptapod.net/pypy/pypy/-/issues/3583 @@ -183,7 +182,7 @@ def test_custom(msg): m.throws5_1() assert msg(excinfo.value) == "MyException5 subclass" - with pytest.raises(m.MyException5) as excinfo: + with pytest.raises(m.MyException5) as excinfo: # noqa: PT012 try: m.throws5() except m.MyException5_1 as err: @@ -212,7 +211,7 @@ def throw_myex5(): m.try_catch(m.MyException5, throw_myex) assert str(excinfo.value) == "nested error" - def pycatch(exctype, f, *args): + def pycatch(exctype, f, *args): # noqa: ARG001 try: f(*args) except m.MyException as e: @@ -303,12 +302,12 @@ def __str__(self): @pytest.mark.parametrize( - "exc_type, exc_value, expected_what", - ( + ("exc_type", "exc_value", "expected_what"), + [ (ValueError, "plain_str", "ValueError: plain_str"), (ValueError, ("tuple_elem",), "ValueError: tuple_elem"), (FlakyException, ("happy",), "FlakyException: FlakyException.__str__"), - ), + ], ) def test_error_already_set_what_with_happy_exceptions( exc_type, exc_value, expected_what @@ -342,10 +341,7 @@ def test_flaky_exception_failure_point_str(): ) assert not py_err_set_after_what lines = what.splitlines() - if env.PYPY and len(lines) == 3: - n = 3 # Traceback is missing. - else: - n = 5 + n = 3 if env.PYPY and len(lines) == 3 else 5 assert ( lines[:n] == [ diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index 120a587c45..04df80260d 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -96,7 +96,7 @@ def test_init_factory_signature(msg): 3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None 4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None - """ # noqa: E501 line too long + """ ) diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 6af6a472d5..fc8af9b77c 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -148,10 +148,7 @@ def _intentional_deadlock(): def _run_in_process(target, *args, **kwargs): - if len(args) == 0: - test_fn = target - else: - test_fn = args[0] + test_fn = target if len(args) == 0 else args[0] # Do not need to wait much, 10s should be more than enough. timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) @@ -178,7 +175,8 @@ def _run_in_process(target, *args, **kwargs): elif test_fn is _intentional_deadlock: assert process.exitcode is None return 0 - elif process.exitcode is None: + + if process.exitcode is None: assert t_delta > 0.9 * timeout msg = "DEADLOCK, most likely, exactly what this test is meant to detect." if env.PYPY and env.WIN: diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 5bbdf6955d..79b2a2b8cc 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -224,9 +224,8 @@ def test_redirect(capfd): assert stream.getvalue() == "" stream = StringIO() - with redirect_stdout(stream): - with m.ostream_redirect(): - m.raw_output(msg) + with redirect_stdout(stream), m.ostream_redirect(): + m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == "" assert stream.getvalue() == msg @@ -244,10 +243,9 @@ def test_redirect_err(capfd): msg2 = "StdErr" stream = StringIO() - with redirect_stderr(stream): - with m.ostream_redirect(stdout=False): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stderr(stream), m.ostream_redirect(stdout=False): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg assert stderr == "" @@ -260,11 +258,9 @@ def test_redirect_both(capfd): stream = StringIO() stream2 = StringIO() - with redirect_stdout(stream): - with redirect_stderr(stream2): - with m.ostream_redirect(): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stdout(stream), redirect_stderr(stream2), m.ostream_redirect(): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == "" assert stderr == "" diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index ab7017886e..7174726fce 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -25,7 +25,7 @@ def test_function_signatures(doc): ) -def test_named_arguments(msg): +def test_named_arguments(): assert m.kw_func0(5, 10) == "x=5, y=10" assert m.kw_func1(5, 10) == "x=5, y=10" @@ -43,8 +43,7 @@ def test_named_arguments(msg): # noinspection PyArgumentList m.kw_func2(x=5, y=10, z=12) assert excinfo.match( - r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))" - + "{3}$" + r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$)){3}$" ) assert m.kw_func4() == "{13 17}" @@ -59,7 +58,7 @@ def test_arg_and_kwargs(): assert m.args_function(*args) == args args = "a1", "a2" - kwargs = dict(arg3="a3", arg4=4) + kwargs = {"arg3": "a3", "arg4": 4} assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) @@ -177,7 +176,7 @@ def test_mixed_args_and_kwargs(msg): assert ( m.args_kwonly_kwargs_defaults.__doc__ - == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" # noqa: E501 line too long + == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" ) assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {}) assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {}) @@ -233,15 +232,15 @@ def test_keyword_only_args(msg): x.method(i=1, j=2) assert ( m.first_arg_kw_only.__init__.__doc__ - == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" # noqa: E501 line too long + == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" ) assert ( m.first_arg_kw_only.method.__doc__ - == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" # noqa: E501 line too long + == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" ) -def test_positional_only_args(msg): +def test_positional_only_args(): assert m.pos_only_all(1, 2) == (1, 2) assert m.pos_only_all(2, 1) == (2, 1) @@ -283,7 +282,7 @@ def test_positional_only_args(msg): # Mix it with args and kwargs: assert ( m.args_kwonly_full_monty.__doc__ - == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" # noqa: E501 line too long + == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" ) assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {}) assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {}) @@ -326,18 +325,18 @@ def test_positional_only_args(msg): # https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987 assert ( m.first_arg_kw_only.pos_only.__doc__ - == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" # noqa: E501 line too long + == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" ) def test_signatures(): - assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__ - assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__ - assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__ - assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__ + assert m.kw_only_all.__doc__ == "kw_only_all(*, i: int, j: int) -> tuple\n" + assert m.kw_only_mixed.__doc__ == "kw_only_mixed(i: int, *, j: int) -> tuple\n" + assert m.pos_only_all.__doc__ == "pos_only_all(i: int, j: int, /) -> tuple\n" + assert m.pos_only_mix.__doc__ == "pos_only_mix(i: int, /, j: int) -> tuple\n" assert ( - "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" - == m.pos_kw_only_mix.__doc__ + m.pos_kw_only_mix.__doc__ + == "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" ) diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 0a2ae1239a..a854685750 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -183,9 +183,9 @@ def test_static_properties(): # Only static attributes can be deleted del m.TestPropertiesOverride.def_readonly_static + assert hasattr(m.TestPropertiesOverride, "def_readonly_static") assert ( - hasattr(m.TestPropertiesOverride, "def_readonly_static") - and m.TestPropertiesOverride.def_readonly_static + m.TestPropertiesOverride.def_readonly_static is m.TestProperties.def_readonly_static ) assert "def_readonly_static" not in m.TestPropertiesOverride.__dict__ @@ -256,10 +256,7 @@ def test_no_mixed_overloads(): @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) def test_property_return_value_policies(access): - if not access.startswith("static"): - obj = m.TestPropRVP() - else: - obj = m.TestPropRVP + obj = m.TestPropRVP() if not access.startswith("static") else m.TestPropRVP ref = getattr(obj, access + "_ref") assert ref.value == 1 diff --git a/tests/test_modules.py b/tests/test_modules.py index c3e8cea6e1..3d73e3fbd2 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,3 +1,5 @@ +import builtins + import pytest import env @@ -86,12 +88,7 @@ def test_builtin_key_type(): Previous versions of pybind11 would add a unicode key in python 2. """ - if hasattr(__builtins__, "keys"): - keys = __builtins__.keys() - else: # this is to make pypy happy since builtins is different there. - keys = __builtins__.__dict__.keys() - - assert {type(k) for k in keys} == {str} + assert all(type(k) == str for k in dir(builtins)) @pytest.mark.xfail("env.PYPY", reason="PyModule_GetName()") diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index cdec9ad60b..070813d3a2 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -22,7 +22,7 @@ def test_dtypes(): ) -@pytest.fixture(scope="function") +@pytest.fixture() def arr(): return np.array([[1, 2, 3], [4, 5, 6]], "=u2") @@ -67,7 +67,7 @@ def test_array_attributes(): @pytest.mark.parametrize( - "args, ret", [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] + ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] ) def test_index_offset(arr, args, ret): assert m.index_at(arr, *args) == ret @@ -93,7 +93,7 @@ def test_dim_check_fail(arr): @pytest.mark.parametrize( - "args, ret", + ("args", "ret"), [ ([], [1, 2, 3, 4, 5, 6]), ([1], [4, 5, 6]), @@ -211,12 +211,14 @@ def assert_references(a, b, base=None): assert b[0, 0] == 1234 a1 = np.array([1, 2], dtype=np.int16) - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="F") - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) @@ -451,13 +453,15 @@ def test_array_resize(): try: m.array_resize3(a, 3, True) except ValueError as e: - assert str(e).startswith("cannot resize an array") + assert str(e).startswith("cannot resize an array") # noqa: PT017 # transposed array doesn't own data b = a.transpose() try: m.array_resize3(b, 3, False) except ValueError as e: - assert str(e).startswith("cannot resize this array: it does not own its data") + assert str(e).startswith( # noqa: PT017 + "cannot resize this array: it does not own its data" + ) # ... but reshape should be fine m.array_reshape2(b) assert b.shape == (8, 8) diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index fcfd587b18..d10457eeb2 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -130,14 +130,10 @@ def test_dtype(simple_dtype): partial_nested_fmt(), "[('a','S3'),('b','S3')]", ( - "{{'names':['a','b','c','d']," - + "'formats':[('S4',(3,)),('" - + e - + "i4',(2,)),('u1',(3,)),('" - + e - + "f4',(4,2))]," - + "'offsets':[0,12,20,24],'itemsize':56}}" - ).format(e=e), + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ), "[('e1','" + e + "i8'),('e2','u1')]", "[('x','i1'),('y','" + e + "u8')]", "[('cflt','" + e + "c8'),('cdbl','" + e + "c16')]", @@ -291,19 +287,17 @@ def test_array_array(): arr = m.create_array_array(3) assert str(arr.dtype).replace(" ", "") == ( - "{{'names':['a','b','c','d']," - + "'formats':[('S4',(3,)),('" - + e - + "i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," - + "'offsets':[0,12,20,24],'itemsize':56}}" - ).format(e=e) + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ) assert m.print_array_array(arr) == [ "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1}," - + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001}," - + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001}," - + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", ] assert arr["a"].tolist() == [ [b"ABCD", b"KLMN", b"UVWX"], diff --git a/tests/test_numpy_vectorize.py b/tests/test_numpy_vectorize.py index 7e8c015c46..f1e8b62540 100644 --- a/tests/test_numpy_vectorize.py +++ b/tests/test_numpy_vectorize.py @@ -149,7 +149,7 @@ def test_docs(doc): doc(m.vectorized_func) == """ vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object - """ # noqa: E501 line too long + """ ) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index a1a80a2933..946b941402 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -15,7 +15,7 @@ def test_obj_class_name(): assert m.obj_class_name([]) == "list" -def test_handle_from_move_only_type_with_operator_PyObject(): # noqa: N802 +def test_handle_from_move_only_type_with_operator_PyObject(): assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() assert m.handle_from_move_only_type_with_operator_PyObject_const() @@ -33,7 +33,7 @@ def test_iterator(doc): @pytest.mark.parametrize( - "pytype, from_iter_func", + ("pytype", "from_iter_func"), [ (frozenset, m.get_frozenset_from_iterable), (list, m.get_list_from_iterable), @@ -87,7 +87,7 @@ def test_list(capture, doc): assert doc(m.print_list) == "print_list(arg0: list) -> None" -def test_none(capture, doc): +def test_none(doc): assert doc(m.get_none) == "get_none() -> None" assert doc(m.print_none) == "print_none(arg0: None) -> None" @@ -182,10 +182,10 @@ def __contains__(self, m): @pytest.mark.parametrize( - "arg,func", + ("arg", "func"), [ (set(), m.anyset_contains), - (dict(), m.dict_contains), + ({}, m.dict_contains), (CustomContains(), m.obj_contains), ], ) @@ -273,7 +273,7 @@ def test_bytes(doc): assert doc(m.bytes_from_str) == "bytes_from_str() -> bytes" -def test_bytearray(doc): +def test_bytearray(): assert m.bytearray_from_char_ssize_t().decode() == "$%" assert m.bytearray_from_char_size_t().decode() == "@$!" assert m.bytearray_from_string().decode() == "foo" @@ -385,7 +385,7 @@ def func(self, x, *args): assert d["implicit_list"] == [1, 2, 3] assert all(x in TestObject.__dict__ for x in d["implicit_dict"]) - assert m.tuple_accessor(tuple()) == (0, 1, 2) + assert m.tuple_accessor(()) == (0, 1, 2) d = m.accessor_assignment() assert d["get"] == 0 @@ -593,7 +593,7 @@ def test_issue2361(): @pytest.mark.parametrize( - "method, args, fmt, expected_view", + ("method", "args", "fmt", "expected_view"), [ (m.test_memoryview_object, (b"red",), "B", b"red"), (m.test_memoryview_buffer_info, (b"green",), "B", b"green"), @@ -651,7 +651,7 @@ def test_memoryview_from_memory(): def test_builtin_functions(): - assert m.get_len([i for i in range(42)]) == 42 + assert m.get_len(list(range(42))) == 42 with pytest.raises(TypeError) as exc_info: m.get_len(i for i in range(42)) assert str(exc_info.value) in [ @@ -695,7 +695,7 @@ def test_pass_bytes_or_unicode_to_string_types(): @pytest.mark.parametrize( - "create_weakref, create_weakref_with_callback", + ("create_weakref", "create_weakref_with_callback"), [ (m.weakref_from_handle, m.weakref_from_handle_and_function), (m.weakref_from_object, m.weakref_from_object_and_function), @@ -710,7 +710,7 @@ class WeaklyReferenced: callback_called = False - def callback(wr): + def callback(_): nonlocal callback_called callback_called = True @@ -730,7 +730,7 @@ def callback(wr): @pytest.mark.parametrize( - "create_weakref, has_callback", + ("create_weakref", "has_callback"), [ (m.weakref_from_handle, False), (m.weakref_from_object, False), @@ -748,10 +748,7 @@ def callback(_): ob = C() # Should raise TypeError on CPython with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext(): - if has_callback: - _ = create_weakref(ob, callback) - else: - _ = create_weakref(ob) + _ = create_weakref(ob, callback) if has_callback else create_weakref(ob) def test_cpp_iterators(): @@ -814,33 +811,36 @@ def test_populate_obj_str_attrs(): @pytest.mark.parametrize( - "a,b", [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))] + ("a", "b"), + [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))], ) def test_inplace_append(a, b): expected = a + b assert m.inplace_append(a, b) == expected -@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))]) +@pytest.mark.parametrize( + ("a", "b"), [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))] +) def test_inplace_subtract(a, b): expected = a - b assert m.inplace_subtract(a, b) == expected -@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), ([1], 3)]) +@pytest.mark.parametrize(("a", "b"), [(3, 2), (3.0, 2.0), ([1], 3)]) def test_inplace_multiply(a, b): expected = a * b assert m.inplace_multiply(a, b) == expected -@pytest.mark.parametrize("a,b", [(6, 3), (6.0, 3.0)]) +@pytest.mark.parametrize(("a", "b"), [(6, 3), (6.0, 3.0)]) def test_inplace_divide(a, b): expected = a / b assert m.inplace_divide(a, b) == expected @pytest.mark.parametrize( - "a,b", + ("a", "b"), [ (False, True), ( @@ -857,7 +857,7 @@ def test_inplace_or(a, b): @pytest.mark.parametrize( - "a,b", + ("a", "b"), [ (True, False), ( @@ -873,13 +873,13 @@ def test_inplace_and(a, b): assert m.inplace_and(a, b) == expected -@pytest.mark.parametrize("a,b", [(8, 1), (-3, 2)]) +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-3, 2)]) def test_inplace_lshift(a, b): expected = a << b assert m.inplace_lshift(a, b) == expected -@pytest.mark.parametrize("a,b", [(8, 1), (-2, 2)]) +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-2, 2)]) def test_inplace_rshift(a, b): expected = a >> b assert m.inplace_rshift(a, b) == expected diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index de486e3e8e..dc129f2bff 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -1,5 +1,5 @@ import pytest -from pytest import approx +from pytest import approx # noqa: PT013 from pybind11_tests import ConstructorStats from pybind11_tests import sequences_and_iterators as m @@ -103,7 +103,8 @@ def test_sequence(): assert "Sequence" in repr(s) assert len(s) == 5 - assert s[0] == 0 and s[3] == 0 + assert s[0] == 0 + assert s[3] == 0 assert 12.34 not in s s[0], s[3] = 12.34, 56.78 assert 12.34 in s @@ -245,7 +246,7 @@ def test_iterator_rvp(): def test_carray_iterator(): """#4100: Check for proper iterator overload with C-Arrays""" - args_gt = list(float(i) for i in range(3)) + args_gt = [float(i) for i in range(3)] arr_h = m.CArrayHolder(*args_gt) args = list(arr_h) assert args_gt == args diff --git a/tests/test_stl.py b/tests/test_stl.py index d30c382113..b7a0d1c0d5 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -14,7 +14,7 @@ def test_vector(doc): assert m.cast_bool_vector() == [True, False] assert m.load_bool_vector([True, False]) - assert m.load_bool_vector(tuple([True, False])) + assert m.load_bool_vector((True, False)) assert doc(m.cast_vector) == "cast_vector() -> List[int]" assert doc(m.load_vector) == "load_vector(arg0: List[int]) -> bool" @@ -23,7 +23,7 @@ def test_vector(doc): assert m.cast_ptr_vector() == ["lvalue", "lvalue"] -def test_deque(doc): +def test_deque(): """std::deque <-> list""" lst = m.cast_deque() assert lst == [1] @@ -95,7 +95,8 @@ def test_recursive_casting(): # Issue #853 test case: z = m.cast_unique_ptr_vector() - assert z[0].value == 7 and z[1].value == 42 + assert z[0].value == 7 + assert z[1].value == 42 def test_move_out_container(): diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 9eb906f065..7dca742a9e 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -186,9 +186,9 @@ def test_map_string_double(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] + assert sorted(um) == ["ua", "ub"] assert list(um.keys()) == list(um) - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] assert list(zip(um.keys(), um.values())) == list(um.items()) assert "UnorderedMapStringDouble" in str(um) @@ -304,11 +304,11 @@ def test_map_delitem(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um) == ["ua", "ub"] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] del um["ua"] - assert sorted(list(um)) == ["ub"] - assert sorted(list(um.items())) == [("ub", 2.6)] + assert sorted(um) == ["ub"] + assert sorted(um.items()) == [("ub", 2.6)] def test_map_view_types(): diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index 4d00d3690d..c17af7df58 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -192,8 +192,7 @@ def test_move_support(): class NCVirtExt(m.NCVirt): def get_noncopyable(self, a, b): # Constructs and returns a new instance: - nc = m.NonCopyable(a * a, b * b) - return nc + return m.NonCopyable(a * a, b * b) def get_movable(self, a, b): # Return a referenced copy @@ -256,7 +255,7 @@ def dispatch(self): assert m.dispatch_issue_go(b) == "Yay.." -def test_recursive_dispatch_issue(msg): +def test_recursive_dispatch_issue(): """#3357: Recursive dispatch fails to find python function override""" class Data(m.Data): @@ -269,7 +268,7 @@ def __call__(self, first, second, visitor): # lambda is a workaround, which adds extra frame to the # current CPython thread. Removing lambda reveals the bug # [https://github.com/pybind/pybind11/issues/3357] - (lambda: visitor(Data(first.value + second.value)))() + (lambda: visitor(Data(first.value + second.value)))() # noqa: PLC3002 class StoreResultVisitor: def __init__(self): diff --git a/tools/codespell_ignore_lines_from_errors.py b/tools/codespell_ignore_lines_from_errors.py index 5403ec3ad0..4ec9add12d 100644 --- a/tools/codespell_ignore_lines_from_errors.py +++ b/tools/codespell_ignore_lines_from_errors.py @@ -17,14 +17,18 @@ def run(args: List[str]) -> None: assert len(args) == 1, "codespell_errors.txt" cache = {} done = set() - for line in sorted(open(args[0]).read().splitlines()): + with open(args[0]) as f: + lines = f.read().splitlines() + + for line in sorted(lines): i = line.find(" ==> ") if i > 0: flds = line[:i].split(":") if len(flds) >= 2: filename, line_num = flds[:2] if filename not in cache: - cache[filename] = open(filename).read().splitlines() + with open(filename) as f: + cache[filename] = f.read().splitlines() supp = cache[filename][int(line_num) - 1] if supp not in done: print(supp) From 3cc7e4258c15a6a19ba5e0b62a220b1a6196d4eb Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 23 Feb 2023 00:59:14 -0500 Subject: [PATCH 084/137] add --version option to pybind11-config (#4526) Without this, it's impossible to get feature parity between detection mechanisms. Both the pkg-config file and the cmake config set their versions, but the python probe script didn't provide an option for this. So you could print the compiler flags for using it, but you could not check what you got. --- pybind11/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index e87b16beb4..180665c23c 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -4,6 +4,7 @@ import sys import sysconfig +from ._version import __version__ from .commands import get_cmake_dir, get_include, get_pkgconfig_dir @@ -25,6 +26,12 @@ def print_includes() -> None: def main() -> None: parser = argparse.ArgumentParser() + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) parser.add_argument( "--includes", action="store_true", From cbb876cc7b02c5f57e715cbc2c46ead3d1fbcf79 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 00:34:41 -0500 Subject: [PATCH 085/137] chore(deps): update pre-commit hooks (#4552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.251 β†’ v0.0.254](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.251...v0.0.254) - [github.com/PyCQA/pylint: v2.16.1 β†’ v2.16.4](https://github.com/PyCQA/pylint/compare/v2.16.1...v2.16.4) - [github.com/pre-commit/mirrors-mypy: v0.991 β†’ v1.0.1](https://github.com/pre-commit/mirrors-mypy/compare/v0.991...v1.0.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2255f2e4d5..7c43161921 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.251 + rev: v0.0.254 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -84,7 +84,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.1" + rev: "v2.16.4" hooks: - id: pylint files: ^pybind11 @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.991" + rev: "v1.0.1" hooks: - id: mypy args: [] From 442261da585536521ff459b1457b2904895f23b4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 9 Mar 2023 13:02:34 -0800 Subject: [PATCH 086/137] Remove test code that does not exercise anything in pybind11, but breaks between Python 3.12alpha3 (still working) and 3.12alpha6 (broken): (#4559) ``` str(OrderedDict([(1, "a"), (2, "b")])) ``` Old: ``` OrderedDict([(1, 'a'), (2, 'b')]) ``` New: ``` OrderedDict({1: 'a', 2: 'b'}) ``` See also: https://github.com/python/cpython/issues/101446 --- tests/test_modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 3d73e3fbd2..2f6d825b79 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -63,7 +63,6 @@ def test_importing(): from pybind11_tests.modules import OD assert OD is OrderedDict - assert str(OD([(1, "a"), (2, "b")])) == "OrderedDict([(1, 'a'), (2, 'b')])" def test_pydoc(): From 04ef4e422980ce1406a76358cbb74e7e3fd23a21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 18:18:44 +0000 Subject: [PATCH 087/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.6.4 to 1.8.1 (#4576) --- .github/workflows/pip.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 7c6fc67a3e..3d4d2c6548 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.1 with: password: ${{ secrets.pypi_password }} - packages_dir: standard/ + packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.1 with: password: ${{ secrets.pypi_password_global }} - packages_dir: global/ + packages-dir: global/ From c4c15d4b109b7870319818d0f17fc0b4357ff631 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Mar 2023 13:01:28 -0700 Subject: [PATCH 088/137] docs: changelog for 2.10.4 (#4532) * docs: changelog for 2.10.4 Signed-off-by: Henry Schreiner * Apply suggestions from code review * Update docs/changelog.rst --------- Signed-off-by: Henry Schreiner --- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bb111c5f26..58cc409839 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,11 +21,47 @@ Changes: ``dec_ref()`` are now enabled by default again. `#4246 `_ +* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()`` + instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete + ``sys.path``. + `#4473 `_ + +* Cast errors now always include Python type information, even if + ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary + sizes slightly (~1.5%) but the error messages are much more informative. + `#4463 `_ + + Build system improvements: * Update clang-tidy to 15 in CI. `#4387 `_ +* Moved the linting framework over to Ruff. + `#4483 `_ + +Version 2.10.4 (Mar 16, 2023) +---------------------------- + +Changes: + +* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and + exits). + `#4526 `_ + +Bug Fixes: + +* Fix a warning when pydebug is enabled on Python 3.11. + `#4461 `_ + +* Ensure ``gil_scoped_release`` RAII is non-copyable. + `#4490 `_ + +* Ensure the tests dir does not show up with new versions of setuptools. + `#4510 `_ + +* Better stacklevel for a warning in setuptools helpers. + `#4516 `_ Version 2.10.3 (Jan 3, 2023) ---------------------------- From cf7d2e6f15be7622515a77466e84bd9414737680 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 16 Mar 2023 14:33:34 -0700 Subject: [PATCH 089/137] Change `always_forkserver_on_unix()` to `use_multiprocessing_forkserver_on_linux()` (#4577) --- tests/conftest.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b60d3c956b..ad5b47b4b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,8 @@ import difflib import gc import multiprocessing -import os import re +import sys import textwrap import traceback @@ -25,8 +25,9 @@ @pytest.fixture(scope="session", autouse=True) -def always_forkserver_on_unix(): - if os.name == "nt": +def use_multiprocessing_forkserver_on_linux(): + if sys.platform != "linux": + # The default on Windows and macOS is "spawn": If it's not broken, don't fix it. return # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 @@ -34,8 +35,6 @@ def always_forkserver_on_unix(): # It is actually a well-known pitfall, unfortunately without guard rails. # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, # visit the issuecomment link above for details). - # Windows does not have fork() and the associated pitfall, therefore it is best left - # running with defaults. multiprocessing.set_start_method("forkserver") From 286873ecf72f03df3a177a45d9dd4e9c678b2bdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:23:52 -0400 Subject: [PATCH 090/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.1 to 1.8.3 (#4584) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.1 to 1.8.3. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.1...v1.8.3) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 3d4d2c6548..d5427c7961 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.1 + uses: pypa/gh-action-pypi-publish@v1.8.3 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.1 + uses: pypa/gh-action-pypi-publish@v1.8.3 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 5bbcba548a27baf8d6bfb7993e814da24b432dbf Mon Sep 17 00:00:00 2001 From: Konstantin Bespalov Date: Fri, 24 Mar 2023 06:01:09 +0100 Subject: [PATCH 091/137] use C++17 syntax to get rid of recursive template instantiations for concatenating type signatures (#4587) --- include/pybind11/detail/descr.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index e7a5e2c145..85259d488e 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -143,11 +143,24 @@ constexpr descr concat(const descr &descr) { return descr; } +#if defined(PYBIND11_CPP17) +template +constexpr descr operator,(const descr &a, + const descr &b) { + return a + const_name(", ") + b; +} + +template +constexpr auto concat(const descr &d, const Args &...args) { + return (d, ..., args); +} +#else template constexpr auto concat(const descr &d, const Args &...args) -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } +#endif template constexpr descr type_descr(const descr &descr) { From 66f12df03b33a3ea32c82b0d0af7c303cf17d50e Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 27 Mar 2023 10:59:56 -0400 Subject: [PATCH 092/137] chore: make #4587 use proper cpp17 feature macro (#4592) --- include/pybind11/detail/descr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index 85259d488e..635614b0d6 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -143,7 +143,7 @@ constexpr descr concat(const descr &descr) { return descr; } -#if defined(PYBIND11_CPP17) +#ifdef __cpp_fold_expressions template constexpr descr operator,(const descr &a, const descr &b) { From 1e8b52a9acea1aa7a322fe6a87049c4828e439f9 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 27 Mar 2023 20:21:06 -0400 Subject: [PATCH 093/137] bugfix: allow noexcept lambdas in C++17. Fix #4565 (#4593) * bugfix: allow noexcept lambdas in CPP17. Fix #4565 * Remove unused code from test case * Fix clang-tidy error * Address reviewer comment --- include/pybind11/detail/common.h | 11 ++++++++++- tests/test_constants_and_functions.cpp | 3 +++ tests/test_constants_and_functions.py | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 1032293d21..129039252f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -754,7 +754,16 @@ template struct remove_class { using type = R(A...); }; - +#ifdef __cpp_noexcept_function_type +template +struct remove_class { + using type = R(A...); +}; +template +struct remove_class { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers template struct intrinsic_type { diff --git a/tests/test_constants_and_functions.cpp b/tests/test_constants_and_functions.cpp index 922375c5ea..312edca9e0 100644 --- a/tests/test_constants_and_functions.cpp +++ b/tests/test_constants_and_functions.cpp @@ -148,4 +148,7 @@ TEST_SUBMODULE(constants_and_functions, m) { py::arg_v("y", 42, ""), py::arg_v("z", default_value)); }); + + // test noexcept(true) lambda (#4565) + m.def("l1", []() noexcept(true) { return 0; }); } diff --git a/tests/test_constants_and_functions.py b/tests/test_constants_and_functions.py index 5da0b84b8e..a1142461c0 100644 --- a/tests/test_constants_and_functions.py +++ b/tests/test_constants_and_functions.py @@ -50,3 +50,7 @@ def __repr__(self): m.register_large_capture_with_invalid_arguments(m) with pytest.raises(RuntimeError): m.register_with_raising_repr(m, RaisingRepr()) + + +def test_noexcept_lambda(): + assert m.l1() == 0 From 654fe92652e6dc0eec80b1877b531aaab3a3e56c Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 27 Mar 2023 17:52:57 -0700 Subject: [PATCH 094/137] Introduce `get_python_state_dict()` for Python 3.12 compatibility. (#4570) * Introduce `get_python_state_dict()` * Conditional version bump for Python 3.12+ * Shuffle subexpressions to make the condition easier to understand (no change to logic). * Make pybind11 ABI version 5 the minimum for Python 3.12+ (as suggested by @Lalaland) * Add back condition for PYPY_VERSION, but keep it open for future PyPy versions. * Fall back to simple `|| defined(PYPY_VERSION)`. `PY_VERSION_HEX` does not appear to be meaningful with PyPy. --- include/pybind11/detail/internals.h | 57 +++++++++++++++++++++++---- include/pybind11/embed.h | 10 ++--- tests/test_embed/test_interpreter.cpp | 22 +++++------ 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 05e36ad18b..8364ebd1e0 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -34,9 +34,18 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 +# if PY_VERSION_HEX >= 0x030C0000 +// Version bump for Python 3.12+, before first 3.12 beta release. +# define PYBIND11_INTERNALS_VERSION 5 +# else +# define PYBIND11_INTERNALS_VERSION 4 +# endif #endif +// This requirement is mainly to reduce the support burden (see PR #4570). +static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, + "pybind11 ABI version 5 is the minimum for Python 3.12+"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ExceptionTranslator = void (*)(std::exception_ptr); @@ -421,6 +430,38 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +inline object get_python_state_dict() { + object state_dict; +#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + } + return state_dict; +} + +inline object get_internals_obj_from_state_dict(handle state_dict) { + return reinterpret_borrow(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); +} + +inline internals **get_internals_pp_from_capsule(handle obj) { + void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); + if (raw_ptr == nullptr) { + raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + } + return static_cast(raw_ptr); +} + /// Return a reference to the current `internals` data PYBIND11_NOINLINE internals &get_internals() { auto **&internals_pp = get_internals_pp(); @@ -445,12 +486,12 @@ PYBIND11_NOINLINE internals &get_internals() { #endif error_scope err_scope; - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_pp = static_cast(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our `error_already_set` + dict state_dict = get_python_state_dict(); + if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) { + internals_pp = get_internals_pp_from_capsule(internals_obj); + } + if (internals_pp && *internals_pp) { + // We loaded the internals through `state_dict`, which means that our `error_already_set` // and `builtin_exception` may be different local classes than the ones set up in the // initial exception translator, below, so add another for our local exception classes. // @@ -484,7 +525,7 @@ PYBIND11_NOINLINE internals &get_internals() { # endif internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 5a175b1341..caa14f4a05 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -243,16 +243,14 @@ inline void initialize_interpreter(bool init_signal_handlers = true, \endrst */ inline void finalize_interpreter() { - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - // Get the internals pointer (without creating it if it doesn't exist). It's possible for the // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()` // during destruction), so we get the pointer-pointer here and check it after Py_Finalize(). detail::internals **internals_ptr_ptr = detail::get_internals_pp(); - // It could also be stashed in builtins, so look there too: - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_ptr_ptr = capsule(builtins[id]); + // It could also be stashed in state_dict, so look there too: + if (object internals_obj + = get_internals_obj_from_state_dict(detail::get_python_state_dict())) { + internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj); } // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index e54e822708..c6c8a22d98 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -255,10 +255,10 @@ TEST_CASE("Add program dir to path using PyConfig") { } #endif -bool has_pybind11_internals_builtin() { - auto builtins = py::handle(PyEval_GetBuiltins()); - return builtins.contains(PYBIND11_INTERNALS_ID); -}; +bool has_state_dict_internals_obj() { + return bool( + py::detail::get_internals_obj_from_state_dict(py::detail::get_python_state_dict())); +} bool has_pybind11_internals_static() { auto **&ipp = py::detail::get_internals_pp(); @@ -268,7 +268,7 @@ bool has_pybind11_internals_static() { TEST_CASE("Restart the interpreter") { // Verify pre-restart state. REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast() == 3); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast() == 123); @@ -285,10 +285,10 @@ TEST_CASE("Restart the interpreter") { REQUIRE(Py_IsInitialized() == 1); // Internals are deleted after a restart. - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); pybind11::detail::get_internals(); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); REQUIRE(reinterpret_cast(*py::detail::get_internals_pp()) == py::module_::import("external_module").attr("internals_at")().cast()); @@ -303,13 +303,13 @@ TEST_CASE("Restart the interpreter") { py::detail::get_internals(); *static_cast(ran) = true; }); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); REQUIRE_FALSE(ran); py::finalize_interpreter(); REQUIRE(ran); py::initialize_interpreter(); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); // C++ modules can be reloaded. @@ -331,7 +331,7 @@ TEST_CASE("Subinterpreter") { REQUIRE(m.attr("add")(1, 2).cast() == 3); } - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); /// Create and switch to a subinterpreter. @@ -341,7 +341,7 @@ TEST_CASE("Subinterpreter") { // Subinterpreters get their own copy of builtins. detail::get_internals() still // works by returning from the static variable, i.e. all interpreters share a single // global pybind11::internals; - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); // Modules tags should be gone. From 4ce05175d51a4685232452bdc1e9cbb13a240a65 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 30 Mar 2023 07:29:25 -0700 Subject: [PATCH 095/137] ci: Python 3.12 optional test job (#4575) * ci: Python 3.12 optional testing Signed-off-by: Henry Schreiner * Skip test_flaky_exception_failure_point_init() for Python 3.12.0a6 (similar to https://github.com/google/pywrapcc/commit/af5c6536aba77b1db99bb16995d3a684351efb93#diff-f46006e3f43ffb1dd5d6862005427f6620f4dcfb1fa2f883d8482550069eeecc). * Disable tests/test_embed/test_interpreter.cpp (broken with Python 3.12.0alpha6) ``` free(): invalid pointer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ test_embed is a Catch v2.13.9 host application. Run with -? for options ------------------------------------------------------------------------------- Custom PyConfig ------------------------------------------------------------------------------- /home/runner/work/pybind11/pybind11/tests/test_embed/test_interpreter.cpp:175 ............................................................................... /home/runner/work/pybind11/pybind11/tests/test_embed/test_interpreter.cpp:179: FAILED: {Unknown expression after the reported line} due to a fatal error condition: SIGABRT - Abort (abnormal termination) signal =============================================================================== test cases: 6 | 5 passed | 1 failed assertions: 1518 | 1517 passed | 1 failed ``` --------- Signed-off-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/upstream.yml | 60 ++++++++++++++++------------------ tests/test_exceptions.py | 5 +++ 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index a15861ee47..b7e14f72e8 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -10,103 +10,101 @@ concurrency: cancel-in-progress: true env: - PIP_ONLY_BINARY: numpy + PIP_ONLY_BINARY: ":all:" # For cmake: VERBOSE: 1 jobs: standard: - name: "🐍 3.11 latest internals β€’ ubuntu-latest β€’ x64" + name: "🐍 3.12 latest β€’ ubuntu-latest β€’ x64" runs-on: ubuntu-latest + # Only runs when the 'python dev' label is selected if: "contains(github.event.pull_request.labels.*.name, 'python dev')" steps: - uses: actions/checkout@v3 - - name: Setup Python 3.11 + - name: Setup Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.11-dev" + python-version: "3.12-dev" - - name: Setup Boost (Linux) - if: runner.os == 'Linux' + - name: Setup Boost run: sudo apt-get install libboost-dev - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 - - name: Prepare env + - name: Run pip installs run: | + python -m pip install --upgrade pip python -m pip install -r tests/requirements.txt - - name: Setup annotations on Linux - if: runner.os == 'Linux' - run: python -m pip install pytest-github-actions-annotate-failures + - name: Show platform info + run: | + python -m platform + cmake --version + pip list # First build - C++11 mode and inplace - name: Configure C++11 run: > - cmake -S . -B . + cmake -S . -B build11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 + -DCMAKE_BUILD_TYPE=Debug - name: Build C++11 - run: cmake --build . -j 2 + run: cmake --build build11 -j 2 - name: Python tests C++11 - run: cmake --build . --target pytest -j 2 + run: cmake --build build11 --target pytest -j 2 - - name: C++11 tests - run: cmake --build . --target cpptest -j 2 + # - name: C++11 tests + # run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 - run: cmake --build . --target test_cmake_build - - - name: Clean directory - run: git clean -fdx + run: cmake --build build11 --target test_cmake_build # Second build - C++17 mode and in a build directory - name: Configure C++17 run: > - cmake -S . -B build2 + cmake -S . -B build17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 - ${{ matrix.args }} - ${{ matrix.args2 }} - name: Build - run: cmake --build build2 -j 2 + run: cmake --build build17 -j 2 - name: Python tests - run: cmake --build build2 --target pytest + run: cmake --build build17 --target pytest - - name: C++ tests - run: cmake --build build2 --target cpptest + # - name: C++ tests + # run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI - name: Configure (unstable ABI) run: > - cmake -S . -B build3 + cmake -S . -B build17max -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" - ${{ matrix.args }} - name: Build (unstable ABI) - run: cmake --build build3 -j 2 + run: cmake --build build17max -j 2 - name: Python tests (unstable ABI) - run: cmake --build build3 --target pytest + run: cmake --build build17max --target pytest - name: Interface test - run: cmake --build build3 --target test_cmake_build + run: cmake --build build17max --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 7f80a5da56..1b4c89add1 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -317,6 +317,11 @@ def test_error_already_set_what_with_happy_exceptions( assert what == expected_what +@pytest.mark.skipif( + # Intentionally very specific: + "sys.version_info == (3, 12, 0, 'alpha', 6)", + reason="WIP: https://github.com/python/cpython/issues/102594", +) @pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") def test_flaky_exception_failure_point_init(): with pytest.raises(RuntimeError) as excinfo: From 7ab88d2e4fbb7fb21473f87b17b68a1f0e51cb4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 10:18:53 -0400 Subject: [PATCH 096/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.3 to 1.8.4 (#4602) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.3 to 1.8.4. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.3...v1.8.4) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index d5427c7961..8ca6fd69ea 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.3 + uses: pypa/gh-action-pypi-publish@v1.8.4 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.3 + uses: pypa/gh-action-pypi-publish@v1.8.4 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From ed466da571fbc1d711351eb818e4bf82adb99eca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 15:52:41 -0400 Subject: [PATCH 097/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.4 to 1.8.5 (#4604) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.4 to 1.8.5. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.4...v1.8.5) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 8ca6fd69ea..7f686b13f9 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.8.5 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.8.5 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 071f35ab85aa5d8d91ee3c5735a601fd3221d783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 12:32:35 -0400 Subject: [PATCH 098/137] chore(deps): bump jwlawson/actions-setup-cmake from 1.13 to 1.14 (#4632) Bumps [jwlawson/actions-setup-cmake](https://github.com/jwlawson/actions-setup-cmake) from 1.13 to 1.14. - [Release notes](https://github.com/jwlawson/actions-setup-cmake/releases) - [Commits](https://github.com/jwlawson/actions-setup-cmake/compare/v1.13...v1.14) --- updated-dependencies: - dependency-name: jwlawson/actions-setup-cmake dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/configure.yml | 2 +- .github/workflows/upstream.yml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8befccdbc2..a6a9a16fa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: run: brew install boost - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Cache wheels if: runner.os == 'macOS' @@ -208,7 +208,7 @@ jobs: debug: ${{ matrix.python-debug }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Valgrind cache if: matrix.valgrind @@ -474,7 +474,7 @@ jobs: run: python3 -m pip install --upgrade pip - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure shell: bash @@ -763,7 +763,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.12.1 @@ -816,7 +816,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.12.1 @@ -867,7 +867,7 @@ jobs: python3 -m pip install -r tests/requirements.txt - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure C++20 run: > @@ -1000,7 +1000,7 @@ jobs: python-version: ${{ matrix.python }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Install ninja-build tool uses: seanmiddleditch/gha-setup-ninja@v3 @@ -1070,7 +1070,7 @@ jobs: run: clang++ --version - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Run pip installs run: | diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 29b041168e..b469a69d11 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -55,7 +55,7 @@ jobs: # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: ${{ matrix.cmake }} diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index b7e14f72e8..be3cd40502 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -33,7 +33,7 @@ jobs: run: sudo apt-get install libboost-dev - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Run pip installs run: | From 07725c28c0b1d68e044ccbc7820920c15c1f21e3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Apr 2023 00:19:21 -0700 Subject: [PATCH 099/137] Introduce `pybind11::detail::is_move_constructible` (#4631) To support the use case captured in the new test_vector_unique_ptr_member.cpp --- include/pybind11/cast.h | 4 +- include/pybind11/detail/init.h | 4 +- include/pybind11/detail/type_caster_base.h | 5 +- tests/CMakeLists.txt | 1 + tests/test_vector_unique_ptr_member.cpp | 56 ++++++++++++++++++++++ tests/test_vector_unique_ptr_member.py | 14 ++++++ 6 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 tests/test_vector_unique_ptr_member.cpp create mode 100644 tests/test_vector_unique_ptr_member.py diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9b013bc39f..9d1ce15d48 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -964,7 +964,7 @@ struct move_always< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -975,7 +975,7 @@ struct move_if_unreferenced< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 9f71278c26..e21171688c 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -175,7 +175,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { template void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); - static_assert(std::is_move_constructible>::value, + static_assert(is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); @@ -190,7 +190,7 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { template void construct(value_and_holder &v_h, Alias &&result, bool) { static_assert( - std::is_move_constructible>::value, + is_move_constructible>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias(std::move(result)); } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 0b710d7e4c..34dcd26ce5 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -827,6 +827,9 @@ using movable_cast_op_type template struct is_copy_constructible : std::is_copy_constructible {}; +template +struct is_move_constructible : std::is_move_constructible {}; + // Specialization for types that appear to be copy constructible but also look like stl containers // (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if // so, copy constructability depends on whether the value_type is copy constructible. @@ -994,7 +997,7 @@ class type_caster_base : public type_caster_generic { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } - template ::value>> + template ::value>> static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b1cb222b4a..1d0eb27c0d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,6 +155,7 @@ set(PYBIND11_TEST_FILES test_tagbased_polymorphic test_thread test_union + test_vector_unique_ptr_member test_virtual_functions) # Invoking cmake with something like: diff --git a/tests/test_vector_unique_ptr_member.cpp b/tests/test_vector_unique_ptr_member.cpp new file mode 100644 index 0000000000..657743e4dd --- /dev/null +++ b/tests/test_vector_unique_ptr_member.cpp @@ -0,0 +1,56 @@ +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace vector_unique_ptr_member { + +struct DataType {}; + +// Reduced from a use case in the wild. +struct VectorOwner { + static std::unique_ptr Create(std::size_t num_elems) { + return std::unique_ptr( + new VectorOwner(std::vector>(num_elems))); + } + + std::size_t data_size() const { return data_.size(); } + +private: + explicit VectorOwner(std::vector> data) : data_(std::move(data)) {} + + const std::vector> data_; +}; + +} // namespace vector_unique_ptr_member +} // namespace pybind11_tests + +namespace pybind11 { +namespace detail { + +template <> +struct is_copy_constructible + : std::false_type {}; + +template <> +struct is_move_constructible + : std::false_type {}; + +} // namespace detail +} // namespace pybind11 + +using namespace pybind11_tests::vector_unique_ptr_member; + +py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } + +// PYBIND11_SMART_HOLDER_TYPE_CASTERS(VectorOwner) + +TEST_SUBMODULE(vector_unique_ptr_member, m) { + py::class_(m, "VectorOwner") + .def_static("Create", &VectorOwner::Create) + .def("data_size", &VectorOwner::data_size); + + m.def("py_cast_VectorOwner_ptr", py_cast_VectorOwner_ptr); +} diff --git a/tests/test_vector_unique_ptr_member.py b/tests/test_vector_unique_ptr_member.py new file mode 100644 index 0000000000..2da3d97c37 --- /dev/null +++ b/tests/test_vector_unique_ptr_member.py @@ -0,0 +1,14 @@ +import pytest + +from pybind11_tests import vector_unique_ptr_member as m + + +@pytest.mark.parametrize("num_elems", range(3)) +def test_create(num_elems): + vo = m.VectorOwner.Create(num_elems) + assert vo.data_size() == num_elems + + +def test_cast(): + vo = m.VectorOwner.Create(0) + assert m.py_cast_VectorOwner_ptr(vo) is vo From ff7f5dfca32224b65524a9cebe3daa8f2d328a35 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 25 Apr 2023 10:25:57 -0700 Subject: [PATCH 100/137] 1. Fully test unstable ABI (#4635) 2. Selectively exercise cmake `-DPYBIND11_TEST_OVERRIDE`: ubuntu, macos, windows Extra work added to quick jobs, based on timings below, to not increase the GHA start-to-last-job-finished time. ``` Duration ^ Number of pytest runs ^ ^ Job identifier ^ ^ ^ 0:03:48.024227 1 1___3___Clang_3.6___C++11___x64.txt 0:03:58.992814 1 2___3___Clang_3.7___C++11___x64.txt 0:04:25.758942 1 1___3.7___Debian___x86____Install.txt 0:04:50.148276 1 4___3___Clang_7___C++11___x64.txt 0:04:55.784558 1 13___3___Clang_15___C++20___x64.txt 0:04:57.048754 1 6___3___Clang_dev___C++11___x64.txt 0:05:00.485181 1 7___3___Clang_5___C++14___x64.txt 0:05:03.744964 1 2___3___almalinux8___x64.txt 0:05:06.222752 1 5___3___Clang_9___C++11___x64.txt 0:05:11.767022 1 2___3___GCC_7___C++17__x64.txt 0:05:18.634930 1 2___3.11__deadsnakes____x64.txt 0:05:22.810995 1 1___3___GCC_7___C++11__x64.txt 0:05:25.275317 1 12___3___Clang_14___C++20___x64.txt 0:05:32.058174 1 5___3___GCC_10___C++17__x64.txt 0:05:39.381351 1 7___3___GCC_12___C++20__x64.txt 0:05:40.502252 1 8___3___Clang_10___C++17___x64.txt 0:05:59.344905 1 3___3___Clang_3.9___C++11___x64.txt 0:06:10.825147 1 6___3___GCC_11___C++20__x64.txt 0:06:20.655443 1 3___3___almalinux9___x64.txt 0:06:22.472061 1 3___3___GCC_8___C++14__x64.txt 0:06:42.647406 1 11___3___Clang_13___C++20___x64.txt 0:06:53.352720 1 1___3.10___CUDA_11.7___Ubuntu_22.04.txt 0:07:07.357801 1 2___3.7___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=14.txt 0:07:09.057603 1 1___3___centos7___x64.txt 0:07:15.546282 1 1___3.8___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=17.txt 0:07:22.566022 1 4___3___GCC_8___C++17__x64.txt 0:08:13.592674 1 2___3.9___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=20.txt 0:08:16.422768 1 9___3___Clang_11___C++20___x64.txt 0:08:21.168457 1 3___3.8___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=17.txt 0:08:27.129468 1 10___3___Clang_12___C++20___x64.txt 0:09:35.045470 1 1___3.10___windows-latest___clang-latest.txt 0:09:57.361843 1 1___3.9___MSVC_2022_C++20___x64.txt 0:10:35.187767 1 1___3.6___MSVC_2019___x86.txt 0:11:14.691200 4 2___3.9___ubuntu-20.04___x64.txt 0:11:37.701167 1 1_macos-latest___brew_install_llvm.txt 0:11:38.688299 4 4___3.11___ubuntu-20.04___x64.txt 0:11:52.720216 1 4___3.9___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=20.txt 0:13:23.456591 4 6___pypy-3.8___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:13:25.863592 2 1___3___ICC_latest___x64.txt 0:13:32.411758 3 9___3.9___windows-2022___x64.txt 0:13:45.473377 4 3___3.10___ubuntu-20.04___x64.txt 0:13:55.366447 4 5___pypy-3.7___ubuntu-20.04___x64.txt 0:13:57.969502 3 10___3.10___windows-2022___x64.txt 0:14:19.837475 3 11___3.11___windows-2022___x64.txt 0:14:33.316770 4 1___3.6___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON_-DCMA.txt 0:15:34.449278 4 22___3.6___windows-2019___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:16:25.189055 2 1___3.9-dbg__deadsnakes____Valgrind___x64.txt 0:17:20.956667 4 15___3.6___macos-latest___x64.txt 0:17:27.513891 4 23___3.9___windows-2019___x64.txt 0:17:58.783286 3 8___3.6___windows-2022___x64.txt 0:18:25.917828 4 7___pypy-3.9___ubuntu-20.04___x64.txt 0:19:17.399820 3 13___pypy-3.8___windows-2022___x64.txt 0:19:45.002122 3 12___pypy-3.7___windows-2022___x64.txt 0:20:03.201926 4 16___3.9___macos-latest___x64.txt 0:20:15.415178 4 17___3.10___macos-latest___x64.txt 0:20:20.263216 4 20___pypy-3.8___macos-latest___x64.txt 0:20:31.998226 3 1___3___windows-latest___mingw64.txt 0:20:40.812286 4 18___3.11___macos-latest___x64.txt 0:22:47.714749 4 19___pypy-3.7___macos-latest___x64.txt 0:23:04.435859 3 2___3___windows-latest___mingw32.txt 0:25:48.719597 3 14___pypy-3.9___windows-2022___x64.txt 0:26:01.211688 4 21___pypy-3.9___macos-latest___x64.txt 0:28:19.971015 1 1___3___CentOS7__PGI_22.9___x64.txt ``` --- .github/workflows/ci.yml | 52 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6a9a16fa6..c88f0797f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,6 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" ${{ matrix.args }} - name: Build (unstable ABI) @@ -497,6 +496,24 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + - name: Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + shell: bash + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial --target pytest # Testing on ICC using the oneAPI apt repo icc: @@ -889,6 +906,21 @@ jobs: - name: Interface test C++20 run: cmake --build build --target test_cmake_build + - name: Configure C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=20 + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest + mingw: name: "🐍 3 β€’ windows-latest β€’ ${{ matrix.sys }}" runs-on: windows-latest @@ -1105,5 +1137,23 @@ jobs: - name: Interface test run: cmake --build . --target test_cmake_build -j 2 + - name: CMake Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest -j 2 + - name: Clean directory run: git clean -fdx From 6de6191a0c56a9b0d31ac1c32778081a9e2b0311 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 25 Apr 2023 14:03:24 -0700 Subject: [PATCH 101/137] Use `std::hash`, `std::equal_to` everywhere **except when libc++ is in use** (#4319) * Try using `std::hash`, `std::equal_to` everywhere. From PR #4316 we know that types in the unnamed namespace in different translation units do not compare equal, as desired. But do types in named namespaces compare equal, as desired? * Revert "Try using `std::hash`, `std::equal_to` everywhere." This reverts commit a06949a9265014b3c581396c4f37c45ccc03dea6. * Use "our own name-based hash and equality functions" for `std::type_index` only under macOS, based on results shown under https://github.com/pybind/pybind11/pull/4316#issuecomment-1305097879 * Patch in PR #4313: Minimal reproducer for clash when binding types defined in the unnamed namespace. * test_unnamed_namespace_b xfail for clang * `PYBIND11_INTERNALS_VERSION 5` * Add a note to docs/classes.rst * For compatibility with Google-internal testing, test_unnamed_namespace_a & test_unnamed_namespace_b need to work when imported in any order. * Trying "__GLIBCXX__ or Windows", based on observations from Google-internal testing. * Try _LIBCPP_VERSION * Account for libc++ behavior in tests and documentation. * Adjust expectations for Windows Clang (and make code less redundant). * Add WindowsClang to ci.yml Added block transferred from PR #4321 * Add clang-latest to name that appears in the GitHub Actions web view. * Tweak the note in classes.rst again. * Add `pip install --upgrade pip`, Show env, cosmetic changes Already tested under PR #4321 * Add macos_brew_install_llvm to ci.yml Added block transferred from PR #4324 * `test_cross_module_exception_translator` xfail 'Homebrew Clang' * Revert back to base version of .github/workflows/ci.yml (the ci.yml changes were merged under #4323 and #4326) * Fixes for ruff * Make updated condition in internals.h dependent on ABI version. * Remove PYBIND11_TEST_OVERRIDE when testing with PYBIND11_INTERNALS_VERSION=10000000 * Selectively exercise cmake `-DPYBIND11_TEST_OVERRIDE`: ubuntu, macos, windows Extra work added to quick jobs, based on timings below, to not increase the GHA start-to-last-job-finished time. ``` Duration ^ Number of pytest runs ^ ^ Job identifier ^ ^ ^ 0:03:48.024227 1 1___3___Clang_3.6___C++11___x64.txt 0:03:58.992814 1 2___3___Clang_3.7___C++11___x64.txt 0:04:25.758942 1 1___3.7___Debian___x86____Install.txt 0:04:50.148276 1 4___3___Clang_7___C++11___x64.txt 0:04:55.784558 1 13___3___Clang_15___C++20___x64.txt 0:04:57.048754 1 6___3___Clang_dev___C++11___x64.txt 0:05:00.485181 1 7___3___Clang_5___C++14___x64.txt 0:05:03.744964 1 2___3___almalinux8___x64.txt 0:05:06.222752 1 5___3___Clang_9___C++11___x64.txt 0:05:11.767022 1 2___3___GCC_7___C++17__x64.txt 0:05:18.634930 1 2___3.11__deadsnakes____x64.txt 0:05:22.810995 1 1___3___GCC_7___C++11__x64.txt 0:05:25.275317 1 12___3___Clang_14___C++20___x64.txt 0:05:32.058174 1 5___3___GCC_10___C++17__x64.txt 0:05:39.381351 1 7___3___GCC_12___C++20__x64.txt 0:05:40.502252 1 8___3___Clang_10___C++17___x64.txt 0:05:59.344905 1 3___3___Clang_3.9___C++11___x64.txt 0:06:10.825147 1 6___3___GCC_11___C++20__x64.txt 0:06:20.655443 1 3___3___almalinux9___x64.txt 0:06:22.472061 1 3___3___GCC_8___C++14__x64.txt 0:06:42.647406 1 11___3___Clang_13___C++20___x64.txt 0:06:53.352720 1 1___3.10___CUDA_11.7___Ubuntu_22.04.txt 0:07:07.357801 1 2___3.7___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=14.txt 0:07:09.057603 1 1___3___centos7___x64.txt 0:07:15.546282 1 1___3.8___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=17.txt 0:07:22.566022 1 4___3___GCC_8___C++17__x64.txt 0:08:13.592674 1 2___3.9___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=20.txt 0:08:16.422768 1 9___3___Clang_11___C++20___x64.txt 0:08:21.168457 1 3___3.8___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=17.txt 0:08:27.129468 1 10___3___Clang_12___C++20___x64.txt 0:09:35.045470 1 1___3.10___windows-latest___clang-latest.txt 0:09:57.361843 1 1___3.9___MSVC_2022_C++20___x64.txt 0:10:35.187767 1 1___3.6___MSVC_2019___x86.txt 0:11:14.691200 4 2___3.9___ubuntu-20.04___x64.txt 0:11:37.701167 1 1_macos-latest___brew_install_llvm.txt 0:11:38.688299 4 4___3.11___ubuntu-20.04___x64.txt 0:11:52.720216 1 4___3.9___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=20.txt 0:13:23.456591 4 6___pypy-3.8___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:13:25.863592 2 1___3___ICC_latest___x64.txt 0:13:32.411758 3 9___3.9___windows-2022___x64.txt 0:13:45.473377 4 3___3.10___ubuntu-20.04___x64.txt 0:13:55.366447 4 5___pypy-3.7___ubuntu-20.04___x64.txt 0:13:57.969502 3 10___3.10___windows-2022___x64.txt 0:14:19.837475 3 11___3.11___windows-2022___x64.txt 0:14:33.316770 4 1___3.6___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON_-DCMA.txt 0:15:34.449278 4 22___3.6___windows-2019___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:16:25.189055 2 1___3.9-dbg__deadsnakes____Valgrind___x64.txt 0:17:20.956667 4 15___3.6___macos-latest___x64.txt 0:17:27.513891 4 23___3.9___windows-2019___x64.txt 0:17:58.783286 3 8___3.6___windows-2022___x64.txt 0:18:25.917828 4 7___pypy-3.9___ubuntu-20.04___x64.txt 0:19:17.399820 3 13___pypy-3.8___windows-2022___x64.txt 0:19:45.002122 3 12___pypy-3.7___windows-2022___x64.txt 0:20:03.201926 4 16___3.9___macos-latest___x64.txt 0:20:15.415178 4 17___3.10___macos-latest___x64.txt 0:20:20.263216 4 20___pypy-3.8___macos-latest___x64.txt 0:20:31.998226 3 1___3___windows-latest___mingw64.txt 0:20:40.812286 4 18___3.11___macos-latest___x64.txt 0:22:47.714749 4 19___pypy-3.7___macos-latest___x64.txt 0:23:04.435859 3 2___3___windows-latest___mingw32.txt 0:25:48.719597 3 14___pypy-3.9___windows-2022___x64.txt 0:26:01.211688 4 21___pypy-3.9___macos-latest___x64.txt 0:28:19.971015 1 1___3___CentOS7__PGI_22.9___x64.txt ``` * Update skipif for Python 3.12a7 (the WIP needs to be handled in a separate PR). --- .github/workflows/upstream.yml | 1 - docs/classes.rst | 10 ++++++++ include/pybind11/detail/internals.h | 3 ++- tests/CMakeLists.txt | 2 ++ tests/test_exceptions.py | 2 +- tests/test_unnamed_namespace_a.cpp | 38 +++++++++++++++++++++++++++++ tests/test_unnamed_namespace_a.py | 34 ++++++++++++++++++++++++++ tests/test_unnamed_namespace_b.cpp | 13 ++++++++++ tests/test_unnamed_namespace_b.py | 5 ++++ 9 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/test_unnamed_namespace_a.cpp create mode 100644 tests/test_unnamed_namespace_a.py create mode 100644 tests/test_unnamed_namespace_b.cpp create mode 100644 tests/test_unnamed_namespace_b.py diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index be3cd40502..be643ddfdb 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -95,7 +95,6 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" - name: Build (unstable ABI) run: cmake --build build17max -j 2 diff --git a/docs/classes.rst b/docs/classes.rst index c0c53135b8..52cd52da3f 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -58,6 +58,16 @@ interactive Python session demonstrating this example is shown below: Static member functions can be bound in the same way using :func:`class_::def_static`. +.. note:: + + Binding C++ types in unnamed namespaces (also known as anonymous namespaces) + works reliably on many platforms, but not all. The `XFAIL_CONDITION` in + tests/test_unnamed_namespace_a.py encodes the currently known conditions. + For background see `#4319 `_. + If portability is a concern, it is therefore not recommended to bind C++ + types in unnamed namespaces. It will be safest to manually pick unique + namespace names. + Keyword and default arguments ============================= It is possible to specify keyword and default arguments using the syntax diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 8364ebd1e0..aaa7f8686e 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -123,7 +123,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1d0eb27c0d..11ac2215b3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,6 +155,8 @@ set(PYBIND11_TEST_FILES test_tagbased_polymorphic test_thread test_union + test_unnamed_namespace_a + test_unnamed_namespace_b test_vector_unique_ptr_member test_virtual_functions) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 1b4c89add1..a92ab59a34 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -319,7 +319,7 @@ def test_error_already_set_what_with_happy_exceptions( @pytest.mark.skipif( # Intentionally very specific: - "sys.version_info == (3, 12, 0, 'alpha', 6)", + "sys.version_info == (3, 12, 0, 'alpha', 7)", reason="WIP: https://github.com/python/cpython/issues/102594", ) @pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") diff --git a/tests/test_unnamed_namespace_a.cpp b/tests/test_unnamed_namespace_a.cpp new file mode 100644 index 0000000000..2152e64bd7 --- /dev/null +++ b/tests/test_unnamed_namespace_a.cpp @@ -0,0 +1,38 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_a, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_a_any_struct"); + } else { + m.attr("unnamed_namespace_a_any_struct") = py::none(); + } + m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION; + m.attr("defined_WIN32_or__WIN32") = +#if defined(WIN32) || defined(_WIN32) + true; +#else + false; +#endif + m.attr("defined___clang__") = +#if defined(__clang__) + true; +#else + false; +#endif + m.attr("defined__LIBCPP_VERSION") = +#if defined(_LIBCPP_VERSION) + true; +#else + false; +#endif + m.attr("defined___GLIBCXX__") = +#if defined(__GLIBCXX__) + true; +#else + false; +#endif +} diff --git a/tests/test_unnamed_namespace_a.py b/tests/test_unnamed_namespace_a.py new file mode 100644 index 0000000000..9d9856c5a4 --- /dev/null +++ b/tests/test_unnamed_namespace_a.py @@ -0,0 +1,34 @@ +import pytest + +from pybind11_tests import unnamed_namespace_a as m +from pybind11_tests import unnamed_namespace_b as mb + +XFAIL_CONDITION = ( + "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))" + " or " + "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32" + " and " + "(m.defined___clang__ or m.defined__LIBCPP_VERSION))" +) +XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=False) +@pytest.mark.parametrize( + "any_struct", [m.unnamed_namespace_a_any_struct, mb.unnamed_namespace_b_any_struct] +) +def test_have_class_any_struct(any_struct): + assert any_struct is not None + + +def test_have_at_least_one_class_any_struct(): + assert ( + m.unnamed_namespace_a_any_struct is not None + or mb.unnamed_namespace_b_any_struct is not None + ) + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=True) +def test_have_both_class_any_struct(): + assert m.unnamed_namespace_a_any_struct is not None + assert mb.unnamed_namespace_b_any_struct is not None diff --git a/tests/test_unnamed_namespace_b.cpp b/tests/test_unnamed_namespace_b.cpp new file mode 100644 index 0000000000..f97757a7de --- /dev/null +++ b/tests/test_unnamed_namespace_b.cpp @@ -0,0 +1,13 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_b, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_b_any_struct"); + } else { + m.attr("unnamed_namespace_b_any_struct") = py::none(); + } +} diff --git a/tests/test_unnamed_namespace_b.py b/tests/test_unnamed_namespace_b.py new file mode 100644 index 0000000000..4bcaa7a6c5 --- /dev/null +++ b/tests/test_unnamed_namespace_b.py @@ -0,0 +1,5 @@ +from pybind11_tests import unnamed_namespace_b as m + + +def test_have_attr_any_struct(): + assert hasattr(m, "unnamed_namespace_b_any_struct") From 3f366ff888c846a3bc39af36615f99dc9e424e51 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 27 Apr 2023 07:24:48 -0700 Subject: [PATCH 102/137] Remove stray comment. (Oversight in PR #4631. Noticed by chance.) (#4641) --- tests/test_vector_unique_ptr_member.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_vector_unique_ptr_member.cpp b/tests/test_vector_unique_ptr_member.cpp index 657743e4dd..680cf9a6d8 100644 --- a/tests/test_vector_unique_ptr_member.cpp +++ b/tests/test_vector_unique_ptr_member.cpp @@ -45,8 +45,6 @@ using namespace pybind11_tests::vector_unique_ptr_member; py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } -// PYBIND11_SMART_HOLDER_TYPE_CASTERS(VectorOwner) - TEST_SUBMODULE(vector_unique_ptr_member, m) { py::class_(m, "VectorOwner") .def_static("Create", &VectorOwner::Create) From 5e946c2fa5b3c3170d702f57311492a8f838444e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 14:32:32 -0400 Subject: [PATCH 103/137] chore(deps): update pre-commit hooks (#4605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/psf/black: 23.1.0 β†’ 23.3.0](https://github.com/psf/black/compare/23.1.0...23.3.0) - [github.com/Lucas-C/pre-commit-hooks: v1.4.2 β†’ v1.5.1](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.4.2...v1.5.1) - [github.com/charliermarsh/ruff-pre-commit: v0.0.254 β†’ v0.0.260](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.254...v0.0.260) - [github.com/PyCQA/pylint: v2.16.4 β†’ v3.0.0a6](https://github.com/PyCQA/pylint/compare/v2.16.4...v3.0.0a6) - [github.com/pre-commit/mirrors-mypy: v1.0.1 β†’ v1.1.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.0.1...v1.1.1) - [github.com/codespell-project/codespell: v2.2.2 β†’ v2.2.4](https://github.com/codespell-project/codespell/compare/v2.2.2...v2.2.4) - [github.com/pre-commit/mirrors-clang-format: v15.0.7 β†’ v16.0.0](https://github.com/pre-commit/mirrors-clang-format/compare/v15.0.7...v16.0.0) * style: pre-commit fixes * style: fix issues * Update tests/test_call_policies.py * Update tests/test_call_policies.py * fix: ignore code in file * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 18 +++---- include/pybind11/detail/type_caster_base.h | 6 +-- pyproject.toml | 1 + tests/test_iostream.py | 58 +++++++++++----------- tests/test_stl.py | 2 +- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c43161921..51b6f0a41b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "23.1.0" # Keep in sync with blacken-docs + rev: "23.3.0" # Keep in sync with blacken-docs hooks: - id: black @@ -51,11 +51,11 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.1.0 # keep in sync with black hook + - black==23.3.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.4.2" + rev: "v1.5.1" hooks: - id: remove-tabs @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.254 + rev: v0.0.263 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -84,7 +84,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.4" + rev: "v3.0.0a6" hooks: - id: pylint files: ^pybind11 @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.0.1" + rev: "v1.1.1" hooks: - id: mypy args: [] @@ -120,11 +120,11 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.2" + rev: "v2.2.4" hooks: - id: codespell exclude: ".supp$" - args: ["-x", ".codespell-ignore-lines"] + args: ["-x.codespell-ignore-lines", "-Lccompiler"] # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.7" + rev: "v16.0.0" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 34dcd26ce5..8f3d0f3791 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -258,9 +258,9 @@ struct value_and_holder { // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; diff --git a/pyproject.toml b/pyproject.toml index 96cf842650..e3655aca74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,3 +93,4 @@ isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests [tool.ruff.per-file-ignores] "tests/**" = ["EM", "N"] +"tests/test_call_policies.py" = ["PLC1901"] diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 79b2a2b8cc..d283eb1520 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -9,16 +9,16 @@ def test_captured(capsys): m.captured_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_err(msg) stdout, stderr = capsys.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg @@ -30,7 +30,7 @@ def test_captured_large_string(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset0(capsys): @@ -40,7 +40,7 @@ def test_captured_utf8_2byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset1(capsys): @@ -50,7 +50,7 @@ def test_captured_utf8_2byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset0(capsys): @@ -60,7 +60,7 @@ def test_captured_utf8_3byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset1(capsys): @@ -70,7 +70,7 @@ def test_captured_utf8_3byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset2(capsys): @@ -80,7 +80,7 @@ def test_captured_utf8_3byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset0(capsys): @@ -90,7 +90,7 @@ def test_captured_utf8_4byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset1(capsys): @@ -100,7 +100,7 @@ def test_captured_utf8_4byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset2(capsys): @@ -110,7 +110,7 @@ def test_captured_utf8_4byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset3(capsys): @@ -120,7 +120,7 @@ def test_captured_utf8_4byte_offset3(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_guard_capture(capsys): @@ -128,7 +128,7 @@ def test_guard_capture(capsys): m.guard_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_series_captured(capture): @@ -145,7 +145,7 @@ def test_flush(capfd): with m.ostream_redirect(): m.noisy_function(msg, flush=False) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout m.noisy_function(msg2, flush=True) stdout, stderr = capfd.readouterr() @@ -164,15 +164,15 @@ def test_not_captured(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" - assert stream.getvalue() == "" + assert not stderr + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream): m.captured_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -182,16 +182,16 @@ def test_err(capfd): with redirect_stderr(stream): m.raw_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stderr(stream): m.captured_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -221,13 +221,13 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream), m.ostream_redirect(): m.raw_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stream.getvalue() == msg stream = StringIO() @@ -235,7 +235,7 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() def test_redirect_err(capfd): @@ -248,7 +248,7 @@ def test_redirect_err(capfd): m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr assert stream.getvalue() == msg2 @@ -262,8 +262,8 @@ def test_redirect_both(capfd): m.raw_output(msg) m.raw_err(msg2) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg assert stream2.getvalue() == msg2 diff --git a/tests/test_stl.py b/tests/test_stl.py index b7a0d1c0d5..2d9dcc89f1 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -367,7 +367,7 @@ def test_issue_1561(): """check fix for issue #1561""" bar = m.Issue1561Outer() bar.list = [m.Issue1561Inner("bar")] - bar.list + assert bar.list assert bar.list[0].data == "bar" From 956390a87f6e2ee6ea20438f3faa05222415018f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 29 Apr 2023 10:15:45 -0400 Subject: [PATCH 104/137] fix(cmake): only define lto if CMAKE's IPO setting is unset (#4643) Signed-off-by: Henry Schreiner --- tools/pybind11Common.cmake | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 0c985bc8e5..e4ff1e4080 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -5,8 +5,8 @@ Adds the following targets:: pybind11::pybind11 - link to headers and pybind11 pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -20,7 +20,7 @@ Adds the following functions:: # CMake 3.10 has an include_guard command, but we can't use that yet # include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::lto) +if(TARGET pybind11::pybind11) return() endif() @@ -372,11 +372,13 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- From da91926295638b2b1c0c85a568a2d66ca3646565 Mon Sep 17 00:00:00 2001 From: biergaizi Date: Mon, 1 May 2023 14:14:52 +0000 Subject: [PATCH 105/137] fix: remove -stdlib=libc++ from setup helpers, not needed on modern Pythons (#4639) * Inject -stdlib=libc++ on macOS only when it's supported, close #4637. On macOS, by default, pybind11 currently unconditionally set the compiler flag "-stdlib=libc++" in Pybind11Extension.__init__(), regardless of which compiler is used. This flag is required for clang, but is invalid for GCC. If GCC is used, it causes compilation failures in all Python projects that use pybind11, with the error message: arm64-apple-darwin22-gcc: error: unrecognized command-line option -stdlib=libc++. This commit uses has_flag() to detect whether "-stdlib=libc++" on macOS, and injects this flag from build_ext.build_extensions(), rather than setting it unconditionally. Signed-off-by: Yifeng Li * revert: just remove flags --------- Signed-off-by: Yifeng Li Co-authored-by: Henry Schreiner --- pybind11/setup_helpers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 0fb4679e49..cb279f27e0 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -144,7 +144,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: @@ -154,11 +153,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property def cxx_std(self) -> int: From dff75a62bd33ff35312aedf696c17f0300301b89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:34:51 -0400 Subject: [PATCH 106/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.5 to 1.8.6 (#4650) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.5 to 1.8.6. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.5...v1.8.6) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 7f686b13f9..6d9be3b1d4 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.5 + uses: pypa/gh-action-pypi-publish@v1.8.6 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.5 + uses: pypa/gh-action-pypi-publish@v1.8.6 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From b3e88ecf894269b6b3fe822a7551c95247a3c47e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:35:21 -0400 Subject: [PATCH 107/137] chore(deps): update pre-commit hooks (#4648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.1.1 β†’ v1.2.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.1.1...v1.2.0) - [github.com/pre-commit/mirrors-clang-format: v16.0.0 β†’ v16.0.2](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.0...v16.0.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51b6f0a41b..aca9b3ca2e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.1.1" + rev: "v1.2.0" hooks: - id: mypy args: [] @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.0" + rev: "v16.0.2" hooks: - id: clang-format types_or: [c++, c, cuda] From f70165463328c218d118204efc13aac93783d17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 5 May 2023 07:39:05 +0200 Subject: [PATCH 108/137] Introduce recursive_container_traits (#4623) * Testing * Similar fix for std::vector * Fix infinite recursion check: 1) Apply to is_copy_assignable additionally 2) Check infinite recursion for map-like types * style: pre-commit fixes * Optional commit that demonstrates the limitations of this PR * Fix positioning of container bindings The bindings were previously in a block that was only activated if numpy was available. * Suggestions from code review: API side * Suggestions from code review: Test side * Suggestions from code review 1) Renaming: is_recursive_container and MutuallyRecursiveContainerPair(MV|VM) 2) Avoid ambiguous specializations of is_recursive_container * Some little fixes * Reordering of structs * Add recursive checks for is_move_constructible * Static testing for pybind11 type traits * More precise checking of recursive types Instead of a trait `is_recursive_container`, use a trait `recursive_container_traits` with dependent type `recursive_container_traits::type_to_check_recursively`. So, instead of just checking if a type is recursive and then trying to somehow deal with it, recursively-defined traits such as is_move_constructible can now directly ask this trait where the recursion should proceed. * Review suggestions 1. Use std::conditional 2. Fix typo * Remove leftover include from test --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/type_caster_base.h | 203 +++++++++++++++--- include/pybind11/stl_bind.h | 8 +- tests/test_copy_move.cpp | 238 +++++++++++++++++++++ tests/test_stl_binders.cpp | 44 ++++ tests/test_stl_binders.py | 18 ++ 5 files changed, 484 insertions(+), 27 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 8f3d0f3791..16387506cf 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -822,26 +822,179 @@ using movable_cast_op_type typename std::add_rvalue_reference>::type, typename std::add_lvalue_reference>::type>>; -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template -struct is_copy_constructible : std::is_copy_constructible {}; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template +struct impl_type_to_check_recursively, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const::type; + using if_not_recursive = std::pair::type, B>; +}; -template -struct is_move_constructible : std::is_move_constructible {}; +/* + * Implementation of `recursive_container_traits` below. + */ +template +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. template -struct is_copy_constructible< +struct impl_recursive_container_traits< Container, - enable_if_t< - all_of, - std::is_same, - // Avoid infinite recursion - negation>>::value>> - : is_copy_constructible {}; + typename std::enable_if::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits::has_recursive_mapped_type + || container_value_type_traits::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template +struct recursive_container_traits : impl_recursive_container_traits {}; + +template +struct is_move_constructible + : all_of, + is_move_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template +struct is_move_constructible> + : all_of, is_move_constructible> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template +struct is_copy_constructible + : all_of, + is_copy_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -852,14 +1005,16 @@ struct is_copy_constructible> : all_of, is_copy_constructible> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template -struct is_copy_assignable : std::is_copy_assignable {}; -template -struct is_copy_assignable, - std::is_same>::value>> - : is_copy_assignable {}; +template +struct is_copy_assignable + : all_of< + std::is_copy_assignable, + is_copy_assignable::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable : std::true_type {}; + template struct is_copy_assignable> : all_of, is_copy_assignable> {}; diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 7cfd996dda..49f1b77821 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -61,9 +61,11 @@ struct is_comparable< /* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ template -struct is_comparable::is_vector>> { - static constexpr const bool value = is_comparable::value; -}; +struct is_comparable::is_vector>> + : is_comparable::type_to_check_recursively> {}; + +template <> +struct is_comparable : std::true_type {}; /* For pairs, recursively check the two data types */ template diff --git a/tests/test_copy_move.cpp b/tests/test_copy_move.cpp index 28c2445644..f54733550a 100644 --- a/tests/test_copy_move.cpp +++ b/tests/test_copy_move.cpp @@ -13,6 +13,8 @@ #include "constructor_stats.h" #include "pybind11_tests.h" +#include + template struct empty { static const derived &get_one() { return instance_; } @@ -293,3 +295,239 @@ TEST_SUBMODULE(copy_move_policies, m) { // Make sure that cast from pytype rvalue to other pytype works m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast(); }); } + +/* + * Rest of the file: + * static_assert based tests for pybind11 adaptations of + * std::is_move_constructible, std::is_copy_constructible and + * std::is_copy_assignable (no adaptation of std::is_move_assignable). + * Difference between pybind11 and std traits: pybind11 traits will also check + * the contained value_types. + */ + +struct NotMovable { + NotMovable() = default; + NotMovable(NotMovable const &) = default; + NotMovable(NotMovable &&) = delete; + NotMovable &operator=(NotMovable const &) = default; + NotMovable &operator=(NotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyable { + NotCopyable() = default; + NotCopyable(NotCopyable const &) = delete; + NotCopyable(NotCopyable &&) = default; + NotCopyable &operator=(NotCopyable const &) = delete; + NotCopyable &operator=(NotCopyable &&) = default; +}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovable { + NotCopyableNotMovable() = default; + NotCopyableNotMovable(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable(NotCopyableNotMovable &&) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct RecursiveVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct RecursiveMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index ca9630bd19..1681760aa8 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -70,6 +70,44 @@ NestMap *times_hundred(int n) { return m; } +/* + * Recursive data structures as test for issue #4623 + */ +struct RecursiveVector : std::vector { + using Parent = std::vector; + using Parent::Parent; +}; + +struct RecursiveMap : std::map { + using Parent = std::map; + using Parent::Parent; +}; + +/* + * Pybind11 does not catch more complicated recursion schemes, such as mutual + * recursion. + * In that case custom recursive_container_traits specializations need to be added, + * thus manually telling pybind11 about the recursion. + */ +struct MutuallyRecursiveContainerPairMV; +struct MutuallyRecursiveContainerPairVM; + +struct MutuallyRecursiveContainerPairMV : std::map {}; +struct MutuallyRecursiveContainerPairVM : std::vector {}; + +namespace pybind11 { +namespace detail { +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(stl_binders, m) { // test_vector_int py::bind_vector>(m, "VectorInt", py::buffer_protocol()); @@ -129,6 +167,12 @@ TEST_SUBMODULE(stl_binders, m) { m, "VectorUndeclStruct", py::buffer_protocol()); }); + // Bind recursive container types + py::bind_vector(m, "RecursiveVector"); + py::bind_map(m, "RecursiveMap"); + py::bind_map(m, "MutuallyRecursiveContainerPairMV"); + py::bind_vector(m, "MutuallyRecursiveContainerPairVM"); + // The rest depends on numpy: try { py::module_::import("numpy"); diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 7dca742a9e..e002f5b678 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -335,3 +335,21 @@ def test_map_view_types(): assert type(unordered_map_string_double.items()) is items_type assert type(map_string_double_const.items()) is items_type assert type(unordered_map_string_double_const.items()) is items_type + + +def test_recursive_vector(): + recursive_vector = m.RecursiveVector() + recursive_vector.append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + # Can't use len() since test_stl_binders.cpp does not include stl.h, + # so the necessary conversion is missing + assert recursive_vector[0].count(m.RecursiveVector()) == 2 + + +def test_recursive_map(): + recursive_map = m.RecursiveMap() + recursive_map[100] = m.RecursiveMap() + recursive_map[100][101] = m.RecursiveMap() + recursive_map[100][102] = m.RecursiveMap() + assert list(recursive_map[100].keys()) == [101, 102] From 90312a6ee8de69f582e165b25338ead4f1a6ccc2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 7 May 2023 10:15:53 -0700 Subject: [PATCH 109/137] Add `type_caster` (#4601) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add `type_caster` (tests are still incomplete). * Fix oversight (`const PyObject *`). * Ensure `type_caster` only works for `PyObject *` * Move `is_same_ignoring_cvref` into `detail` namespace. * Add test_cast_nullptr * Change is_same_ignoring_cvref from variable template to using. ``` test_type_caster_pyobject_ptr.cpp:8:23: error: variable templates only available with β€˜-std=c++14’ or β€˜-std=gnu++14’ [-Werror] 8 | static constexpr bool is_same_ignoring_cvref = std::is_same, U>::value; | ^~~~~~~~~~~~~~~~~~~~~~ ``` * Remove `return_value_policy::reference_internal` `keep_alive` feature (because of doubts about it actually being useful). * Add missing test, fix bug (missing `throw error_already_set();`), various cosmetic changes. * Move `type_caster` from test to new include (pybind11/type_caster_pyobject_ptr.h) * Add new header file to CMakeLists.txt and tests/extra_python_package/test_files.py * Backport changes from https://github.com/google/pywrapcc/pull/30021 to https://github.com/pybind/pybind11/pull/4601 * Fix oversight in test (to resolve a valgrind leak detection error) and add a related comment in cast.h. No production code changes. Make tests more sensitive by using `ValueHolder` instead of empty tuples and dicts. Manual leak checks with `while True:` & top command repeated for all tests. * Add tests for interop with stl.h `list_caster` (No production code changes.) * Bug fix in test. Minor comment enhancements. * Change `type_caster::name` to `object`, as suggested by @Skylion007 * Expand comment for the new `T cast(const handle &handle)` [`T` = `PyObject *`] * Add `T cast(object &&obj)` overload as suggested by @Skylion007 The original suggestion leads to `error: call to 'cast' is ambiguous` (full error message below), therefore SFINAE guarding is needed. ``` clang++ -o pybind11/tests/test_type_caster_pyobject_ptr.os -c -std=c++17 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.10 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:1: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:12: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:13: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/detail/class.h:12: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/attr.h:14: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1165:12: error: call to 'cast' is ambiguous return pybind11::cast(std::move(*this)); ^~~~~~~~~~~~~~~~~ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:109:70: note: in instantiation of function template specialization 'pybind11::object::cast<_object *>' requested here return hfunc.f(std::forward(args)...).template cast(); ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:103:16: note: in instantiation of member function 'pybind11::detail::type_caster>::load(pybind11::handle, bool)::func_wrapper::operator()' requested here struct func_wrapper { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1456:47: note: in instantiation of member function 'pybind11::detail::type_caster>::load' requested here if ((... || !std::get(argcasters).load(call.args[Is], call.args_convert[Is]))) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1434:50: note: in instantiation of function template specialization 'pybind11::detail::argument_loader &, int>::load_impl_sequence<0UL, 1UL>' requested here bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:227:33: note: in instantiation of member function 'pybind11::detail::argument_loader &, int>::load_args' requested here if (!args_converter.load_args(call)) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:101:9: note: in instantiation of function template specialization 'pybind11::cpp_function::initialize<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), _object *, const std::function<_object *(int)> &, int, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::return_value_policy>' requested here initialize( ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1163:22: note: in instantiation of function template specialization 'pybind11::cpp_function::cpp_function<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), pybind11::name, pybind11::scope, pybind11::sibling, pybind11::return_value_policy, void>' requested here cpp_function func(std::forward(f), ^ /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:48:7: note: in instantiation of function template specialization 'pybind11::module_::def<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), pybind11::return_value_policy>' requested here m.def( ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1077:3: note: candidate function [with T = _object *, $1 = 0] T cast(object &&obj) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1149:1: note: candidate function [with T = _object *] cast(object &&object) { ^ 1 error generated. ``` --- CMakeLists.txt | 3 +- include/pybind11/cast.h | 34 ++++- include/pybind11/detail/common.h | 4 + include/pybind11/type_caster_pyobject_ptr.h | 61 +++++++++ tests/CMakeLists.txt | 1 + tests/extra_python_package/test_files.py | 1 + tests/test_type_caster_pyobject_ptr.cpp | 130 ++++++++++++++++++++ tests/test_type_caster_pyobject_ptr.py | 104 ++++++++++++++++ 8 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 include/pybind11/type_caster_pyobject_ptr.h create mode 100644 tests/test_type_caster_pyobject_ptr.cpp create mode 100644 tests/test_type_caster_pyobject_ptr.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d93203881..25a7d14984 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,8 @@ set(PYBIND11_HEADERS include/pybind11/pytypes.h include/pybind11/stl.h include/pybind11/stl_bind.h - include/pybind11/stl/filesystem.h) + include/pybind11/stl/filesystem.h + include/pybind11/type_caster_pyobject_ptr.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9d1ce15d48..db39341180 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1041,7 +1041,11 @@ make_caster load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template ::value, int> = 0> +template ::value + && !detail::is_same_ignoring_cvref::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference::value, @@ -1055,6 +1059,34 @@ T cast(const handle &handle) { return T(reinterpret_borrow(handle)); } +// Note that `cast(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template ::value, int> = 0> object cast(T &&value, diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 129039252f..1ff0df09fe 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -661,6 +661,10 @@ template using remove_cvref_t = typename remove_cvref::type; #endif +/// Example usage: is_same_ignoring_cvref::value +template +using is_same_ignoring_cvref = std::is_same, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 0000000000..aa914f9e15 --- /dev/null +++ b/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template ::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref::value, + "Invalid C++ type T for to-Python conversion (type_caster)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster::cast(): unsupported return_value_policy: " + + std::to_string(static_cast(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow(src); + return true; + } + + template + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 11ac2215b3..5574663b63 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -154,6 +154,7 @@ set(PYBIND11_TEST_FILES test_stl_binders test_tagbased_polymorphic test_thread + test_type_caster_pyobject_ptr test_union test_unnamed_namespace_a test_unnamed_namespace_b diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index dd6393bf0b..e5982f962e 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -43,6 +43,7 @@ "include/pybind11/pytypes.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/type_caster_pyobject_ptr.h", } detail_headers = { diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp new file mode 100644 index 0000000000..1667ea1266 --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.cpp @@ -0,0 +1,130 @@ +#include +#include +#include + +#include "pybind11_tests.h" + +#include +#include + +namespace { + +std::vector make_vector_pyobject_ptr(const py::object &ValueHolder) { + std::vector vec_obj; + for (int i = 1; i < 3; i++) { + vec_obj.push_back(ValueHolder(i * 93).release().ptr()); + } + // This vector now owns the refcounts. + return vec_obj; +} + +} // namespace + +TEST_SUBMODULE(type_caster_pyobject_ptr, m) { + m.def("cast_from_pyobject_ptr", []() { + PyObject *ptr = PyLong_FromLongLong(6758L); + return py::cast(ptr, py::return_value_policy::take_ownership); + }); + m.def("cast_handle_to_pyobject_ptr", [](py::handle obj) { + auto rc1 = obj.ref_count(); + auto *ptr = py::cast(obj); + auto rc2 = obj.ref_count(); + if (rc2 != rc1 + 1) { + return -1; + } + return 100 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_object_to_pyobject_ptr", [](py::object obj) { + py::handle hdl = obj; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(obj)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 300 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_list_to_pyobject_ptr", [](py::list lst) { + // This is to cover types implicitly convertible to object. + py::handle hdl = lst; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(lst)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 400 - static_cast(py::len(py::reinterpret_steal(ptr))); + }); + + m.def( + "return_pyobject_ptr", + []() { return PyLong_FromLongLong(2314L); }, + py::return_value_policy::take_ownership); + m.def("pass_pyobject_ptr", [](PyObject *ptr) { + return 200 - py::reinterpret_borrow(ptr).attr("value").cast(); + }); + + m.def("call_callback_with_object_return", + [](const std::function &cb, int value) { return cb(value); }); + m.def( + "call_callback_with_pyobject_ptr_return", + [](const std::function &cb, int value) { return cb(value); }, + py::return_value_policy::take_ownership); + m.def( + "call_callback_with_pyobject_ptr_arg", + [](const std::function &cb, py::handle obj) { return cb(obj.ptr()); }, + py::arg("cb"), // This triggers return_value_policy::automatic_reference + py::arg("obj")); + + m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) { + if (set_error) { + PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling."); + } + PyObject *ptr = nullptr; + py::cast(ptr); + }); + + m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() { + PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling."); + py::cast(Py_None); + }); + + m.def("pass_list_pyobject_ptr", [](const std::vector &vec_obj) { + int acc = 0; + for (const auto &ptr : vec_obj) { + acc = acc * 1000 + py::reinterpret_borrow(ptr).attr("value").cast(); + } + return acc; + }); + + m.def("return_list_pyobject_ptr_take_ownership", + make_vector_pyobject_ptr, + // Ownership is transferred one-by-one when the vector is converted to a Python list. + py::return_value_policy::take_ownership); + + m.def("return_list_pyobject_ptr_reference", + make_vector_pyobject_ptr, + // Ownership is not transferred. + py::return_value_policy::reference); + + m.def("dec_ref_each_pyobject_ptr", [](const std::vector &vec_obj) { + std::size_t i = 0; + for (; i < vec_obj.size(); i++) { + py::handle h(vec_obj[i]); + if (static_cast(h.ref_count()) < 2) { + break; // Something is badly wrong. + } + h.dec_ref(); + } + return i; + }); + + m.def("pass_pyobject_ptr_and_int", [](PyObject *, int) {}); + +#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing. + { + PyObject *ptr = nullptr; + (void) py::cast(*ptr); + } +#endif +} diff --git a/tests/test_type_caster_pyobject_ptr.py b/tests/test_type_caster_pyobject_ptr.py new file mode 100644 index 0000000000..1f1ece2baf --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.py @@ -0,0 +1,104 @@ +import pytest + +from pybind11_tests import type_caster_pyobject_ptr as m + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below. +class ValueHolder: + def __init__(self, value): + self.value = value + + +def test_cast_from_pyobject_ptr(): + assert m.cast_from_pyobject_ptr() == 6758 + + +def test_cast_handle_to_pyobject_ptr(): + assert m.cast_handle_to_pyobject_ptr(ValueHolder(24)) == 76 + + +def test_cast_object_to_pyobject_ptr(): + assert m.cast_object_to_pyobject_ptr(ValueHolder(43)) == 257 + + +def test_cast_list_to_pyobject_ptr(): + assert m.cast_list_to_pyobject_ptr([1, 2, 3, 4, 5]) == 395 + + +def test_return_pyobject_ptr(): + assert m.return_pyobject_ptr() == 2314 + + +def test_pass_pyobject_ptr(): + assert m.pass_pyobject_ptr(ValueHolder(82)) == 118 + + +@pytest.mark.parametrize( + "call_callback", + [ + m.call_callback_with_object_return, + m.call_callback_with_pyobject_ptr_return, + ], +) +def test_call_callback_with_object_return(call_callback): + def cb(value): + if value < 0: + raise ValueError("Raised from cb") + return ValueHolder(1000 - value) + + assert call_callback(cb, 287).value == 713 + + with pytest.raises(ValueError, match="^Raised from cb$"): + call_callback(cb, -1) + + +def test_call_callback_with_pyobject_ptr_arg(): + def cb(obj): + return 300 - obj.value + + assert m.call_callback_with_pyobject_ptr_arg(cb, ValueHolder(39)) == 261 + + +@pytest.mark.parametrize("set_error", [True, False]) +def test_cast_to_python_nullptr(set_error): + expected = { + True: r"^Reflective of healthy error handling\.$", + False: ( + r"^Internal error: pybind11::error_already_set called " + r"while Python error indicator not set\.$" + ), + }[set_error] + with pytest.raises(RuntimeError, match=expected): + m.cast_to_pyobject_ptr_nullptr(set_error) + + +def test_cast_to_python_non_nullptr_with_error_set(): + with pytest.raises(SystemError) as excinfo: + m.cast_to_pyobject_ptr_non_nullptr_with_error_set() + assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()" + assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling." + + +def test_pass_list_pyobject_ptr(): + acc = m.pass_list_pyobject_ptr([ValueHolder(842), ValueHolder(452)]) + assert acc == 842452 + + +def test_return_list_pyobject_ptr_take_ownership(): + vec_obj = m.return_list_pyobject_ptr_take_ownership(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + + +def test_return_list_pyobject_ptr_reference(): + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + +def test_type_caster_name_via_incompatible_function_arguments_type_error(): + with pytest.raises(TypeError, match=r"1\. \(arg0: object, arg1: int\) -> None"): + m.pass_pyobject_ptr_and_int(ValueHolder(101), ValueHolder(202)) From e9b961d9b913575c07ba28c038c3706731768da6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 8 May 2023 10:13:54 -0700 Subject: [PATCH 110/137] Elide to-python conversion of setter return values (#4621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reproducer for property setter with return type that is not wrapped. * Use `py::class_()` to work around the return value conversion issue. * WIP drop_return_value * Remove struct drop_return_value * Introduce `return_value_policy::return_none` for use by setters. * Add `is_setter` to attr.h and use from `.def_property()` * Merge the new test into test_methods_and_attributes * Remove return_none return_value_policy again. * Fix oversight (NOLINTNEXTLINE placement). * Simplification (for the better) found while searching for a way to resolve GCC build failures. Example of failure resolved by this change: g++ (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0 ``` cd /build/tests && /usr/bin/c++ -DPYBIND11_TEST_EIGEN -Dpybind11_tests_EXPORTS -I/mounted_pybind11/include -isystem /usr/include/python3.8 -isystem /build/_deps/eigen-src -g -std=c++17 -fPIC -fvisibility=hidden -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -MD -MT tests/CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o -MF CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o.d -o CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o -c /mounted_pybind11/tests/test_buffers.cpp In file included from /mounted_pybind11/include/pybind11/stl.h:12, from /mounted_pybind11/tests/test_buffers.cpp:10: /mounted_pybind11/include/pybind11/pybind11.h: In instantiation of β€˜pybind11::class_& pybind11::class_::def_property(const char*, const Getter&, const Setter&, const Extra& ...) [with Getter = pybind11::cpp_function; Setter = std::nullptr_t; Extra = {pybind11::return_value_policy}; type_ = pybind11::buffer_info; options = {}]’: /mounted_pybind11/include/pybind11/pybind11.h:1716:58: required from β€˜pybind11::class_& pybind11::class_::def_property_readonly(const char*, const pybind11::cpp_function&, const Extra& ...) [with Extra = {pybind11::return_value_policy}; type_ = pybind11::buffer_info; options = {}]’ /mounted_pybind11/include/pybind11/pybind11.h:1684:9: required from β€˜pybind11::class_& pybind11::class_::def_readonly(const char*, const D C::*, const Extra& ...) [with C = pybind11::buffer_info; D = long int; Extra = {}; type_ = pybind11::buffer_info; options = {}]’ /mounted_pybind11/tests/test_buffers.cpp:209:61: required from here /mounted_pybind11/include/pybind11/pybind11.h:1740:25: error: call of overloaded β€˜cpp_function(std::nullptr_t&, pybind11::is_setter)’ is ambiguous 1740 | name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /mounted_pybind11/include/pybind11/pybind11.h:101:5: note: candidate: β€˜pybind11::cpp_function::cpp_function(Func&&, const Extra& ...) [with Func = std::nullptr_t&; Extra = {pybind11::is_setter}; = void]’ 101 | cpp_function(Func &&f, const Extra &...extra) { | ^~~~~~~~~~~~ In file included from /mounted_pybind11/include/pybind11/stl.h:12, from /mounted_pybind11/tests/test_buffers.cpp:10: /mounted_pybind11/include/pybind11/pybind11.h:87:5: note: candidate: β€˜pybind11::cpp_function::cpp_function(std::nullptr_t, const Extra& ...) [with Extra = {pybind11::is_setter}; std::nullptr_t = std::nullptr_t]’ 87 | cpp_function(std::nullptr_t, const Extra &...) {} | ^~~~~~~~~~~~ ``` * Bug fix: obvious in hindsight. I thought the original version was incrementing the reference count for None, but no. Discovered via many failing tests in the wild (10s of thousands). It is very tricky to construct a meaningful unit test for this bug specifically. It's unlikely to come back, because 10s of thousands of tests will fail again. --- include/pybind11/attr.h | 16 +++++++++++-- include/pybind11/pybind11.h | 18 ++++++++++---- tests/test_methods_and_attributes.cpp | 34 +++++++++++++++++++++++++++ tests/test_methods_and_attributes.py | 9 +++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index b5e3b7b22c..1044db94d9 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -26,6 +26,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -188,8 +191,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +233,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -426,6 +432,12 @@ struct process_attribute : process_attribute_default { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute : process_attribute_default { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 6205effd61..28ebc22297 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -84,6 +84,7 @@ class cpp_function : public function { cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template @@ -244,10 +245,16 @@ class cpp_function : public function { using Guard = extract_guard_t; /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call(cap->f), - policy, - call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes::postcall(call, result); @@ -1729,7 +1736,8 @@ class class_ : public detail::generic_type { template class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); } template class_ &def_property(const char *name, diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 815dd5e98a..31d46eb7ed 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -177,6 +177,38 @@ struct RValueRefParam { std::size_t func4(std::string &&s) const & { return s.size(); } }; +namespace pybind11_tests { +namespace exercise_is_setter { + +struct FieldBase { + int int_value() const { return int_value_; } + + FieldBase &SetIntValue(int int_value) { + int_value_ = int_value; + return *this; + } + +private: + int int_value_ = -99; +}; + +struct Field : FieldBase {}; + +void add_bindings(py::module &m) { + py::module sm = m.def_submodule("exercise_is_setter"); + // NOTE: FieldBase is not wrapped, therefore ... + py::class_(sm, "Field") + .def(py::init<>()) + .def_property( + "int_value", + &Field::int_value, + &Field::SetIntValue // ... the `FieldBase &` return value here cannot be converted. + ); +} + +} // namespace exercise_is_setter +} // namespace pybind11_tests + TEST_SUBMODULE(methods_and_attributes, m) { // test_methods_and_attributes py::class_ emna(m, "ExampleMandA"); @@ -456,4 +488,6 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def("func2", &RValueRefParam::func2) .def("func3", &RValueRefParam::func3) .def("func4", &RValueRefParam::func4); + + pybind11_tests::exercise_is_setter::add_bindings(m); } diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index a854685750..955a85f67c 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -522,3 +522,12 @@ def test_rvalue_ref_param(): assert r.func2("1234") == 4 assert r.func3("12345") == 5 assert r.func4("123456") == 6 + + +def test_is_setter(): + fld = m.exercise_is_setter.Field() + assert fld.int_value == -99 + setter_return = fld.int_value = 100 + assert isinstance(setter_return, int) + assert setter_return == 100 + assert fld.int_value == 100 From cca4c51ca463ea02fa504331ff21bc313c80c7f3 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Tue, 9 May 2023 08:04:20 -0600 Subject: [PATCH 111/137] Update errors in string "Explicit conversions" docs (#4658) `PyUnicode_DecodeLatin1` requires you to pass in the `error` parameter. The code as it is in the docs didn't compile. There is a reference leak in the example code. `PyUnicode_DecodeLatin1` returns a new reference. Calling `py::str(PyObject*)` calls `PyObject_Str`, which also returns a new reference. That reference is managed by the `py::str` constructor (which correctly steals the reference, using the `stolen_t` constructor), but the original reference returned by `PyUnicode_DecodeLatin1` is never decref'd: it never makes it into an `object`, and it's never manually decremented. This fixes both of those issues. The code compiles, and I viewed the sphinx docs locally. --- docs/advanced/cast/strings.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/advanced/cast/strings.rst b/docs/advanced/cast/strings.rst index e246c5219a..271716b4b3 100644 --- a/docs/advanced/cast/strings.rst +++ b/docs/advanced/cast/strings.rst @@ -101,8 +101,11 @@ conversion has the same overhead as implicit conversion. m.def("str_output", []() { std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1 - py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length()); - return py_s; + py::handle py_s = PyUnicode_DecodeLatin1(s.data(), s.length(), nullptr); + if (!py_s) { + throw py::error_already_set(); + } + return py::reinterpret_steal(py_s); } ); @@ -113,7 +116,8 @@ conversion has the same overhead as implicit conversion. The `Python C API `_ provides -several built-in codecs. +several built-in codecs. Note that these all return *new* references, so +use :cpp:func:`reinterpret_steal` when converting them to a :cpp:class:`str`. One could also use a third party encoding library such as libiconv to transcode From d72ffb448c58b4ffb08b5ad629bc788646e2d59e Mon Sep 17 00:00:00 2001 From: Joyce Date: Mon, 15 May 2023 14:02:25 -0300 Subject: [PATCH 112/137] ci: set minimal permissions to github workflows (#4665) * set ci.yml minimal permissions Signed-off-by: Joyce * set configure.yml minimal permissions Signed-off-by: Joyce * set format.yml minimal permissions Signed-off-by: Joyce * set pip.yml minimal permissions Signed-off-by: Joyce * set upstream.yml minimal permissions Signed-off-by: Joyce * set labeler.yml minimal permissions Signed-off-by: Joyce * Update ci.yml to read all Signed-off-by: Joyce * test labeler.yml Signed-off-by: Joyce * restore the if at labeler.yml Signed-off-by: Joyce --------- Signed-off-by: Joyce --- .github/workflows/ci.yml | 2 ++ .github/workflows/configure.yml | 3 +++ .github/workflows/format.yml | 3 +++ .github/workflows/labeler.yml | 5 +++++ .github/workflows/pip.yml | 3 +++ .github/workflows/upstream.yml | 3 +++ 6 files changed, 19 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c88f0797f2..8c2aba343a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,8 @@ on: - stable - v* +permissions: read-all + concurrency: group: test-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index b469a69d11..4ae22281c6 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -9,6 +9,9 @@ on: - stable - v* +permissions: + contents: read + env: # For cmake: VERBOSE: 1 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 46489feb31..b8242ee52c 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -12,6 +12,9 @@ on: - stable - "v*" +permissions: + contents: read + env: FORCE_COLOR: 3 # For cmake: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 165a2fd87b..858a4a0e26 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,10 +3,15 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: label: name: Labeler runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: actions/labeler@main diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 6d9be3b1d4..c1feb6fe10 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -12,6 +12,9 @@ on: types: - published +permissions: + contents: read + env: PIP_ONLY_BINARY: numpy diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index be643ddfdb..4acfbfce75 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -5,6 +5,9 @@ on: workflow_dispatch: pull_request: +permissions: + contents: read + concurrency: group: upstream-${{ github.ref }} cancel-in-progress: true From 19816f0db7b89468104f5ed284c0ec9d7e312c44 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 22 May 2023 13:44:03 -0700 Subject: [PATCH 113/137] chore: update changelog, with target date for v2.11.0 release (#4677) * Remove .dev1 from version number. * [skip ci] Update changelog.rst * [ci skip] Fix pre-commit: rst ``code`` is two backticks * Apply fix suggested by @henryiii * [ci skip] Set target date for the release to June 2, 2023 * [ci skip] Apply more fixes suggested by @henryiii (I missed those before). * [ci skip] Revert "Remove .dev1 from version number." This reverts commit afc80134cb1035be2913fd8153a16a864181c52c. --- docs/changelog.rst | 54 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 58cc409839..32f6606aad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,10 +10,25 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. -IN DEVELOPMENT --------------- +Version 2.11.0 (June 2, 2023) +----------------------------- + +New features: + +* ``pybind11::detail::is_move_constructible`` can now be specialized for cases + in which ``std::is_move_constructible`` does not work as needed. This is + very similar to the long-established + ``pybind11::detail::is_copy_constructible``. + `#4631 `_ -Changes will be summarized here periodically. +* Introduce ``recursive_container_traits``. + `#4623 `_ + +* ``pybind11/type_caster_pyobject_ptr.h`` was added to support automatic + wrapping of APIs that make use of ``PyObject *``. This header needs to + included explicitly (i.e. it is not included implicitly + with ``pybind/pybind11.h``). + `#4601 `_ Changes: @@ -31,6 +46,28 @@ Changes: sizes slightly (~1.5%) but the error messages are much more informative. `#4463 `_ +* Setter return values (which are inaccessible for all practical purposes) are + no longer converted to Python (only to be discarded). + `#4621 `_ + +* Allow lambda specified to function definition to be ``noexcept(true)`` + in C++17. + `#4593 `_ + +* Get rid of recursive template instantiations for concatenating type + signatures on C++17 and higher. + `#4587 `_ + +* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11 + ABI version for Python 3.12 is version 5. (The default ABI version + for Python versions up to and including 3.11 is still version 4.). + `#4570 `_ + +* With ``PYBIND11_INTERNALS_VERSION 5`` (default for Python 3.12+), MSVC builds + use ``std::hash`` and ``std::equal_to`` + instead of string-based type comparisons. This resolves issues when binding + types defined in the unnamed namespace. + `#4319 `_ Build system improvements: @@ -40,8 +77,17 @@ Build system improvements: * Moved the linting framework over to Ruff. `#4483 `_ +* Skip lto checks and target generation when + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. + `#4643 `_ + +* No longer inject ``-stdlib=libc++``, not needed for modern Pythons + (macOS 10.9+). + `#4639 `_ + + Version 2.10.4 (Mar 16, 2023) ----------------------------- +----------------------------- Changes: From ce9bbc0a213bfb21b77ada8de5346be5e350e3c4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 May 2023 10:03:33 -0700 Subject: [PATCH 114/137] Python 3.11+: Add `__notes__` to `error_already_set::what()` output. (#4678) * First version adding `__notes__` to `error_already_set::what()` output. * Fix trivial oversight (missing adjustment in existing test). * Minor enhancements of new code. * Re-enable `cmake --target cpptest -j 2` * Revert "Re-enable `cmake --target cpptest -j 2`" This reverts commit 60816285e9e7b95b7d33a60805888b80b2bec641. * Add general comment explaining why the `error_fetch_and_normalize` code is so unusual. --- include/pybind11/pytypes.h | 70 +++++++++++++++++++++++++++++++++----- tests/test_exceptions.py | 36 +++++++++++++++----- 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f11ed5da78..f5d3f34f38 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -471,13 +471,24 @@ inline const char *obj_class_name(PyObject *obj) { std::string error_string(); +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. struct error_fetch_and_normalize { - // Immediate normalization is long-established behavior (starting with - // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 - // from Sep 2016) and safest. Normalization could be deferred, but this could mask - // errors elsewhere, the performance gain is very minor in typical situations - // (usually the dominant bottleneck is EH unwinding), and the implementation here - // would be more complex. + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); if (!m_type) { @@ -492,6 +503,14 @@ struct error_fetch_and_normalize { "of the original active exception type."); } m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else // PyErr_NormalizeException() may change the exception type if there are cascading // failures. This can potentially be extremely confusing. PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); @@ -506,12 +525,12 @@ struct error_fetch_and_normalize { + " failed to obtain the name " "of the normalized active exception type."); } -#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 // This behavior runs the risk of masking errors in the error handling, but avoids a // conflict with PyPy, which relies on the normalization here to change OSError to // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). m_lazy_error_string = exc_type_name_norm; -#else +# else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -523,6 +542,7 @@ struct error_fetch_and_normalize { msg += ": " + format_value_and_trace(); pybind11_fail(msg); } +# endif #endif } @@ -558,6 +578,40 @@ struct error_fetch_and_normalize { } } } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } + } +#endif } else { result = ""; } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index a92ab59a34..ccac4536d6 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -317,13 +317,7 @@ def test_error_already_set_what_with_happy_exceptions( assert what == expected_what -@pytest.mark.skipif( - # Intentionally very specific: - "sys.version_info == (3, 12, 0, 'alpha', 7)", - reason="WIP: https://github.com/python/cpython/issues/102594", -) -@pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") -def test_flaky_exception_failure_point_init(): +def _test_flaky_exception_failure_point_init_before_py_3_12(): with pytest.raises(RuntimeError) as excinfo: m.error_already_set_what(FlakyException, ("failure_point_init",)) lines = str(excinfo.value).splitlines() @@ -337,7 +331,33 @@ def test_flaky_exception_failure_point_init(): # Checking the first two lines of the traceback as formatted in error_string(): assert "test_exceptions.py(" in lines[3] assert lines[3].endswith("): __init__") - assert lines[4].endswith("): test_flaky_exception_failure_point_init") + assert lines[4].endswith( + "): _test_flaky_exception_failure_point_init_before_py_3_12" + ) + + +def _test_flaky_exception_failure_point_init_py_3_12(): + # Behavior change in Python 3.12: https://github.com/python/cpython/issues/102594 + what, py_err_set_after_what = m.error_already_set_what( + FlakyException, ("failure_point_init",) + ) + assert not py_err_set_after_what + lines = what.splitlines() + assert lines[0].endswith("ValueError[WITH __notes__]: triggered_failure_point_init") + assert lines[1] == "__notes__ (len=1):" + assert "Normalization failed:" in lines[2] + assert "FlakyException" in lines[2] + + +@pytest.mark.skipif( + "env.PYPY and sys.version_info[:2] < (3, 12)", + reason="PyErr_NormalizeException Segmentation fault", +) +def test_flaky_exception_failure_point_init(): + if sys.version_info[:2] < (3, 12): + _test_flaky_exception_failure_point_init_before_py_3_12() + else: + _test_flaky_exception_failure_point_init_py_3_12() def test_flaky_exception_failure_point_str(): From 6e6bcca5b28461cd16b2af70bba837a4b24482c0 Mon Sep 17 00:00:00 2001 From: Joyce Date: Tue, 23 May 2023 14:05:25 -0300 Subject: [PATCH 115/137] Create s Security Policy (#4671) * Create SECURITY.md * Update test_files.py to include SECURITY.md file * Update MANIFEST.in to include SECURITY.md file --- MANIFEST.in | 2 +- SECURITY.md | 13 +++++++++++++ tests/extra_python_package/test_files.py | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 SECURITY.md diff --git a/MANIFEST.in b/MANIFEST.in index 31632acc33..7ce83c5527 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg +include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..3d74611f2d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e5982f962e..5b9e145694 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -111,6 +111,7 @@ "MANIFEST.in", "README.rst", "PKG-INFO", + "SECURITY.md", } local_sdist_files = { From 8e1f9d5c40f74632799c8a44287d32d18a6c8630 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 May 2023 10:49:32 -0700 Subject: [PATCH 116/137] Add `format_descriptor<>` & `npy_format_descriptor<>` `PyObject *` specializations. (#4674) * Add `npy_format_descriptor` to enable `py::array_t` to/from-python conversions. * resolve clang-tidy warning * Use existing constructor instead of adding a static method. Thanks @Skylion007 for pointing out. * Add `format_descriptor` Trivial addition, but still in search for a meaningful test. * Add test_format_descriptor_format * Ensure the Eigen `type_caster`s do not segfault when loading arrays with dtype=object * Use `static_assert()` `!std::is_pointer<>` to replace runtime guards. * Add comments to explain how to check for ref-count bugs. (NO code changes.) * Make the "Pointer types ... are not supported" message Eigen-specific, as suggested by @Lalaland. Move to new pybind11/eigen/common.h header. * Change "format_descriptor_format" implementation as suggested by @Lalaland. Additional tests meant to ensure consistency between py::format_descriptor<>, np.array, np.format_parser turn out to be useful only to highlight long-standing inconsistencies. * resolve clang-tidy warning * Account for np.float128, np.complex256 not being available on Windows, in a future-proof way. * Fully address i|q|l ambiguity (hopefully). * Remove the new `np.format_parser()`-based test, it's much more distracting than useful. * Use bi.itemsize to disambiguate "l" or "L" * Use `py::detail::compare_buffer_info::compare()` to validate the `format_descriptor::format()` strings. * Add `buffer_info::compare` to make `detail::compare_buffer_info::compare` more visible & accessible. * silence clang-tidy warning * pytest-compatible access to np.float128, np.complex256 * Revert "pytest-compatible access to np.float128, np.complex256" This reverts commit e9a289c50fc07199806d14ded644215ab6f03afa. * Use `sizeof(long double) == sizeof(double)` instead of `std::is_same<>` * Report skipped `long double` tests. * Change the name of the new `buffer_info` member function to `item_type_is_equivalent_to`. Add comment defining "equivalent" by example. * Change `item_type_is_equivalent_to<>()` from `static` function to member function, as suggested by @Lalaland --- CMakeLists.txt | 1 + include/pybind11/buffer_info.h | 17 +++++- include/pybind11/detail/common.h | 9 +++ include/pybind11/eigen/common.h | 9 +++ include/pybind11/eigen/matrix.h | 13 +++++ include/pybind11/eigen/tensor.h | 5 ++ include/pybind11/numpy.h | 18 ++++-- tests/extra_python_package/test_files.py | 1 + tests/test_buffers.cpp | 35 ++++++++++++ tests/test_buffers.py | 57 +++++++++++++++++++ tests/test_numpy_array.cpp | 26 +++++++++ tests/test_numpy_array.py | 71 ++++++++++++++++++++++++ 12 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 include/pybind11/eigen/common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 25a7d14984..0ad74db2bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,7 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h include/pybind11/eigen/matrix.h include/pybind11/eigen/tensor.h include/pybind11/embed.h diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 06120d5563..b99ee8bef4 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector f_strides(const std::vector &shape, ssize_t return strides; } +template +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -150,6 +153,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + private: struct private_ctr_tag {}; @@ -170,9 +184,10 @@ struct buffer_info { PYBIND11_NAMESPACE_BEGIN(detail) -template +template struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 1ff0df09fe..c2e6f9ec89 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1025,6 +1025,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in template struct format_descriptor {}; +template +struct format_descriptor< + T, + detail::enable_if_t::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; diff --git a/include/pybind11/eigen/common.h b/include/pybind11/eigen/common.h new file mode 100644 index 0000000000..24f56d1584 --- /dev/null +++ b/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index bd533bed37..8d4342f81b 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -10,6 +10,7 @@ #pragma once #include "../numpy.h" +#include "common.h" /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: @@ -287,6 +288,8 @@ handle eigen_encapsulate(Type *src) { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using props = EigenProps; bool load(handle src, bool convert) { @@ -405,6 +408,9 @@ struct type_caster::value>> { // Base class for casting reference/map/block/etc. objects back to python. template struct eigen_map_caster { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + private: using props = EigenProps; @@ -457,6 +463,8 @@ struct type_caster< using Type = Eigen::Ref; using props = EigenProps; using Scalar = typename props::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::Map; using Array = array_t struct type_caster::value>> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + protected: using Matrix = Eigen::Matrix; @@ -632,6 +643,8 @@ struct type_caster::value>> { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using StorageIndex = remove_reference_t().outerIndexPtr())>; using Index = typename Type::Index; static constexpr bool rowMajor = Type::IsRowMajor; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index de7dcba89d..25d12baca1 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -8,6 +8,7 @@ #pragma once #include "../numpy.h" +#include "common.h" #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); @@ -164,6 +165,8 @@ PYBIND11_WARNING_POP template struct type_caster::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using Helper = eigen_tensor_helper; static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); @@ -359,6 +362,8 @@ struct get_storage_pointer_type struct type_caster, typename eigen_tensor_helper>::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper>; diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 854d6e87fa..36077ec04d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -564,6 +564,8 @@ class dtype : public object { m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType explicit dtype(int typenum) : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { if (m_ptr == nullptr) { @@ -1283,12 +1285,16 @@ struct npy_format_descriptor< public: static constexpr int value = values[detail::is_fmt_numeric::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template +struct npy_format_descriptor::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 5b9e145694..57387dd8bc 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -57,6 +57,7 @@ } eigen_headers = { + "include/pybind11/eigen/common.h", "include/pybind11/eigen/matrix.h", "include/pybind11/eigen/tensor.h", } diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 6b6e8cba7f..b5b8c355b3 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -7,12 +7,47 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "constructor_stats.h" #include "pybind11_tests.h" TEST_SUBMODULE(buffers, m) { + m.attr("long_double_and_double_have_same_size") = (sizeof(long double) == sizeof(double)); + + m.def("format_descriptor_format_buffer_info_equiv", + [](const std::string &cpp_name, const py::buffer &buffer) { + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *format_table = new std::map; + static auto *equiv_table + = new std::map; + if (format_table->empty()) { +#define PYBIND11_ASSIGN_HELPER(...) \ + (*format_table)[#__VA_ARGS__] = py::format_descriptor<__VA_ARGS__>::format(); \ + (*equiv_table)[#__VA_ARGS__] = &py::buffer_info::item_type_is_equivalent_to<__VA_ARGS__>; + PYBIND11_ASSIGN_HELPER(PyObject *) + PYBIND11_ASSIGN_HELPER(bool) + PYBIND11_ASSIGN_HELPER(std::int8_t) + PYBIND11_ASSIGN_HELPER(std::uint8_t) + PYBIND11_ASSIGN_HELPER(std::int16_t) + PYBIND11_ASSIGN_HELPER(std::uint16_t) + PYBIND11_ASSIGN_HELPER(std::int32_t) + PYBIND11_ASSIGN_HELPER(std::uint32_t) + PYBIND11_ASSIGN_HELPER(std::int64_t) + PYBIND11_ASSIGN_HELPER(std::uint64_t) + PYBIND11_ASSIGN_HELPER(float) + PYBIND11_ASSIGN_HELPER(double) + PYBIND11_ASSIGN_HELPER(long double) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) +#undef PYBIND11_ASSIGN_HELPER + } + return std::pair( + (*format_table)[cpp_name], (buffer.request().*((*equiv_table)[cpp_name]))()); + }); + // test_from_python / test_to_python: class Matrix { public: diff --git a/tests/test_buffers.py b/tests/test_buffers.py index eb58c4675e..63d9d869fd 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -10,6 +10,63 @@ np = pytest.importorskip("numpy") +if m.long_double_and_double_have_same_size: + # Determined by the compiler used to build the pybind11 tests + # (e.g. MSVC gets here, but MinGW might not). + np_float128 = None + np_complex256 = None +else: + # Determined by the compiler used to build numpy (e.g. MinGW). + np_float128 = getattr(np, *["float128"] * 2) + np_complex256 = getattr(np, *["complex256"] * 2) + +CPP_NAME_FORMAT_NP_DTYPE_TABLE = [ + ("PyObject *", "O", object), + ("bool", "?", np.bool_), + ("std::int8_t", "b", np.int8), + ("std::uint8_t", "B", np.uint8), + ("std::int16_t", "h", np.int16), + ("std::uint16_t", "H", np.uint16), + ("std::int32_t", "i", np.int32), + ("std::uint32_t", "I", np.uint32), + ("std::int64_t", "q", np.int64), + ("std::uint64_t", "Q", np.uint64), + ("float", "f", np.float32), + ("double", "d", np.float64), + ("long double", "g", np_float128), + ("std::complex", "Zf", np.complex64), + ("std::complex", "Zd", np.complex128), + ("std::complex", "Zg", np_complex256), +] +CPP_NAME_FORMAT_TABLE = [ + (cpp_name, format) + for cpp_name, format, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE + if np_dtype is not None +] +CPP_NAME_NP_DTYPE_TABLE = [ + (cpp_name, np_dtype) for cpp_name, _, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE +] + + +@pytest.mark.parametrize(("cpp_name", "np_dtype"), CPP_NAME_NP_DTYPE_TABLE) +def test_format_descriptor_format_buffer_info_equiv(cpp_name, np_dtype): + if np_dtype is None: + pytest.skip( + f"cpp_name=`{cpp_name}`: `long double` and `double` have same size." + ) + if isinstance(np_dtype, str): + pytest.skip(f"np.{np_dtype} does not exist.") + np_array = np.array([], dtype=np_dtype) + for other_cpp_name, expected_format in CPP_NAME_FORMAT_TABLE: + format, np_array_is_matching = m.format_descriptor_format_buffer_info_equiv( + other_cpp_name, np_array + ) + assert format == expected_format + if other_cpp_name == cpp_name: + assert np_array_is_matching + else: + assert not np_array_is_matching + def test_from_python(): with pytest.raises(RuntimeError) as excinfo: diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index b118e2c6cc..8c122a8658 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -523,4 +523,30 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("test_fmt_desc_const_double", [](const py::array_t &) {}); sm.def("round_trip_float", [](double d) { return d; }); + + sm.def("pass_array_pyobject_ptr_return_sum_str_values", + [](const py::array_t &objs) { + std::string sum_str_values; + for (const auto &obj : objs) { + sum_str_values += py::str(obj.attr("value")); + } + return sum_str_values; + }); + + sm.def("pass_array_pyobject_ptr_return_as_list", + [](const py::array_t &objs) -> py::list { return objs; }); + + sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) { + py::size_t arr_size = py::len(objs); + py::array_t arr_from_list(static_cast(arr_size)); + PyObject **data = arr_from_list.mutable_data(); + for (py::size_t i = 0; i < arr_size; i++) { + assert(data[i] == nullptr); + data[i] = py::cast(objs[i].attr("value")); + } + return arr_from_list; + }); + + sm.def("return_array_pyobject_ptr_from_list", + [](const py::list &objs) -> py::array_t { return objs; }); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 070813d3a2..12e7d17d15 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -595,3 +595,74 @@ def test_round_trip_float(): arr = np.zeros((), np.float64) arr[()] = 37.2 assert m.round_trip_float(arr) == 37.2 + + +# HINT: An easy and robust way (although only manual unfortunately) to check for +# ref-count leaks in the test_.*pyobject_ptr.* functions below is to +# * temporarily insert `while True:` (one-by-one), +# * run this test, and +# * run the Linux `top` command in another shell to visually monitor +# `RES` for a minute or two. +# If there is a leak, it is usually evident in seconds because the `RES` +# value increases without bounds. (Don't forget to Ctrl-C the test!) + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below: +# * Ref-count leaks will be immediately evident. +# * Sanitizers are much more likely to detect heap-use-after-free due to +# other ref-count bugs. +class PyValueHolder: + def __init__(self, value): + self.value = value + + +def WrapWithPyValueHolder(*values): + return [PyValueHolder(v) for v in values] + + +def UnwrapPyValueHolder(vhs): + return [vh.value for vh in vhs] + + +def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object) + ) + == "-3four5.0" + ) + + +def test_pass_array_pyobject_ptr_return_sum_str_values_list(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + WrapWithPyValueHolder(2, "three", -4.0) + ) + == "2three-4.0" + ) + + +def test_pass_array_pyobject_ptr_return_as_list(): + # Intentionally all temporaries, do not change. + assert UnwrapPyValueHolder( + m.pass_array_pyobject_ptr_return_as_list( + np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object) + ) + ) == [-1, "two", 3.0] + + +@pytest.mark.parametrize( + ("return_array_pyobject_ptr", "unwrap"), + [ + (m.return_array_pyobject_ptr_cpp_loop, list), + (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder), + ], +) +def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap): + # Intentionally all temporaries, do not change. + arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0)) + assert isinstance(arr_from_list, np.ndarray) + assert arr_from_list.dtype == np.dtype("O") + assert unwrap(arr_from_list) == [6, "seven", -8.0] From d0232b119f5f0cf2bf6cecacaf099c301c811ca8 Mon Sep 17 00:00:00 2001 From: "T.Yamada" Date: Thu, 25 May 2023 13:39:36 +0900 Subject: [PATCH 117/137] Use annotated for array (#4679) * use Annotated for std::array docstring * add tests * fix test * style: pre-commit fixes * fix valarray annotation * style: pre-commit fixes * refix test * add FixedSize * style: pre-commit fixes * Update test_stl.py * style: pre-commit fixes --------- Co-authored-by: Taiju Yamada Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/stl.h | 8 ++++---- tests/test_stl.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 84c5171081..f39f44f7c9 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -273,11 +273,11 @@ struct array_caster { } PYBIND11_TYPE_CASTER(ArrayType, - const_name("List[") + value_conv::name + const_name(const_name(""), const_name("Annotated[")) + + const_name("List[") + value_conv::name + const_name("]") + const_name(const_name(""), - const_name("[") + const_name() - + const_name("]")) - + const_name("]")); + const_name(", FixedSize(") + + const_name() + const_name(")]"))); }; template diff --git a/tests/test_stl.py b/tests/test_stl.py index 2d9dcc89f1..8a614f8b87 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -39,8 +39,11 @@ def test_array(doc): assert m.load_array(lst) assert m.load_array(tuple(lst)) - assert doc(m.cast_array) == "cast_array() -> List[int[2]]" - assert doc(m.load_array) == "load_array(arg0: List[int[2]]) -> bool" + assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]" + assert ( + doc(m.load_array) + == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool" + ) def test_valarray(doc): From 29487dee8f86c984605570c298b21c79ea3bada3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 7 Jun 2023 12:00:37 -0700 Subject: [PATCH 118/137] =?UTF-8?q?Disable=20=F0=9F=90=8D=203=20=E2=80=A2?= =?UTF-8?q?=20CentOS7=20/=20PGI=2022.9=20=E2=80=A2=20x64=20(#4691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2aba343a..91144241f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -400,6 +400,7 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: + if: ${{ false }} # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690 runs-on: ubuntu-latest name: "🐍 3 β€’ CentOS7 / PGI 22.9 β€’ x64" container: centos:7 From 3617f3554aa5096e2148281c14be99afe9c138e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 12:57:38 -0700 Subject: [PATCH 119/137] chore(deps): update pre-commit hooks (#4689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.263 β†’ v0.0.270](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.263...v0.0.270) - [github.com/pre-commit/mirrors-mypy: v1.2.0 β†’ v1.3.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.2.0...v1.3.0) - [github.com/shellcheck-py/shellcheck-py: v0.9.0.2 β†’ v0.9.0.5](https://github.com/shellcheck-py/shellcheck-py/compare/v0.9.0.2...v0.9.0.5) - [github.com/pre-commit/mirrors-clang-format: v16.0.2 β†’ v16.0.4](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.2...v16.0.4) * Resolve new ruff error: ``` tests/test_pytypes.py:478:20: PLW0130 Duplicate value `3` in set ``` The reduction from `{3, 3}` to `{3}` never happened in pybind11 code, therefore this change is essentially a no-op for the purpose of unit-testing pybind11. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 8 ++++---- tests/test_pytypes.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aca9b3ca2e..bcc3f27232 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.263 + rev: v0.0.270 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.2.0" + rev: "v1.3.0" hooks: - id: mypy args: [] @@ -128,7 +128,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.9.0.2" + rev: "v0.9.0.5" hooks: - id: shellcheck @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.2" + rev: "v16.0.4" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 946b941402..afb7a1ce8a 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -475,7 +475,7 @@ def test_pybind11_str_raw_str(): assert cvt({}) == "{}" assert cvt({3: 4}) == "{3: 4}" assert cvt(set()) == "set()" - assert cvt({3, 3}) == "{3}" + assert cvt({3}) == "{3}" valid_orig = "Η±" valid_utf8 = valid_orig.encode("utf-8") From c679a9209557ef0324a19f4618c556b60e251009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:49:14 -0700 Subject: [PATCH 120/137] chore(deps): bump deadsnakes/action from 3.0.0 to 3.0.1 (#4687) Bumps [deadsnakes/action](https://github.com/deadsnakes/action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/deadsnakes/action/releases) - [Commits](https://github.com/deadsnakes/action/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: deadsnakes/action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91144241f4..b844d81336 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -203,7 +203,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v3.0.0 + uses: deadsnakes/action@v3.0.1 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} From 0e43fcc75e6b7429e3511dfb44343ec05a0ab843 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 10 Jun 2023 10:14:08 -0700 Subject: [PATCH 121/137] Python 3.12b2 testing (#4695) * Uncomment `Interface test` sections (test_embed) * fix: setuptools has been removed from default installs in 3.12 --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 6 +++--- .github/workflows/upstream.yml | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b844d81336..45a6738f75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,9 @@ jobs: # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools if: "!(matrix.runs-on == 'windows-2022')" @@ -242,8 +244,6 @@ jobs: python -m pip install -r tests/requirements.txt - name: Configure - env: - SETUPTOOLS_USE_DISTUTILS: stdlib run: > cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 4acfbfce75..d4220be228 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -1,4 +1,3 @@ - name: Upstream on: @@ -65,8 +64,8 @@ jobs: - name: Python tests C++11 run: cmake --build build11 --target pytest -j 2 - # - name: C++11 tests - # run: cmake --build build11 --target cpptest -j 2 + - name: C++11 tests + run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 run: cmake --build build11 --target test_cmake_build @@ -80,14 +79,14 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 - - name: Build + - name: Build C++17 run: cmake --build build17 -j 2 - - name: Python tests + - name: Python tests C++17 run: cmake --build build17 --target pytest - # - name: C++ tests - # run: cmake --build build17 --target cpptest + - name: C++17 tests + run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI - name: Configure (unstable ABI) @@ -105,10 +104,12 @@ jobs: - name: Python tests (unstable ABI) run: cmake --build build17max --target pytest - - name: Interface test + - name: Interface test (unstable ABI) run: cmake --build build17max --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools From 86f60a0c072e5bd5cab79233f712c321fe8deca7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 16 Jun 2023 14:09:40 -0700 Subject: [PATCH 122/137] pre-commit markdown-it-py<3 (for Python 3.7 compatibility) (#4704) * markdown-it-py dropped Python 3.7 support, but we are type checking for Python 3.7 from a newer version of Python. Keep using markdown-it-py<3 as long as we support Python 3.7. * Shuffle order of pre-commit checks for better agility. --- .pre-commit-config.yaml | 91 +++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcc3f27232..2d508276e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,48 @@ ci: exclude: ^tools/JoinPaths.cmake$ repos: + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v16.0.4" + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: "23.3.0" # Keep in sync with blacken-docs + hooks: + - id: black + +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.270 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.3.0" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: + - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. + - nox + - rich + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v4.4.0" @@ -39,12 +81,6 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: "23.3.0" # Keep in sync with blacken-docs - hooks: - - id: black - # Also code format the docs - repo: https://github.com/asottile/blacken-docs rev: "1.13.0" @@ -66,13 +102,6 @@ repos: - id: fix-ligatures - id: fix-smartquotes -# Ruff, the Python auto-correcting linter written in Rust -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 - hooks: - - id: ruff - args: ["--fix", "--show-fixes"] - # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks rev: "v1.10.0" @@ -81,32 +110,6 @@ repos: - id: rst-directive-colons - id: rst-inline-touching-normal - -# PyLint has native support - not always usable, but works for us -- repo: https://github.com/PyCQA/pylint - rev: "v3.0.0a6" - hooks: - - id: pylint - files: ^pybind11 - -# CMake formatting -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: "v0.6.13" - hooks: - - id: cmake-format - additional_dependencies: [pyyaml] - types: [file] - files: (\.cmake|CMakeLists.txt)(.in)?$ - -# Check static types with mypy -- repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" - hooks: - - id: mypy - args: [] - exclude: ^(tests|docs)/ - additional_dependencies: [nox, rich] - # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest rev: "0.49" @@ -141,9 +144,9 @@ repos: entry: PyBind|Numpy|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ -# Clang format the codebase automatically -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.4" +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v3.0.0a6" hooks: - - id: clang-format - types_or: [c++, c, cuda] + - id: pylint + files: ^pybind11 From 849322806cd4b3697ad1d35eedd6d0352c5f267a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 17 Jun 2023 07:02:23 -0700 Subject: [PATCH 123/137] Systematically add `PIP_BREAK_SYSTEM_PACKAGES` to all .yml files from which pip is called. (#4705) * Systematically add PIP_BREAK_SYSTEM_PACKAGES to all .yml files from which pip is called. * Try gcc:10-bullseye (because gcc:10 is broken: https://github.com/docker-library/gcc/issues/95) * bug fix (matrix did not work as hoped) --- .github/workflows/ci.yml | 17 +++++++++-------- .github/workflows/configure.yml | 1 + .github/workflows/pip.yml | 1 + .github/workflows/upstream.yml | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45a6738f75..1d5d502b00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ concurrency: cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy FORCE_COLOR: 3 PYTEST_TIMEOUT: 300 @@ -455,16 +456,16 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11 } - - { gcc: 7, std: 17 } - - { gcc: 8, std: 14 } - - { gcc: 8, std: 17 } - - { gcc: 10, std: 17 } - - { gcc: 11, std: 20 } - - { gcc: 12, std: 20 } + - { gcc: 7, std: 11, container_suffix: "" } + - { gcc: 7, std: 17, container_suffix: "" } + - { gcc: 8, std: 14, container_suffix: "" } + - { gcc: 8, std: 17, container_suffix: "" } + - { gcc: 10, std: 17, container_suffix: "-bullseye" } + - { gcc: 11, std: 20, container_suffix: "" } + - { gcc: 12, std: 20, container_suffix: "" } name: "🐍 3 β€’ GCC ${{ matrix.gcc }} β€’ C++${{ matrix.std }}β€’ x64" - container: "gcc:${{ matrix.gcc }}" + container: "gcc:${{ matrix.gcc }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 4ae22281c6..82a4939004 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -13,6 +13,7 @@ permissions: contents: read env: + PIP_BREAK_SYSTEM_PACKAGES: 1 # For cmake: VERBOSE: 1 diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index c1feb6fe10..04d95a28c5 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -16,6 +16,7 @@ permissions: contents: read env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy jobs: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index d4220be228..dd8a1c9606 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -12,6 +12,7 @@ concurrency: cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: ":all:" # For cmake: VERBOSE: 1 From bc1bcf7c05b6097c2f0c993a776a975d7332f279 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 21 Jun 2023 13:25:10 -0400 Subject: [PATCH 124/137] chore: 3.12 + cleanup (#4713) Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 2 ++ .pre-commit-config.yaml | 1 + pybind11/setup_helpers.py | 8 ++++---- pyproject.toml | 18 ++++++++++-------- setup.cfg | 1 + setup.py | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d5d502b00..1d61ad6a9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: - '3.9' - '3.10' - '3.11' + - '3.12' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -74,6 +75,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Setup Boost (Linux) # Can't use boost + define _ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d508276e1..9a829d72b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,6 +54,7 @@ repos: - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. - nox - rich + - types-setuptools # CMake formatting - repo: https://github.com/cheshirekow/cmake-format-precommit diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index cb279f27e0..aeeee9dcfa 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -66,8 +66,8 @@ from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import build_ext as _build_ext # type: ignore[assignment] + from distutils.extension import Extension as _Extension # type: ignore[assignment] import distutils.ccompiler import distutils.errors @@ -84,7 +84,7 @@ # directory into your path if it sits beside your setup.py. -class Pybind11Extension(_Extension): # type: ignore[misc] +class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you @@ -266,7 +266,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]: raise RuntimeError(msg) -class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search diff --git a/pyproject.toml b/pyproject.toml index e3655aca74..59c15ea636 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = ["setuptools>=42", "cmake>=3.18", "ninja"] build-backend = "setuptools.build_meta" + [tool.check-manifest] ignore = [ "tests/**", @@ -15,6 +16,7 @@ ignore = [ "noxfile.py", ] + [tool.mypy] files = ["pybind11"] python_version = "3.6" @@ -24,7 +26,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true [[tool.mypy.overrides]] -module = ["ghapi.*", "setuptools.*"] +module = ["ghapi.*"] ignore_missing_imports = true @@ -55,10 +57,11 @@ messages_control.disable = [ "unused-argument", # covered by Ruff ARG ] + [tool.ruff] select = [ "E", "F", "W", # flake8 - "B", "B904", # flake8-bugbear + "B", # flake8-bugbear "I", # isort "N", # pep8-naming "ARG", # flake8-unused-arguments @@ -77,14 +80,13 @@ select = [ "YTT", # flake8-2020 ] ignore = [ - "PLR", # Design related pylint - "E501", # Line too long (Black is enough) - "PT011", # Too broad with raises in pytest - "PT004", # Fixture that doesn't return needs underscore (no, it is fine) - "SIM118",# iter(x) is not always the same as iter(x.keys()) + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118", # iter(x) is not always the same as iter(x.keys()) ] target-version = "py37" -typing-modules = ["scikit_build_core._compat.typing"] src = ["src"] unfixable = ["T20"] exclude = [] diff --git a/setup.cfg b/setup.cfg index c35dd07ae6..92e6c953a9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython diff --git a/setup.py b/setup.py index 68573519c1..9fea7d35c7 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def get_and_replace( # Use our input files instead when making the SDist (and anything that depends # on it, like a wheel) -class SDist(setuptools.command.sdist.sdist): # type: ignore[misc] +class SDist(setuptools.command.sdist.sdist): def make_release_tree(self, base_dir: str, files: List[str]) -> None: super().make_release_tree(base_dir, files) From e10da79b6ee2554be364ef14df1c988f94df02ea Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 24 Jun 2023 12:12:35 -0700 Subject: [PATCH 125/137] Undo ci.yml gcc10 workaround after docker-library/gcc#95 was resolved. (#4717) The gcc10 workaround was introduced with PR #4705. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d61ad6a9c..0d64014123 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -458,16 +458,16 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11, container_suffix: "" } - - { gcc: 7, std: 17, container_suffix: "" } - - { gcc: 8, std: 14, container_suffix: "" } - - { gcc: 8, std: 17, container_suffix: "" } - - { gcc: 10, std: 17, container_suffix: "-bullseye" } - - { gcc: 11, std: 20, container_suffix: "" } - - { gcc: 12, std: 20, container_suffix: "" } + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } name: "🐍 3 β€’ GCC ${{ matrix.gcc }} β€’ C++${{ matrix.std }}β€’ x64" - container: "gcc:${{ matrix.gcc }}${{ matrix.container_suffix }}" + container: "gcc:${{ matrix.gcc }}" steps: - uses: actions/checkout@v3 From 2fb3d7cbde264a0b3f921e802f287195387e8263 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 27 Jun 2023 15:08:32 -0700 Subject: [PATCH 126/137] Trivial refactoring to make the capsule API more user friendly. (#4720) * Trivial refactoring to make the capsule API more user friendly. * Use new API in production code. Thanks @Lalaland for pointing this out. --- include/pybind11/pybind11.h | 2 +- include/pybind11/pytypes.h | 51 ++++++++++++++++++++++--------------- tests/test_pytypes.cpp | 9 +++++++ tests/test_pytypes.py | 13 ++++++++++ 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 28ebc22297..3bce1a01ba 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -508,8 +508,8 @@ class cpp_function : public function { rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), [](void *ptr) { destruct((detail::function_record *) ptr); }); - rec_capsule.set_name(detail::get_function_record_capsule_name()); guarded_strdup.release(); object scope_module; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f5d3f34f38..c93e3d3b92 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1925,28 +1925,13 @@ class capsule : public object { } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast(value), nullptr, [](PyObject *o) { - // guard if destructor called while err indicator is set - error_scope error_guard; - auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr && PyErr_Occurred()) { - throw error_already_set(); - } - const char *name = get_name_in_error_scope(o); - void *ptr = PyCapsule_GetPointer(o, name); - if (ptr == nullptr) { - throw error_already_set(); - } - - if (destructor != nullptr) { - destructor(ptr); - } - }); + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { - throw error_already_set(); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { @@ -2014,6 +1999,32 @@ class capsule : public object { return name; } + + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 1028bb58e8..b4ee642891 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -260,6 +260,15 @@ TEST_SUBMODULE(pytypes, m) { }); }); + m.def("return_capsule_with_destructor_3", []() { + py::print("creating capsule"); + auto cap = py::capsule((void *) 1233, "oname", [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + py::print("original name: {}"_s.format(cap.name())); + return cap; + }); + m.def("return_renamed_capsule_with_destructor_2", []() { py::print("creating capsule"); auto cap = py::capsule((void *) 1234, [](void *ptr) { diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index afb7a1ce8a..eda7a20a9d 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -319,6 +319,19 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_destructor_3() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + destructing capsule: 1233 + original name: oname + """ + ) + with capture: a = m.return_renamed_capsule_with_destructor_2() del a From 5ccb9e412d8974e8eeac1b061d9077ac0bd365e1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 4 Jul 2023 08:38:14 -0400 Subject: [PATCH 127/137] chore: ruff moved to astral-sh (#4726) * chore: ruff moved to astral-sh Committed via https://github.com/asottile/all-repos * ci: bump CUDA version --- .github/workflows/ci.yml | 4 ++-- .pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d64014123..39b4c237e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,8 +337,8 @@ jobs: # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest - name: "🐍 3.10 β€’ CUDA 11.7 β€’ Ubuntu 22.04" - container: nvidia/cuda:11.7.0-devel-ubuntu22.04 + name: "🐍 3.10 β€’ CUDA 12.2 β€’ Ubuntu 22.04" + container: nvidia/cuda:12.2.0-devel-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a829d72b1..d406624f3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: - id: black # Ruff, the Python auto-correcting linter written in Rust -- repo: https://github.com/charliermarsh/ruff-pre-commit +- repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.0.270 hooks: - id: ruff From 782b6281a2aec358fb54dc1badc4b8ae1988bd41 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 9 Jul 2023 09:40:47 -0700 Subject: [PATCH 128/137] Drop PyPy 3.7 from GitHub Actions (ci.yml) (#4728) * Remove `pypy-3.7` from ci.yml * Update .github/workflows/ci.yml --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39b4c237e8..48f7c5e934 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,9 +37,9 @@ jobs: - '3.10' - '3.11' - '3.12' - - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' + - 'pypy-3.10' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the From d462dd91a58116a043974b198462864a527e84ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:43:20 -0700 Subject: [PATCH 129/137] chore(deps): bump scipy from 1.8.0 to 1.10.0 in /tests (#4731) Bumps [scipy](https://github.com/scipy/scipy) from 1.8.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.8.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 04aafa8cf9..4ba1011196 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,4 +6,4 @@ numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" pytest==7.0.0 pytest-timeout scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" -scipy==1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" +scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" From 47dc0c4bd16b7a97edb2072e93e6960647c3872f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:31:43 -0700 Subject: [PATCH 130/137] chore(deps): update pre-commit hooks (#4727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v16.0.4 β†’ v16.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.4...v16.0.6) - https://github.com/charliermarsh/ruff-pre-commit β†’ https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.270 β†’ v0.0.276](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.270...v0.0.276) - [github.com/pre-commit/mirrors-mypy: v1.3.0 β†’ v1.4.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.3.0...v1.4.1) - [github.com/asottile/blacken-docs: 1.13.0 β†’ 1.14.0](https://github.com/asottile/blacken-docs/compare/1.13.0...1.14.0) - [github.com/codespell-project/codespell: v2.2.4 β†’ v2.2.5](https://github.com/codespell-project/codespell/compare/v2.2.4...v2.2.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d406624f3c..86ac965d96 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.4" + rev: "v16.0.6" hooks: - id: clang-format types_or: [c++, c, cuda] @@ -38,14 +38,14 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.276 hooks: - id: ruff args: ["--fix", "--show-fixes"] # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" + rev: "v1.4.1" hooks: - id: mypy args: [] @@ -84,7 +84,7 @@ repos: # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "1.13.0" + rev: "1.14.0" hooks: - id: blacken-docs additional_dependencies: @@ -124,7 +124,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.4" + rev: "v2.2.5" hooks: - id: codespell exclude: ".supp$" From b2732c6e1a96eb97e79359c049871c41e2cd34c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 08:55:18 -0700 Subject: [PATCH 131/137] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 (#4718) * chore(deps): bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.6 to 1.8.7. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.6...v1.8.7) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Apply suggestions from code review --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 04d95a28c5..d6687b441c 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -102,13 +102,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 6d22dba82f1789f11a8eb2c2debbcbd4d2d8a969 Mon Sep 17 00:00:00 2001 From: Luc de Jonckheere Date: Wed, 12 Jul 2023 17:56:40 +0200 Subject: [PATCH 132/137] Warning on comparing wrapper enums with is (#4732) * Warning on comparing wrapper enums with is * backticks for quoting and link to related issue --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- docs/classes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/classes.rst b/docs/classes.rst index 52cd52da3f..4f2167dac1 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -549,3 +549,7 @@ The ``name`` property returns the name of the enum value as a unicode string. ... By default, these are omitted to conserve space. + +.. warning:: + + Contrary to Python customs, enum values from the wrappers should not be compared using ``is``, but with ``==`` (see `#1177 `_ for background). From 2e5f5c4cf8efdd62acbeb4632e8988b2148f6caf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 12 Jul 2023 13:10:24 -0500 Subject: [PATCH 133/137] fix: support CMake 3.27, drop 3.4 (#4719) * fix: support CMake 3.27, drop 3.4 Signed-off-by: Henry Schreiner * Update upgrade.rst * Update upgrade.rst * Update upgrade.rst --------- Signed-off-by: Henry Schreiner --- .github/workflows/configure.yml | 12 ++++++++---- CMakeLists.txt | 8 ++++---- docs/advanced/embedding.rst | 2 +- docs/compiling.rst | 9 ++++++--- docs/faq.rst | 3 ++- docs/upgrade.rst | 14 ++++++++++++++ tests/CMakeLists.txt | 11 ++++------- tests/test_cmake_build/CMakeLists.txt | 3 --- .../installed_embed/CMakeLists.txt | 8 ++++---- .../installed_function/CMakeLists.txt | 8 ++++---- .../installed_target/CMakeLists.txt | 8 ++++---- .../subdirectory_embed/CMakeLists.txt | 8 ++++---- .../subdirectory_function/CMakeLists.txt | 8 ++++---- .../subdirectory_target/CMakeLists.txt | 8 ++++---- tools/pybind11Common.cmake | 10 +++++++++- tools/pybind11Config.cmake.in | 4 +++- 16 files changed, 75 insertions(+), 49 deletions(-) diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 82a4939004..ec7cd612de 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -26,20 +26,24 @@ jobs: matrix: runs-on: [ubuntu-20.04, macos-latest, windows-latest] arch: [x64] - cmake: ["3.23"] + cmake: ["3.26"] include: - runs-on: ubuntu-20.04 arch: x64 - cmake: 3.4 + cmake: "3.5" + + - runs-on: ubuntu-20.04 + arch: x64 + cmake: "3.27" - runs-on: macos-latest arch: x64 - cmake: 3.7 + cmake: "3.7" - runs-on: windows-2019 arch: x64 # x86 compilers seem to be missing on 2019 image - cmake: 3.18 + cmake: "3.18" name: 🐍 3.7 β€’ CMake ${{ matrix.cmake }} β€’ ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ad74db2bb..87ec103468 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,15 +5,15 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.22) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.22) + cmake_policy(VERSION 3.26) endif() # Avoid infinite recursion if tests include this as a subdirectory diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dd980d483a..e6a1686f87 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` diff --git a/docs/compiling.rst b/docs/compiling.rst index 2b543be0be..1fd098bec4 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -241,7 +241,7 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) add_subdirectory(pybind11) @@ -261,6 +261,9 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. versionchanged:: 2.6 CMake 3.4+ is required. +.. versionchanged:: 2.11 + CMake 3.5+ is required. + Further information can be found at :doc:`cmake/index`. pybind11_add_module @@ -495,7 +498,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -553,7 +556,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) diff --git a/docs/faq.rst b/docs/faq.rst index 28498e7dfc..1eb00efada 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -284,7 +284,8 @@ There are three possible solutions: COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead of the old, deprecated search tools, and these modules are much better at - finding the correct Python. + finding the correct Python. If FindPythonLibs/Interp are not available + (CMake 3.27+), then this will be ignored and FindPython will be used. 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. However, you will have to use the target-based system, and do more setup yourself, because it does not know about or include things that depend on diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 6a9db2d08f..b13d21f5ec 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,20 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.11: + +v2.11 +===== + +* The minimum version of CMake is now 3.5. A future version will likely move to + requiring something like CMake 3.15. Note that CMake 3.27 is removing the + long-deprecated support for ``FindPythonInterp`` if you set 3.27 as the + minimum or maximum supported version. To prepare for that future, CMake 3.15+ + using ``FindPython`` or setting ``PYBIND11_FINDPYTHON`` is highly recommended, + otherwise pybind11 will automatically switch to using ``FindPython`` if + ``FindPythonInterp`` is not available. + + .. _upgrade-guide-2.9: v2.9 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5574663b63..80ee9c1f11 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,20 +5,17 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.21) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.21) + cmake_policy(VERSION 3.26) endif() -# Only needed for CMake < 3.5 support -include(CMakeParseArguments) - # Filter out items; print an optional message if any items filtered. This ignores extensions. # # Usage: diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 8bfaa386ae..e5aa975cfc 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -1,6 +1,3 @@ -# Built-in in CMake 3.5+ -include(CMakeParseArguments) - add_custom_target(test_cmake_build) function(pybind11_add_build_test name) diff --git a/tests/test_cmake_build/installed_embed/CMakeLists.txt b/tests/test_cmake_build/installed_embed/CMakeLists.txt index f7d6939982..d9dcb45e42 100644 --- a/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_embed CXX) diff --git a/tests/test_cmake_build/installed_function/CMakeLists.txt b/tests/test_cmake_build/installed_function/CMakeLists.txt index d7ca4db55d..2f4f642753 100644 --- a/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,13 +1,13 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) project(test_installed_module CXX) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_function CXX) diff --git a/tests/test_cmake_build/installed_target/CMakeLists.txt b/tests/test_cmake_build/installed_target/CMakeLists.txt index bc5e101f1d..a981e236f5 100644 --- a/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_target CXX) diff --git a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index 58cdd7cfd1..f286746b95 100644 --- a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_embed CXX) diff --git a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index 01557c439a..275a75c0b5 100644 --- a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_function CXX) diff --git a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index ba82fdee2e..37bb2c56e7 100644 --- a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_target CXX) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index e4ff1e4080..308d1b70d1 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -163,11 +163,19 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) elseif( - PYBIND11_FINDPYTHON + _pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON OR Python_FOUND OR Python2_FOUND OR Python3_FOUND) diff --git a/tools/pybind11Config.cmake.in b/tools/pybind11Config.cmake.in index 9383e8c671..5734f437b3 100644 --- a/tools/pybind11Config.cmake.in +++ b/tools/pybind11Config.cmake.in @@ -63,7 +63,9 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). New FindPython mode ^^^^^^^^^^^^^^^^^^^ From b33d06f61543878a81a433ed2d472b61d3fca0ca Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 12 Jul 2023 16:20:08 -0400 Subject: [PATCH 134/137] bugfix: fixes a test suite bug in the __new__ example (#4698) * bugfix: fixes a test suite bug in the __new__ example * See https://github.com/pybind/pybind11/pull/4698#discussion_r1227107682 --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- tests/test_class.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_class.cpp b/tests/test_class.cpp index ca925917e6..7241bc8818 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -85,7 +85,7 @@ TEST_SUBMODULE(class_, m) { .def_static("new_instance", &NoConstructor::new_instance, "Return an instance"); py::class_(m, "NoConstructorNew") - .def(py::init([](const NoConstructorNew &self) { return self; })) // Need a NOOP __init__ + .def(py::init([]() { return nullptr; })) // Need a NOOP __init__ .def_static("__new__", [](const py::object &) { return NoConstructorNew::new_instance(); }); From 1a917f1852eb7819b671fc3fa862840f4c491a07 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 14 Jul 2023 08:20:38 -0700 Subject: [PATCH 135/137] docs: preparation for v2.11.0 release (#4744) * Remove .dev1 from version number. * Update Changelog (starting from `nox -s make_changelog` output) * Miscellaneous minor fixes from proofreading in GitHub web view. * docs: minor changelog updates Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner Co-authored-by: Henry Schreiner --- docs/changelog.rst | 47 +++++++++++++++++++++++++++----- docs/release.rst | 10 ++++--- include/pybind11/detail/common.h | 4 +-- pybind11/_version.py | 2 +- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 32f6606aad..7bb115b90f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,15 +10,15 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. -Version 2.11.0 (June 2, 2023) +Version 2.11.0 (July 14, 2023) ----------------------------- New features: -* ``pybind11::detail::is_move_constructible`` can now be specialized for cases - in which ``std::is_move_constructible`` does not work as needed. This is - very similar to the long-established - ``pybind11::detail::is_copy_constructible``. +* The newly added ``pybind11::detail::is_move_constructible`` trait can be + specialized for cases in which ``std::is_move_constructible`` does not work + as needed. This is very similar to the long-established + ``pybind11::detail::is_copy_constructible``. `#4631 `_ * Introduce ``recursive_container_traits``. @@ -30,6 +30,19 @@ New features: with ``pybind/pybind11.h``). `#4601 `_ +* ``format_descriptor<>`` & ``npy_format_descriptor<>`` ``PyObject *`` + specializations were added. The latter enables ``py::array_t`` + to/from-python conversions. + `#4674 `_ + +* ``buffer_info`` gained an ``item_type_is_equivalent_to()`` member + function. + `#4674 `_ + +* The ``capsule`` API gained a user-friendly constructor + (``py::capsule(ptr, "name", dtor)``). + `#4720 `_ + Changes: * ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & @@ -46,6 +59,12 @@ Changes: sizes slightly (~1.5%) but the error messages are much more informative. `#4463 `_ +* The docstring generation for the ``std::array``-list caster was fixed. + Previously, signatures included the size of the list in a non-standard, + non-spec compliant way. The new format conforms to PEP 593. + **Tooling for processing the docstrings may need to be updated accordingly.** + `#4679 `_ + * Setter return values (which are inaccessible for all practical purposes) are no longer converted to Python (only to be discarded). `#4621 `_ @@ -58,7 +77,7 @@ Changes: signatures on C++17 and higher. `#4587 `_ -* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11 +* Compatibility with Python 3.12 (beta). Note that the minimum pybind11 ABI version for Python 3.12 is version 5. (The default ABI version for Python versions up to and including 3.11 is still version 4.). `#4570 `_ @@ -69,15 +88,23 @@ Changes: types defined in the unnamed namespace. `#4319 `_ +* Python exception ``__notes__`` (introduced with Python 3.11) are now added to + the ``error_already_set::what()`` output. + `#4678 `_ + Build system improvements: +* CMake 3.27 support was added, CMake 3.4 support was dropped. + FindPython will be used if ``FindPythonInterp`` is not present. + `#4719 `_ + * Update clang-tidy to 15 in CI. `#4387 `_ * Moved the linting framework over to Ruff. `#4483 `_ -* Skip lto checks and target generation when +* Skip ``lto`` checks and target generation when ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. `#4643 `_ @@ -85,6 +112,12 @@ Build system improvements: (macOS 10.9+). `#4639 `_ +* PyPy 3.10 support was added, PyPy 3.7 support was dropped. + `#4728 `_ + +* Testing with Python 3.12 beta releases was added. + `#4713 `_ + Version 2.10.4 (Mar 16, 2023) ----------------------------- diff --git a/docs/release.rst b/docs/release.rst index e761cdf7a6..4950c3b887 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -33,10 +33,12 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use - Run ``nox -s tests_packaging`` to ensure this was done correctly. - Ensure that all the information in ``setup.cfg`` is up-to-date, like supported Python versions. - - Add release date in ``docs/changelog.rst``. - - Check to make sure - `needs-changelog `_ - issues are entered in the changelog (clear the label when done). + - Add release date in ``docs/changelog.rst`` and integrate the output of + ``nox -s make_changelog``. + - Note that the ``make_changelog`` command inspects + `needs changelog `_. + - Manually clear the ``needs changelog`` labels using the GitHub web + interface (very easy: start by clicking the link above). - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it fails due to a known flake issue, either ignore or restart CI.) - Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index c2e6f9ec89..27c426db5b 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_PATCH 0 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020B0000 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 1cb51fc5c2..1c310e0c95 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.11.0" version_info = tuple(_to_int(s) for s in __version__.split(".")) From 2965fa8de3cf9e82c789f906a525a76197b186c1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 17 Jul 2023 09:50:35 -0700 Subject: [PATCH 136/137] Preparation for v2.11.1 patch release (#4752) * Update README.rst - Add missing comma in the list of acknowlegements (#4750) * Disable `PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF` generally for PyPy (not just PyPy Windows). (#4751) * Update changelog * Provide `PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF` as an option (#4753) * Remove GIL checks * Update common.h * Add flag * style: pre-commit fixes * Update pytypes.h * style: pre-commit fixes * Update common.h * style: pre-commit fixes * Update pytypes.h * style: pre-commit fixes * Update common.h * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update changelog: PR #4753 --------- Co-authored-by: bzaar Co-authored-by: Ethan Steinberg Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- README.rst | 2 +- docs/changelog.rst | 15 +++++++++++++++ include/pybind11/detail/common.h | 7 ++----- include/pybind11/pytypes.h | 7 ++++++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 3c75edb575..80213a4062 100644 --- a/README.rst +++ b/README.rst @@ -135,7 +135,7 @@ This project was created by `Wenzel Jakob `_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz MiΔ…sko, +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz MiΔ…sko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris SchΓ€ling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. diff --git a/docs/changelog.rst b/docs/changelog.rst index 7bb115b90f..add3fd66b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,21 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. +Version 2.11.1 (July 17, 2023) +----------------------------- + +Changes: + +* ``PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF`` is now provided as an option + for disabling the default-on ``PyGILState_Check()``'s in + ``pybind11::handle``'s ``inc_ref()`` & ``dec_ref()``. + `#4753 `_ + +* ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF`` was disabled for PyPy in general + (not just PyPy Windows). + `#4751 `_ + + Version 2.11.0 (July 14, 2023) ----------------------------- diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 27c426db5b..bf424371e5 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -324,11 +324,8 @@ PYBIND11_WARNING_POP #endif // See description of PR #4246: -#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) \ - && !(defined(PYPY_VERSION) \ - && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the \ - process (see PR #4268) */ \ - && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ + && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) # define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index c93e3d3b92..64aad63476 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -303,7 +303,12 @@ class handle : public detail::object_api { stderr, "%s is being called while the GIL is either not held or invalid. Please see " "https://pybind11.readthedocs.io/en/stable/advanced/" - "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" + "If you are convinced there is no bug in your code, you can #define " + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF" + "to disable this check. In that case you have to ensure this #define is consistently " + "used for all translation units linked into a given pybind11 extension, otherwise " + "there will be ODR violations.", function_name.c_str()); fflush(stderr); if (Py_TYPE(m_ptr)->tp_name != nullptr) { From 8a099e44b3d5f85b20f05828d919d2332a8de841 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 17 Jul 2023 10:24:25 -0700 Subject: [PATCH 137/137] Fix version number mishap: actually update 0 to 1 (#4756) --- include/pybind11/detail/common.h | 4 ++-- pybind11/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index bf424371e5..31a54c773a 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0 +#define PYBIND11_VERSION_PATCH 1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B0000 +#define PYBIND11_VERSION_HEX 0x020B0100 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 1c310e0c95..9280fa054e 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0" +__version__ = "2.11.1" version_info = tuple(_to_int(s) for s in __version__.split("."))