Merge branch 'pybind:master' into master

.github/workflows/emscripten.yaml

@@ -0,0 +1,30 @@
+name: WASM
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+    - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-wasm-emscripten:
+    name: Pyodide wheel
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+        fetch-depth: 0
+
+    - uses: pypa/[email protected]
+      env:
+        PYODIDE_BUILD_EXPORTS: whole_archive
+        CFLAGS: -fexceptions
+        LDFLAGS: -fexceptions
+      with:
+        package-dir: tests
+        only: cp312-pyodide_wasm32

CMakeLists.txt

@@ -154,6 +154,7 @@ set(PYBIND11_HEADERS
     include/pybind11/detail/internals.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
+    include/pybind11/detail/value_and_holder.h
     include/pybind11/attr.h
     include/pybind11/buffer_info.h
     include/pybind11/cast.h

include/pybind11/cast.h

@@ -794,11 +794,11 @@ struct copyable_holder_caster : public type_caster_base<type> {
         }
     }
 
-    bool load_value(value_and_holder &&v_h) {
+    void load_value(value_and_holder &&v_h) {
         if (v_h.holder_constructed()) {
             value = v_h.value_ptr();
             holder = v_h.template holder<holder_type>();
-            return true;
+            return;
         }
         throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) "
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)

include/pybind11/detail/common.h

@@ -462,6 +462,22 @@ PYBIND11_WARNING_POP
                 return "Hello, World!";
             });
         }
+
+    The third macro argument is optional (available since 2.13.0), and can be used to
+    mark the extension module as safe to run without the GIL under a free-threaded CPython
+    interpreter. Passing this argument has no effect on other interpreters.
+
+    .. code-block:: cpp
+
+        PYBIND11_MODULE(example, m, py::mod_gil_not_used()) {
+            m.doc() = "pybind11 example module safe to run without the GIL";
+
+            // Add bindings here
+            m.def("foo", []() {
+                return "Hello, Free-threaded World!";
+            });
+        }
+
 \endrst */
 #define PYBIND11_MODULE(name, variable, ...)                                                      \
     static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name)            \

include/pybind11/detail/init.h

@@ -128,11 +128,13 @@ void construct(value_and_holder &v_h, Cpp<Class> *ptr, bool need_alias) {
         // the holder and destruction happens when we leave the C++ scope, and the holder
         // class gets to handle the destruction however it likes.
         v_h.value_ptr() = ptr;
-        v_h.set_instance_registered(true);          // To prevent init_instance from registering it
-        v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder
+        v_h.set_instance_registered(true); // Trick to prevent init_instance from registering it
+        // DANGER ZONE BEGIN: exceptions will leave v_h in an invalid state.
+        v_h.type->init_instance(v_h.inst, nullptr);                        // Set up the holder
         Holder<Class> temp_holder(std::move(v_h.holder<Holder<Class>>())); // Steal the holder
         v_h.type->dealloc(v_h); // Destroys the moved-out holder remains, resets value ptr to null
         v_h.set_instance_registered(false);
+        // DANGER ZONE END.
 
         construct_alias_from_cpp<Class>(is_alias_constructible<Class>{}, v_h, std::move(*ptr));
     } else {

include/pybind11/detail/type_caster_base.h

@@ -14,6 +14,7 @@
 #include "descr.h"
 #include "internals.h"
 #include "typeid.h"
+#include "value_and_holder.h"
 
 #include <cstdint>
 #include <iterator>
@@ -259,67 +260,6 @@ PYBIND11_NOINLINE handle find_registered_python_instance(void *src,
     });
 }
 
-struct value_and_holder {
-    instance *inst = nullptr;
-    size_t index = 0u;
-    const detail::type_info *type = nullptr;
-    void **vh = nullptr;
-
-    // Main constructor for a found value/holder:
-    value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index)
-        : inst{i}, index{index}, type{type},
-          vh{inst->simple_layout ? inst->simple_value_holder
-                                 : &inst->nonsimple.values_and_holders[vpos]} {}
-
-    // Default constructor (used to signal a value-and-holder not found by get_value_and_holder())
-    value_and_holder() = default;
-
-    // Used for past-the-end iterator
-    explicit value_and_holder(size_t index) : index{index} {}
-
-    template <typename V = void>
-    V *&value_ptr() const {
-        return reinterpret_cast<V *&>(vh[0]);
-    }
-    // True if this `value_and_holder` has a non-null value pointer
-    explicit operator bool() const { return value_ptr() != nullptr; }
-
-    template <typename H>
-    H &holder() const {
-        return reinterpret_cast<H &>(vh[1]);
-    }
-    bool holder_constructed() const {
-        return inst->simple_layout
-                   ? inst->simple_holder_constructed
-                   : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u;
-    }
-    // NOLINTNEXTLINE(readability-make-member-function-const)
-    void set_holder_constructed(bool v = true) {
-        if (inst->simple_layout) {
-            inst->simple_holder_constructed = v;
-        } else if (v) {
-            inst->nonsimple.status[index] |= instance::status_holder_constructed;
-        } else {
-            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed;
-        }
-    }
-    bool instance_registered() const {
-        return inst->simple_layout
-                   ? inst->simple_instance_registered
-                   : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0);
-    }
-    // NOLINTNEXTLINE(readability-make-member-function-const)
-    void set_instance_registered(bool v = true) {
-        if (inst->simple_layout) {
-            inst->simple_instance_registered = v;
-        } else if (v) {
-            inst->nonsimple.status[index] |= instance::status_instance_registered;
-        } else {
-            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered;
-        }
-    }
-};
-
 // Container for accessing and iterating over an instance's values/holders
 struct values_and_holders {
 private:

include/pybind11/detail/value_and_holder.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2016-2024 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "common.h"
+
+#include <cstddef>
+#include <typeinfo>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+struct value_and_holder {
+    instance *inst = nullptr;
+    size_t index = 0u;
+    const detail::type_info *type = nullptr;
+    void **vh = nullptr;
+
+    // Main constructor for a found value/holder:
+    value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index)
+        : inst{i}, index{index}, type{type},
+          vh{inst->simple_layout ? inst->simple_value_holder
+                                 : &inst->nonsimple.values_and_holders[vpos]} {}
+
+    // Default constructor (used to signal a value-and-holder not found by get_value_and_holder())
+    value_and_holder() = default;
+
+    // Used for past-the-end iterator
+    explicit value_and_holder(size_t index) : index{index} {}
+
+    template <typename V = void>
+    V *&value_ptr() const {
+        return reinterpret_cast<V *&>(vh[0]);
+    }
+    // True if this `value_and_holder` has a non-null value pointer
+    explicit operator bool() const { return value_ptr() != nullptr; }
+
+    template <typename H>
+    H &holder() const {
+        return reinterpret_cast<H &>(vh[1]);
+    }
+    bool holder_constructed() const {
+        return inst->simple_layout
+                   ? inst->simple_holder_constructed
+                   : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u;
+    }
+    // NOLINTNEXTLINE(readability-make-member-function-const)
+    void set_holder_constructed(bool v = true) {
+        if (inst->simple_layout) {
+            inst->simple_holder_constructed = v;
+        } else if (v) {
+            inst->nonsimple.status[index] |= instance::status_holder_constructed;
+        } else {
+            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed;
+        }
+    }
+    bool instance_registered() const {
+        return inst->simple_layout
+                   ? inst->simple_instance_registered
+                   : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0);
+    }
+    // NOLINTNEXTLINE(readability-make-member-function-const)
+    void set_instance_registered(bool v = true) {
+        if (inst->simple_layout) {
+            inst->simple_instance_registered = v;
+        } else if (v) {
+            inst->nonsimple.status[index] |= instance::status_instance_registered;
+        } else {
+            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered;
+        }
+    }
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/CMakeLists.txt

@@ -88,7 +88,12 @@ set(PYBIND11_TEST_FILTER
 if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
   # We're being loaded directly, i.e. not via add_subdirectory, so make this
   # work as its own project and load the pybind11Config to get the tools we need
-  find_package(pybind11 REQUIRED CONFIG)
+
+  if(SKBUILD)
+    add_subdirectory(.. pybind11_src)
+  else()
+    find_package(pybind11 REQUIRED CONFIG)
+  endif()
 endif()
 
 if(NOT CMAKE_BUILD_TYPE AND NOT DEFINED CMAKE_CONFIGURATION_TYPES)
@@ -490,6 +495,9 @@ foreach(target ${test_targets})
       endforeach()
     endif()
   endif()
+  if(SKBUILD)
+    install(TARGETS ${target} LIBRARY DESTINATION .)
+  endif()
 endforeach()
 
 # Provide nice organisation in IDEs

tests/extra_python_package/test_files.py

@@ -58,6 +58,7 @@
     "include/pybind11/detail/internals.h",
     "include/pybind11/detail/type_caster_base.h",
     "include/pybind11/detail/typeid.h",
+    "include/pybind11/detail/value_and_holder.h",
 }
 
 eigen_headers = {

tests/pyproject.toml

@@ -0,0 +1,17 @@
+# Warning: this is currently used for pyodide, and is not a general out-of-tree
+# builder for the tests (yet). Specifically, wheels can't be built from SDists.
+
+[build-system]
+requires = ["scikit-build-core"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "pybind11_tests"
+version = "0.0.1"
+dependencies = ["pytest", "pytest-timeout", "numpy", "scipy"]
+
+[tool.scikit-build.cmake.define]
+PYBIND11_FINDPYTHON = true
+
+[tool.cibuildwheel]
+test-command = "pytest -o timeout=0 -p no:cacheprovider {project}/tests/test_*.py"

tests/test_async.py

@@ -1,10 +1,15 @@
 from __future__ import annotations
 
+import sys
+
 import pytest
 
 asyncio = pytest.importorskip("asyncio")
 m = pytest.importorskip("pybind11_tests.async_module")
 
+if sys.platform.startswith("emscripten"):
+    pytest.skip("Can't run a new event_loop in pyodide", allow_module_level=True)
+
 
 @pytest.fixture()
 def event_loop():

tests/test_callbacks.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import sys
 import time
 from threading import Thread
 
@@ -153,6 +154,7 @@ def test_python_builtins():
     assert m.test_sum_builtin(sum, []) == 0
 
 
+@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads")
 def test_async_callbacks():
     # serves as state for async callback
     class Item:
@@ -176,6 +178,7 @@ def gen_f():
     assert sum(res) == sum(x + 3 for x in work)
 
 
+@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads")
 def test_async_async_callbacks():
     t = Thread(target=test_async_callbacks)
     t.start()

tests/test_exceptions.py

@@ -75,7 +75,7 @@ def test_cross_module_exceptions(msg):
 
 # TODO: FIXME
 @pytest.mark.xfail(
-    "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang'))",
+    "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang')) or sys.platform.startswith('emscripten')",
     raises=RuntimeError,
     reason="See Issue #2847, PR #2999, PR #4324",
 )