diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6120255b4..2141414cb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,35 @@ jobs: pytest tests/extra_setuptools if: "!(matrix.runs-on == 'windows-2022')" + manylinux: + name: Manylinux on 🐍 3.13t • GIL + runs-on: ubuntu-latest + timeout-minutes: 40 + container: quay.io/pypa/musllinux_1_2_x86_64:latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Prepare venv + run: python3.13t -m venv .venv + + - name: Install Python deps + run: .venv/bin/pip install -r tests/requirements.txt + + - name: Configure C++11 + run: > + cmake -S. -Bbuild + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPython_ROOT_DIR=.venv + + - name: Build C++11 + run: cmake --build build -j2 + + - name: Python tests C++11 + run: cmake --build build --target pytest -j2 deadsnakes: strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec67488779..a81a8e0aa2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: # Ruff, the Python auto-correcting linter/formatter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.5 + rev: v0.4.7 hooks: - id: ruff args: ["--fix", "--show-fixes"] diff --git a/CMakeLists.txt b/CMakeLists.txt index 4840a0e484..6897f70b8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF) if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION) add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION) @@ -306,6 +307,7 @@ if(PYBIND11_INSTALL) tools/pybind11Common.cmake tools/pybind11Tools.cmake tools/pybind11NewTools.cmake + tools/pybind11GuessPythonExtSuffix.cmake DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) if(NOT PYBIND11_EXPORT_NAME) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 48fc02cb39..8308ba7107 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -388,7 +388,11 @@ inline void clear_patients(PyObject *self) { auto *instance = reinterpret_cast(self); auto &internals = get_internals(); auto pos = internals.patients.find(self); - assert(pos != internals.patients.end()); + + if (pos == internals.patients.end()) { + pybind11_fail("FATAL: Internal consistency check failed: Invalid clear_patients() call."); + } + // Clearing the patients can cause more Python code to run, which // can invalidate the iterator. Extract the vector of patients // from the unordered_map first. diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 39d44fda64..99606f8d1d 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -3040,7 +3040,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * PyObject *locals = PyEval_GetFrameLocals(); # else PyObject *locals = PyEval_GetLocals(); - Py_INCREF(locals); + Py_XINCREF(locals); # endif if (locals != nullptr) { # if PY_VERSION_HEX >= 0x030b0000 diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index d5f6af8e02..3d2d0c2dac 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -183,7 +183,15 @@ class object_api : public pyobject_tag { str_attr_accessor doc() const; /// Return the object's current reference count - int ref_count() const { return static_cast(Py_REFCNT(derived().ptr())); } + ssize_t ref_count() const { +#ifdef PYPY_VERSION + // PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT + // Following pybind11 2.12.1 and older behavior and removing this part + return static_cast(static_cast(Py_REFCNT(derived().ptr()))); +#else + return Py_REFCNT(derived().ptr()); +#endif + } // TODO PYBIND11_DEPRECATED( // "Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()") @@ -2175,6 +2183,11 @@ class list : public object { throw error_already_set(); } } + void clear() /* py-non-const */ { + if (PyList_SetSlice(m_ptr, 0, PyList_Size(m_ptr), nullptr) == -1) { + throw error_already_set(); + } + } }; class args : public tuple { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7fd7149582..d8e050eca8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -358,7 +358,7 @@ if(Boost_FOUND) add_library(Boost::headers IMPORTED INTERFACE) if(TARGET Boost::boost) # Classic FindBoost - set_property(TARGET Boost::boost PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost) + set_property(TARGET Boost::headers PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost) else() # Very old FindBoost, or newer Boost than CMake in older CMakes set_property(TARGET Boost::headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index a8d9bc5257..95b3e967db 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -80,6 +80,7 @@ "share/cmake/pybind11/pybind11Common.cmake", "share/cmake/pybind11/pybind11Config.cmake", "share/cmake/pybind11/pybind11ConfigVersion.cmake", + "share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake", "share/cmake/pybind11/pybind11NewTools.cmake", "share/cmake/pybind11/pybind11Targets.cmake", "share/cmake/pybind11/pybind11Tools.cmake", diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 81869ebe21..15cbf74545 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,8 @@ PYBIND11_MODULE(pybind11_tests, m) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + // Free threaded Python uses UINT32_MAX for immortal objects. + m.attr("PYBIND11_REFCNT_IMMORTAL") = UINT32_MAX; m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) true; diff --git a/tests/test_class.py b/tests/test_class.py index 73a48309e8..edaa5b3ca4 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -3,7 +3,7 @@ import pytest import env -from pybind11_tests import ConstructorStats, UserType +from pybind11_tests import PYBIND11_REFCNT_IMMORTAL, ConstructorStats, UserType from pybind11_tests import class_ as m @@ -377,7 +377,9 @@ class PyDog(m.Dog): refcount_3 = getrefcount(cls) assert refcount_1 == refcount_3 - assert refcount_2 > refcount_1 + assert (refcount_2 > refcount_1) or ( + refcount_2 == refcount_1 == PYBIND11_REFCNT_IMMORTAL + ) def test_reentrant_implicit_conversion_failure(msg): diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt index 09a3693999..9b539cd42d 100644 --- a/tests/test_embed/CMakeLists.txt +++ b/tests/test_embed/CMakeLists.txt @@ -7,6 +7,13 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STR return() endif() +if(TARGET Python::Module AND NOT TARGET Python::Python) + message(STATUS "Skipping embed test since no embed libs found") + add_custom_target(cpptest) # Dummy target since embedding is not supported. + set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}") + return() +endif() + find_package(Catch 2.13.9) if(CATCH_FOUND) diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 2f40aca6af..bc76ec7c2d 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -150,10 +150,13 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // test_args_refcount // PyPy needs a garbage collection to get the reference count values to match CPython's behaviour +// PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT, so truncate #ifdef PYPY_VERSION # define GC_IF_NEEDED ConstructorStats::gc() +# define REFCNT(x) (int) Py_REFCNT(x) #else # define GC_IF_NEEDED +# define REFCNT(x) Py_REFCNT(x) #endif m.def("arg_refcount_h", [](py::handle h) { GC_IF_NEEDED; @@ -172,7 +175,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { py::tuple t(a.size()); for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); } return t; }); @@ -182,7 +185,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { t[0] = o.ref_count(); for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i + 1] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i + 1] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); } return t; }); diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 0e3bbdb765..d7bc7379b1 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -381,7 +381,7 @@ def test_args_refcount(): arguments""" refcount = m.arg_refcount_h - myval = 54321 + myval = object() expected = refcount(myval) assert m.arg_refcount_h(myval) == expected assert m.arg_refcount_o(myval) == expected + 1 @@ -420,6 +420,7 @@ def test_args_refcount(): # for the `py::args`; in the previous case, we could simply inc_ref and pass on Python's input # tuple without having to inc_ref the individual elements, but here we can't, hence the extra # refs. - assert m.mixed_args_refcount(myval, myval, myval) == (exp3 + 3, exp3 + 3, exp3 + 3) + exp3_3 = exp3 + 3 + assert m.mixed_args_refcount(myval, myval, myval) == (exp3_3, exp3_3, exp3_3) assert m.class_default_argument() == "" diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 12e6151812..f3709d40c6 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -135,6 +135,7 @@ TEST_SUBMODULE(pytypes, m) { m.def("list_size_t", []() { return py::list{(py::size_t) 0}; }); m.def("list_insert_ssize_t", [](py::list *l) { return l->insert((py::ssize_t) 1, 83); }); m.def("list_insert_size_t", [](py::list *l) { return l->insert((py::size_t) 3, 57); }); + m.def("list_clear", [](py::list *l) { l->clear(); }); m.def("get_list", []() { py::list list; list.append("value"); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index f6271a628a..38edfd9998 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -65,6 +65,8 @@ def test_list(capture, doc): assert lins == [1, 83, 2] m.list_insert_size_t(lins) assert lins == [1, 83, 2, 57] + m.list_clear(lins) + assert lins == [] with capture: lst = m.get_list() @@ -631,7 +633,8 @@ def test_memoryview(method, args, fmt, expected_view): ], ) def test_memoryview_refcount(method): - buf = b"\x0a\x0b\x0c\x0d" + # Avoiding a literal to avoid an immortal object in free-threaded builds + buf = "\x0a\x0b\x0c\x0d".encode("ascii") ref_before = sys.getrefcount(buf) view = method(buf) ref_after = sys.getrefcount(buf) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index 8275b9d5aa..0640a1915f 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -205,7 +205,7 @@ endif() # Make sure the Python has the same pointer-size as the chosen compiler # Skip if CMAKE_SIZEOF_VOID_P is not defined # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN) -if(NOT CMAKE_CROSSCOMPILING +if(NOT _PYBIND11_CROSSCOMPILING AND CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}")) if(PythonLibsNew_FIND_REQUIRED) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 57721aeb16..d8e18e67bf 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS "${pybind11_INCLUDE_DIR}" CACHE INTERNAL "Include directory for pybind11 (Python not requested)") +if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING) + set(_PYBIND11_CROSSCOMPILING + ON + CACHE INTERNAL "") +else() + set(_PYBIND11_CROSSCOMPILING + OFF + CACHE INTERNAL "") +endif() + # --------------------- Shared targets ---------------------------- # Build an interface library target: @@ -195,7 +205,7 @@ endif() # --------------------- pybind11_find_import ------------------------------- -if(NOT _pybind11_nopython) +if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING) # Check to see if modules are importable. Use REQUIRED to force an error if # one of the modules is not found. _FOUND will be set if the # package was found (underscores replace dashes if present). QUIET will hide diff --git a/tools/pybind11GuessPythonExtSuffix.cmake b/tools/pybind11GuessPythonExtSuffix.cmake new file mode 100644 index 0000000000..c5fb3b42c9 --- /dev/null +++ b/tools/pybind11GuessPythonExtSuffix.cmake @@ -0,0 +1,86 @@ +cmake_minimum_required(VERSION 3.5) + +function(pybind11_guess_python_module_extension python) + + # The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX}) + message( + STATUS + "Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}" + ) + set(PYTHON_MODULE_EXT_SUFFIX + "$ENV{SETUPTOOLS_EXT_SUFFIX}" + CACHE + STRING + "Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)") + endif() + # If that didn't work, use the Python_SOABI variable: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI) + message( + STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}") + # The final extension depends on the system + set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}") + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(_PY_BUILD_EXTENSION ".pyd") + endif() + # If the SOABI already has an extension, use it as the full suffix + # (used for debug versions of Python on Windows) + if(${python}_SOABI MATCHES "\\.") + set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}") + # If the SOABI is empty, this is usually a bug, but we generate a + # correct extension anyway, which is the best we can do + elseif("${${python}_SOABI}" STREQUAL "") + message( + WARNING + "${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly." + ) + set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}") + # Otherwise, add the system-dependent extension to it + else() + set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}") + endif() + endif() + + # If we could not deduce the extension suffix, unset the results: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX) + unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE) + unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE) + unset(PYTHON_IS_DEBUG PARENT_SCOPE) + return() + endif() + + # Sanity checks: + if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI + OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\.")) + message( + WARNING + "Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})." + ) + endif() + + # Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT) + + # Try to deduce the debug ABI from the extension suffix: + if(NOT DEFINED _PYTHON_IS_DEBUG) + if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-" + OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d") + set(_PYTHON_IS_DEBUG On) + else() + set(_PYTHON_IS_DEBUG Off) + endif() + endif() + + # Return results + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + PARENT_SCOPE) + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + PARENT_SCOPE) + set(PYTHON_IS_DEBUG + "${_PYTHON_IS_DEBUG}" + PARENT_SCOPE) + +endfunction() diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 9fe2eb08dc..f2ec347561 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -32,6 +32,13 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(Python_ROOT_DIR "$ENV{pythonLocation}") endif() + # Interpreter should not be found when cross-compiling + if(_PYBIND11_CROSSCOMPILING) + set(_pybind11_interp_component "") + else() + set(_pybind11_interp_component Interpreter) + endif() + # Development.Module support (required for manylinux) started in 3.18 if(CMAKE_VERSION VERSION_LESS 3.18) set(_pybind11_dev_component Development) @@ -48,8 +55,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) endif() endif() - find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component} - ${_pybind11_quiet} ${_pybind11_global_keyword}) + find_package( + Python 3.6 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component} + ${_pybind11_quiet} ${_pybind11_global_keyword}) # If we are in submodule mode, export the Python targets to global targets. # If this behavior is not desired, FindPython _before_ pybind11. @@ -59,7 +67,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) if(TARGET Python::Python) set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE) endif() - set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE) + if(TARGET Python::Interpreter) + set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE) + endif() if(TARGET Python::Module) set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE) endif() @@ -100,69 +110,89 @@ if(PYBIND11_MASTER_PROJECT) endif() endif() -# If a user finds Python, they may forget to include the Interpreter component -# and the following two steps require it. It is highly recommended by CMake -# when finding development libraries anyway, so we will require it. -if(NOT DEFINED ${_Python}_EXECUTABLE) - message( - FATAL_ERROR - "${_Python} was found without the Interpreter component. Pybind11 requires this component.") - -endif() - -if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL - PYBIND11_PYTHON_EXECUTABLE_LAST) - # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed - unset(PYTHON_IS_DEBUG CACHE) - unset(PYTHON_MODULE_EXTENSION CACHE) -endif() - -set(PYBIND11_PYTHON_EXECUTABLE_LAST - "${${_Python}_EXECUTABLE}" - CACHE INTERNAL "Python executable during the last CMake run") - -if(NOT DEFINED PYTHON_IS_DEBUG) - # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter - execute_process( - COMMAND "${${_Python}_EXECUTABLE}" "-c" - "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))" - RESULT_VARIABLE _PYTHON_IS_DEBUG) - set(PYTHON_IS_DEBUG - "${_PYTHON_IS_DEBUG}" - CACHE INTERNAL "Python debug status") -endif() - -# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is -# required for PyPy3 (as of 7.3.1) -if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) - execute_process( - COMMAND - "${${_Python}_EXECUTABLE}" "-c" - "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" - OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX - ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR - OUTPUT_STRIP_TRAILING_WHITESPACE) - - if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "") +if(NOT _PYBIND11_CROSSCOMPILING) + # If a user finds Python, they may forget to include the Interpreter component + # and the following two steps require it. It is highly recommended by CMake + # when finding development libraries anyway, so we will require it. + if(NOT DEFINED ${_Python}_EXECUTABLE) message( - FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'" - "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}" + FATAL_ERROR + "${_Python} was found without the Interpreter component. Pybind11 requires this component." ) + endif() - # This needs to be available for the pybind11_extension function - if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) - get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) - set(PYTHON_MODULE_DEBUG_POSTFIX - "${_PYTHON_MODULE_DEBUG_POSTFIX}" - CACHE INTERNAL "") + if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL + PYBIND11_PYTHON_EXECUTABLE_LAST) + # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed + unset(PYTHON_IS_DEBUG CACHE) + unset(PYTHON_MODULE_EXTENSION CACHE) endif() - if(NOT DEFINED PYTHON_MODULE_EXTENSION) - get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) - set(PYTHON_MODULE_EXTENSION - "${_PYTHON_MODULE_EXTENSION}" - CACHE INTERNAL "") + set(PYBIND11_PYTHON_EXECUTABLE_LAST + "${${_Python}_EXECUTABLE}" + CACHE INTERNAL "Python executable during the last CMake run") + + if(NOT DEFINED PYTHON_IS_DEBUG) + # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter + execute_process( + COMMAND "${${_Python}_EXECUTABLE}" "-c" + "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))" + RESULT_VARIABLE _PYTHON_IS_DEBUG) + set(PYTHON_IS_DEBUG + "${_PYTHON_IS_DEBUG}" + CACHE INTERNAL "Python debug status") + endif() + + # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is + # required for PyPy3 (as of 7.3.1) + if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + execute_process( + COMMAND + "${${_Python}_EXECUTABLE}" "-c" + "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" + OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX + ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "") + message( + FATAL_ERROR + "pybind11 could not query the module file extension, likely the 'distutils'" + "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}") + endif() + + # This needs to be available for the pybind11_extension function + if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + CACHE INTERNAL "") + endif() + + if(NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + CACHE INTERNAL "") + endif() + endif() +else() + if(NOT DEFINED PYTHON_IS_DEBUG + OR NOT DEFINED PYTHON_MODULE_EXTENSION + OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake") + pybind11_guess_python_module_extension("${_Python}") + endif() + # When cross-compiling, we cannot query the Python interpreter, so we require + # the user to set these variables explicitly. + if(NOT DEFINED PYTHON_IS_DEBUG + OR NOT DEFINED PYTHON_MODULE_EXTENSION + OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + message( + FATAL_ERROR + "When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \ + variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)") endif() endif() diff --git a/tools/test-pybind11GuessPythonExtSuffix.cmake b/tools/test-pybind11GuessPythonExtSuffix.cmake new file mode 100644 index 0000000000..0de2c0169b --- /dev/null +++ b/tools/test-pybind11GuessPythonExtSuffix.cmake @@ -0,0 +1,161 @@ +cmake_minimum_required(VERSION 3.5) + +# Tests for pybind11_guess_python_module_extension +# Run using `cmake -P tools/test-pybind11GuessPythonExtSuffix.cmake` + +include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake") + +macro(expect_streq actual expected) + if(NOT "${actual}" STREQUAL "${expected}") + message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: '${expected}'") + endif() +endmacro() + +macro(expect_false actual) + if("${actual}") + message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: false") + endif() +endmacro() + +macro(expect_true actual) + if(NOT "${actual}") + message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: true") + endif() +endmacro() + +# Windows +set(CMAKE_SYSTEM_NAME "Windows") +set(CMAKE_SHARED_MODULE_SUFFIX ".dll") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cp311-win_arm64") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cp311d-win_arm64") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311d-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "pypy310-pp73-win_amd64") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-win_amd64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "_d.cp311-win_amd64.pyd") # This is a quirk of FindPython3 +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_amd64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "_d") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +unset(Python3_SOABI) +set(ENV{SETUPTOOLS_EXT_SUFFIX} ".cp39-win_arm64.pyd") # Set by cibuildwheel +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp39-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) +unset(ENV{SETUPTOOLS_EXT_SUFFIX}) + +set(Python3_SOABI "cp311-win_arm64") +set(ENV{SETUPTOOLS_EXT_SUFFIX} "") # Should not be used +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) +unset(ENV{SETUPTOOLS_EXT_SUFFIX}) + +# macOS +set(CMAKE_SYSTEM_NAME "Darwin") +set(CMAKE_SHARED_MODULE_SUFFIX ".so") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312-darwin") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-darwin.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312d-darwin") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-darwin.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +# Linux +set(CMAKE_SYSTEM_NAME "Linux") +set(CMAKE_SHARED_MODULE_SUFFIX ".so") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312-arm-linux-gnueabihf") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-arm-linux-gnueabihf.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312d-arm-linux-gnueabihf") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-arm-linux-gnueabihf.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "pypy310-pp73-x86_64-linux-gnu") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-x86_64-linux-gnu.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "pypy310d-pp73-x86_64-linux-gnu") +# TODO: I'm not sure if this is the right SOABI for PyPy debug builds +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310d-pp73-x86_64-linux-gnu.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE)