From 0e0752431c5e10bd9bd1de88b02406148aef84dd Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Wed, 22 Jan 2025 19:13:06 +0000 Subject: [PATCH 1/4] remove OOAPI specifier from enum to allow windows debug builds to link successfully --- libopenage/util/enum.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libopenage/util/enum.h b/libopenage/util/enum.h index 394db95b93..c6a9b4666b 100644 --- a/libopenage/util/enum.h +++ b/libopenage/util/enum.h @@ -1,4 +1,4 @@ -// Copyright 2015-2024 the openage authors. See copying.md for legal info. +// Copyright 2015-2025 the openage authors. See copying.md for legal info. #pragma once @@ -44,7 +44,7 @@ namespace util { * NumericType numeric */ template -class OAAPI EnumValue { +class EnumValue { public: constexpr EnumValue(const char *value_name, NumericType numeric_value) : name(value_name), numeric(numeric_value) {} @@ -124,7 +124,7 @@ class OAAPI EnumValue { * bool operator >=(Enum[DerivedType] other) except + */ template -class OAAPI Enum { +class Enum { using this_type = Enum; public: From bdd88b4b64803d1cb2b4a2005dc1ae6b8bd460d4 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 24 Jan 2025 12:26:48 +0000 Subject: [PATCH 2/4] working wondows debug build --- buildsystem/cythonize.py | 19 +++++++++++++++++++ buildsystem/python.cmake | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/buildsystem/cythonize.py b/buildsystem/cythonize.py index 9e8ba8e234..89b499a29f 100755 --- a/buildsystem/cythonize.py +++ b/buildsystem/cythonize.py @@ -88,13 +88,32 @@ def cythonize_wrapper(modules, **kwargs): with redirect_stdout(cython_filter): if src_modules: cythonize(src_modules, **kwargs) + windows_include_python_debug_build_wrapper(src_modules, bin_dir) if bin_modules: os.chdir(bin_dir) cythonize(bin_modules, **kwargs) + windows_include_python_debug_build_wrapper(bin_modules, bin_dir) os.chdir(src_dir) +def windows_include_python_debug_build_wrapper(modules, path): + for module in modules: + module = str(module) + if (path): + module = path + "\\" + module + module = module.removesuffix(".py").removesuffix(".pyx") + module = module + ".cpp" + with open(module, "r") as file: + text = file.read() + text = text.replace("#include \"Python.h\"", + "#ifdef _DEBUG\n#define _DEBUG_WAS_DEFINED\n#undef _DEBUG\n#endif\n\ + #include \"Python.h\"\n#ifdef _DEBUG_WAS_DEFINED\n#define _DEBUG\n\ + #undef _DEBUG_WAS_DEFINED\n#endif", 1) + with open(module, "w") as file: + file.write(text) + + def main(): """ CLI entry point """ cli = argparse.ArgumentParser() diff --git a/buildsystem/python.cmake b/buildsystem/python.cmake index f5c4d84de5..a8e1fed72e 100644 --- a/buildsystem/python.cmake +++ b/buildsystem/python.cmake @@ -128,8 +128,7 @@ function(add_cython_modules) if(MINGW) set_target_properties("${TARGETNAME}" PROPERTIES LINK_FLAGS "-municode") endif() - - target_link_libraries("${TARGETNAME}" PRIVATE ${PYEXT_LIBRARY}) + target_link_libraries("${TARGETNAME}" PRIVATE C:/vcpkg/installed/x64-windows/lib/python311.lib) else() set_property(GLOBAL APPEND PROPERTY SFT_CYTHON_MODULES "${source}") add_library("${TARGETNAME}" MODULE "${CPPNAME}") @@ -140,7 +139,7 @@ function(add_cython_modules) ) if(WIN32) - target_link_libraries("${TARGETNAME}" PRIVATE ${PYEXT_LIBRARY}) + target_link_libraries("${TARGETNAME}" PRIVATE C:/vcpkg/installed/x64-windows/lib/python311.lib) endif() endif() @@ -514,6 +513,7 @@ function(python_finalize) ${INPLACEMODULES_LISTFILE} "$" ) + message(hello jeremiah, ${INPLACEMODULES_LISTFILE}) set(INPLACEMODULES_TIMEFILE "${CMAKE_BINARY_DIR}/py/inplacemodules_timefile") add_custom_command(OUTPUT "${INPLACEMODULES_TIMEFILE}" COMMAND ${INPLACEMODULES_INVOCATION} From b48b3fcbf023af419c91bb4a1ab2a819118694c8 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 24 Jan 2025 12:33:16 +0000 Subject: [PATCH 3/4] working windows debug build --- buildsystem/HandlePythonOptions.cmake | 13 ++++++++++- buildsystem/cythonize.py | 33 +++++++++++++++++++-------- buildsystem/python.cmake | 12 ++++++---- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/buildsystem/HandlePythonOptions.cmake b/buildsystem/HandlePythonOptions.cmake index ef7da71f9b..d6888c0673 100644 --- a/buildsystem/HandlePythonOptions.cmake +++ b/buildsystem/HandlePythonOptions.cmake @@ -1,4 +1,4 @@ -# Copyright 2015-2023 the openage authors. See copying.md for legal info. +# Copyright 2015-2025 the openage authors. See copying.md for legal info. # finds the python interpreter, install destination and extension flags. @@ -39,6 +39,17 @@ if(PYTHON_VER VERSION_GREATER_EQUAL 3.8 AND PYTHON_VERSION VERSION_LESS 3.9) endif() set(PYEXT_LIBRARY "${PYTHON_LIBRARIES}") + +#Windows always uses optimized version of Python lib +if(WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + #get index of string "optimized" and increment it by 1 so index points at the path of the optimized lib + list (FIND PYEXT_LIBRARY "optimized" _index) + if (${_index} GREATER -1) + MATH(EXPR _index "${_index}+1") + list(GET PYEXT_LIBRARY ${_index} PYEXT_LIBRARY) + endif() +endif() + set(PYEXT_INCLUDE_DIRS "${PYTHON_INCLUDE_DIRS};${NUMPY_INCLUDE_DIR}") if(NOT CMAKE_PY_INSTALL_PREFIX) diff --git a/buildsystem/cythonize.py b/buildsystem/cythonize.py index 89b499a29f..5d7cd492fe 100755 --- a/buildsystem/cythonize.py +++ b/buildsystem/cythonize.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2015-2021 the openage authors. See copying.md for legal info. +# Copyright 2015-2025 the openage authors. See copying.md for legal info. """ Runs Cython on all modules that were listed via add_cython_module. @@ -73,7 +73,7 @@ def remove_if_exists(filename): filename.unlink() -def cythonize_wrapper(modules, **kwargs): +def cythonize_wrapper(modules, force_optimized_lib = False, **kwargs): """ Calls cythonize, filtering useless warnings """ bin_dir, bin_modules = kwargs['build_dir'], [] src_dir, src_modules = Path.cwd(), [] @@ -88,29 +88,40 @@ def cythonize_wrapper(modules, **kwargs): with redirect_stdout(cython_filter): if src_modules: cythonize(src_modules, **kwargs) - windows_include_python_debug_build_wrapper(src_modules, bin_dir) + if sys.platform == 'win32' and force_optimized_lib: + windows_use_optimized_lib_python(src_modules, bin_dir) if bin_modules: os.chdir(bin_dir) cythonize(bin_modules, **kwargs) - windows_include_python_debug_build_wrapper(bin_modules, bin_dir) + if sys.platform == 'win32' and force_optimized_lib: + windows_use_optimized_lib_python(bin_modules, bin_dir) os.chdir(src_dir) -def windows_include_python_debug_build_wrapper(modules, path): +def windows_use_optimized_lib_python(modules, path): + """ + Add an #ifdef statement in cythonized .cpp files to temporarily undefine _DEBUG before + #include "Python.h" + + This function modifies the generated C++ files from Cython to prevent linking to + the debug version of the Python library on Windows. The debug version of the + Python library cannot import Python libraries that contain extension modules. + """ + for module in modules: module = str(module) - if (path): + if path: module = path + "\\" + module module = module.removesuffix(".py").removesuffix(".pyx") module = module + ".cpp" - with open(module, "r") as file: + with open(module, "r", encoding='utf8') as file: text = file.read() text = text.replace("#include \"Python.h\"", "#ifdef _DEBUG\n#define _DEBUG_WAS_DEFINED\n#undef _DEBUG\n#endif\n\ #include \"Python.h\"\n#ifdef _DEBUG_WAS_DEFINED\n#define _DEBUG\n\ #undef _DEBUG_WAS_DEFINED\n#endif", 1) - with open(module, "w") as file: + with open(module, "w", encoding='utf8') as file: file.write(text) @@ -144,6 +155,8 @@ def main(): )) cli.add_argument("--threads", type=int, default=cpu_count(), help="number of compilation threads to use") + cli.add_argument("--force_optimized_lib", action="store_true", + help= "edit compiled .cpp files to link to optimized version of python libary") args = cli.parse_args() # cython emits warnings on using absolute paths to modules @@ -178,12 +191,12 @@ def main(): # writing funny lines at the head of each file. cythonize_args['language'] = 'c++' - cythonize_wrapper(modules, **cythonize_args) + cythonize_wrapper(modules, args.force_optimized_lib, **cythonize_args) # build standalone executables that embed the py interpreter Options.embed = "main" - cythonize_wrapper(embedded_modules, **cythonize_args) + cythonize_wrapper(embedded_modules, args.force_optimized_lib, **cythonize_args) # verify depends from Cython.Build.Dependencies import _dep_tree diff --git a/buildsystem/python.cmake b/buildsystem/python.cmake index a8e1fed72e..a46f7106f3 100644 --- a/buildsystem/python.cmake +++ b/buildsystem/python.cmake @@ -1,4 +1,4 @@ -# Copyright 2014-2020 the openage authors. See copying.md for legal info. +# Copyright 2014-2025 the openage authors. See copying.md for legal info. # provides macros for defining python extension modules and pxdgen sources. # and a 'finalize' function that must be called in the end. @@ -128,7 +128,8 @@ function(add_cython_modules) if(MINGW) set_target_properties("${TARGETNAME}" PROPERTIES LINK_FLAGS "-municode") endif() - target_link_libraries("${TARGETNAME}" PRIVATE C:/vcpkg/installed/x64-windows/lib/python311.lib) + + target_link_libraries("${TARGETNAME}" PRIVATE ${PYEXT_LIBRARY}) else() set_property(GLOBAL APPEND PROPERTY SFT_CYTHON_MODULES "${source}") add_library("${TARGETNAME}" MODULE "${CPPNAME}") @@ -139,7 +140,7 @@ function(add_cython_modules) ) if(WIN32) - target_link_libraries("${TARGETNAME}" PRIVATE C:/vcpkg/installed/x64-windows/lib/python311.lib) + target_link_libraries("${TARGETNAME}" PRIVATE ${PYEXT_LIBRARY}) endif() endif() @@ -374,6 +375,9 @@ function(python_finalize) # cythonize (.pyx -> .cpp) + if(WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(force_optimized_lib "--force_optimized_lib") + endif() get_property(cython_modules GLOBAL PROPERTY SFT_CYTHON_MODULES) write_on_change("${CMAKE_BINARY_DIR}/py/cython_modules" "${cython_modules}") @@ -392,6 +396,7 @@ function(python_finalize) "${CMAKE_BINARY_DIR}/py/cython_modules" "${CMAKE_BINARY_DIR}/py/cython_modules_embed" "${CMAKE_BINARY_DIR}/py/pxd_list" + ${force_optimized_lib} "--build-dir" "${CMAKE_BINARY_DIR}" COMMAND "${CMAKE_COMMAND}" -E touch "${CYTHONIZE_TIMEFILE}" DEPENDS @@ -513,7 +518,6 @@ function(python_finalize) ${INPLACEMODULES_LISTFILE} "$" ) - message(hello jeremiah, ${INPLACEMODULES_LISTFILE}) set(INPLACEMODULES_TIMEFILE "${CMAKE_BINARY_DIR}/py/inplacemodules_timefile") add_custom_command(OUTPUT "${INPLACEMODULES_TIMEFILE}" COMMAND ${INPLACEMODULES_INVOCATION} From ee550f5a9d931845404d83888eaa4c9cfe3312ea Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 24 Jan 2025 18:10:33 +0000 Subject: [PATCH 4/4] CI-CD:use debug config for CI --- .github/workflows/windows-server-2019.yml | 2 +- .github/workflows/windows-server-2022.yml | 2 +- buildsystem/HandlePythonOptions.cmake | 1 + buildsystem/cythonize.py | 30 +++++++++++++++++------ buildsystem/python.cmake | 5 +--- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/windows-server-2019.yml b/.github/workflows/windows-server-2019.yml index bf3ff7856b..406d303f27 100644 --- a/.github/workflows/windows-server-2019.yml +++ b/.github/workflows/windows-server-2019.yml @@ -51,7 +51,7 @@ jobs: mkdir build cd build cmake -DCMAKE_TOOLCHAIN_FILE="$TOOLCHAIN_FILE" -DCMAKE_BUILD_TYPE=Release -DCMAKE_TRY_COMPILE_CONFIGURATION=Release -DCMAKE_CXX_FLAGS='/Zc:__cplusplus /permissive- /EHsc' -DCMAKE_EXE_LINKER_FLAGS='' -DCMAKE_MODULE_LINKER_FLAGS='' -DCMAKE_SHARED_LINKER_FLAGS='' -DDOWNLOAD_NYAN=YES -DCXX_OPTIMIZATION_LEVEL=auto -DCXX_SANITIZE_FATAL=False -DCXX_SANITIZE_MODE=none -DWANT_BACKTRACE=if_available -DWANT_GPERFTOOLS_PROFILER=if_available -DWANT_GPERFTOOLS_TCMALLOC=False -DWANT_INOTIFY=if_available -DWANT_NCURSES=if_available -DWANT_OPENGL=if_available -DWANT_VULKAN=if_available -DFLEX_EXECUTABLE="$FLEX_PATH" -G "Visual Studio 16 2019" -A x64 ../source - cmake --build . --config RelWithDebInfo -- -nologo -maxCpuCount + cmake --build . --config Debug -- -nologo -maxCpuCount shell: pwsh - name: Package run: | diff --git a/.github/workflows/windows-server-2022.yml b/.github/workflows/windows-server-2022.yml index a721db5a52..10a6a52c59 100644 --- a/.github/workflows/windows-server-2022.yml +++ b/.github/workflows/windows-server-2022.yml @@ -51,7 +51,7 @@ jobs: mkdir build cd build cmake -DCMAKE_TOOLCHAIN_FILE="$TOOLCHAIN_FILE" -DCMAKE_BUILD_TYPE=Release -DCMAKE_TRY_COMPILE_CONFIGURATION=Release -DCMAKE_CXX_FLAGS='/Zc:__cplusplus /permissive- /EHsc' -DCMAKE_EXE_LINKER_FLAGS='' -DCMAKE_MODULE_LINKER_FLAGS='' -DCMAKE_SHARED_LINKER_FLAGS='' -DDOWNLOAD_NYAN=YES -DCXX_OPTIMIZATION_LEVEL=auto -DCXX_SANITIZE_FATAL=False -DCXX_SANITIZE_MODE=none -DWANT_BACKTRACE=if_available -DWANT_GPERFTOOLS_PROFILER=if_available -DWANT_GPERFTOOLS_TCMALLOC=False -DWANT_INOTIFY=if_available -DWANT_NCURSES=if_available -DWANT_OPENGL=if_available -DWANT_VULKAN=if_available -DFLEX_EXECUTABLE="$FLEX_PATH" -G "Visual Studio 17 2022" -A x64 ../source - cmake --build . --config RelWithDebInfo -- -nologo -maxCpuCount + cmake --build . --config Debug -- -nologo -maxCpuCount shell: pwsh - name: Package run: | diff --git a/buildsystem/HandlePythonOptions.cmake b/buildsystem/HandlePythonOptions.cmake index d6888c0673..20a35f3d61 100644 --- a/buildsystem/HandlePythonOptions.cmake +++ b/buildsystem/HandlePythonOptions.cmake @@ -47,6 +47,7 @@ if(WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") if (${_index} GREATER -1) MATH(EXPR _index "${_index}+1") list(GET PYEXT_LIBRARY ${_index} PYEXT_LIBRARY) + set(force_optimized_lib_flag "--force_optimized_lib") endif() endif() diff --git a/buildsystem/cythonize.py b/buildsystem/cythonize.py index 5d7cd492fe..e7ac86ae6f 100755 --- a/buildsystem/cythonize.py +++ b/buildsystem/cythonize.py @@ -89,17 +89,17 @@ def cythonize_wrapper(modules, force_optimized_lib = False, **kwargs): if src_modules: cythonize(src_modules, **kwargs) if sys.platform == 'win32' and force_optimized_lib: - windows_use_optimized_lib_python(src_modules, bin_dir) + win_use_optimized_lib_python(src_modules, bin_dir) if bin_modules: os.chdir(bin_dir) cythonize(bin_modules, **kwargs) if sys.platform == 'win32' and force_optimized_lib: - windows_use_optimized_lib_python(bin_modules, bin_dir) + win_use_optimized_lib_python(bin_modules, bin_dir) os.chdir(src_dir) -def windows_use_optimized_lib_python(modules, path): +def win_use_optimized_lib_python(modules, path): """ Add an #ifdef statement in cythonized .cpp files to temporarily undefine _DEBUG before #include "Python.h" @@ -107,6 +107,8 @@ def windows_use_optimized_lib_python(modules, path): This function modifies the generated C++ files from Cython to prevent linking to the debug version of the Python library on Windows. The debug version of the Python library cannot import Python libraries that contain extension modules. + see: https://github.com/python/cpython/issues/127619 (To unserstand the problem) + see: https://stackoverflow.com/a/59566420 (To understand the soloution) """ for module in modules: @@ -117,10 +119,24 @@ def windows_use_optimized_lib_python(modules, path): module = module + ".cpp" with open(module, "r", encoding='utf8') as file: text = file.read() - text = text.replace("#include \"Python.h\"", - "#ifdef _DEBUG\n#define _DEBUG_WAS_DEFINED\n#undef _DEBUG\n#endif\n\ - #include \"Python.h\"\n#ifdef _DEBUG_WAS_DEFINED\n#define _DEBUG\n\ - #undef _DEBUG_WAS_DEFINED\n#endif", 1) + if not text.count("OPENAGE: UNDEF_DEBUG_INSERTED"): + text = text.replace( + '#include "Python.h"', + ( + "\n\n// OPENAGE: UNDEF_DEBUG_INSERTED\n" + "// Avoid linking to the debug version of the Python library on Windows\n\n" + "#ifdef _DEBUG\n" + "#define _DEBUG_WAS_DEFINED\n" + "#undef _DEBUG\n#endif\n" + "#include \"Python.h\"\n" + "#ifdef _DEBUG_WAS_DEFINED\n" + "#define _DEBUG\n" + "#undef _DEBUG_WAS_DEFINED\n" + "#endif\n\n" + "// OPENAGE: UNDEF_DEBUG_INSERTED\n\n" + ), + 1 + ) with open(module, "w", encoding='utf8') as file: file.write(text) diff --git a/buildsystem/python.cmake b/buildsystem/python.cmake index a46f7106f3..90117baadc 100644 --- a/buildsystem/python.cmake +++ b/buildsystem/python.cmake @@ -375,9 +375,6 @@ function(python_finalize) # cythonize (.pyx -> .cpp) - if(WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") - set(force_optimized_lib "--force_optimized_lib") - endif() get_property(cython_modules GLOBAL PROPERTY SFT_CYTHON_MODULES) write_on_change("${CMAKE_BINARY_DIR}/py/cython_modules" "${cython_modules}") @@ -396,7 +393,7 @@ function(python_finalize) "${CMAKE_BINARY_DIR}/py/cython_modules" "${CMAKE_BINARY_DIR}/py/cython_modules_embed" "${CMAKE_BINARY_DIR}/py/pxd_list" - ${force_optimized_lib} + ${force_optimized_lib_flag} "--build-dir" "${CMAKE_BINARY_DIR}" COMMAND "${CMAKE_COMMAND}" -E touch "${CYTHONIZE_TIMEFILE}" DEPENDS