Merge pull request #489 from STEllAR-GROUP/flag_for_intel_gpu_workaround

Flag for intel gpu workaround

Author: G-071

.jenkins/lsu/Jenkinsfile-Node-Level

@@ -66,7 +66,7 @@ pipeline {
                                ' +kokkos simd_extension=SCALAR simd_library=KOKKOS  %gcc@11^[email protected] max_cpu_count=128 malloc=jemalloc %gcc@11 ^kokkos %gcc@11 ^hpx-kokkos %gcc@11 ^cppuddle %gcc@11 ^silo~mpi ;with-CC-with-kokkos-with-scalar;gcc/11;medusa --exclusive;--fresh',\
                                ' +kokkos simd_extension=AVX512 simd_library=KOKKOS  %gcc@11^[email protected] max_cpu_count=128 malloc=jemalloc %gcc@11 ^kokkos %gcc@11 ^hpx-kokkos %gcc@11 ^cppuddle %gcc@11 ^silo~mpi ;with-CC-with-kokkos-with-avx512;gcc/11;medusa --exclusive;--fresh',\
                                ' +kokkos +kokkos_hpx_kernels multipole_host_tasks=16 monopole_host_tasks=4 hydro_host_tasks=4 simd_extension=AVX512 simd_library=KOKKOS  %gcc@11^[email protected] max_cpu_count=128 malloc=jemalloc %gcc@11 ^kokkos %gcc@11 ^hpx-kokkos %gcc@11 ^cppuddle %gcc@11 ^silo~mpi ;with-CC-with-kokkos-with-avx512-with-hpx-kokkos-kernels;gcc/11;medusa --exclusive;--fresh',\
-                               ' -cuda -rocm +sycl  %gcc@11 ^[email protected] use_unsupported_sycl_arch=70 ^[email protected] sycl_target_arch=70;with-CC-with-kokkos-with-sycl;gcc/11 cuda/12;cuda-V100 -w diablo --gpus=1 --exclusive;--fresh'
+                               ' -cuda -rocm +sycl  %gcc@11 ^[email protected] use_unsupported_sycl_arch=70 ^[email protected] sycl_target_arch=70;with-CC-with-kokkos-with-sycl;gcc/11 cuda/12;cuda-V100 -w diablo --gpus=1 -c 10 --exclusive;--fresh'
                     }
                 }
                 stages {

CMakeLists.txt

@@ -51,6 +51,9 @@ option(OCTOTIGER_WITH_QUADMATH "Enable sections using GCC Quad-Precision Math Li
 option(OCTOTIGER_WITH_Vc "" ON)
 # Re-enables Vc datatypes in the octotiger parts - use with caution
 option(OCTOTIGER_WITH_LEGACY_Vc "" OFF)
+# Use workaround to make Intel GPUs work as intended
+# May cause rare segfault during cleanup when using the KOKKOS SYCL backend on NVIDIA hardware
+option(OCTOTIGER_WITH_INTEL_GPU_WORKAROUND "This activates a workaround that enables SYCL builds to run on Intel GPUs with Octotiger (but cause problems for SYCL builds with CUDA" OFF)
 
 # TODO Can these safely be removed? They should be deprecated
 option(OCTOTIGER_WITH_AVX "" OFF)
@@ -742,6 +745,13 @@ elseif(OCTOTIGER_WITH_Vc AND USE_AVX AND OCTOTIGER_WITH_AVX)
   set(OCTOTIGER_WITH_AVX512 OFF CACHE BOOL "" FORCE)
 endif()
 
+# Intel GPU workarounds...
+if(OCTOTIGER_WITH_INTEL_GPU_WORKAROUND)
+  message(WARNING " Compiling with SYCL Intel GPU workaround (might cause cleanup segfaults when using the SYCL CUDA backend)")
+  target_compile_definitions(octolib PUBLIC OCTOTIGER_HAVE_INTEL_GPU_WORKAROUND)
+  target_compile_definitions(hydrolib PUBLIC OCTOTIGER_HAVE_INTEL_GPU_WORKAROUND)
+endif()
+
 # Handle CUDA
 if(OCTOTIGER_WITH_CUDA)
   message(INFO " CUDA support is turned on!")

src/monopole_interactions/monopole_kernel_interface.cpp

@@ -30,11 +30,15 @@
 #include "octotiger/options.hpp"
 
 #if defined(OCTOTIGER_HAVE_KOKKOS) && defined(KOKKOS_ENABLE_SYCL)
+#if defined(OCTOTIGER_HAVE_INTEL_GPU_WORKAROUND)
 #include "octotiger/sycl_initialization_guard.hpp"
 static const char module_identifier_monopoles[] = "gravity_solver_monopoles";
 /// Dummy variable to ensure the touch_sycl_device_by_running_a_dummy_kernel is being run
 const int init_sycl_device_monopoles =
     octotiger::sycl_util::touch_sycl_device_by_running_a_dummy_kernel<module_identifier_monopoles>();
+#else
+#pragma message "SYCL builds without OCTOTIGER_WITH_INTEL_GPU_WORKAROUND=ON may break on Intel GPUs"
+#endif
 #endif
 
 #if defined(OCTOTIGER_HAVE_KOKKOS)

src/multipole_interactions/multipole_kernel_interface.cpp

@@ -30,12 +30,16 @@
 #include "octotiger/options.hpp"
 
 #if defined(OCTOTIGER_HAVE_KOKKOS) && defined(KOKKOS_ENABLE_SYCL)
+#if defined(OCTOTIGER_HAVE_INTEL_GPU_WORKAROUND)
 #include "octotiger/sycl_initialization_guard.hpp"
 static const char module_identifier_multipoles[] = "gravity_solver_multipoles";
 /// Dummy variable to ensure the touch_sycl_device_by_running_a_dummy_kernel is being run
 const int init_sycl_device_multipoles =
     octotiger::sycl_util::touch_sycl_device_by_running_a_dummy_kernel<
         module_identifier_multipoles>();
+#else
+#pragma message "SYCL builds without OCTOTIGER_WITH_INTEL_GPU_WORKAROUND=ON may break on Intel GPUs"
+#endif
 #endif
 
 #ifdef OCTOTIGER_HAVE_KOKKOS

src/unitiger/hydro_impl/hydro_kernel_interface.cpp

@@ -15,13 +15,19 @@
 #ifdef OCTOTIGER_HAVE_KOKKOS
 #include "octotiger/unitiger/hydro_impl/hydro_kokkos_kernel.hpp"
 #endif
-#if defined(OCTOTIGER_HAVE_KOKKOS) && defined(KOKKOS_ENABLE_SYCL)
+
+#if defined(OCTOTIGER_HAVE_KOKKOS) && defined(KOKKOS_ENABLE_SYCL) 
+#if defined(OCTOTIGER_HAVE_INTEL_GPU_WORKAROUND)
 #include "octotiger/sycl_initialization_guard.hpp"
 static const char module_identifier_hydro[] = "hydro_solver";
 /// Dummy variable to ensure the touch_sycl_device_by_running_a_dummy_kernel is being run
 const int init_sycl_device_hydro =
     octotiger::sycl_util::touch_sycl_device_by_running_a_dummy_kernel<module_identifier_hydro>();
+#else
+#pragma message "SYCL builds without OCTOTIGER_WITH_INTEL_GPU_WORKAROUND=ON may break on Intel GPUs"
 #endif
+#endif
+
 #if defined(OCTOTIGER_HAVE_KOKKOS)
 hpx::once_flag init_hydro_kokkos_pool_flag;
 #if defined(KOKKOS_ENABLE_CUDA)