diff --git a/cmake/experiments/cuda_module_loading_lazy.cmake b/cmake/experiments/cuda_module_loading_lazy.cmake
index 531efb52daf8b..d1e07f57cb045 100644
--- a/cmake/experiments/cuda_module_loading_lazy.cmake
+++ b/cmake/experiments/cuda_module_loading_lazy.cmake
@@ -23,8 +23,8 @@ if(LINUX)
     )
     return()
   endif()
-  if(NOT ${CUDA_FOUND})
-    message("EXP_CUDA_MODULE_LOADING_LAZY only works with CUDA")
+  if(NOT WITH_GPU)
+    message("EXP_CUDA_MODULE_LOADING_LAZY only works with GPU")
     return()
   endif()
   if(${CUDA_VERSION} VERSION_GREATER_EQUAL "11.7")
diff --git a/tools/nvcc_lazy.sh b/tools/nvcc_lazy.sh
index efb0223ae6c25..7795836f8c199 100755
--- a/tools/nvcc_lazy.sh
+++ b/tools/nvcc_lazy.sh
@@ -36,6 +36,7 @@ echo -e >> $1
 echo "## CUDA_MODULE_LOADING=EAGER,DEFAULT,LAZY" >> $1
 echo -e >> $1
 echo "# set cicc PATH for Centos" >> $1
+echo "export PATH=\$PATH:$2/bin" >> $1
 echo "export PATH=\$PATH:$2/nvvm/bin" >> $1
 echo -e >> $1
 echo "# check nvcc version, if nvcc >= 11.7, just run nvcc itself" >> $1