diff --git a/container-images/scripts/build_llama_and_whisper.sh b/container-images/scripts/build_llama_and_whisper.sh
index 2a573634..bc2252bd 100644
--- a/container-images/scripts/build_llama_and_whisper.sh
+++ b/container-images/scripts/build_llama_and_whisper.sh
@@ -31,9 +31,12 @@ dnf_install() {
       "${rpm_list[@]}"
   elif [ "$containerfile" = "rocm" ]; then
     dnf install -y rocm-dev hipblas-devel rocblas-devel
-  else
+  elif [ "$containerfile" = "cuda" ]; then
     dnf install -y "${rpm_list[@]}"
   fi
+
+  # For Vulkan image, we don't need to install anything extra but rebuild with
+  # -DGGML_VULKAN
 }
 
 cmake_steps() {