feat: support QNN

.github/workflows/ci.yml

@@ -1,9 +1,16 @@
 name: CI
 on: push
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4

CMakeLists.txt

@@ -5,6 +5,9 @@ cmake_policy(SET CMP0042 NEW)
 project (llama-node)
 
 set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED true)
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_C_STANDARD_REQUIRED true)
 
 if(NOT DEFINED napi_build_version)
   set(napi_build_version 6)
@@ -68,7 +71,7 @@ find_program(PATCH patch REQUIRED)
 
 add_custom_target(
   patch ALL
-  COMMAND ${PATCH} -p1 -n -i ${CMAKE_SOURCE_DIR}/patches/llama.patch
+  COMMAND ${PATCH} -p1 -N -i ${CMAKE_SOURCE_DIR}/patches/llama.patch
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/llama.cpp
   COMMENT "Applying patches"
 )
@@ -92,6 +95,63 @@ file(
     "src/SaveSessionWorker.h"
 )
 
+if (LLAMA_QNN)
+  if (PLATFORM STREQUAL "linux" AND ARCH STREQUAL "x64")
+    set(QNN_PLATFORM "x86_64-linux-clang")
+  elseif (PLATFORM STREQUAL "linux" AND ARCH STREQUAL "arm64")
+    set(QNN_PLATFORM "aarch64-ubuntu-gcc7.5")
+  elseif (PLATFORM STREQUAL "win32" AND ARCH STREQUAL "x64")
+    set(QNN_PLATFORM "x86_64-windows-msvc")
+  elseif (PLATFORM STREQUAL "win32" AND ARCH STREQUAL "arm64")
+    set(QNN_PLATFORM "aarch64-windows-msvc")
+  endif()
+
+  if (NOT QNN_PLATFORM)
+    message(FATAL_ERROR "QNN is not supported on this platform")
+  endif()
+  set(QNN_LIB_PATH ${QNN_ROOT}/lib/${QNN_PLATFORM})
+  message(STATUS "QNN_LIB_PATH: ${QNN_LIB_PATH}")
+
+  file(
+    GLOB QNN_SO_FILES
+      "${QNN_LIB_PATH}/libc++*"
+      "${QNN_LIB_PATH}/libQnn*.so"
+      "${QNN_LIB_PATH}/Htp*.dll"
+      "${QNN_LIB_PATH}/Qnn*"
+  )
+
+  file(COPY ${QNN_SO_FILES} DESTINATION ${PLATFORM_BINARY_DIR})
+
+  file(GLOB QNN_EXTRA_FILES "${QNN_ROOT}/lib/hexagon-v*/unsigned/libQnn*Skel.so")
+
+  if (PLATFORM STREQUAL "win32")
+    list(APPEND QNN_EXTRA_FILES "${QNN_ROOT}/lib/hexagon-v73/unsigned/libqnnhtpv73.cat")
+  endif()
+
+  file(COPY ${QNN_EXTRA_FILES} DESTINATION ${PLATFORM_BINARY_DIR})
+
+  list(APPEND LINKS ${QNN_SO_FILES})
+
+  file(
+    GLOB QNN_HEADER_FILES
+      "src/ggml-qnn/ggml-qnn.h"
+  )
+
+  file(
+    GLOB QNN_SOURCE_FILES
+      "src/ggml-qnn/pthread-shim.h"
+      "src/ggml-qnn/ggml-qnn.cpp"
+  )
+
+  target_compile_definitions(ggml PUBLIC GGML_USE_QNN)
+  target_include_directories(ggml PUBLIC ${QNN_ROOT}/include ${QNN_ROOT}/include/QNN)
+  target_sources(ggml PRIVATE ${QNN_SOURCE_FILES} ${QNN_HEADER_FILES})
+  target_include_directories(llama PRIVATE "src/ggml-qnn")
+  set_target_properties(ggml PROPERTIES CXX_STANDARD 17)
+  set_target_properties(ggml PROPERTIES CXX_STANDARD_REQUIRED ON)
+  set_target_properties(ggml PROPERTIES C_STANDARD 11)
+endif()
+
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
 target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common)

package.json

@@ -39,6 +39,7 @@
   },
   "files": [
     "bin/**/*",
+    "patches/*",
     "src/**/*.{c,cc,cpp,h,hh,hpp,txt,cmake}",
     "lib/*.js",
     "lib/*.ts",

patches/llama.patch

@@ -1,3 +1,67 @@
+diff --git a/ggml-backend.c b/ggml-backend.c
+index e91d97cd..39d4efec 100644
+--- a/ggml-backend.c
++++ b/ggml-backend.c
+@@ -445,6 +445,11 @@ GGML_CALL static void ggml_backend_registry_init(void) {
+     extern GGML_CALL void ggml_backend_kompute_reg_devices(void);
+     ggml_backend_kompute_reg_devices();
+ #endif
++
++#ifdef GGML_USE_QNN
++    extern GGML_CALL void ggml_backend_qnn_reg_devices(void);
++    ggml_backend_qnn_reg_devices();
++#endif
+ }
+
+ GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {
+diff --git a/llama.cpp b/llama.cpp
+index a25d115c..7dedb2a1 100644
+--- a/llama.cpp
++++ b/llama.cpp
+@@ -17,6 +17,8 @@
+ #  include "ggml-sycl.h"
+ #elif defined(GGML_USE_KOMPUTE)
+ #   include "ggml-kompute.h"
++#elif defined(GGML_USE_QNN)
++#   include "ggml-qnn.h"
+ #endif
+
+ #ifdef GGML_USE_METAL
+@@ -1658,6 +1660,8 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) {
+     buft = ggml_backend_opencl_buffer_type();
+ #elif defined(GGML_USE_KOMPUTE)
+     buft = ggml_backend_kompute_buffer_type(gpu);
++#elif defined(GGML_USE_QNN)
++    buft = ggml_backend_qnn_buffer_type(gpu);
+     if (buft == nullptr) {
+         LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu);
+     }
+@@ -14916,7 +14920,7 @@ bool llama_supports_mlock(void) {
+
+ bool llama_supports_gpu_offload(void) {
+ #if defined(GGML_USE_CUDA) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \
+-    defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE)
++    defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_QNN)
+     // Defined when llama.cpp is compiled with support for offloading model layers to GPU.
+     return true;
+ #else
+@@ -15143,6 +15147,16 @@ struct llama_context * llama_new_context_with_model(
+                 ctx->backends.push_back(backend);
+             }
+         }
++#elif defined(GGML_USE_QNN)
++        if (model->n_gpu_layers > 0) {
++            auto * backend = ggml_backend_qnn_init(model->main_gpu);
++            if (backend == nullptr) {
++                LLAMA_LOG_ERROR("%s: failed to initialize Kompute backend\n", __func__);
++                llama_free(ctx);
++                return nullptr;
++            }
++            ctx->backends.push_back(backend);
++        }
+ #elif defined(GGML_USE_VULKAN)
+         if (model->split_mode == LLAMA_SPLIT_MODE_ROW) {
+             LLAMA_LOG_ERROR("%s: Row split not supported. Failed to initialize Vulkan backend\n", __func__);
 diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp
 index 1736ab73..55831936 100644
 --- a/ggml-vulkan.cpp

src/LlamaContext.cpp

@@ -57,6 +57,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   params.use_mmap = get_option<bool>(options, "use_mmap", true);
   params.numa =
       static_cast<ggml_numa_strategy>(get_option<uint32_t>(options, "numa", 0));
+  params.main_gpu = get_option<int32_t>(options, "main_gpu", 0);
 
   llama_backend_init();
   llama_numa_init(params.numa);