From 759c7dce63bb40cc60fbb1094b1355c103df2a89 Mon Sep 17 00:00:00 2001
From: zepan <ze.pan@intel.com>
Date: Tue, 11 Jun 2024 17:37:12 +0800
Subject: [PATCH 1/3] Enable vllm backend.

Signed-off-by: zepan <ze.pan@intel.com>
---
 examples/.config/vllm_optimize.json | 28 ++++++++++
 examples/vllm/run_tuning.sh         | 81 +++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 examples/.config/vllm_optimize.json
 create mode 100644 examples/vllm/run_tuning.sh

diff --git a/examples/.config/vllm_optimize.json b/examples/.config/vllm_optimize.json
new file mode 100644
index 00000000000..1d44309de4b
--- /dev/null
+++ b/examples/.config/vllm_optimize.json
@@ -0,0 +1,28 @@
+{
+  "chatglm2_6b_gen_ipex_static": {
+    "working_dir": "examples/vllm",
+    "tune": {
+      "cmd": "bash run_tuning.sh",
+      "params": {
+        "topology": "chatglm2_6b",
+        "task": "generation",
+        "approach": "static",
+        "output_model": "saved_results"
+      }
+    },
+    "benchmark": {
+      "cmd": "bash run_benchmark.sh",
+      "params": {
+        "topology": "chatglm2_6b",
+        "task": "generation",
+        "approach": "static",
+        "backend": "ipex",
+        "mode": "benchmark",
+        "batch_size": "112",
+        "iters": "100",
+        "int8": "false",
+        "config": "saved_results"
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/examples/vllm/run_tuning.sh b/examples/vllm/run_tuning.sh
new file mode 100644
index 00000000000..ef198d1b527
--- /dev/null
+++ b/examples/vllm/run_tuning.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+set -x
+
+function main {
+
+  init_params "$@"
+  run_tuning
+
+}
+
+# init params
+function init_params {
+  topology="gpt_j"
+  tuned_checkpoint="saved_results"
+  DATASET_NAME="NeelNanda/pile-10k"
+  model_name_or_path="EleutherAI/gpt-j-6b"
+  extra_cmd=""
+  batch_size=8
+  approach="PostTrainingStatic"
+  script="vllm_acceleration_example.py"
+  alpha=0.5
+  weight_dtype="int4_clip"
+  scheme="asym"
+  for var in "$@"
+  do
+    case $var in
+      --topology=*)
+          topology=$(echo $var |cut -f2 -d=)
+      ;;
+      --dataset_location=*)
+          dataset_location=$(echo $var |cut -f2 -d=)
+      ;;
+      --input_model=*)
+          input_model=$(echo $var |cut -f2 -d=)
+      ;;
+       --output_model=*)
+           tuned_checkpoint=$(echo $var |cut -f2 -d=)
+       ;;
+       --task=*)
+           task=$(echo $var |cut -f2 -d=)
+       ;;
+       --approach=*)
+           approach=$(echo $var |cut -f2 -d=)
+       ;;
+       --weight_dtype=*)
+           weight_dtype=$(echo $var |cut -f2 -d=)
+       ;;
+       --bits=*)
+           bits=$(echo $var |cut -f2 -d=)
+       ;;
+       --scheme=*)
+           scheme=$(echo $var |cut -f2 -d=)
+       ;;
+      *)
+          echo "Error: No such parameter: ${var}"
+          exit 1
+      ;;
+    esac
+  done
+
+}
+
+# run_tuning
+function run_tuning {
+    if [ "${topology}" = "chatglm2_6b" ]; then
+        model_name_or_path="THUDM/chatglm2-6b"
+        script="vllm_acceleration_example.py"
+    fi
+
+    if [ ${script} = "vllm_acceleration_example.py" ];then
+        pip install requirement.txt
+        python -u ./${script} \
+            --model ${model_name_or_path} \
+            --prompt=你好
+    else
+        echo "Error: Please provide the correct script."
+        exit 1
+    fi
+}
+
+main "$@"

From ad7369d297a580ff1efc55a6c55e401a6f6abebf Mon Sep 17 00:00:00 2001
From: zepan <ze.pan@intel.com>
Date: Wed, 12 Jun 2024 22:21:50 +0800
Subject: [PATCH 2/3] Modify the working_dir for vllm chatglm2_6b.

Signed-off-by: zepan <ze.pan@intel.com>
---
 examples/.config/vllm_optimize.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/.config/vllm_optimize.json b/examples/.config/vllm_optimize.json
index 1d44309de4b..3b0507d3ebb 100644
--- a/examples/.config/vllm_optimize.json
+++ b/examples/.config/vllm_optimize.json
@@ -1,6 +1,6 @@
 {
   "chatglm2_6b_gen_ipex_static": {
-    "working_dir": "examples/vllm",
+    "working_dir": "vllm",
     "tune": {
       "cmd": "bash run_tuning.sh",
       "params": {

From 37f4411136ddcfd91ac2ae1da7800104e81ce4ac Mon Sep 17 00:00:00 2001
From: zepan <ze.pan@intel.com>
Date: Mon, 17 Jun 2024 22:00:21 +0800
Subject: [PATCH 3/3] modify vllm_optimize.json

Signed-off-by: zepan <ze.pan@intel.com>
---
 examples/.config/vllm_optimize.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/.config/vllm_optimize.json b/examples/.config/vllm_optimize.json
index 3b0507d3ebb..bfabe9e8bcf 100644
--- a/examples/.config/vllm_optimize.json
+++ b/examples/.config/vllm_optimize.json
@@ -1,5 +1,5 @@
 {
-  "chatglm2_6b_gen_ipex_static": {
+  "chatglm2_6b": {
     "working_dir": "vllm",
     "tune": {
       "cmd": "bash run_tuning.sh",