Update tests for transformers 4.36

.github/workflows/llm-harness-evaluation.yml

@@ -164,12 +164,6 @@ jobs:
         shell: bash
         run: |
           pip install --upgrade datasets==2.14.6 
-          if [ "${{ matrix.model_name }}" = "Mistral-7B-v0.1" ]; then
-            pip install --upgrade transformers==4.36
-          else
-            pip install --upgrade transformers==4.31
-          fi
-
 
       - name: Run harness
         shell: bash

.github/workflows/llm-ppl-evaluation.yml

@@ -144,16 +144,11 @@ jobs:
           echo "MODEL_PATH=${ORIGIN_DIR}/${{ matrix.model_name }}/" >> "$GITHUB_ENV"
           MODEL_PATH=${ORIGIN_DIR}/${{ matrix.model_name }}/
           wget -r -nH -nc --no-verbose --cut-dirs=1 ${LLM_FTP_URL}/llm/${{ matrix.model_name }} -P ${ORIGIN_DIR}
-          
+
       - name: Upgrade packages
         shell: bash
         run: |
-          pip install --upgrade datasets==2.14.6 
-          if [ "${{ matrix.model_name }}" = "Mistral-7B-v0.1" ]; then
-            pip install --upgrade transformers==4.36
-          else
-            pip install --upgrade transformers==4.31
-          fi
+          pip install --upgrade datasets==2.14.6
 
       - name: Run perplexity
         shell: bash

.github/workflows/llm_tests_for_stable_version_on_spr.yml

@@ -13,9 +13,9 @@ on:
   # pull_request:
   #   branches: [main]
   #   paths:
-  #     - ".github/workflows/llm_performance_tests.yml"
-  #     - "python/llm/test/benchmark/**"
-  #     - "python/llm/dev/benchmark/all-in-one/**"
+  #    - ".github/workflows/llm_performance_tests.yml"
+  #    - "python/llm/test/benchmark/**"
+  #    - "python/llm/dev/benchmark/all-in-one/**"
   workflow_dispatch:
   workflow_call:
 

.github/workflows/llm_unit_tests.yml

@@ -99,7 +99,7 @@ jobs:
           echo "LLAMA_ORIGIN_PATH=${ORIGIN_DIR}/llama-7b-hf" >> "$GITHUB_ENV"
           echo "BLOOM_ORIGIN_PATH=${ORIGIN_DIR}/bloom-7b1" >> "$GITHUB_ENV"
           echo "ORIGINAL_CHATGLM2_6B_PATH=${ORIGIN_DIR}/chatglm2-6b" >> "$GITHUB_ENV"
-          echo "ORIGINAL_REPLIT_CODE_PATH=${ORIGIN_DIR}/replit-code-v1-3b" >> "$GITHUB_ENV"
+          echo "ORIGINAL_CODESHELL_7B_PATH=${ORIGIN_DIR}/CodeShell-7B-Chat" >> "$GITHUB_ENV"
           echo "ORIGINAL_WHISPER_TINY_PATH=${ORIGIN_DIR}/whisper-tiny" >> "$GITHUB_ENV"
           echo "MISTRAL_ORIGIN_PATH=${ORIGIN_DIR}/Mistral-7B-v0.1" >> "$GITHUB_ENV"
           echo "LLAMA2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Llama-2-7b-chat-hf" >> "$GITHUB_ENV"
@@ -157,13 +157,13 @@ jobs:
           # fi
           if [ ! -d $ORIGINAL_CHATGLM2_6B_PATH ]; then
             echo "Directory $ORIGINAL_CHATGLM2_6B_PATH not found. Downloading from FTP server..."
-            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/chatglm2-6b -P $ORIGIN_DIR"            
+            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/chatglm2-6b -P $ORIGIN_DIR"
             wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/chatglm2-6b -P $ORIGIN_DIR
           fi
-          if [ ! -d $ORIGINAL_REPLIT_CODE_PATH ]; then
-            echo "Directory $ORIGINAL_REPLIT_CODE_PATH not found. Downloading from FTP server..."
-            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/replit-code-v1-3b -P $ORIGIN_DIR"
-            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/replit-code-v1-3b -P $ORIGIN_DIR
+          if [ ! -d $ORIGINAL_CODESHELL_7B_PATH ]; then
+            echo "Directory $ORIGINAL_CODESHELL_7B_PATH not found. Downloading from FTP server..."
+            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/CodeShell-7B-Chat -P $ORIGIN_DIR"
+            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/CodeShell-7B-Chat -P $ORIGIN_DIR
           fi
           if [ ! -d $ORIGINAL_WHISPER_TINY_PATH ]; then
             echo "Directory $ORIGINAL_WHISPER_TINY_PATH not found. Downloading from FTP server..."
@@ -226,14 +226,15 @@ jobs:
         shell: bash
         run: |
           pip install llama-index-readers-file llama-index-vector-stores-postgres llama-index-embeddings-huggingface
-          pip install transformers==4.36.0
+          pip install transformers==4.36.2
           pip install "pydantic>=2.0.0"
           bash python/llm/test/run-llm-llamaindex-tests.sh
       - name: Run sentence-transformers uninstallation
         if: ${{ always() }}
         shell: bash
         run: |
           pip uninstall sentence-transformers -y || true
+
   llm-unit-test-on-arc:
     needs: [setup-python-version, llm-cpp-build]
     strategy:
@@ -364,8 +365,6 @@ jobs:
           fi
           python -m pip install datasets librosa soundfile einops tiktoken transformers_stream_generator
           bash python/llm/test/run-llm-inference-tests-gpu.sh
-          python -m pip install transformers==4.34.0 
-          bash python/llm/test/run-llm-inference-tests-gpu-434.sh
 
       - name: Run LLM example tests
         shell: bash
@@ -428,7 +427,7 @@ jobs:
             pip install --pre --upgrade ipex-llm[xpu_2.0] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
             source /home/arda/intel/oneapi/setvars.sh
           fi
-          pip install transformers==4.36.0
+          pip install transformers==4.36.2
           pip install "pydantic>=2.0.0"
           bash python/llm/test/run-llm-llamaindex-tests-gpu.sh
       - name: Run sentence-transformers uninstallation

python/llm/setup.py

@@ -53,7 +53,7 @@
 
 cpu_torch_version = ["torch==2.1.2+cpu;platform_system=='Linux'", "torch==2.1.2;platform_system=='Windows'"]
 CONVERT_DEP = ['numpy == 1.26.4', # lastet 2.0.0b1 will cause error
-               'transformers == 4.31.0', 'sentencepiece', 'tokenizers == 0.13.3',
+               'transformers == 4.36.2', 'sentencepiece', 'tokenizers == 0.15.2',
                # TODO: Support accelerate 0.22.0
                'accelerate == 0.21.0', 'tabulate'] + cpu_torch_version
 
@@ -279,10 +279,9 @@ def setup_package():
 
     # Add internal requires for llama-index
     llama_index_requires = copy.deepcopy(all_requires)
-    for exclude_require in ['transformers == 4.31.0', 'tokenizers == 0.13.3'] + cpu_torch_version:
+    for exclude_require in cpu_torch_version:
         llama_index_requires.remove(exclude_require)
     llama_index_requires += ["torch<2.2.0",
-                             "transformers>=4.34.0,<4.39.0",
                              "sentence-transformers~=2.6.1"]
 
 

python/llm/src/ipex_llm/optimize.py

@@ -47,7 +47,8 @@ def _save_low_bit(self, save_dir, *args, **kwargs):
     if isinstance(self, PreTrainedModel):
         # We borrowed this method to adapt to Transformer model cases
         # as much as possible, and later we may merge these two situations
-        self.save_pretrained(save_dir)
+        kwargs['safe_serialization'] = False
+        self.save_pretrained(save_dir, *args, **kwargs)
     else:
         # TODO: For the lowbit model still larger than 8GB,
         #       save it into shards.

python/llm/test/benchmark/arc-perf-test.yaml

@@ -10,13 +10,14 @@ repo_id:
   - 'databricks/dolly-v1-6b'
   - 'databricks/dolly-v2-7b'
   - 'databricks/dolly-v2-12b'
-  - 'internlm/internlm-chat-7b-8k'
+  - 'internlm/internlm-chat-7b'
   - 'Qwen/Qwen-7B-Chat'
   - 'BAAI/AquilaChat-7B'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
   - 'bigscience/bloomz-7b1'
-  - 'fnlp/moss-moon-003-sft-4bit'
+#  - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
+  - 'mistralai/Mistral-7B-v0.1'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -31,7 +32,7 @@ test_api:
   - "transformer_int4_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
-  - 'fnlp/moss-moon-003-sft-4bit:1024'
-  - 'fnlp/moss-moon-003-sft-4bit:2048'
+#  - 'fnlp/moss-moon-003-sft-4bit:1024'
+#  - 'fnlp/moss-moon-003-sft-4bit:2048'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
   - 'bigscience/bloomz-7b1:2048'

python/llm/test/benchmark/igpu-perf/1024-128.yaml

@@ -12,10 +12,11 @@ repo_id:
   - 'WisdomShell/CodeShell-7B-Chat'
   - 'tiiuae/falcon-7b-instruct-with-patch'
   - 'mosaicml/mpt-7b-chat'
-  - 'liuhaotian/llava-v1.5-7b'
+#  - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+
   - 'RWKV/rwkv-4-world-7b'
   - 'RWKV/rwkv-5-world-7b'
   - 'IEITYuan/Yuan2-2B-hf'
+  - 'mistralai/Mistral-7B-Instruct-v0.1'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml

@@ -12,10 +12,11 @@ repo_id:
   - 'WisdomShell/CodeShell-7B-Chat'
   - 'tiiuae/falcon-7b-instruct-with-patch'
   - 'mosaicml/mpt-7b-chat'
-  - 'liuhaotian/llava-v1.5-7b'
+#  - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+
   # - 'RWKV/rwkv-4-world-7b'
   # - 'RWKV/rwkv-5-world-7b'
   - 'IEITYuan/Yuan2-2B-hf'
+  - 'mistralai/Mistral-7B-Instruct-v0.1'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml

@@ -12,10 +12,11 @@ repo_id:
   - 'WisdomShell/CodeShell-7B-Chat'
   - 'tiiuae/falcon-7b-instruct-with-patch'
   - 'mosaicml/mpt-7b-chat'
-  - 'liuhaotian/llava-v1.5-7b'
+#  - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+
   - 'RWKV/rwkv-4-world-7b'
   - 'RWKV/rwkv-5-world-7b'
   - 'IEITYuan/Yuan2-2B-hf'
+  - 'mistralai/Mistral-7B-Instruct-v0.1'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

python/llm/test/benchmark/igpu-perf/2048-256.yaml

@@ -12,10 +12,11 @@ repo_id:
   - 'WisdomShell/CodeShell-7B-Chat'
   - 'tiiuae/falcon-7b-instruct-with-patch'
   - 'mosaicml/mpt-7b-chat'
-  - 'liuhaotian/llava-v1.5-7b'
+#  - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+
   - 'RWKV/rwkv-4-world-7b'
   - 'RWKV/rwkv-5-world-7b'
   - 'IEITYuan/Yuan2-2B-hf'
+  - 'mistralai/Mistral-7B-Instruct-v0.1'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

python/llm/test/benchmark/igpu-perf/32-32.yaml

@@ -12,10 +12,11 @@ repo_id:
   - 'WisdomShell/CodeShell-7B-Chat'
   - 'tiiuae/falcon-7b-instruct-with-patch'
   - 'mosaicml/mpt-7b-chat'
-  - 'liuhaotian/llava-v1.5-7b'
+#  - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+
   - 'RWKV/rwkv-4-world-7b'
   - 'RWKV/rwkv-5-world-7b'
   - 'IEITYuan/Yuan2-2B-hf'
+  - 'mistralai/Mistral-7B-Instruct-v0.1'
 local_model_hub: 'path to your local model hub'
 warm_up: 3
 num_trials: 5

python/llm/test/benchmark/stable-version-cpu-perf-test.yaml

@@ -6,7 +6,7 @@ repo_id:
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat'
   - 'Qwen/Qwen-14B-Chat'
-local_model_hub: '/models'
+local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
 num_beams: 1 # default to greedy search

python/llm/test/benchmark/stable-version-cpu-stress-test.yaml

@@ -6,7 +6,7 @@ repo_id:
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat'
   - 'Qwen/Qwen-14B-Chat'
-local_model_hub: '/models'
+local_model_hub: '/mnt/disk1/models'
 warm_up: 3
 num_trials: 50
 num_beams: 1 # default to greedy search