Skip to content

Commit 6db8957

Browse files
JinBridgerJinBridger
committed
Fix NPU LLM example save/load tokenizer
1 parent ab01753 commit 6db8957

File tree

11 files changed

+33
-11
lines changed

11 files changed

+33
-11
lines changed

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/Pipeline-Models/baichuan2.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
7979
transpose_value_cache=not args.disable_transpose_value_cache,
8080
trust_remote_code=True,
8181
save_directory=args.save_directory)
82+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
83+
tokenizer.save_pretrained(args.save_directory)
8284
else:
8385
model = AutoModelForCausalLM.load_low_bit(
8486
args.save_directory,
@@ -90,8 +92,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
9092
transpose_value_cache=not args.disable_transpose_value_cache,
9193
trust_remote_code=True
9294
)
95+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
9396

94-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
9597

9698
if args.disable_streaming:
9799
streamer = None

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/Pipeline-Models/llama2.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
7878
attn_implementation="eager",
7979
transpose_value_cache=not args.disable_transpose_value_cache,
8080
save_directory=args.save_directory)
81+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
82+
tokenizer.save_pretrained(args.save_directory)
8183
else:
8284
model = AutoModelForCausalLM.load_low_bit(
8385
args.save_directory,
@@ -88,8 +90,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
8890
pipeline=True,
8991
transpose_value_cache=not args.disable_transpose_value_cache,
9092
)
93+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
9194

92-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
9395

9496
if args.disable_streaming:
9597
streamer = None

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/Pipeline-Models/llama3.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]],
8484
attn_implementation="eager",
8585
transpose_value_cache=not args.disable_transpose_value_cache,
8686
save_directory=args.save_directory)
87+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
88+
tokenizer.save_pretrained(args.save_directory)
8789
else:
8890
model = AutoModelForCausalLM.load_low_bit(
8991
args.save_directory,
@@ -94,8 +96,8 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]],
9496
pipeline=True,
9597
transpose_value_cache=not args.disable_transpose_value_cache,
9698
)
99+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
97100

98-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
99101

100102
if args.disable_streaming:
101103
streamer = None

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/Pipeline-Models/minicpm.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@
6666
transpose_value_cache=not args.disable_transpose_value_cache,
6767
trust_remote_code=True,
6868
save_directory=args.save_directory)
69+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
70+
tokenizer.save_pretrained(args.save_directory)
6971
else:
7072
model = AutoModelForCausalLM.load_low_bit(
7173
args.save_directory,
@@ -77,8 +79,8 @@
7779
transpose_value_cache=not args.disable_transpose_value_cache,
7880
trust_remote_code=True
7981
)
82+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
8083

81-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
8284

8385
if args.disable_streaming:
8486
streamer = None

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/Pipeline-Models/qwen.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@
7070
mixed_precision=True,
7171
trust_remote_code=True,
7272
save_directory=args.save_directory)
73+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
74+
tokenizer.save_pretrained(args.save_directory)
7375
else:
7476
model = AutoModelForCausalLM.load_low_bit(
7577
args.save_directory,
@@ -79,8 +81,8 @@
7981
max_prompt_len=args.max_prompt_len,
8082
pipeline=True,
8183
transpose_value_cache=not args.disable_transpose_value_cache)
84+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
8285

83-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
8486

8587
if args.disable_streaming:
8688
streamer = None

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/baichuan2.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
7979
transpose_value_cache=not args.disable_transpose_value_cache,
8080
save_directory=args.save_directory
8181
)
82+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
83+
tokenizer.save_pretrained(args.save_directory)
8284
else:
8385
model = AutoModelForCausalLM.load_low_bit(
8486
args.save_directory,
@@ -90,8 +92,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
9092
transpose_value_cache=not args.disable_transpose_value_cache,
9193
trust_remote_code=True,
9294
)
95+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
9396

94-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
9597

9698
DEFAULT_SYSTEM_PROMPT = """\
9799
"""

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/generate.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
args = parser.parse_args()
4444
model_path = args.repo_id_or_model_path
4545

46-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
4746

4847
if not args.lowbit_path or not os.path.exists(args.lowbit_path):
4948
model = AutoModelForCausalLM.from_pretrained(
@@ -52,13 +51,16 @@
5251
load_in_low_bit=args.load_in_low_bit,
5352
attn_implementation="eager"
5453
)
54+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
55+
tokenizer.save_pretrained(args.lowbit_path)
5556
else:
5657
model = AutoModelForCausalLM.load_low_bit(
5758
args.lowbit_path,
5859
trust_remote_code=True,
5960
bigdl_transformers_low_bit=args.load_in_low_bit,
6061
attn_implementation="eager"
6162
)
63+
tokenizer = AutoTokenizer.from_pretrained(args.lowbit_path, trust_remote_code=True)
6264

6365
print(model)
6466

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama2.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
7979
transpose_value_cache=not args.disable_transpose_value_cache,
8080
save_directory=args.save_directory
8181
)
82+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
83+
tokenizer.save_pretrained(args.save_directory)
8284
else:
8385
model = AutoModelForCausalLM.load_low_bit(
8486
args.save_directory,
@@ -89,8 +91,8 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]],
8991
max_prompt_len=args.max_prompt_len,
9092
transpose_value_cache=not args.disable_transpose_value_cache,
9193
)
94+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
9295

93-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
9496

9597
DEFAULT_SYSTEM_PROMPT = """\
9698
"""

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama3.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]],
8080
transpose_value_cache=not args.disable_transpose_value_cache,
8181
save_directory=args.save_directory
8282
)
83+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
84+
tokenizer.save_pretrained(args.save_directory)
8385
else:
8486
model = AutoModelForCausalLM.load_low_bit(
8587
args.save_directory,
@@ -90,8 +92,8 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]],
9092
max_prompt_len=args.max_prompt_len,
9193
transpose_value_cache=not args.disable_transpose_value_cache,
9294
)
95+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
9396

94-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
9597

9698
DEFAULT_SYSTEM_PROMPT = """\
9799
"""

‎python/llm/example/NPU/HF-Transformers-AutoModels/LLM/minicpm.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@
6565
transpose_value_cache=not args.disable_transpose_value_cache,
6666
save_directory=args.save_directory
6767
)
68+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
69+
tokenizer.save_pretrained(args.save_directory)
6870
else:
6971
model = AutoModelForCausalLM.load_low_bit(
7072
args.save_directory,
@@ -76,7 +78,7 @@
7678
transpose_value_cache=not args.disable_transpose_value_cache,
7779
trust_remote_code=True,
7880
)
79-
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
81+
tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True)
8082

8183
print("-" * 80)
8284
print("done")

0 commit comments

Comments
 (0)