Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Removed fallback for lm_head op #1482

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Removed fallback for lm_head op #1482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
aef6112
Removed fallback for lm_head op
PenghuiCheng Apr 15, 2024
0cbaa50
Fixed load issue
PenghuiCheng Apr 15, 2024
78a7fc1
update_script
PenghuiCheng Apr 15, 2024
57a65ec
Update run_generation_gpu_woq.py
airMeng Apr 16, 2024
1f5c68d
Update requirements_GPU.txt
airMeng Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/huggingface/pytorch/text-generation/quantization/requirements_GPU.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ protobuf
sentencepiece != 0.1.92
--extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
torch==2.1.0a0
transformers
transformers==4.35
optimum-intel
bitsandbytes #baichuan
transformers_stream_generator
Expand Down
10 changes: 4 additions & 6 deletions examples/huggingface/pytorch/text-generation/quantization/run_generation_gpu_woq.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,7 @@
user_model = None

# tokenizer
if config.model_type == "llama":
from transformers import LlamaTokenizer
tokenizer = LlamaTokenizer.from_pretrained(args.model)
else:
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=args.trust_remote_code)
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=args.trust_remote_code)

quantization_config = None
if args.woq:
Expand Down Expand Up @@ -253,7 +249,9 @@
dtype=amp_dtype if amp_enabled else None,
):
for i in range(num_iter + num_warmup):
with torch.autograd.profiler_legacy.profile(enabled=args.do_profiling, use_xpu=(args.device=="xpu"), record_shapes=False) as prof:
# workaround for Windows
# with torch.autograd.profiler_legacy.profile(enabled=args.do_profiling, use_xpu=(args.device=="xpu"), record_shapes=False) as prof:
if True
input_ids = tokenizer(
prompt, return_tensors="pt").input_ids.to(args.device)
tic = time.time()
Expand Down
15 changes: 1 addition & 14 deletions intel_extension_for_transformers/transformers/llm/quantization/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,7 @@ def replace_linear(
empty_weights=False,
):
if modules_to_not_convert is None:
# output_layer is chatglm last layer name
# embed_out is dolly_v2 last layer name
modules_to_not_convert = ["lm_head", "output_layer", "embed_out"]
modules_to_not_convert = []
if quantization_config.llm_int8_skip_modules:
modules_to_not_convert = modules_to_not_convert.extend(
quantization_config.llm_int8_skip_modules
Expand Down Expand Up @@ -517,17 +515,6 @@ def default_calib_func(model):
},
},
},
op_name_dict={
".*lm_head": { # re.match
"weight": {"dtype": "fp32"},
},
".*output_layer": { # re.match
"weight": {"dtype": "fp32"},
},
".*embed_out": { # re.match
"weight": {"dtype": "fp32"},
},
},
recipes=recipes,
)
# TEQ: set calib_func=None, use default training func as calib_func
Expand Down
Loading