change mixed_precision to q6_k (#11706)

intel-analytics · Aug 2, 2024 · aa98ef9 · aa98ef9
1 parent 1baa3ef
commit aa98ef9
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/python/llm/src/ipex_llm/transformers/convert.py b/python/llm/src/ipex_llm/transformers/convert.py
@@ -394,7 +394,7 @@ def _replace_with_low_bit_linear(model, qtype, modules_to_not_convert=None,
                     if mixed_precision and is_lm_head(name, model_config, out_features):
                         if cur_qtype in [ggml_tensor_qtype["sym_int4"],
                                          ggml_tensor_qtype["asym_int4"]]:
-                            cur_qtype = ggml_tensor_qtype["sym_int8"]
+                            cur_qtype = ggml_tensor_qtype["q6_k"]
 
                     # check hidden size whether is a multiple of 256
                     cur_qtype = check_hidden_size(cur_qtype, in_features)