Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix Llava conversion for LlavaQwen2ForCausalLM with Clip vision tower #33613

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+7 −4
Open

Fix Llava conversion for LlavaQwen2ForCausalLM with Clip vision tower #33613

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions src/transformers/models/llava/convert_llava_weights_to_hf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ def load_original_state_dict(model_id):
if "lm_head.weight" not in original_state_dict:
original_state_dict["lm_head.weight"] = original_state_dict["model.embed_tokens.weight"].clone()

del original_state_dict["model.image_newline"] # not used in the original implementation because "merge_type=flat"
if "model.image_newline" in original_state_dict:
# not used in the original implementation because "merge_type=flat"
del original_state_dict["model.image_newline"]
return original_state_dict


Expand Down Expand Up @@ -107,7 +109,7 @@ def convert_llava_llama_to_hf(text_model_id, vision_model_id, output_hub_path, o
image_processor = AutoImageProcessor.from_pretrained(vision_model_id)
processor = LlavaProcessor(tokenizer=tokenizer, image_processor=image_processor)

if "Qwen" in text_model_id:
if "siglip" in vision_model_id:
vision_config = SiglipVisionConfig(
hidden_size=1152,
image_size=384,
Expand All @@ -128,8 +130,9 @@ def convert_llava_llama_to_hf(text_model_id, vision_model_id, output_hub_path, o
# llms-lab interleeave models do not use any selection startegy except for last hidden state
if "Qwen" in text_model_id:
config.image_token_index = 151646
config.vision_feature_select_strategy = "full"
config.vision_feature_layer = -1
if "siglip" in vision_model_id:
config.vision_feature_select_strategy = "full"
config.vision_feature_layer = -1
else:
config.pad_token_id = 32001
config.image_token_index = 32000
Expand Down