diff --git a/src/transformers/models/blip_2/convert_blip_2_original_to_pytorch.py b/src/transformers/models/blip_2/convert_blip_2_original_to_pytorch.py
index f571be20e13577..0ddcbbdea4eff2 100644
--- a/src/transformers/models/blip_2/convert_blip_2_original_to_pytorch.py
+++ b/src/transformers/models/blip_2/convert_blip_2_original_to_pytorch.py
@@ -150,13 +150,13 @@ def convert_blip2_checkpoint(
     if "opt" in model_name:
         tokenizer = AutoTokenizer.from_pretrained("facebook/opt-2.7b")
     elif "itm" in model_name:
-        tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", truncation_side="right")
+        tokenizer = BertTokenizer.from_pretrained(
+            "bert-base-uncased", truncation_side="right", model_input_names=["input_ids", "attention_mask"]
+        )
         tokenizer.add_special_tokens({"bos_token": "[DEC]"})
     else:
         tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xl")
 
-    tokenizer.model_input_names = ["input_ids", "attention_mask"]
-
     if "itm" in model_name:
         eos_token_id = None
     else: