Skip to content

Commit

Permalink
add BC args order change idefics2 processor and update doc
Browse files Browse the repository at this point in the history
  • Loading branch information
yonigozlan committed Aug 12, 2024
1 parent 79bd90a commit 9799303
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 2 deletions.
2 changes: 1 addition & 1 deletion src/transformers/models/idefics2/modeling_idefics2.py
Original file line number Diff line number Diff line change
Expand Up @@ -1558,7 +1558,7 @@ def forward(
... "In which city is that bridge located?<image>",
... ]
>>> images = [[image1, image2], [image3]]
>>> inputs = processor(text=prompts, images=images, padding=True, return_tensors="pt").to("cuda")
>>> inputs = processor(images=images, text=prompts, padding=True, return_tensors="pt").to("cuda")
>>> # Generate
>>> generated_ids = model.generate(**inputs, bad_words_ids=BAD_WORDS_IDS, max_new_tokens=20)
Expand Down
21 changes: 20 additions & 1 deletion src/transformers/models/idefics2/processing_idefics2.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"""

import sys
import warnings
from typing import TYPE_CHECKING, List, Optional, Union

from ...feature_extraction_utils import BatchFeature
Expand Down Expand Up @@ -149,7 +150,7 @@ def __call__(
... "<image>In this image, we see",
... "bla bla bla<image>",
... ]
>>> outputs = processor(text=text, images=images, return_tensors="pt", padding=True)
>>> outputs = processor(images=images, text=text, return_tensors="pt", padding=True)
>>> input_ids = outputs.input_ids
>>> input_tokens = processor.tokenizer.batch_decode(input_ids)
>>> print(input_tokens)
Expand All @@ -169,6 +170,24 @@ def __call__(
`<fake_token_around_image>` + `<image>` * `image_seq_len` * <fake_token_around_image>`.
"""
if text is None and images is None:
raise ValueError("You must provide either `text` or `images`.")
# check if images and text inputs are reversed for BC
if (
text is not None
and not isinstance(text[0], str)
or images is not None
and not (
is_image_or_image_url(images)
or is_image_or_image_url(images[0])
or (isinstance(images[0], list) and is_image_or_image_url(images[0][0]))
)
):
warnings.warn(
"It looks like you are passing the inputs in the wrong order. You should pass the images input first and the text input second."
"Images and text inputs will be swapped."
)
images, text = text, images

output_kwargs = self._merge_kwargs(
Idefics2ProcessorKwargs,
Expand Down

0 comments on commit 9799303

Please sign in to comment.