Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@leloykun
leloykun committed Sep 24, 2024
1 parent 3e797c3 commit 964be32
Show file tree
Hide file tree
Showing 3 changed files with 271 additions and 29 deletions.
132 changes: 132 additions & 0 deletions tests/models/layoutlmv2/test_processor_layoutlmv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,138 @@ def test_model_specific_kwargs(self):
apply_ocr=False,
)

def test_image_processor_defaults_preserved_by_image_kwargs(self):
"""
We use do_rescale=True, rescale_factor=-1 to ensure that image_processor kwargs are preserved in the processor.
We then check that the mean of the pixel_values is less than or equal to 0 after processing.
Since the original pixel_values are in [0, 255], this is a good indicator that the rescale_factor is indeed applied.
"""
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor_components["image_processor"] = self.get_component(
"image_processor", do_rescale=True, rescale_factor=-1
)
processor_components["tokenizer"] = self.get_component("tokenizer", max_length=117, padding="max_length")

processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, return_tensors="pt")
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)

def test_kwargs_overrides_default_image_processor_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor_components["image_processor"] = self.get_component(
"image_processor", do_rescale=True, rescale_factor=1
)
processor_components["tokenizer"] = self.get_component("tokenizer", max_length=117, padding="max_length")

processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, do_rescale=True, rescale_factor=-1, return_tensors="pt")
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)

def test_unstructured_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()
max_length = 76
inputs = processor(
text=input_str,
images=image_input,
return_tensors="pt",
do_rescale=True,
rescale_factor=-1,
padding="max_length",
max_length=max_length,
)

self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

def test_unstructured_kwargs_batched(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = ["lower newer", "upper older longer string"]
image_input = self.prepare_image_inputs() * 2
inputs = processor(
text=input_str,
images=image_input,
return_tensors="pt",
do_rescale=True,
rescale_factor=-1,
padding="longest",
max_length=76,
)

self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertTrue(
len(inputs[self.text_input_name][0]) == len(inputs[self.text_input_name][1])
and len(inputs[self.text_input_name][1]) < 76
)

def test_structured_kwargs_nested(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
all_kwargs = {
"common_kwargs": {"return_tensors": "pt"},
"images_kwargs": {"do_rescale": True, "rescale_factor": -1},
"text_kwargs": {"padding": "max_length", "max_length": 76},
}

inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.skip_processor_without_typed_kwargs(processor)

self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

def test_structured_kwargs_nested_from_dict(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
all_kwargs = {
"common_kwargs": {"return_tensors": "pt"},
"images_kwargs": {"do_rescale": True, "rescale_factor": -1},
"text_kwargs": {"padding": "max_length", "max_length": 76},
}

inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)


# different use cases tests
@require_torch
Expand Down
132 changes: 132 additions & 0 deletions tests/models/layoutxlm/test_processor_layoutxlm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,138 @@ def test_model_specific_kwargs(self):
apply_ocr=False,
)

def test_image_processor_defaults_preserved_by_image_kwargs(self):
"""
We use do_rescale=True, rescale_factor=-1 to ensure that image_processor kwargs are preserved in the processor.
We then check that the mean of the pixel_values is less than or equal to 0 after processing.
Since the original pixel_values are in [0, 255], this is a good indicator that the rescale_factor is indeed applied.
"""
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor_components["image_processor"] = self.get_component(
"image_processor", do_rescale=True, rescale_factor=-1
)
processor_components["tokenizer"] = self.get_component("tokenizer", max_length=117, padding="max_length")

processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, return_tensors="pt")
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)

def test_kwargs_overrides_default_image_processor_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor_components["image_processor"] = self.get_component(
"image_processor", do_rescale=True, rescale_factor=1
)
processor_components["tokenizer"] = self.get_component("tokenizer", max_length=117, padding="max_length")

processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, do_rescale=True, rescale_factor=-1, return_tensors="pt")
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)

def test_unstructured_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()
max_length = 76
inputs = processor(
text=input_str,
images=image_input,
return_tensors="pt",
do_rescale=True,
rescale_factor=-1,
padding="max_length",
max_length=max_length,
)

self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

def test_unstructured_kwargs_batched(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = ["lower newer", "upper older longer string"]
image_input = self.prepare_image_inputs() * 2
inputs = processor(
text=input_str,
images=image_input,
return_tensors="pt",
do_rescale=True,
rescale_factor=-1,
padding="longest",
max_length=76,
)

self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertTrue(
len(inputs[self.text_input_name][0]) == len(inputs[self.text_input_name][1])
and len(inputs[self.text_input_name][1]) < 76
)

def test_structured_kwargs_nested(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
all_kwargs = {
"common_kwargs": {"return_tensors": "pt"},
"images_kwargs": {"do_rescale": True, "rescale_factor": -1},
"text_kwargs": {"padding": "max_length", "max_length": 76},
}

inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.skip_processor_without_typed_kwargs(processor)

self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

def test_structured_kwargs_nested_from_dict(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
processor_components = self.prepare_components()
processor = self.processor_class(**processor_components)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
all_kwargs = {
"common_kwargs": {"return_tensors": "pt"},
"images_kwargs": {"do_rescale": True, "rescale_factor": -1},
"text_kwargs": {"padding": "max_length", "max_length": 76},
}

inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)


# different use cases tests
@require_sentencepiece
Expand Down
36 changes: 7 additions & 29 deletions tests/test_processing_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
require_torch,
require_vision,
)
from transformers.utils import is_torch_available, is_vision_available
from transformers.utils import is_vision_available


try:
Expand All @@ -36,10 +36,6 @@
from typing_extensions import Unpack


if is_torch_available():
import torch


if is_vision_available():
from PIL import Image

Expand Down Expand Up @@ -178,10 +174,7 @@ def test_image_processor_defaults_preserved_by_image_kwargs(self):
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, return_tensors="pt")
if inputs[self.images_input_name].dtype == torch.uint8:
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
else:
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)

def test_kwargs_overrides_default_tokenizer_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
Expand Down Expand Up @@ -214,10 +207,7 @@ def test_kwargs_overrides_default_image_processor_kwargs(self):
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, do_rescale=True, rescale_factor=-1, return_tensors="pt")
if inputs[self.images_input_name].dtype == torch.uint8:
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
else:
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)

def test_unstructured_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
Expand All @@ -239,10 +229,7 @@ def test_unstructured_kwargs(self):
max_length=max_length,
)

if inputs[self.images_input_name].dtype == torch.uint8:
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
else:
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

def test_unstructured_kwargs_batched(self):
Expand All @@ -264,10 +251,7 @@ def test_unstructured_kwargs_batched(self):
max_length=76,
)

if inputs[self.images_input_name].dtype == torch.uint8:
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
else:
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertTrue(
len(inputs[self.text_input_name][0]) == len(inputs[self.text_input_name][1])
and len(inputs[self.text_input_name][1]) < 76
Expand Down Expand Up @@ -311,10 +295,7 @@ def test_structured_kwargs_nested(self):
inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.skip_processor_without_typed_kwargs(processor)

if inputs[self.images_input_name].dtype == torch.uint8:
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
else:
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

def test_structured_kwargs_nested_from_dict(self):
Expand All @@ -334,10 +315,7 @@ def test_structured_kwargs_nested_from_dict(self):
}

inputs = processor(text=input_str, images=image_input, **all_kwargs)
if inputs[self.images_input_name].dtype == torch.uint8:
self.assertLessEqual(inputs[self.images_input_name].shape[-1], 224)
else:
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
self.assertEqual(inputs[self.text_input_name].shape[-1], 76)

# TODO: the same test, but for audio + text processors that have strong overlap in kwargs
Expand Down

0 comments on commit 964be32

Please sign in to comment.