Skip to content

Commit

Permalink
add comment on get_component
Browse files Browse the repository at this point in the history
  • Loading branch information
leloykun committed Aug 19, 2024
1 parent c4f5474 commit 984d83d
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions tests/models/chameleon/test_processor_chameleon.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ class ChameleonProcessorTest(ProcessorTesterMixin, unittest.TestCase):

def get_component(self, attribute, **kwargs):
assert attribute in self.processor_class.attributes
if attribute != "tokenizer":
return super().get_component(attribute, **kwargs)
# We use the fast tokenizer by default as the slow tokenizer expects the vocab file to be present in the loading directory.
# This vocab file is neither in the official repo nor does it get saved when we save the processor in `setUp` below.
component_class_name = getattr(self.processor_class, f"{attribute}_class")
if isinstance(component_class_name, tuple):
if "_fast" in component_class_name[0]:
Expand Down

0 comments on commit 984d83d

Please sign in to comment.