remove unnecessary assertNotEqual from t5 (and siglip), add copied from)

NielsRogge · Aug 26, 2024 · bf4f6db · bf4f6db
1 parent 8ba6e0b
commit bf4f6db
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 1 deletion.
diff --git a/tests/models/siglip/test_tokenization_siglip.py b/tests/models/siglip/test_tokenization_siglip.py
@@ -385,6 +385,7 @@ def test_tokenizer_integration(self):
             input_ids = tokenizer(text, padding="max_length").input_ids
             self.assertListEqual(input_ids, expected)
 
+    # Copied from tests.models.t5.test_tokenization_t5.T5TokenizationTest.test_some_edge_cases with T5->Siglip
     def test_some_edge_cases(self):
         tokenizer = SiglipTokenizer.from_pretrained("google/siglip-base-patch16-224", legacy=False)
 

diff --git a/tests/models/t5/test_tokenization_t5.py b/tests/models/t5/test_tokenization_t5.py
@@ -400,7 +400,6 @@ def test_some_edge_cases(self):
         sp_tokens = tokenizer.sp_model.encode("</s>>", out_type=str)
         self.assertEqual(sp_tokens, ["<", "/", "s", ">", ">"])
         tokens = tokenizer.tokenize("</s>>")
-        self.assertNotEqual(sp_tokens, tokens)
         self.assertEqual(tokens, ["</s>", ">"])
 
         tokens = tokenizer.tokenize("")