Improve implementation

src/transformers/models/markuplm/processing_markuplm.py

@@ -30,17 +30,17 @@ class MarkupLMProcessor(ProcessorMixin):
     [`MarkupLMProcessor`] offers all the functionalities you need to prepare data for the model.
 
     It first uses [`MarkupLMFeatureExtractor`] to get nodes and corresponding xpaths from one or more HTML strings. Next,
-    these are provided to [`MarkupLMTokenizer`], which turns them into token-level `input_ids`, `attention_mask`,
+    these are provided to [`MarkupLMTokenizer`] or [`MarkupLMTokenizerFast`], which turns them into token-level `input_ids`, `attention_mask`,
     `token_type_ids`, `xpath_tags_seq` and `xpath_subs_seq`.
 
     Args:
         feature_extractor (`MarkupLMFeatureExtractor`):
             An instance of [`MarkupLMFeatureExtractor`]. The feature extractor is a required input.
-        tokenizer (`MarkupLMTokenizer`):
-            An instance of [`MarkupLMTokenizer`]. The tokenizer is a required input.
+        tokenizer (`MarkupLMTokenizer` or `MarkupLMTokenizerFast`):
+            An instance of [`MarkupLMTokenizer`] or [`MarkupLMTokenizerFast`]. The tokenizer is a required input.
     """
     feature_extractor_class = "MarkupLMFeatureExtractor"
-    tokenizer_class = ("MarkupLMTokenizer")
+    tokenizer_class = ("MarkupLMTokenizer", "MarkupLMTokenizerFast")
 
     def __call__(
         self,
@@ -95,4 +95,4 @@ def __call__(
             **kwargs,
         )
 
-        return encoded_inputs
+        return encoded_inputs

src/transformers/models/markuplm/test_tokenizer.py

@@ -1,25 +1,31 @@
-from transformers import MarkupLMTokenizer, MarkupLMTokenizerFast
 import torch
 
-slow_tokenizer = MarkupLMTokenizer.from_pretrained("microsoft/markuplm-base")
+from transformers import MarkupLMTokenizer, MarkupLMTokenizerFast, LayoutLMv2TokenizerFast
 
-slow_encoding = slow_tokenizer(
-    ["hello", "world"],
-    xpaths=["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span"],
-    padding="max_length",
-    max_length=20,
-    return_tensors="pt",
-)
+
+# slow_tokenizer = MarkupLMTokenizer.from_pretrained("microsoft/markuplm-base")
+
+# slow_encoding = slow_tokenizer(
+#     ["hello", "world"],
+#     xpaths=["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span"],
+#     padding="max_length",
+#     max_length=20,
+#     return_tensors="pt",
+# )
 
 fast_tokenizer = MarkupLMTokenizerFast.from_pretrained("microsoft/markuplm-base")
 
-fast_encoding = fast_tokenizer(
-    ["hello", "world"],
-    xpaths=["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span"],
+fast_tokenizer_bis = LayoutLMv2TokenizerFast.from_pretrained("microsoft/layoutlmv2-base-uncased")
+
+fast_encoding = fast_tokenizer_bis(
+    ["hello", "world", "how", "are", "you"],
+    #xpaths=["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span", "html/body", "html/body/div"],
+    nodes=[[1,2,3,4] for _ in range(5)],
     padding="max_length",
-    max_length=20,
+    max_length=2,
     return_tensors="pt",
+    return_overflowing_tokens=True,
 )
 
-for k in slow_encoding.keys():
-    assert torch.allclose(slow_encoding[k], fast_encoding[k])
+# for k in slow_encoding.keys():
+#     assert torch.allclose(slow_encoding[k], fast_encoding[k])

src/transformers/models/markuplm/tokenization_markuplm_fast.py

@@ -582,7 +582,9 @@ def _batch_encode_plus(
 
         if not isinstance(batch_text_or_text_pairs, list):
             raise TypeError(f"batch_text_or_text_pairs has to be a list (got {type(batch_text_or_text_pairs)})")
-
+
+        print("Max length:", max_length)
+
         # Set the truncation and padding strategy and restore the initial configuration
         self.set_truncation_and_padding(
             padding_strategy=padding_strategy,
@@ -595,12 +597,18 @@ def _batch_encode_plus(
         if is_pair:
             batch_text_or_text_pairs = [(text.split(), text_pair) for text, text_pair in batch_text_or_text_pairs]
 
+        print("Batch text or text pairs:", batch_text_or_text_pairs)
+
         encodings = self._tokenizer.encode_batch(
             batch_text_or_text_pairs,
             add_special_tokens=add_special_tokens,
             is_pretokenized=True,  # we set this to True as MarkupLM always expects pretokenized inputs
         )
 
+        print("Encodings:", encodings)
+        print(len(encodings))
+        print(encodings[0].overflowing)
+
         # Convert encoding to dict
         # `Tokens` has type: Tuple[
         #                       List[Dict[str, List[List[int]]]] or List[Dict[str, 2D-Tensor]],

tests/test_processor_markuplm.py

@@ -19,16 +19,19 @@
 import unittest
 from typing import List
 
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerBase, PreTrainedTokenizerFast
+from transformers import (
+    MarkupLMFeatureExtractor,
+    MarkupLMProcessor,
+    MarkupLMTokenizer,
+    PreTrainedTokenizer,
+    PreTrainedTokenizerBase,
+    PreTrainedTokenizerFast,
+)
 from transformers.file_utils import FEATURE_EXTRACTOR_NAME, cached_property
-from transformers import MarkupLMTokenizer
 from transformers.models.markuplm.tokenization_markuplm import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_torch, slow
 
 
-from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor
-
-
 @require_tokenizers
 class MarkupLMProcessorTest(unittest.TestCase):
     tokenizer_class = MarkupLMTokenizer
@@ -402,4 +405,4 @@ def test_processor_case_5(self):
 
             # verify bbox
             expected_bbox = [[6, 7, 4, 2], [3, 9, 2, 4], [1, 1, 2, 3], [1, 1, 2, 3], [1000, 1000, 1000, 1000]]
-            self.assertListEqual(input_processor.bbox[1].tolist()[-5:], expected_bbox)
+            self.assertListEqual(input_processor.bbox[1].tolist()[-5:], expected_bbox)

tests/test_tokenization_markuplm.py

@@ -1392,7 +1392,7 @@ def test_training_new_tokenizer(self):
 
         # Test we can use the new tokenizer with something not seen during training
         text = [["this", "is", "the"], ["how", "are", "you"]]
-        xpaths = [["html/body"]*3, ["html/body"]*3]
+        xpaths = [["html/body"] * 3, ["html/body"] * 3]
         inputs = new_tokenizer(text, xpaths=xpaths)
         self.assertEqual(len(inputs["input_ids"]), 2)
         decoded_input = new_tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
@@ -1510,7 +1510,7 @@ def test_training_new_tokenizer_with_special_tokens_change(self):
 
         # Test we can use the new tokenizer with something not seen during training
         nodes = [["this", "is"], ["hello", "🤗"]]
-        xpaths = [["html/body"]*2, ["html/body"]*2]
+        xpaths = [["html/body"] * 2, ["html/body"] * 2]
         inputs = new_tokenizer(nodes, xpaths=xpaths)
         self.assertEqual(len(inputs["input_ids"]), 2)
         decoded_input = new_tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
@@ -1595,16 +1595,23 @@ def test_batch_encode_dynamic_overflowing(self):
 
                 # Single example
                 nodes, xpaths = self.get_nodes_and_xpaths()
+                print("Nodes:", nodes)
+                print("Xpaths:", xpaths)
                 tokens = tokenizer.encode_plus(
                     nodes,
                     xpaths=xpaths,
-                    max_length=6,
+                    max_length=1,
                     padding=True,
                     truncation=True,
                     return_tensors=returned_tensor,
                     return_overflowing_tokens=True,
                 )
 
+                for k, v in tokens.items():
+                    print(k, v.shape)
+
+                print(tokenizer.decode(tokens.input_ids.squeeze().tolist()))
+
                 for key in filter(lambda x: "overflow_to_sample_mapping" not in x, tokens.keys()):
                     self.assertEqual(len(tokens[key].shape), 2)