Additional explanation for truncated tokenization warning (#130)

Added an optional text to be printed in the warning. Can be used to
describe the tokenizer op caller

---------

Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>
Co-authored-by: VADIM RATNER [email protected] <[email protected]>

Author: 9 people

fusedrug/data/tokenizer/ops/modular_tokenizer_ops.py

@@ -193,6 +193,7 @@ def __call__(
         on_unknown: Optional[str] = "warn",
         verbose: Optional[int] = 1,
         validate_ends_with_eos: Optional[bool] = None,
+        additional_caller_info_text: Optional[str] = "",
     ) -> NDict:
         """_summary_
 
@@ -297,7 +298,7 @@ def __call__(
             len(encoded.overflowing) > 0
         ):  # note, encoded.overflowing may have multiple items, and each item can contain multiple items
             print(
-                f"Warning: FastModularTokenizer (pid={os.getpid()}) had to truncate sequence: [{overflow_info}]  \
+                f"Warning: FastModularTokenizer (pid={os.getpid()}, {additional_caller_info_text}) had to truncate sequence: [{overflow_info}]  \
                     for tokenizer: {self._tokenizer_path} for sample_id {get_sample_id(sample_dict)}"
             )