feat(detector): support and test fr, de, en exist at the same time.

fix(splitter): add para `lang_map` allow mapping different language to same language for better result, if you know the range of your target languages

Author: DoodleBears

langsplit/__init__.py

@@ -1 +1,2 @@
-from .split.splitter import split
+from .split.splitter import split, SentenceSplitter, SubString
+from .detect_lang.detector import LANG_MAP

langsplit/detect_lang/detector.py

@@ -1,12 +1,16 @@
 from langdetect import detect
 import fast_langdetect
 
-lang_map = {
+LANG_MAP = {
     "zh": "zh",
     "zh-cn": "zh",
     "zh-tw": "x",
     "ko": "ko",
     "ja": "ja",
+    "de": "de",
+    "fr": "fr",
+    "en": "en",
+    "x": "en",
 }
 
 

langsplit/split/splitter.py

@@ -1,10 +1,10 @@
-from typing import List
+from typing import List, Dict
 from dataclasses import dataclass
 
 from langdetect.lang_detect_exception import LangDetectException
 from wtpsplit import SaT, WtP
 
-from langsplit.detect_lang.detector import detect_lang, fast_detect_lang, lang_map
+from langsplit.detect_lang.detector import detect_lang, fast_detect_lang, LANG_MAP
 
 
 @dataclass
@@ -29,33 +29,42 @@ def split(self, text: str, threshold: float = 5e-5, verbose=False):
 def split(
     text: str,
     threshold: float = 5e-5,
+    lang_map=None,
     verbose=False,
     splitter: SentenceSplitter = default_sentence_splitter,
-):
+) -> List[SubString]:
     """using
     1. `wtpsplit` to split sentences into 'small' substring
     2. concat substring based on language using `fasttext` and `langdetect`
 
     Args:
         text (str): text to split
-        threshold (float, optional): the lower the more separated (more) substring will return. Defaults to 5e-5.
+        threshold (float, optional): the lower the more separated (more) substring will return. Defaults to 5e-5 (if your text contains no Chinese, Japanese, Korean, 1e-3 is suggested)
+        lang_map (_type_, optional): mapping different language to same language for better result, if you know the range of your target languages. Defaults to None.
+        verbose (bool, optional): print the process. Defaults to False.
+        splitter (SentenceSplitter, optional): sentence splitter. Defaults to default_sentence_splitter.
+
+    Returns:
+        List[SubString]: substring with .lang and .text
     """
     substr_list = splitter.split(text=text, threshold=threshold, verbose=verbose)
     if verbose:
         print(f"substr_list: {substr_list}")
-    substr_list = _init_substr_lang(substr_list)
+    substr_list = _init_substr_lang(substr=substr_list, lang_map=lang_map)
     if verbose:
         print(f"substr_list: {substr_list}")
-    substr_list = _smart_concat(substr_list)
+    substr_list = _smart_concat(substr_list=substr_list, lang_map=lang_map)
     if verbose:
         print(f"split_result: {substr_list}")
     return substr_list
 
 
-def _smart_concat(substr_list: List[SubString]):
+def _smart_concat(substr_list: List[SubString], lang_map=None):
+    if lang_map is None:
+        lang_map = LANG_MAP
     is_concat_complete = False
     while is_concat_complete is False:
-        substr_list = _smart_concat_logic(substr_list)
+        substr_list = _smart_concat_logic(substr_list, lang_map=lang_map)
         is_concat_complete = True
         for index, block in enumerate(substr_list):
             if block.lang == "x":
@@ -68,15 +77,20 @@ def _smart_concat(substr_list: List[SubString]):
     return substr_list
 
 
-def _init_substr_lang(substr: List[str]) -> List[SubString]:
+def _init_substr_lang(substr: List[str], lang_map=None) -> List[SubString]:
     concat_result = []
     lang = ""
+    if lang_map is None:
+        lang_map = LANG_MAP
     for block in substr:
         try:
             cur_lang = detect_lang(block)
         except LangDetectException:
             cur_lang = lang
-        cur_lang = lang_map.get(cur_lang, "en")
+        cur_lang = lang_map.get(cur_lang, "x")
+        if cur_lang == "x":
+            cur_lang = fast_detect_lang(block)
+            cur_lang = lang_map.get(cur_lang, "x")
         concat_result.append(SubString(cur_lang, block))
         lang = cur_lang
     return concat_result
@@ -143,7 +157,7 @@ def _find_nearest_lang_with_direction(
         for i in range(1, len(concat_result)):
             if index + i < len(concat_result) and concat_result[index + i].lang != "x":
                 return concat_result[index + i].lang
-    return "en"
+    return "x"
 
 
 def _get_find_direction(substr_list: List[SubString], index: int) -> bool:
@@ -179,13 +193,15 @@ def _merge_blocks(concat_result: List[SubString]):
     return smart_concat_result
 
 
-def _check_languages(lang_text_list: List[SubString]):
+def _check_languages(lang_text_list: List[SubString], lang_map=None):
+    if lang_map is None:
+        lang_map = LANG_MAP
     for index, block in enumerate(lang_text_list):
         try:
             cur_lang = fast_detect_lang(block.text)
         except LangDetectException:
-            cur_lang = "en"
-        cur_lang = lang_map.get(cur_lang, "en")
+            cur_lang = "x"
+        cur_lang = lang_map.get(cur_lang, "x")
         if cur_lang == "ko":
             fast_lang = fast_detect_lang(block.text, text_len_threshold=0)
             if fast_lang != "ko":
@@ -198,13 +214,13 @@ def _check_languages(lang_text_list: List[SubString]):
     return lang_text_list
 
 
-def _smart_concat_logic(lang_text_list: List[SubString]):
+def _smart_concat_logic(lang_text_list: List[SubString], lang_map=None):
     lang_text_list = _merge_middle_substr_to_two_side(lang_text_list)
     lang_text_list = _merge_blocks(lang_text_list)
-    lang_text_list = _check_languages(lang_text_list)
+    lang_text_list = _check_languages(lang_text_list=lang_text_list, lang_map=lang_map)
     lang_text_list = _merge_middle_substr_to_two_side(lang_text_list)
     lang_text_list = _fill_missing_languages(lang_text_list)
     lang_text_list = _merge_two_side_substr_to_near(lang_text_list)
     lang_text_list = _merge_blocks(lang_text_list)
-    lang_text_list = _check_languages(lang_text_list)
+    lang_text_list = _check_languages(lang_text_list=lang_text_list, lang_map=lang_map)
     return lang_text_list

tests/test_split_multi_lang.py

@@ -0,0 +1,66 @@
+from langsplit import split
+
+texts = [
+    "我是 VGroupChatBot，一个旨在支持多人通信的助手，通过可视化消息来帮助团队成员更好地交流。我可以帮助团队成员更好地整理和共享信息，特别是在讨论、会议和Brainstorming等情况下。你好我的名字是西野くまですmy name is bob很高兴认识你どうぞよろしくお願いいたします「こんにちは」是什么意思。",
+    "你好，我的名字是西野くまです。I am from Tokyo, 日本の首都。今天的天气非常好，sky is clear and sunny。おはようございます、皆さん！我们一起来学习吧。Learning languages can be fun and exciting。昨日はとても忙しかったので、今日は少しリラックスしたいです。Let's take a break and enjoy some coffee。中文、日本語、and English are three distinct languages, each with its own unique charm。希望我们能一起进步，一起成长。Let's keep studying and improving our language skills together. ありがとう！",
+    "你好，今日はどこへ行きますか？",
+    "我的名字是田中さんです。",
+    "我喜欢吃寿司和拉面おいしいです。",
+    "今天の天気はとてもいいですね。",
+    "我在学习日本語少し難しいです。",
+    "日语真是おもしろい啊",
+    "你喜欢看アニメ吗？",
+    "我想去日本旅行、特に京都に行きたいです。",
+    "昨天見た映画はとても感動的でした。" "我朋友是日本人、彼はとても優しいです。",
+    "我们一起去カラオケ吧、楽しそうです。",
+    "你今天吃了什么、朝ごはんは何ですか？",
+    "我的家在北京、でも、仕事で東京に住んでいます。",
+    "我在学做日本料理、日本料理を作るのを習っています。",
+    "你会说几种语言、何ヶ国語話せますか？",
+    "我昨天看了一本书、その本はとても面白かったです。",
+    "我们一起去逛街、買い物に行きましょう。",
+    "你最近好吗、最近どうですか？",
+    "我在学做日本料理와 한국 요리、日本料理を作るのを習っています。",
+    "你会说几种语言、何ヶ国語話せますか？몇 개 언어를 할 수 있어요？",
+    "我昨天看了一本书、その本はとても面白かったです。어제 책을 읽었는데, 정말 재미있었어요。",
+    "我们一起去逛街와 쇼핑、買い物に行きましょう。쇼핑하러 가요。",
+    "你最近好吗、最近どうですか？요즘 어떻게 지내요？",
+]
+
+texts_2 = [
+    "Ich liebe Paris, c'est une belle ville, and the food is amazing!",
+    "Berlin ist wunderbar, je veux y retourner, and explore more.",
+    "Bonjour, wie geht's dir today?",
+    "Die Musik hier ist fantastisch, la musique est superbe, and I enjoy it a lot.",
+    "Guten Morgen, je t'aime, have a great day!",
+    "Das Wetter ist heute schön, il fait beau aujourd'hui, and it's perfect for a walk.",
+    "Ich mag dieses Buch, ce livre est intéressant, and it has a great story.",
+    "Vielen Dank, merci beaucoup, for your help.",
+    "Wir reisen nach Deutschland, nous voyageons en Allemagne, and we are excited.",
+    "Ich bin müde, je suis fatigué, and I need some rest.",
+]
+
+new_lang_map = {
+    "zh": "zh",
+    "zh-cn": "zh",
+    "zh-tw": "x",
+    "ko": "ko",
+    "ja": "ja",
+    "de": "de",
+    "fr": "fr",
+    "en": "en",
+    "x": "en",
+}
+
+
+for text in texts:
+    substr_list = split(text=text, verbose=False, lang_map=new_lang_map, threshold=5e-5)
+    for index, substr in enumerate(substr_list):
+        print(f"{substr.lang}|{index}: {substr.text}")
+    print("----------------------")
+
+for text in texts_2:
+    substr_list = split(text=text, verbose=False, lang_map=new_lang_map, threshold=1e-3)
+    for index, substr in enumerate(substr_list):
+        print(f"{substr.lang}|{index}: {substr.text}")
+    print("----------------------")