修复 scws 分词不完整的问题

vanry · Aug 1, 2018 · ece71f7 · ece71f7
1 parent fe55b17
commit ece71f7
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/src/Tokenizers/ScwsTokenizer.php b/src/Tokenizers/ScwsTokenizer.php
@@ -17,9 +17,13 @@ public function getTokens($text)
     {
         $this->scws->sendText($text);
 
-        $result = $this->scws->getResult();
+        $tokens = [];
 
-        return $result === false ? [] : array_column($result, 'word');
+        while ($result = $this->scws->getResult()) {
+            $tokens = array_merge($tokens, array_column($result, 'word'));
+        }
+
+        return $tokens;
     }
 
     public function getScws()