Skip to content

Commit

Permalink
o pona e sona ni: nasin nanpa seme li wile lon open
Browse files Browse the repository at this point in the history
  • Loading branch information
gregdan3 committed May 3, 2024
1 parent c13afad commit 1ea6431
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ from sonatoki.Filters import (
ProperName,
Punctuations,
)
from sonatoki.Scorers import Scaling
from sonatoki.Scorers import SoftScaling
from sonatoki.Cleaners import ConsecutiveDuplicates
from sonatoki.Tokenizers import word_tokenize_tok
from sonatoki.Preprocessors import URLs, DiscordEmotes
Expand All @@ -41,11 +41,12 @@ def main():
ignoring_filters=[Numerics, Punctuations],
scoring_filters=[NimiLinku, Syllabic, ProperName, Alphabetic],
cleaners=[ConsecutiveDuplicates],
scorer=Scaling,
scorer=SoftScaling,
tokenizer=word_tokenize_tok,
)
ilo.is_toki_pona("imagine how is touch the sky") # False
ilo.is_toki_pona("o pilin insa e ni: sina pilin e sewi") # True
ilo.is_toki_pona("I Think I Can Evade Detection") # False

if __name__ == "__main__":
main()
Expand Down

0 comments on commit 1ea6431

Please sign in to comment.