[Rust] Bump huggingface tokenizer to 0.20.0 (#49)

This PR bumps the huggignface tokenizer dependency to
version 0.20.0 to address the tokenizer issue in some latest models
with latest trained tokenizers.

Author: MasterJH5574

rust/Cargo.toml

@@ -8,6 +8,6 @@ crate-type = ["staticlib"]
 
 [dependencies]
 
-tokenizers = { version = "0.19.1", default-features = false, features = ["onig"] }
+tokenizers = { version = "0.20.0", default-features = false, features = ["onig"] }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"

rust/src/lib.rs

@@ -77,8 +77,8 @@ impl TokenizerWrapper {
         );
         let mut tokenizer = Tokenizer::new(BPE::new(vocab, merges));
         tokenizer
-            .with_pre_tokenizer(byte_level)
-            .with_decoder(byte_level);
+            .with_pre_tokenizer(Some(byte_level))
+            .with_decoder(Some(byte_level));
         TokenizerWrapper {
             tokenizer: tokenizer,
             decode_str: String::new(),