Always update data files

Hendrik van Antwerpen · Hendrik van Antwerpen · commit 9350e9a4965c · 2024-09-25T11:36:50.000+02:00
diff --git a/crates/bpe/src/byte_pair_encoding.rs b/crates/bpe/src/byte_pair_encoding.rs
@@ -597,7 +597,6 @@ mod data {
     }
 
     #[test]
-    #[ignore = "run manually to update data files"]
     fn update_token_dicts() {
         serialize_tokens(
             &cl100k_base().expect("tiktoken initialization must not fail!"),

Original file line number	Diff line number	Diff line change
`@@ -597,7 +597,6 @@ mod data {`
`597`	`597`	`}`
`598`	`598`
`599`	`599`	`#[test]`
`600`		`- #[ignore = "run manually to update data files"]`
`601`	`600`	`fn update_token_dicts() {`
`602`	`601`	`serialize_tokens(`
`603`	`602`	`&cl100k_base().expect("tiktoken initialization must not fail!"),`