Restore parameters

github · Oct 18, 2024 · 9931ddc · 9931ddc
1 parent 67a088e
commit 9931ddc
Showing 1 changed file with 3 additions and 15 deletions.
diff --git a/crates/bpe/benchmarks/performance.rs b/crates/bpe/benchmarks/performance.rs
@@ -200,23 +200,15 @@ fn worstcase_comparison_benchmark(c: &mut Criterion) {
                 &bytes,
                 |b, bytes| {
                     b.iter_batched(
-                        || {
-                            let text = select_test_string(&text, *bytes);
-                            assert!(tok.split(text).nth(1).is_none());
-                            text
-                        },
+                        || select_test_string(&text, *bytes),
                         |text| tok.encode(text),
                         criterion::BatchSize::SmallInput,
                     )
                 },
             );
             group.bench_with_input(BenchmarkId::new("tiktoken", bytes), &bytes, |b, bytes| {
                 b.iter_batched(
-                    || {
-                        let text = select_test_string(&text, *bytes);
-                        assert!(tok.split(text).nth(1).is_none());
-                        text
-                    },
+                    || select_test_string(&text, *bytes),
                     |text| tiktoken.encode_ordinary(text),
                     criterion::BatchSize::SmallInput,
                 )
@@ -226,11 +218,7 @@ fn worstcase_comparison_benchmark(c: &mut Criterion) {
                 &bytes,
                 |b, bytes| {
                     b.iter_batched(
-                        || {
-                            let text = select_test_string(&text, *bytes);
-                            assert!(tok.split(text).nth(1).is_none());
-                            text
-                        },
+                        || select_test_string(&text, *bytes),
                         |text| huggingface.encode_fast(text, false).unwrap(),
                         criterion::BatchSize::SmallInput,
                     )