From 9931ddc42d6c3555b607913908381e933b619375 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Wed, 16 Oct 2024 15:23:39 +0200 Subject: [PATCH] Restore parameters --- crates/bpe/benchmarks/performance.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/crates/bpe/benchmarks/performance.rs b/crates/bpe/benchmarks/performance.rs index 98f0357..d192225 100644 --- a/crates/bpe/benchmarks/performance.rs +++ b/crates/bpe/benchmarks/performance.rs @@ -200,11 +200,7 @@ fn worstcase_comparison_benchmark(c: &mut Criterion) { &bytes, |b, bytes| { b.iter_batched( - || { - let text = select_test_string(&text, *bytes); - assert!(tok.split(text).nth(1).is_none()); - text - }, + || select_test_string(&text, *bytes), |text| tok.encode(text), criterion::BatchSize::SmallInput, ) @@ -212,11 +208,7 @@ fn worstcase_comparison_benchmark(c: &mut Criterion) { ); group.bench_with_input(BenchmarkId::new("tiktoken", bytes), &bytes, |b, bytes| { b.iter_batched( - || { - let text = select_test_string(&text, *bytes); - assert!(tok.split(text).nth(1).is_none()); - text - }, + || select_test_string(&text, *bytes), |text| tiktoken.encode_ordinary(text), criterion::BatchSize::SmallInput, ) @@ -226,11 +218,7 @@ fn worstcase_comparison_benchmark(c: &mut Criterion) { &bytes, |b, bytes| { b.iter_batched( - || { - let text = select_test_string(&text, *bytes); - assert!(tok.split(text).nth(1).is_none()); - text - }, + || select_test_string(&text, *bytes), |text| huggingface.encode_fast(text, false).unwrap(), criterion::BatchSize::SmallInput, )