From 35d539049db3bc67fc55824fb294a8ac388870b5 Mon Sep 17 00:00:00 2001 From: Yunfan Yang Date: Fri, 31 Jul 2026 01:00:47 -0400 Subject: [PATCH] TUI: add retained transcript performance benchmarks --- Cargo.lock | 1 + crates/warp_tui/Cargo.toml | 7 + crates/warp_tui/benches/transcript_bench.rs | 100 ++++++ crates/warp_tui/src/benchmark_support.rs | 323 ++++++++++++++++++++ crates/warp_tui/src/lib.rs | 5 +- crates/warp_tui/src/test_fixtures.rs | 1 + 6 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 crates/warp_tui/benches/transcript_bench.rs create mode 100644 crates/warp_tui/src/benchmark_support.rs diff --git a/Cargo.lock b/Cargo.lock index 7e77ed7daea..a8480ff7297 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16011,6 +16011,7 @@ dependencies = [ "chrono", "clap", "command", + "criterion", "dirs 6.0.0", "fs4", "futures", diff --git a/crates/warp_tui/Cargo.toml b/crates/warp_tui/Cargo.toml index d239bdb91be..dfe5015256c 100644 --- a/crates/warp_tui/Cargo.toml +++ b/crates/warp_tui/Cargo.toml @@ -106,6 +106,7 @@ arboard = { workspace = true } arboard = { workspace = true, features = ["wayland-data-control"] } [dev-dependencies] +criterion = { version = "0.5.1", features = ["html_reports"] } tempfile.workspace = true # `test-util` exposes `Appearance::mock()`, used to register the `Appearance` # singleton that the editor-backed TUI input view depends on in tests. @@ -117,8 +118,14 @@ warp_editor = { workspace = true, features = ["test-util"] } command.workspace = true pathfinder_color.workspace = true warp = { workspace = true, features = ["tui", "test-util"] } +[[bench]] +name = "transcript_bench" +harness = false +required-features = ["test-util"] [features] +# Exposes deterministic, production-shaped transcript fixtures to benchmarks. +test-util = ["warp/test-util", "warp_core/test-util"] # Declared so the `warp_channel_config::load_config!` macro's # `cfg(feature = "release_bundle")` branch is a known cfg in this crate. Today it # stays off and the local bin uses the runtime generator; wiring the embedded diff --git a/crates/warp_tui/benches/transcript_bench.rs b/crates/warp_tui/benches/transcript_bench.rs new file mode 100644 index 00000000000..16d70db35f2 --- /dev/null +++ b/crates/warp_tui/benches/transcript_bench.rs @@ -0,0 +1,100 @@ +use std::hint::black_box; +use std::time::Duration; + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use warp_tui::benchmark_support::{TranscriptBenchmark, TranscriptDataset}; + +fn benchmark_many_small_blocks(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("tui_transcript/many_small_blocks"); + for blocks in [100, 1_000, 10_000] { + let mut benchmark = + TranscriptBenchmark::new(TranscriptDataset::ManySmallBlocks { blocks }, 120, 50); + group.bench_with_input( + BenchmarkId::new("retained_end_frame", blocks), + &blocks, + |b, _| b.iter(|| black_box(benchmark.present())), + ); + benchmark.scroll_to_row(blocks / 2); + group.bench_with_input( + BenchmarkId::new("retained_middle_frame", blocks), + &blocks, + |b, _| b.iter(|| black_box(benchmark.present())), + ); + } + group.finish(); +} + +fn benchmark_long_agent_response(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("tui_transcript/long_agent_response"); + for rows in [1_000, 10_000] { + let mut benchmark = + TranscriptBenchmark::new(TranscriptDataset::LongAgentResponse { rows }, 120, 50); + group.bench_with_input( + BenchmarkId::new("retained_end_frame", rows), + &rows, + |b, _| b.iter(|| black_box(benchmark.present())), + ); + benchmark.scroll_to_row(rows / 2); + group.bench_with_input( + BenchmarkId::new("retained_middle_frame", rows), + &rows, + |b, _| b.iter(|| black_box(benchmark.present())), + ); + group.bench_with_input( + BenchmarkId::new("invalidated_middle_frame", rows), + &rows, + |b, _| { + b.iter(|| { + benchmark.invalidate_all(); + black_box(benchmark.present()) + }) + }, + ); + benchmark.scroll_to_end(); + } + group.finish(); +} + +fn benchmark_offscreen_streaming_tail(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("tui_transcript/offscreen_streaming_tail"); + for tail_rows in [1_000, 10_000] { + let mut benchmark = TranscriptBenchmark::new( + TranscriptDataset::OffscreenStreamingTail { + preceding_rows: 100, + tail_rows, + }, + 120, + 50, + ); + group.bench_with_input( + BenchmarkId::new("retained_end_frame", tail_rows), + &tail_rows, + |b, _| b.iter(|| black_box(benchmark.present())), + ); + benchmark.scroll_to_row(0); + group.bench_with_input( + BenchmarkId::new("dirty_fixed_viewport_frame", tail_rows), + &tail_rows, + |b, _| { + b.iter(|| { + benchmark.invalidate_streaming_tail(); + black_box(benchmark.present()) + }) + }, + ); + } + group.finish(); +} + +criterion_group! { + name = benches; + config = Criterion::default() + .sample_size(10) + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_secs(1)); + targets = + benchmark_many_small_blocks, + benchmark_long_agent_response, + benchmark_offscreen_streaming_tail +} +criterion_main!(benches); diff --git a/crates/warp_tui/src/benchmark_support.rs b/crates/warp_tui/src/benchmark_support.rs new file mode 100644 index 00000000000..c81edce12dc --- /dev/null +++ b/crates/warp_tui/src/benchmark_support.rs @@ -0,0 +1,323 @@ +//! Production-shaped fixtures for retained TUI transcript benchmarks. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::Arc; + +use parking_lot::FairMutex; +use warp::tui_export::{ + AIAgentExchangeId, AIAgentInput, AIAgentOutput, AIAgentOutputMessage, AIAgentOutputMessageType, + AIAgentText, AIAgentTextSection, AIBlockModel, AIBlockOutputStatus, AIConversationId, + AIRequestType, Appearance, LLMId, MessageId, OutputStatusUpdateCallback, RichContentItem, + RichContentType, ServerOutputId, Shared, TerminalModel, +}; +use warpui::platform::WindowStyle; +use warpui::{ + AddWindowOptions, App, AppContext, Entity, EntityId, EntityIdSet, TuiView, TypedActionView, + ViewContext, ViewHandle, WindowInvalidation, +}; +use warpui_core::elements::tui::{ + TuiElement, TuiRect, TuiViewportPosition, TuiViewportVerticalAlignment, TuiViewportedList, + TuiViewportedListState, +}; +use warpui_core::presenter::tui::TuiPresenter; + +use crate::agent_block::TuiAIBlock; +use crate::test_fixtures::add_test_action_model_and_events; +use crate::tui_block_list_viewport_source::{ + AgentBlockRegistry, CLISubagentBlockRegistry, HandoffBlockRegistry, TuiBlockListViewportSource, +}; +use crate::tui_builder::TuiUiBuilder; + +/// Shape of the retained transcript fixture. +#[derive(Clone, Copy, Debug)] +pub enum TranscriptDataset { + /// Many independently indexed terminal blocks. + ManySmallBlocks { blocks: usize }, + /// One rich agent response containing `rows` hard lines. + LongAgentResponse { rows: usize }, + /// A streaming rich response below `preceding_rows` of fixed history. + OffscreenStreamingTail { + preceding_rows: usize, + tail_rows: usize, + }, +} + +/// One production-shaped retained transcript benchmark. +pub struct TranscriptBenchmark { + app: App, + root: ViewHandle, + model: Arc>, + agent_block_id: Option, + presenter: TuiPresenter, + area: TuiRect, +} + +impl TranscriptBenchmark { + /// Builds and primes a benchmark at `width × height`. + pub fn new(dataset: TranscriptDataset, width: u16, height: u16) -> Self { + App::test((), move |mut app| async move { + app.add_singleton_model(|_| Appearance::mock()); + let terminal_model = Arc::new(FairMutex::new(TerminalModel::mock(None, None))); + if let TranscriptDataset::ManySmallBlocks { blocks } = dataset { + let mut model = terminal_model.lock(); + for index in 0..blocks { + let command = format!("printf block-{index}"); + let output = format!("output-{index}\r\n"); + model.simulate_block(command.as_str(), output.as_str()); + } + } + if let TranscriptDataset::OffscreenStreamingTail { preceding_rows, .. } = dataset { + let output = "history row\r\n".repeat(preceding_rows); + terminal_model + .lock() + .simulate_block("history", output.as_str()); + } + + let agent_blocks = AgentBlockRegistry::new(RefCell::new(HashMap::new())); + let cli_subagent_blocks = CLISubagentBlockRegistry::new(RefCell::new(HashMap::new())); + let handoff_blocks = HandoffBlockRegistry::new(RefCell::new(HashMap::new())); + let model_for_root = terminal_model.clone(); + let agent_blocks_for_root = agent_blocks.clone(); + let cli_subagent_blocks_for_root = cli_subagent_blocks.clone(); + let handoff_blocks_for_root = handoff_blocks.clone(); + let (window_id, root) = app.update(|ctx| { + ctx.add_tui_window( + AddWindowOptions { + window_style: WindowStyle::NotStealFocus, + ..Default::default() + }, + move |_| BenchmarkTranscriptView { + model: model_for_root, + agent_blocks: agent_blocks_for_root, + cli_subagent_blocks: cli_subagent_blocks_for_root, + handoff_blocks: handoff_blocks_for_root, + viewport: TuiViewportedListState::new_at_end(), + }, + ) + }); + + let agent_block_id = match dataset { + TranscriptDataset::ManySmallBlocks { .. } => None, + TranscriptDataset::LongAgentResponse { rows } + | TranscriptDataset::OffscreenStreamingTail { + tail_rows: rows, .. + } => { + let (action_model, model_events) = add_test_action_model_and_events(&mut app); + let text = long_response(rows); + let status = match dataset { + TranscriptDataset::LongAgentResponse { .. } => completed_text_status(text), + TranscriptDataset::OffscreenStreamingTail { .. } => { + streaming_text_status(text) + } + TranscriptDataset::ManySmallBlocks { .. } => unreachable!(), + }; + let terminal_model_for_block = terminal_model.clone(); + let agent_block = app.update(|ctx| { + ctx.add_typed_action_tui_view(window_id, move |ctx| { + TuiAIBlock::new( + (AIConversationId::new(), AIAgentExchangeId::new()), + Rc::new(BenchmarkAgentBlockModel { status }), + action_model, + &model_events, + terminal_model_for_block, + false, + ctx, + ) + }) + }); + let view_id = agent_block.id(); + agent_blocks.borrow_mut().insert(view_id, agent_block); + terminal_model.lock().block_list_mut().append_rich_content( + RichContentItem::new(Some(RichContentType::AIBlock), view_id, None, false), + false, + ); + Some(view_id) + } + }; + + let mut benchmark = Self { + app, + root, + model: terminal_model, + agent_block_id, + presenter: TuiPresenter::new(), + area: TuiRect::new(0, 0, width, height), + }; + benchmark.invalidate_all(); + benchmark.present(); + benchmark + }) + } + + /// Paints one frame from retained view elements and returns a cheap checksum. + pub fn present(&mut self) -> u64 { + let frame = self + .app + .update(|ctx| self.presenter.present(ctx, &self.root, self.area)); + frame.buffer.content.iter().fold(0u64, |checksum, cell| { + checksum.wrapping_add(cell.symbol().len() as u64) + }) + } + + /// Re-renders the root and long agent block before the next frame. + pub fn invalidate_all(&mut self) { + let mut updated = EntityIdSet::from_iter([self.root.id()]); + if let Some(agent_block_id) = self.agent_block_id { + updated.insert(agent_block_id); + } + let invalidation = WindowInvalidation { + updated, + ..Default::default() + }; + self.app.read(|ctx| { + self.presenter + .invalidate(&invalidation, ctx, self.root.window_id(ctx)); + }); + } + + /// Marks the rich tail dirty and re-renders its retained view. + pub fn invalidate_streaming_tail(&mut self) { + let agent_block_id = self + .agent_block_id + .expect("streaming-tail fixture has an agent block"); + self.model + .lock() + .block_list_mut() + .mark_rich_content_dirty(agent_block_id); + self.invalidate_all(); + } + + /// Moves the retained viewport to an absolute transcript row. + pub fn scroll_to_row(&mut self, row: usize) { + self.root.update(&mut self.app, |view, _| { + view.viewport + .set_position(TuiViewportPosition::RowsFromTop(row)); + }); + } + + /// Moves the retained viewport back to the transcript end. + pub fn scroll_to_end(&mut self) { + self.root.update(&mut self.app, |view, _| { + view.viewport.scroll_to_end(); + }); + } +} + +struct BenchmarkTranscriptView { + model: Arc>, + agent_blocks: AgentBlockRegistry, + cli_subagent_blocks: CLISubagentBlockRegistry, + handoff_blocks: HandoffBlockRegistry, + viewport: TuiViewportedListState, +} + +impl Entity for BenchmarkTranscriptView { + type Event = (); +} + +impl TypedActionView for BenchmarkTranscriptView { + type Action = (); +} + +impl TuiView for BenchmarkTranscriptView { + fn ui_name() -> &'static str { + "BenchmarkTranscriptView" + } + + fn child_view_ids(&self, _app: &AppContext) -> Vec { + self.agent_blocks.borrow().keys().copied().collect() + } + + fn render(&self, app: &AppContext) -> Box { + let source = TuiBlockListViewportSource::new_with_rich_content( + self.model.clone(), + self.agent_blocks.clone(), + self.cli_subagent_blocks.clone(), + self.handoff_blocks.clone(), + ); + TuiViewportedList::new( + self.viewport.clone(), + source, + TuiUiBuilder::from_app(app).selection_style(), + ) + .with_vertical_alignment(TuiViewportVerticalAlignment::GrowFromBottom) + .finish() + } +} + +struct BenchmarkAgentBlockModel { + status: AIBlockOutputStatus, +} + +impl AIBlockModel for BenchmarkAgentBlockModel { + type View = TuiAIBlock; + + fn status(&self, _app: &AppContext) -> AIBlockOutputStatus { + self.status.clone() + } + + fn server_output_id(&self, _app: &AppContext) -> Option { + None + } + + fn model_id(&self, _app: &AppContext) -> Option { + None + } + + fn base_model<'a>(&'a self, _app: &'a AppContext) -> Option<&'a LLMId> { + None + } + + fn inputs_to_render<'a>(&'a self, _app: &'a AppContext) -> &'a [AIAgentInput] { + &[] + } + + fn conversation_id(&self, _app: &AppContext) -> Option { + None + } + + fn on_updated_output( + &self, + _callback: OutputStatusUpdateCallback, + _ctx: &mut ViewContext, + ) { + } + + fn request_type(&self, _app: &AppContext) -> AIRequestType { + AIRequestType::Active + } +} + +fn completed_text_status(text: String) -> AIBlockOutputStatus { + AIBlockOutputStatus::Complete { + output: Shared::new(AIAgentOutput { + messages: vec![AIAgentOutputMessage { + id: MessageId::new("benchmark-response".to_owned()), + message: AIAgentOutputMessageType::Text(AIAgentText { + sections: vec![AIAgentTextSection::PlainText { text: text.into() }], + }), + citations: Vec::new(), + }], + ..Default::default() + }), + } +} + +fn streaming_text_status(text: String) -> AIBlockOutputStatus { + let AIBlockOutputStatus::Complete { output } = completed_text_status(text) else { + unreachable!() + }; + AIBlockOutputStatus::PartiallyReceived { output } +} + +fn long_response(rows: usize) -> String { + let mut text = String::with_capacity(rows.saturating_mul(48)); + for row in 0..rows { + text.push_str("benchmark transcript response row "); + text.push_str(&(row % 10_000).to_string()); + text.push('\n'); + } + text +} diff --git a/crates/warp_tui/src/lib.rs b/crates/warp_tui/src/lib.rs index 233a40d0cbd..e048d2d6435 100644 --- a/crates/warp_tui/src/lib.rs +++ b/crates/warp_tui/src/lib.rs @@ -14,6 +14,9 @@ mod alt_screen_view; mod api_keys_menu; mod attachment_bar; mod autoupdate; +#[cfg(feature = "test-util")] +#[doc(hidden)] +pub mod benchmark_support; mod cli_agent_osc_event_publisher; mod clipboard; mod cloud_run; @@ -63,7 +66,7 @@ mod terminal_block; mod terminal_content_element; mod terminal_session_view; mod terminal_use; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] mod test_fixtures; mod tool_call_labels; mod transcript_view; diff --git a/crates/warp_tui/src/test_fixtures.rs b/crates/warp_tui/src/test_fixtures.rs index 5a8f37e1f1d..2a1191bf0d0 100644 --- a/crates/warp_tui/src/test_fixtures.rs +++ b/crates/warp_tui/src/test_fixtures.rs @@ -1,4 +1,5 @@ //! Shared fixtures for `warp_tui` unit tests. +#![cfg_attr(not(test), allow(dead_code))] use std::any::Any; use std::sync::Arc;