From df279458aaaa2f5b64af2948705abddde4bc5019 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Thu, 25 Jun 2026 15:55:46 +0800 Subject: [PATCH 1/2] tweak whisper default for better long form audio decoding --- .../python/src/transcribe_cpp/__init__.py | 6 +- bindings/rust/transcribe-cpp/src/session.rs | 2 +- bindings/rust/transcribe-cpp/src/types.rs | 8 +- .../swift/Sources/TranscribeCpp/Options.swift | 5 +- bindings/typescript/src/index.ts | 221 ++++++++++++++---- bindings/typescript/src/types.ts | 1 + examples/cli/main.cpp | 4 +- scripts/whisper_long_form_parity.py | 11 +- src/arch/whisper/model.cpp | 14 +- src/transcribe.cpp | 10 + tests/api_smoke.c | 2 +- tests/whisper_e2e_smoke.cpp | 7 +- 12 files changed, 228 insertions(+), 63 deletions(-) diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index bffb5096..41ddf060 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -975,7 +975,7 @@ def _resolve_family(self, family: "FamilyExtension", slot: str): def run(self, pcm: PCMLike, *, task: Task = "transcribe", language: str | None = None, target_language: str | None = None, - timestamps: Timestamps = "none", + timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, family: FamilyExtension | None = None) -> Result: @@ -1011,7 +1011,7 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", language: str | None = None, target_language: str | None = None, - timestamps: Timestamps = "none", + timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, family: FamilyExtension | None = None, @@ -1339,7 +1339,7 @@ def transcribe( task: Task = "transcribe", language: str | None = None, target_language: str | None = None, - timestamps: Timestamps = "none", + timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, family: FamilyExtension | None = None, diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index a1647458..e58a9265 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -46,7 +46,7 @@ impl Default for RunOptions { fn default() -> Self { RunOptions { task: Task::Transcribe, - timestamps: TimestampKind::None, + timestamps: TimestampKind::Auto, pnc: Pnc::Default, itn: Itn::Default, language: None, diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index 02c370ce..5c27fa14 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -29,10 +29,14 @@ impl Task { /// Requested (or returned) timestamp granularity. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum TimestampKind { - /// Text only, no alignment data. The default for a run. - #[default] + /// Text only, no alignment data. None, /// "Richest the model supports" — resolved per-family at run time. + /// The default for a run: matches the wider Whisper ecosystem + /// (whisper.cpp / OpenAI / HF decode with timestamps by default) and + /// avoids Whisper's fragile no-timestamps path. No-timestamp families + /// resolve this back to [`None`](Self::None), so it is a safe default. + #[default] Auto, /// Segment-level start/end times. Segment, diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index 95b325ff..4401fda3 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -126,7 +126,10 @@ public struct RunOptions: Sendable { public init( task: TranscriptionTask = .transcribe, - timestamps: TimestampKind = .none, + // .auto ("richest the model supports", per-family) mirrors the C + // transcribe_run_params_init default: whisper resolves it to .segment + // (its robust path), no-timestamp families resolve to .none. + timestamps: TimestampKind = .auto, pnc: Pnc = .default, itn: Itn = .default, language: String? = nil, diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index 684a85a6..5ef90e29 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -6,7 +6,12 @@ * backend discovery, log routing, and cooperative cancellation. */ -import { native, setLogHandler, type LogHandler, type Native } from "./native.js"; +import { + native, + setLogHandler, + type LogHandler, + type Native, +} from "./native.js"; import { resolveLibrary } from "./loader.js"; import * as g from "./_generated.js"; import { @@ -67,7 +72,10 @@ const KV_TYPES: Record = { f32: g.TRANSCRIBE_KV_TYPE_F32, f16: g.TRANSCRIBE_KV_TYPE_F16, }; -const TASKS = { transcribe: g.TRANSCRIBE_TASK_TRANSCRIBE, translate: g.TRANSCRIBE_TASK_TRANSLATE }; +const TASKS = { + transcribe: g.TRANSCRIBE_TASK_TRANSCRIBE, + translate: g.TRANSCRIBE_TASK_TRANSLATE, +}; const TIMESTAMPS: Record = { none: g.TRANSCRIBE_TIMESTAMPS_NONE, auto: g.TRANSCRIBE_TIMESTAMPS_AUTO, @@ -89,7 +97,11 @@ const FEATURES: Record = { // ---- helpers --------------------------------------------------------------- -function lookup(map: Record, key: T, what: string): number { +function lookup( + map: Record, + key: T, + what: string, +): number { const v = map[key]; if (v === undefined) { throw new TranscribeError( @@ -196,7 +208,9 @@ function deferFree(lock: Mutex, fn: () => void, after?: () => void): void { function callAsync(fn: any, ...args: any[]): Promise { return new Promise((resolve, reject) => - fn.async(...args, (err: Error | null, res: T) => (err ? reject(err) : resolve(res))), + fn.async(...args, (err: Error | null, res: T) => + err ? reject(err) : resolve(res), + ), ); } @@ -212,11 +226,15 @@ function toFloat32(pcm: PcmLike): Float32Array { else if (ArrayBuffer.isView(pcm)) { const b = pcm as Buffer; if (b.byteLength % 4 !== 0) { - throw new TranscribeError("PCM byte length must be a multiple of 4 (float32)"); + throw new TranscribeError( + "PCM byte length must be a multiple of 4 (float32)", + ); } out = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4); } else { - throw new TranscribeError("PCM must be a Float32Array, number[], ArrayBuffer, or Buffer"); + throw new TranscribeError( + "PCM must be a Float32Array, number[], ArrayBuffer, or Buffer", + ); } if (out.length === 0) throw new TranscribeError("PCM is empty"); return out; @@ -261,9 +279,17 @@ class Mutex { // one exception: `headerHash` below is the compile-time PUBLIC_HEADER_HASH and // is the value the binding *expects*, not one read from the loaded library. -export function version(): { version: string; commit: string; headerHash: string } { +export function version(): { + version: string; + commit: string; + headerHash: string; +} { const n = native(); - return { version: n.F.version(), commit: n.F.versionCommit(), headerHash: g.PUBLIC_HEADER_HASH }; + return { + version: n.F.version(), + commit: n.F.versionCommit(), + headerHash: g.PUBLIC_HEADER_HASH, + }; } export function libraryPath(): string { @@ -375,7 +401,10 @@ function batchAccessors(n: Native, h: any, i: number): Accessors { }; } -function materialize(n: Native, acc: Accessors): Omit { +function materialize( + n: Native, + acc: Accessors, +): Omit { const F = n.F; const segments: Segment[] = []; @@ -510,7 +539,11 @@ const FAMILY: Record = { kind: g.TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM, type: "transcribe_parakeet_buffered_stream_ext", init: "parakeetBufferedStreamExtInit", - map: (o) => ({ left_ms: o.leftMs, chunk_ms: o.chunkMs, right_ms: o.rightMs }), + map: (o) => ({ + left_ms: o.leftMs, + chunk_ms: o.chunkMs, + right_ms: o.rightMs, + }), }, voxtral: { slot: "stream", @@ -530,16 +563,26 @@ const FAMILY: Record = { * accepts the kind. The returned buffer must be kept alive (held via the params * object) until the native call returns. */ -function buildFamily(n: Native, modelHandle: any, family: FamilyExtension, slot: ExtSlot): any { +function buildFamily( + n: Native, + modelHandle: any, + family: FamilyExtension, + slot: ExtSlot, +): any { const reg = FAMILY[family.kind]; - if (!reg) throw new InvalidArgument(`unknown family extension kind ${JSON.stringify(family.kind)}`); + if (!reg) + throw new InvalidArgument( + `unknown family extension kind ${JSON.stringify(family.kind)}`, + ); if (reg.slot !== slot) { throw new InvalidArgument( `family "${family.kind}" is a ${reg.slot} extension, not valid for a ${slot} call`, ); } if (!n.F.modelAcceptsExtKind(modelHandle, SLOT[reg.slot], reg.kind)) { - throw new UnsupportedRequest(`this model does not accept the "${family.kind}" ${reg.slot} extension`); + throw new UnsupportedRequest( + `this model does not accept the "${family.kind}" ${reg.slot} extension`, + ); } const ext: any = {}; n.F[reg.init](ext); // defaults + struct_size + kind @@ -574,7 +617,10 @@ function toStreamUpdate(u: any): StreamUpdate { * owning Session within this module. The closures capture the Stream; WeakMap * ephemeron semantics keep that from pinning it in memory. */ -const STREAM_TEARDOWN = new WeakMap(); +const STREAM_TEARDOWN = new WeakMap< + Stream, + { deactivate(): void; invalidate(): void; releaseLease(): void } +>(); interface SessionControl { enterCompute(kind: string): void; @@ -640,7 +686,9 @@ export class Session { /** Reads touch the session; forbidden while a worker call is in flight. */ #assertNotComputing(what: string): void { if (this.#inFlight) { - throw new TranscribeError(`cannot read session ${what} while ${this.#inFlight} is in flight; await it first`); + throw new TranscribeError( + `cannot read session ${what} while ${this.#inFlight} is in flight; await it first`, + ); } } @@ -662,7 +710,10 @@ export class Session { * reads it on a worker thread while this runs, so do not mutate the buffer * (e.g. reuse a scratch array) until the returned promise resolves. */ - async run(pcm: PcmLike, opts: TranscribeOptions = {}): Promise { + async run( + pcm: PcmLike, + opts: TranscribeOptions = {}, + ): Promise { const n = this.#n; const F = n.F; const h = this.handle; @@ -671,7 +722,8 @@ export class Session { const p = this.#buildRunParams(opts); return this.#lock.run(async () => { - if (this.#disposed) throw new TranscribeError("session has been disposed"); + if (this.#disposed) + throw new TranscribeError("session has been disposed"); if (this.#lock.streamActive) throw busyError("run"); const cancel = this.#installAbort(opts.signal); let status: number; @@ -683,7 +735,10 @@ export class Session { cancel?.(); } - if (status === g.TRANSCRIBE_ERR_ABORTED || status === g.TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { + if ( + status === g.TRANSCRIBE_ERR_ABORTED || + status === g.TRANSCRIBE_ERR_OUTPUT_TRUNCATED + ) { const partial: TranscriptionResult = { ...materialize(n, singleAccessors(n, h)), aborted: F.wasAborted(h), @@ -698,7 +753,11 @@ export class Session { } check(n, status, "transcribe_run"); - return { ...materialize(n, singleAccessors(n, h)), aborted: F.wasAborted(h), truncated: F.wasTruncated(h) }; + return { + ...materialize(n, singleAccessors(n, h)), + aborted: F.wasAborted(h), + truncated: F.wasTruncated(h), + }; }); } @@ -707,12 +766,18 @@ export class Session { const p: any = {}; n.F.runParamsInit(p); p.task = lookup(TASKS, opts.task ?? "transcribe", "task"); - p.timestamps = lookup(TIMESTAMPS, opts.timestamps ?? "none", "timestamps"); + // Default "auto" mirrors the C transcribe_run_params_init default: + // whisper resolves it to "segment" (its robust path), no-timestamp + // families resolve to "none". + p.timestamps = lookup(TIMESTAMPS, opts.timestamps ?? "auto", "timestamps"); if (opts.language !== undefined) p.language = opts.language; - if (opts.targetLanguage !== undefined) p.target_language = opts.targetLanguage; - if (opts.keepSpecialTags !== undefined) p.keep_special_tags = opts.keepSpecialTags; + if (opts.targetLanguage !== undefined) + p.target_language = opts.targetLanguage; + if (opts.keepSpecialTags !== undefined) + p.keep_special_tags = opts.keepSpecialTags; if (opts.specKDrafts !== undefined) p.spec_k_drafts = opts.specKDrafts; - if (opts.family) p.family = buildFamily(n, this.#model.handle, opts.family, "run"); + if (opts.family) + p.family = buildFamily(n, this.#model.handle, opts.family, "run"); return p; } @@ -721,23 +786,35 @@ export class Session { * Inputs are borrowed, not copied: native code reads them on a worker thread * while this runs, so do not mutate them until the returned promise resolves. */ - async runBatch(pcms: PcmLike[], opts: TranscribeOptions = {}): Promise { + async runBatch( + pcms: PcmLike[], + opts: TranscribeOptions = {}, + ): Promise { const n = this.#n; const F = n.F; const h = this.handle; - if (pcms.length === 0) throw new InvalidArgument("runBatch requires at least one input"); + if (pcms.length === 0) + throw new InvalidArgument("runBatch requires at least one input"); const arrays = pcms.map(toFloat32); const counts = Int32Array.from(arrays, (a) => a.length); const p = this.#buildRunParams(opts); return this.#lock.run(async () => { - if (this.#disposed) throw new TranscribeError("session has been disposed"); + if (this.#disposed) + throw new TranscribeError("session has been disposed"); if (this.#lock.streamActive) throw busyError("runBatch"); const cancel = this.#installAbort(opts.signal); let status: number; this.#inFlight = "runBatch()"; try { - status = await callAsync(F.runBatch, h, arrays, counts, arrays.length, p); + status = await callAsync( + F.runBatch, + h, + arrays, + counts, + arrays.length, + p, + ); } finally { this.#inFlight = null; cancel?.(); @@ -753,12 +830,23 @@ export class Session { if (st === g.TRANSCRIBE_OK) { out.push({ ok: true, - result: { ...materialize(n, batchAccessors(n, h, i)), aborted: false, truncated: false }, + result: { + ...materialize(n, batchAccessors(n, h, i)), + aborted: false, + truncated: false, + }, }); } else { - const error = exceptionForStatus(st, F.statusString(st), `utterance ${i}`); + const error = exceptionForStatus( + st, + F.statusString(st), + `utterance ${i}`, + ); error.utteranceIndex = i; - if (st === g.TRANSCRIBE_ERR_ABORTED || st === g.TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { + if ( + st === g.TRANSCRIBE_ERR_ABORTED || + st === g.TRANSCRIBE_ERR_OUTPUT_TRUNCATED + ) { error.partialResult = { ...materialize(n, batchAccessors(n, h, i)), aborted: st === g.TRANSCRIBE_ERR_ABORTED, @@ -787,16 +875,22 @@ export class Session { }); const sp: any = {}; F.streamParamsInit(sp); - sp.commit_policy = lookup(COMMIT_POLICIES, opts.commitPolicy ?? "auto", "commitPolicy"); + sp.commit_policy = lookup( + COMMIT_POLICIES, + opts.commitPolicy ?? "auto", + "commitPolicy", + ); if (opts.stablePrefixAgreementN !== undefined) { sp.stable_prefix_agreement_n = opts.stablePrefixAgreementN; } - if (opts.family) sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); + if (opts.family) + sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); return this.#lock.run(async () => { // Recheck inside the lock: dispose() may have run after we captured `h` // but before this queued body — don't begin a stream on a dead session. - if (this.#disposed) throw new TranscribeError("session has been disposed"); + if (this.#disposed) + throw new TranscribeError("session has been disposed"); if (this.#lock.streamActive) throw busyError("begin a stream"); check(n, F.streamBegin(h, rp, sp), "transcribe_stream_begin"); this.#lock.streamActive = true; // claim the lease for the whole stream lifetime @@ -826,7 +920,10 @@ export class Session { flag.aborted = true; }; signal.addEventListener("abort", onAbort, { once: true }); - const cbPtr = n.koffi.register(() => flag.aborted, n.koffi.pointer(n.abortProto)); + const cbPtr = n.koffi.register( + () => flag.aborted, + n.koffi.pointer(n.abortProto), + ); n.F.setAbortCallback(this.handle, cbPtr, null); return () => { n.F.setAbortCallback(this.handle, null, null); @@ -860,7 +957,11 @@ export class Session { const n = this.#n; const h = this.#h; this.#h = null; - deferFree(this.#lock, () => n.F.sessionFree(h), () => teardown?.releaseLease()); + deferFree( + this.#lock, + () => n.F.sessionFree(h), + () => teardown?.releaseLease(), + ); } [Symbol.dispose](): void { @@ -887,7 +988,8 @@ export class Stream { this.#session = session; this.#lock = lock; const sessionControl = SESSION_CONTROL.get(session); - if (!sessionControl) throw new TranscribeError("session control is missing"); + if (!sessionControl) + throw new TranscribeError("session control is missing"); this.#sessionControl = sessionControl; this.#keepalive = keepalive; // Expose the teardown surface ONLY to the owning Session, via the @@ -923,7 +1025,9 @@ export class Stream { #assertCurrent(what: string): void { if (this.#stale) { - throw new TranscribeError(`cannot ${what}: stream is no longer current for this session`); + throw new TranscribeError( + `cannot ${what}: stream is no longer current for this session`, + ); } } @@ -949,7 +1053,13 @@ export class Stream { this.#inFlight = true; this.#sessionControl.enterCompute("feed()/finalize()"); try { - const status = await callAsync(n.F.streamFeed, h, samples, samples.length, u); + const status = await callAsync( + n.F.streamFeed, + h, + samples, + samples.length, + u, + ); if (status !== g.TRANSCRIBE_OK) { // Native feed failures leave the stream in FAILED, which is no longer // an active stream in the C API. Keep the wrapper readable for @@ -1104,7 +1214,10 @@ export class TranscribeModel { ensureExitHook(); } - static async load(path: string, opts: ModelOptions = {}): Promise { + static async load( + path: string, + opts: ModelOptions = {}, + ): Promise { const n = native(); const p: any = {}; n.F.modelLoadParamsInit(p); @@ -1114,7 +1227,8 @@ export class TranscribeModel { const out: any[] = [null]; const st = await callAsync(n.F.modelLoadFile, path, p, out); check(n, st, `loading model ${path}`); - if (!out[0]) throw new ModelLoadError(`model load returned a null handle for ${path}`); + if (!out[0]) + throw new ModelLoadError(`model load returned a null handle for ${path}`); return new TranscribeModel(n, out[0]); } @@ -1134,14 +1248,20 @@ export class TranscribeModel { const out: any[] = [null]; check(n, n.F.sessionInit(this.handle, p, out), "opening session"); - if (!out[0]) throw new TranscribeError("session init returned a null handle"); - const session = new Session(n, this, out[0], this.#lock, (s) => this.#sessions.delete(s)); + if (!out[0]) + throw new TranscribeError("session init returned a null handle"); + const session = new Session(n, this, out[0], this.#lock, (s) => + this.#sessions.delete(s), + ); this.#sessions.add(session); return session; } /** Convenience: one session, one run, disposed after. */ - async transcribe(pcm: PcmLike, opts: TranscribeOptions = {}): Promise { + async transcribe( + pcm: PcmLike, + opts: TranscribeOptions = {}, + ): Promise { const session = this.createSession(); try { return await session.run(pcm, opts); @@ -1176,7 +1296,10 @@ export class TranscribeModel { } supports(feature: Feature): boolean { - return this.#n.F.modelSupports(this.handle, lookup(FEATURES, feature, "feature")); + return this.#n.F.modelSupports( + this.handle, + lookup(FEATURES, feature, "feature"), + ); } /** Whether this model accepts the given family extension on its slot. */ @@ -1195,7 +1318,9 @@ export class TranscribeModel { const buf = new Int32Array(cap); const r = F.tokenize(this.handle, text, buf, cap); if (r === INT_MIN) { - throw new NotImplementedByModel("this model's tokenizer does not support encode"); + throw new NotImplementedByModel( + "this model's tokenizer does not support encode", + ); } if (r >= 0) return buf.subarray(0, r); cap = -r; // buffer too small; -r is the count needed @@ -1219,7 +1344,11 @@ export class TranscribeModel { get device(): BackendInfo { const dev: any = {}; this.#n.F.backendDeviceInit(dev); - check(this.#n, this.#n.F.modelGetDevice(this.handle, dev), "reading model device"); + check( + this.#n, + this.#n.F.modelGetDevice(this.handle, dev), + "reading model device", + ); return deviceFromRaw(dev); } diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 9df5cbb6..81312120 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -130,6 +130,7 @@ export interface TranscribeOptions { task?: Task; language?: string; targetLanguage?: string; + /** Default "auto" (richest the model supports, per-family). */ timestamps?: TimestampKind; keepSpecialTags?: boolean; /** Speculative-decode draft count; -1 = family default. */ diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index da5acc36..f521249b 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -132,7 +132,7 @@ struct cli_args { transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; int gpu_device = 0; // --device N: 0 = auto, >0 = registry index - transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_NONE; + transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; // Whisper-family knobs. Ignored for non-Whisper models. std::string initial_prompt; // --initial-prompt TEXT @@ -209,7 +209,7 @@ void print_usage(const char * argv0) { " (default: auto)\n" " --device N GPU device index from --list-devices: 0 = auto\n" " (first of kind), >0 selects that registry index\n" - " --timestamps TYPE timestamps: auto, none, segment, word, token (default: none)\n" + " --timestamps TYPE timestamps: auto, none, segment, word, token (default: auto)\n" " --batch FILE batch mode: FILE has one wav path per line\n" " --batch-jsonl output one JSON line per file (for batch)\n" " --batch-size N group N utterances into one transcribe_run_batch\n" diff --git a/scripts/whisper_long_form_parity.py b/scripts/whisper_long_form_parity.py index 1acc7f61..4ec21466 100644 --- a/scripts/whisper_long_form_parity.py +++ b/scripts/whisper_long_form_parity.py @@ -219,7 +219,16 @@ def main() -> int: help="transcribe-cli binary path") p.add_argument("--language", default="en") p.add_argument("--initial-prompt", - help="initial prompt text / glossary to pass to both HF and C++") + help="initial prompt text / glossary to pass to both HF and C++. " + "NOTE: with --prompt-condition first (the default), the C++ " + "engine DELIBERATELY diverges from HF — it primes the prompt " + "on the first window only (whisper.cpp/OpenAI behavior) " + "instead of every window. HF re-primes every window, which " + "can collapse interior windows on out-of-distribution " + "prompts; the C++ behavior is the robust drop-in. So a " + "prompted run with prompt-condition=first is EXPECTED to " + "show non-zero WER vs HF and is not a regression. Use " + "--prompt-condition all for a like-for-like HF comparison.") p.add_argument("--prompt-condition", choices=["first", "all"], default="first", help="prompt placement when --initial-prompt is set") p.add_argument("--no-condition", action="store_true", diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index e7552444..19404e8f 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -1947,9 +1947,16 @@ transcribe_status whisper_run( // skip_ending_double_timestamps: if history's last two ids are // both timestamps, drop the last one (PR #34537 / #35750). // elif initial prompt is set: - // prev_tokens = [<|startofprev|>, prompt_text_ids...] (every chunk) + // prev_tokens = [<|startofprev|>, prompt_text_ids...] + // ALL_SEGMENTS: every chunk + // FIRST_SEGMENT: first chunk only // else: // prev_tokens = empty (Stage 2 behavior) + // + // HF (_prepare_decoder_input_ids) re-applies the prompt on + // EVERY chunk here regardless of prompt_condition. We deliberately + // diverge for FIRST_SEGMENT (the default) to match whisper.cpp / + // OpenAI, which prime only the first window. std::vector prev_tokens; if (do_condition_on_prev_tokens && !prev_history_segments.empty() && prev_sot_id >= 0) @@ -1978,7 +1985,10 @@ transcribe_status whisper_run( max_prev_cap); prev_tokens.insert(prev_tokens.end(), hist.end() - cap, hist.end()); - } else if (!prompt_text_ids.empty() && prev_sot_id >= 0) { + } else if (!prompt_text_ids.empty() && prev_sot_id >= 0 && + (is_first_chunk || + wp->prompt_condition == TRANSCRIBE_WHISPER_PROMPT_ALL_SEGMENTS)) { + // FIRST_SEGMENT primes the initial prompt on the first window prev_tokens.push_back(prev_sot_id); prev_tokens.insert(prev_tokens.end(), prompt_text_ids.begin(), diff --git a/src/transcribe.cpp b/src/transcribe.cpp index b03038a3..6b129e16 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -497,6 +497,16 @@ extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { std::memset(p, 0, sizeof(*p)); p->struct_size = sizeof(*p); p->spec_k_drafts = -1; // family default + // Default to AUTO ("richest the model supports", resolved per-family at + // run time) rather than the memset NONE. AUTO matches the wider Whisper + // ecosystem (whisper.cpp no_timestamps=false, OpenAI without_timestamps= + // False, HF return_timestamps), which decode WITH timestamps by default. + // Whisper's no-timestamps path is fragile: with an initial_prompt set it + // can collapse to a degenerate early-EOT result (reproducible identically + // in HF transformers and whisper.cpp). AUTO routes Whisper through its + // SEGMENT path (no DTW), while no-timestamp families (cohere, sensevoice, + // canary, granite_nar) resolve AUTO back to NONE — no behavior change. + p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; } extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * p) { diff --git a/tests/api_smoke.c b/tests/api_smoke.c index 0a3566a4..33d614fb 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -238,7 +238,7 @@ static void test_init_macros(void) { transcribe_run_params_init(&rp_macro); CHECK(rp_macro.struct_size == sizeof(struct transcribe_run_params)); CHECK(rp_macro.task == TRANSCRIBE_TASK_TRANSCRIBE); - CHECK(rp_macro.timestamps == TRANSCRIBE_TIMESTAMPS_NONE); + CHECK(rp_macro.timestamps == TRANSCRIBE_TIMESTAMPS_AUTO); CHECK(rp_macro.pnc == TRANSCRIBE_PNC_MODE_DEFAULT); CHECK(rp_macro.itn == TRANSCRIBE_ITN_MODE_DEFAULT); CHECK(rp_macro.language == NULL); diff --git a/tests/whisper_e2e_smoke.cpp b/tests/whisper_e2e_smoke.cpp index 556d042b..51e98715 100644 --- a/tests/whisper_e2e_smoke.cpp +++ b/tests/whisper_e2e_smoke.cpp @@ -132,13 +132,12 @@ int main() { CHECK(st == TRANSCRIBE_OK); CHECK(std::strstr(transcribe_full_text(ctx), "country") != nullptr); CHECK_EQ_INT(transcribe_returned_timestamp_kind(ctx), - TRANSCRIBE_TIMESTAMPS_NONE); - CHECK_EQ_INT(transcribe_n_segments(ctx), 1); + TRANSCRIBE_TIMESTAMPS_SEGMENT); + CHECK(transcribe_n_segments(ctx) >= 1); { transcribe_segment seg; transcribe_segment_init(&seg); CHECK_EQ_INT(transcribe_get_segment(ctx, 0, &seg), TRANSCRIBE_OK); - CHECK_EQ_INT(seg.t0_ms, 0); - CHECK_EQ_INT(seg.t1_ms, 0); + CHECK(seg.t1_ms >= seg.t0_ms); } } From 19424123a5c8285905bedb7d43fe4813a33ad543 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Thu, 25 Jun 2026 15:58:40 +0800 Subject: [PATCH 2/2] clean up comments --- bindings/rust/transcribe-cpp/src/types.rs | 4 ---- src/transcribe.cpp | 9 +-------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index 5c27fa14..00e6a0fe 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -32,10 +32,6 @@ pub enum TimestampKind { /// Text only, no alignment data. None, /// "Richest the model supports" — resolved per-family at run time. - /// The default for a run: matches the wider Whisper ecosystem - /// (whisper.cpp / OpenAI / HF decode with timestamps by default) and - /// avoids Whisper's fragile no-timestamps path. No-timestamp families - /// resolve this back to [`None`](Self::None), so it is a safe default. #[default] Auto, /// Segment-level start/end times. diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 6b129e16..0b30593a 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -498,14 +498,7 @@ extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { p->struct_size = sizeof(*p); p->spec_k_drafts = -1; // family default // Default to AUTO ("richest the model supports", resolved per-family at - // run time) rather than the memset NONE. AUTO matches the wider Whisper - // ecosystem (whisper.cpp no_timestamps=false, OpenAI without_timestamps= - // False, HF return_timestamps), which decode WITH timestamps by default. - // Whisper's no-timestamps path is fragile: with an initial_prompt set it - // can collapse to a degenerate early-EOT result (reproducible identically - // in HF transformers and whisper.cpp). AUTO routes Whisper through its - // SEGMENT path (no DTW), while no-timestamp families (cohere, sensevoice, - // canary, granite_nar) resolve AUTO back to NONE — no behavior change. + // run time) rather than the memset NONE. p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; }