From c1951f262a63240917ee871d4ca3018df935e08a Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Wed, 15 Jul 2026 11:31:08 +0000 Subject: [PATCH 1/2] fix: bound streaming VAD frame buffers --- funasr/frontends/wav_frontend.py | 56 ++- funasr/models/fsmn_vad_streaming/model.py | 127 +++++-- tests/test_fsmn_vad_streaming_buffers.py | 432 ++++++++++++++++++++++ 3 files changed, 588 insertions(+), 27 deletions(-) create mode 100644 tests/test_fsmn_vad_streaming_buffers.py diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py index 072bebf82..02cbf6c36 100644 --- a/funasr/frontends/wav_frontend.py +++ b/funasr/frontends/wav_frontend.py @@ -262,6 +262,8 @@ def forward_lfr_cmvn( class WavFrontendOnline(nn.Module): """Conventional frontend structure for streaming ASR/VAD.""" + supports_aligned_waveforms = True + def __init__( self, cmvn_file: str = None, @@ -514,17 +516,30 @@ def forward(self, input: torch.Tensor, input_lengths: torch.Tensor, **kwargs): cache = kwargs.get("cache", {}) if len(cache) == 0: self.init_cache(cache) + cache.setdefault("waveform_buffer", None) + cache.setdefault("waveform_buffer_start_sample", 0) + cache.setdefault("emitted_lfr_frames", 0) + cache["aligned_waveforms"] = torch.empty(0) + return_waveform = kwargs.get("return_waveform", False) batch_size = input.shape[0] assert ( batch_size == 1 ), "we support to extract feature online only when the batch size is equal to 1 now" + # VAD opts in to exact score-aligned waveform spans. + if return_waveform and input.shape[1] > 0: + cache["waveform_buffer"] = ( + input.clone() + if cache["waveform_buffer"] is None + else torch.cat((cache["waveform_buffer"], input), dim=1) + ) waveforms, feats, feats_lengths = self.forward_fbank( input, input_lengths, cache=cache ) # input shape: B T D + has_fbank_frames = bool(feats.shape[0]) - if feats.shape[0]: + if has_fbank_frames: cache["waveforms"] = torch.cat((cache["reserve_waveforms"], waveforms), dim=1) @@ -587,6 +602,41 @@ def forward(self, input: torch.Tensor, input_lengths: torch.Tensor, **kwargs): feats, feats_lengths, _ = self.forward_lfr_cmvn( feats, feats_lengths, is_final, cache=cache ) + + if return_waveform and feats.ndim == 3 and feats.shape[1] > 0: + score_frame_shift = self.lfr_n * self.frame_shift_sample_length + aligned_start = cache["emitted_lfr_frames"] * score_frame_shift + aligned_sample_count = ( + (feats.shape[1] - 1) * score_frame_shift + self.frame_sample_length + ) + aligned_end = aligned_start + aligned_sample_count + waveform_buffer = cache["waveform_buffer"] + buffer_start = cache["waveform_buffer_start_sample"] + buffer_end = buffer_start + ( + 0 if waveform_buffer is None else waveform_buffer.shape[1] + ) + if ( + waveform_buffer is None + or aligned_start < buffer_start + or aligned_end > buffer_end + ): + raise RuntimeError( + "Frontend emitted LFR frames without enough waveform samples: " + f"need [{aligned_start}, {aligned_end}), " + f"have [{buffer_start}, {buffer_end})" + ) + local_start = aligned_start - buffer_start + local_end = aligned_end - buffer_start + cache["aligned_waveforms"] = waveform_buffer[ + :, local_start:local_end + ].clone() + cache["emitted_lfr_frames"] += feats.shape[1] + + next_frame_start = cache["emitted_lfr_frames"] * score_frame_shift + next_buffer_start = min(next_frame_start, buffer_end) + drop_samples = next_buffer_start - buffer_start + cache["waveform_buffer"] = waveform_buffer[:, drop_samples:].clone() + cache["waveform_buffer_start_sample"] = next_buffer_start # if is_final: # self.init_cache(cache) return feats, feats_lengths @@ -603,6 +653,10 @@ def init_cache(self, cache: dict = None): cache["input_cache"] = torch.empty(0) cache["lfr_splice_cache"] = [] cache["waveforms"] = None + cache["aligned_waveforms"] = torch.empty(0) + cache["waveform_buffer"] = None + cache["waveform_buffer_start_sample"] = 0 + cache["emitted_lfr_frames"] = 0 cache["fbanks"] = None cache["fbanks_lens"] = None return cache diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index ae5f43aaa..744724d1d 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -403,6 +403,35 @@ def __init__( self.encoder = encoder self.encoder_conf = encoder_conf + def DropCachedFrames(self, drop_frames: int, cache: dict = None) -> None: + """Release score, decibel, and waveform history before an absolute frame.""" + if cache is None: + cache = {} + stats = cache["stats"] + if drop_frames > stats.frm_cnt: + raise RuntimeError( + f"Cannot drop through frame {drop_frames}; only {stats.frm_cnt} frames exist" + ) + + real_drop_frames = drop_frames - stats.last_drop_frames + if real_drop_frames <= 0: + return + + frame_shift_length = int( + self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 + ) + drop_samples = real_drop_frames * frame_shift_length + stats.data_buf_all = stats.data_buf_all[drop_samples:].clone() + stats.decibel = stats.decibel[real_drop_frames:] + stats.scores = stats.scores[:, real_drop_frames:, :].clone() + stats.last_drop_frames = drop_frames + + data_buf_offset = max( + stats.data_buf_start_frame - stats.last_drop_frames, + 0, + ) * frame_shift_length + stats.data_buf = stats.data_buf_all[data_buf_offset:] + def ResetDetection(self, cache: dict = None): """Resetdetection. @@ -424,44 +453,80 @@ def ResetDetection(self, cache: dict = None): if cache["stats"].output_data_buf: assert cache["stats"].output_data_buf[-1].contain_seg_end_point == True drop_frames = int(cache["stats"].output_data_buf[-1].end_ms / self.vad_opts.frame_in_ms) - real_drop_frames = drop_frames - cache["stats"].last_drop_frames - cache["stats"].last_drop_frames = drop_frames - cache["stats"].data_buf_all = cache["stats"].data_buf_all[ - real_drop_frames - * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) : - ] - cache["stats"].decibel = cache["stats"].decibel[real_drop_frames:] - cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :] - - def ComputeDecibel(self, cache: dict = None) -> None: + self.DropCachedFrames(drop_frames, cache=cache) + + def ComputeDecibel( + self, cache: dict = None, frame_count: Optional[int] = None + ) -> None: """Computedecibel. Args: + frame_count: Number of VAD score frames emitted for this waveform. cache: State cache dict for streaming inference. """ if cache is None: cache = {} frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000) frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) - if cache["stats"].data_buf_all is None: - cache["stats"].data_buf_all = cache["stats"].waveform[ - 0 - ] # cache["stats"].data_buf is pointed to cache["stats"].waveform[0] - cache["stats"].data_buf = cache["stats"].data_buf_all + if frame_count is not None and int(frame_count) <= 0: + return + stats = cache["stats"] + waveform = stats.waveform[0] + + frame_count_was_provided = frame_count is not None + available_frames = max( + (waveform.numel() - frame_sample_length) // frame_shift_length + 1, + 0, + ) + frame_count = available_frames if frame_count is None else int(frame_count) + if frame_count <= 0: + return + + # The frontend provides the exact waveform span for these VAD score frames. + aligned_sample_count = ( + (frame_count - 1) * frame_shift_length + frame_sample_length + ) + if waveform.numel() != aligned_sample_count: + if frame_count_was_provided or waveform.numel() < aligned_sample_count: + raise RuntimeError( + "VAD score frames and waveform samples are not aligned: " + f"expected {aligned_sample_count}, got {waveform.numel()}" + ) + waveform = waveform[:aligned_sample_count] + + aligned_waveform = waveform + + if stats.data_buf_all is None: + stats.data_buf_all = aligned_waveform.clone() else: - cache["stats"].data_buf_all = torch.cat( - (cache["stats"].data_buf_all, cache["stats"].waveform[0]) + new_sample_count = frame_count * frame_shift_length + stats.data_buf_all = torch.cat( + (stats.data_buf_all, aligned_waveform[-new_sample_count:]) ) - - waveform_numpy = cache["stats"].waveform.numpy() - offsets = np.arange(0, waveform_numpy.shape[1] - frame_sample_length + 1, frame_shift_length) - frames = waveform_numpy[0, offsets[:, np.newaxis] + np.arange(frame_sample_length)] + data_buf_offset = max( + stats.data_buf_start_frame - stats.last_drop_frames, + 0, + ) * frame_shift_length + stats.data_buf = stats.data_buf_all[data_buf_offset:] - decibel_numpy = 10 * np.log10(np.sum(np.square(frames), axis=1) + 0.000001) + waveform_numpy = aligned_waveform.detach().cpu().numpy() + + offsets = np.arange( + 0, + waveform_numpy.shape[0] - frame_sample_length + 1, + frame_shift_length, + ) + frames = waveform_numpy[ + offsets[:, np.newaxis] + np.arange(frame_sample_length) + ] + + decibel_numpy = 10 * np.log10( + np.sum(np.square(frames), axis=1) + 0.000001 + ) decibel_numpy = decibel_numpy.tolist() - cache["stats"].decibel.extend(decibel_numpy) + stats.decibel.extend(decibel_numpy) def ComputeScores(self, feats: torch.Tensor, cache: dict = None) -> None: @@ -710,7 +775,6 @@ def GetFrameState(self, t: int, cache: dict = None): # for each frame, calc log posterior probability of each state if cur_decibel < self.vad_opts.decibel_thres: frame_state = FrameState.kFrameStateSil - self.DetectOneFrame(frame_state, t, False, cache=cache) return frame_state sum_score = 0.0 @@ -777,17 +841,20 @@ def forward( """ if cache is None: cache = {} + if feats.numel() == 0: + return [] # if len(cache) == 0: # self.AllResetDetection() # self.waveform = waveform # compute decibel for each frame cache["stats"].waveform = waveform is_streaming_input = kwargs.get("is_streaming_input", True) - self.ComputeDecibel(cache=cache) + self.ComputeDecibel(frame_count=feats.shape[1], cache=cache) self.ComputeScores(feats, cache=cache) if not is_final: self.DetectCommonFrames(cache=cache) else: self.DetectLastFrames(cache=cache) + self.DropCachedFrames(cache["stats"].data_buf_start_frame, cache=cache) segments = [] for batch_num in range(0, feats.shape[0]): # only support batch_size = 1 now segment_batch = [] @@ -946,6 +1013,9 @@ def inference( speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres accumulated_ms = cache.get("_dynamic_accumulated_ms", 0) in_speech = cache.get("_dynamic_in_speech", False) + frontend_inference_kwargs = {} + if getattr(frontend, "supports_aligned_waveforms", False): + frontend_inference_kwargs["return_waveform"] = True for i in range(n): kwargs["is_final"] = _is_final and i == n - 1 @@ -973,6 +1043,7 @@ def inference( frontend=frontend, cache=cache["frontend"], is_final=kwargs["is_final"], + **frontend_inference_kwargs, ) time3 = time.perf_counter() meta_data["extract_feat"] = f"{time3 - time2:0.3f}" @@ -982,9 +1053,13 @@ def inference( speech = speech.to(device=kwargs["device"]) speech_lengths = speech_lengths.to(device=kwargs["device"]) + waveform = cache["frontend"].get("aligned_waveforms") + if waveform is None or waveform.numel() == 0: + # Custom frontends may expose an exact span under the legacy key. + waveform = cache["frontend"]["waveforms"] batch = { "feats": speech, - "waveform": cache["frontend"]["waveforms"], + "waveform": waveform, "is_final": kwargs["is_final"], "cache": cache, "is_streaming_input": is_streaming_input, diff --git a/tests/test_fsmn_vad_streaming_buffers.py b/tests/test_fsmn_vad_streaming_buffers.py new file mode 100644 index 000000000..e31b1925a --- /dev/null +++ b/tests/test_fsmn_vad_streaming_buffers.py @@ -0,0 +1,432 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from funasr.frontends.wav_frontend import WavFrontendOnline +from funasr.models.fsmn_vad_streaming import model as vad_model + + +class DummyEncoder(torch.nn.Module): + def forward(self, feats, cache=None): + return torch.zeros(feats.shape[0], feats.shape[1], 2) + + +class DummyWindowDetector: + def Reset(self): + return None + + +def build_vad_and_cache(): + vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(vad) + vad.encoder = DummyEncoder() + vad.vad_opts = SimpleNamespace( + frame_length_ms=25, + frame_in_ms=10, + sample_rate=16000, + nn_eval_block_size=0, + ) + vad.DetectCommonFrames = lambda cache=None: 0 + vad.DetectLastFrames = lambda cache=None: 0 + + cache = { + "encoder": {}, + "stats": vad_model.Stats( + sil_pdf_ids=[], + max_end_sil_frame_cnt_thresh=0, + speech_noise_thres=0.5, + ), + "windows_detector": DummyWindowDetector(), + } + return vad, cache + + +def run_aligned_frontend_windows(vad, cache): + source = torch.arange(4000, dtype=torch.float32) / 4000.0 + calls = ( + (source[0:560], 2), + (source[320:1520], 6), + (source[1280:2480], 6), + ) + for waveform, frame_count in calls: + vad.forward( + feats=torch.zeros(1, frame_count, 4), + waveform=waveform.unsqueeze(0), + cache=cache, + is_final=False, + ) + return source + + +def test_streaming_buffers_only_append_audio_aligned_with_score_frames(): + vad, cache = build_vad_and_cache() + source = run_aligned_frontend_windows(vad, cache) + stats = cache["stats"] + + expected_waveform = source[:2480] + expected_frames = expected_waveform.unfold(0, 400, 160) + expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001) + + assert stats.scores.shape[1] == 14 + assert len(stats.decibel) == stats.scores.shape[1] + assert torch.allclose(torch.tensor(stats.decibel), expected_decibel) + assert torch.equal(stats.data_buf_all, expected_waveform) + assert torch.equal(stats.data_buf, expected_waveform) + + +def test_reset_detection_drops_completed_aligned_frames(): + vad, cache = build_vad_and_cache() + source = run_aligned_frontend_windows(vad, cache) + stats = cache["stats"] + + segment = vad_model.E2EVadSpeechBufWithDoa() + segment.Reset() + segment.end_ms = 80 + segment.contain_seg_end_point = True + stats.output_data_buf.append(segment) + stats.data_buf_start_frame = 8 + + vad.ResetDetection(cache=cache) + + assert stats.last_drop_frames == 8 + assert stats.scores.shape[1] == 6 + assert len(stats.decibel) == 6 + assert torch.equal(stats.data_buf_all, source[1280:2480]) + assert torch.equal(stats.data_buf, stats.data_buf_all) + + vad.forward( + feats=torch.zeros(1, 2, 4), + waveform=source[2240:2800].unsqueeze(0), + cache=cache, + is_final=False, + ) + + expected_waveform = source[1280:2800] + expected_frames = expected_waveform.unfold(0, 400, 160) + expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001) + assert stats.scores.shape[1] == 8 + assert len(stats.decibel) == 8 + assert torch.allclose(torch.tensor(stats.decibel), expected_decibel) + assert torch.equal(stats.data_buf_all, expected_waveform) + assert torch.equal(stats.data_buf, expected_waveform) + + +@pytest.mark.parametrize( + ("final_size", "expected_frame_counts", "expected_ranges", "expected_sample_count"), + ( + (160, (2, 6, 6, 3), ((0, 560), (320, 1520), (1280, 2480), (2240, 2960)), 2960), + (0, (2, 6, 6, 2), ((0, 560), (320, 1520), (1280, 2480), (2240, 2800)), 2800), + ), +) +def test_real_frontend_exposes_exact_waveforms_for_final_chunks( + final_size, + expected_frame_counts, + expected_ranges, + expected_sample_count, +): + frontend = WavFrontendOnline( + fs=16000, + n_mels=80, + frame_length=25, + frame_shift=10, + lfr_m=5, + lfr_n=1, + dither=0.0, + upsacle_samples=False, + ) + frontend_cache = {} + vad, vad_cache = build_vad_and_cache() + source = torch.arange(4000, dtype=torch.float32) / 4000.0 + offset = 0 + + for index, (chunk_size, frame_count, sample_range) in enumerate( + zip((960, 960, 960, final_size), expected_frame_counts, expected_ranges) + ): + chunk = source[offset : offset + chunk_size] + offset += chunk_size + is_final = index == len(expected_frame_counts) - 1 + feats, _ = frontend( + chunk.unsqueeze(0), + torch.tensor([chunk.numel()]), + cache=frontend_cache, + is_final=is_final, + return_waveform=True, + ) + aligned_waveform = frontend_cache["aligned_waveforms"] + + assert feats.shape[1] == frame_count + assert torch.equal(aligned_waveform[0], source[slice(*sample_range)]) + assert frontend_cache["waveform_buffer"].shape[1] <= 640 + + vad.forward( + feats=feats, + waveform=aligned_waveform, + cache=vad_cache, + is_final=is_final, + ) + + expected_waveform = source[:expected_sample_count] + expected_frames = expected_waveform.unfold(0, 400, 160) + expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001) + stats = vad_cache["stats"] + + assert stats.scores.shape[1] == sum(expected_frame_counts) + assert len(stats.decibel) == stats.scores.shape[1] + assert torch.allclose(torch.tensor(stats.decibel), expected_decibel) + assert torch.equal(stats.data_buf_all, expected_waveform) + + +def test_compute_decibel_keeps_positional_cache_call_compatible(): + vad, cache = build_vad_and_cache() + waveform = torch.arange(560, dtype=torch.float32) / 560.0 + cache["stats"].waveform = waveform.unsqueeze(0) + + vad.ComputeDecibel(cache) + + assert len(cache["stats"].decibel) == 2 + assert torch.equal(cache["stats"].data_buf_all, waveform) + + +def test_real_frontend_tracks_initial_padding_across_small_chunks(): + frontend = WavFrontendOnline( + fs=16000, + n_mels=80, + frame_length=25, + frame_shift=10, + lfr_m=5, + lfr_n=1, + dither=0.0, + upsacle_samples=False, + ) + cache = {} + source = torch.arange(2000, dtype=torch.float32) / 2000.0 + emitted_frames = 0 + input_offset = 0 + + for index, (chunk_size, expected_frames) in enumerate( + zip((480, 480, 480, 160), (0, 2, 3, 3)) + ): + chunk = source[input_offset : input_offset + chunk_size] + input_offset += chunk_size + feats, _ = frontend( + chunk.unsqueeze(0), + torch.tensor([chunk.numel()]), + cache=cache, + is_final=index == 3, + return_waveform=True, + ) + + assert (feats.shape[1] if feats.ndim == 3 else 0) == expected_frames + assert cache["waveform_buffer"].shape[1] <= 640 + if expected_frames == 0: + assert cache["aligned_waveforms"].numel() == 0 + continue + + aligned_sample_count = (expected_frames - 1) * 160 + 400 + expected_waveform = source[ + emitted_frames * 160 : emitted_frames * 160 + aligned_sample_count + ] + assert torch.equal(cache["aligned_waveforms"][0], expected_waveform) + emitted_frames += expected_frames + + assert emitted_frames == 8 + + +def test_frontend_cursor_does_not_advance_past_received_audio(): + frontend = WavFrontendOnline( + fs=16000, + n_mels=80, + frame_length=25, + frame_shift=10, + lfr_m=7, + lfr_n=6, + dither=0.0, + upsacle_samples=False, + ) + cache = {} + source = torch.arange(2400, dtype=torch.float32) / 2400.0 + + first_feats, _ = frontend( + source[:880].unsqueeze(0), + torch.tensor([880]), + cache=cache, + is_final=False, + return_waveform=True, + ) + + assert first_feats.shape[1] == 1 + assert torch.equal(cache["aligned_waveforms"][0], source[:400]) + assert cache["waveform_buffer_start_sample"] == 880 + + second_feats, _ = frontend( + source[880:1840].unsqueeze(0), + torch.tensor([960]), + cache=cache, + is_final=False, + return_waveform=True, + ) + + assert second_feats.shape[1] == 1 + assert torch.equal(cache["aligned_waveforms"][0], source[960:1360]) + + +def test_zero_score_frontend_batch_is_a_model_noop(): + frontend = WavFrontendOnline( + fs=16000, + n_mels=80, + frame_length=25, + frame_shift=10, + lfr_m=5, + lfr_n=1, + dither=0.0, + upsacle_samples=False, + ) + frontend_cache = {} + source = torch.arange(480, dtype=torch.float32) / 480.0 + feats, _ = frontend( + source.unsqueeze(0), + torch.tensor([source.numel()]), + cache=frontend_cache, + is_final=False, + return_waveform=True, + ) + vad, cache = build_vad_and_cache() + + assert feats.numel() == 0 + assert ( + vad.forward( + feats=feats, + waveform=frontend_cache["aligned_waveforms"], + cache=cache, + is_final=False, + ) + == [] + ) + assert cache["stats"].data_buf_all is None + + +def test_consumed_silence_history_is_compacted_without_an_endpoint(): + vad, cache = build_vad_and_cache() + + def consume_all_frames(cache=None): + cache["stats"].data_buf_start_frame = cache["stats"].frm_cnt + + vad.DetectCommonFrames = consume_all_frames + source = torch.arange(100 * 960 + 400, dtype=torch.float32) / 100000.0 + + for index in range(100): + start = index * 960 + vad.forward( + feats=torch.zeros(1, 6, 4), + waveform=source[start : start + 1200].unsqueeze(0), + cache=cache, + is_final=False, + ) + + stats = cache["stats"] + assert stats.last_drop_frames == stats.frm_cnt == 600 + assert stats.data_buf_all.numel() == 240 + assert stats.data_buf.numel() == 240 + assert stats.scores.shape[1] == 0 + assert stats.decibel == [] + + +def test_waveform_alignment_is_opt_in_for_other_frontend_consumers(): + frontend = WavFrontendOnline( + fs=16000, + n_mels=80, + frame_length=25, + frame_shift=10, + lfr_m=5, + lfr_n=1, + dither=0.0, + upsacle_samples=False, + ) + cache = {} + waveform = torch.zeros(1, 960) + + frontend( + waveform, + torch.tensor([waveform.shape[1]]), + cache=cache, + is_final=False, + ) + + assert cache["waveform_buffer"] is None + assert cache["aligned_waveforms"].numel() == 0 + + +def test_misaligned_waveform_fails_before_mutating_model_buffers(): + vad, cache = build_vad_and_cache() + cache["stats"].waveform = torch.zeros(1, 880) + + with pytest.raises(RuntimeError, match="score frames and waveform samples"): + vad.ComputeDecibel(cache=cache, frame_count=2) + + assert cache["stats"].data_buf_all is None + assert cache["stats"].decibel == [] + + +def test_low_decibel_frame_advances_detector_once_with_absolute_index(): + vad, cache = build_vad_and_cache() + stats = cache["stats"] + stats.decibel = [-100.0] + stats.scores = torch.zeros(1, 1, 2) + stats.frm_cnt = 101 + stats.last_drop_frames = 100 + vad.vad_opts.decibel_thres = 0.0 + vad.vad_opts.nn_eval_block_size = 1 + detected_frames = [] + vad.DetectOneFrame = lambda frame_state, frame_index, is_final, cache=None: ( + detected_frames.append(frame_index) + ) + + vad_model.FsmnVADStreaming.DetectCommonFrames(vad, cache=cache) + + assert detected_frames == [100] + + +@pytest.mark.parametrize("supports_alignment", (False, True)) +def test_inference_negotiates_aligned_waveform_support(supports_alignment): + vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + vad.vad_opts = SimpleNamespace(speech_to_sil_time_thres=100) + vad.forward = lambda **batch: [] + cache = { + "frontend": {}, + "prev_samples": torch.empty(0), + "encoder": {}, + "stats": SimpleNamespace(), + } + frontend = SimpleNamespace(fs=16000, frame_shift=10, lfr_n=1) + if supports_alignment: + frontend.supports_aligned_waveforms = True + extract_kwargs = [] + + def fake_extract_fbank(*args, **kwargs): + extract_kwargs.append(kwargs) + cache["frontend"]["waveforms"] = torch.zeros(1, 560) + return torch.zeros(1, 2, 4), torch.tensor([2]) + + with ( + patch.object( + vad_model, + "load_audio_text_image_video", + return_value=[torch.zeros(960)], + ), + patch.object(vad_model, "extract_fbank", side_effect=fake_extract_fbank), + ): + vad_model.FsmnVADStreaming.inference( + vad, + torch.zeros(960), + frontend=frontend, + cache=cache, + key=["utt"], + chunk_size=60, + is_final=False, + device="cpu", + dynamic_silence=False, + ) + + assert extract_kwargs[0].get("return_waveform", False) is supports_alignment From 30c2fa9e372cc42090bf47969c4b543f18ffda7c Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Wed, 15 Jul 2026 12:03:53 +0000 Subject: [PATCH 2/2] fix: validate missing streaming VAD waveforms --- funasr/models/fsmn_vad_streaming/model.py | 7 +++- tests/test_fsmn_vad_streaming_buffers.py | 44 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index 744724d1d..63e181de2 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -1056,7 +1056,12 @@ def inference( waveform = cache["frontend"].get("aligned_waveforms") if waveform is None or waveform.numel() == 0: # Custom frontends may expose an exact span under the legacy key. - waveform = cache["frontend"]["waveforms"] + waveform = cache["frontend"].get("waveforms") + if speech.numel() > 0 and (waveform is None or waveform.numel() == 0): + raise RuntimeError( + "Streaming VAD frontend must provide aligned_waveforms or waveforms " + "when emitting feature frames" + ) batch = { "feats": speech, "waveform": waveform, diff --git a/tests/test_fsmn_vad_streaming_buffers.py b/tests/test_fsmn_vad_streaming_buffers.py index e31b1925a..1f97f01ef 100644 --- a/tests/test_fsmn_vad_streaming_buffers.py +++ b/tests/test_fsmn_vad_streaming_buffers.py @@ -430,3 +430,47 @@ def fake_extract_fbank(*args, **kwargs): ) assert extract_kwargs[0].get("return_waveform", False) is supports_alignment + + +def test_inference_rejects_missing_waveform_before_forward(): + vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + vad.vad_opts = SimpleNamespace(speech_to_sil_time_thres=100) + forward_batches = [] + vad.forward = lambda **batch: forward_batches.append(batch) or [] + cache = { + "frontend": {}, + "prev_samples": torch.empty(0), + "encoder": {}, + "stats": SimpleNamespace(), + } + frontend = SimpleNamespace(fs=16000, frame_shift=10, lfr_n=1) + + with ( + patch.object( + vad_model, + "load_audio_text_image_video", + return_value=[torch.zeros(960)], + ), + patch.object( + vad_model, + "extract_fbank", + return_value=(torch.zeros(1, 2, 4), torch.tensor([2])), + ), + pytest.raises( + RuntimeError, + match="must provide aligned_waveforms or waveforms", + ), + ): + vad_model.FsmnVADStreaming.inference( + vad, + torch.zeros(960), + frontend=frontend, + cache=cache, + key=["utt"], + chunk_size=60, + is_final=False, + device="cpu", + dynamic_silence=False, + ) + + assert forward_batches == []