From a6d87c34c6dc4bfa84f85a1b88a7b7ab563fca80 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 04:11:13 +0800 Subject: [PATCH 01/14] fix(player): pause MP2 audio during buffering --- web-ui/src/mpegts/audio/pcm-audio-player.ts | 102 ++++++++++++++++++-- 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/web-ui/src/mpegts/audio/pcm-audio-player.ts b/web-ui/src/mpegts/audio/pcm-audio-player.ts index 97df8c43..76304c15 100644 --- a/web-ui/src/mpegts/audio/pcm-audio-player.ts +++ b/web-ui/src/mpegts/audio/pcm-audio-player.ts @@ -133,6 +133,7 @@ export class PCMAudioPlayer { private driftLogCounter = 0; private controlTimer: ReturnType | null = null; + private isBuffering: boolean = false; private isSeeking: boolean = false; /** Last `timeupdate` playback position, used to measure seek delta (seeking may already show the target time). */ private lastKnownVideoTime: number = 0; @@ -147,6 +148,10 @@ export class PCMAudioPlayer { private boundOnVolumeChange: (() => void) | null = null; private boundOnTimeUpdate: (() => void) | null = null; private boundOnRateChange: (() => void) | null = null; + private boundOnVideoWaiting: (() => void) | null = null; + private boundOnVideoStalled: (() => void) | null = null; + private boundOnVideoPlaying: (() => void) | null = null; + private boundOnVideoCanPlay: (() => void) | null = null; /** Called when AudioContext is blocked by autoplay policy (needs user interaction). */ onSuspended: (() => void) | null = null; @@ -187,7 +192,7 @@ export class PCMAudioPlayer { this.context.onstatechange = () => { Log.v(TAG, `AudioContext state changed to: ${this.context?.state}`); - if (this.context?.state === "running") { + if (this.context?.state === "running" && this.canScheduleAudio()) { this.resyncFromBuffer(this.videoElement?.currentTime ?? 0); } }; @@ -225,6 +230,14 @@ export class PCMAudioPlayer { // no audible interruption. this.controlTick(); }; + this.boundOnVideoWaiting = () => this.enterBuffering("waiting"); + this.boundOnVideoStalled = () => { + if (video.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { + this.enterBuffering("stalled"); + } + }; + this.boundOnVideoPlaying = () => this.maybeExitBuffering(); + this.boundOnVideoCanPlay = () => this.maybeExitBuffering(); video.addEventListener("seeking", this.boundOnVideoSeeking); video.addEventListener("seeked", this.boundOnVideoSeeked); @@ -233,6 +246,10 @@ export class PCMAudioPlayer { video.addEventListener("volumechange", this.boundOnVolumeChange); video.addEventListener("timeupdate", this.boundOnTimeUpdate); video.addEventListener("ratechange", this.boundOnRateChange); + video.addEventListener("waiting", this.boundOnVideoWaiting); + video.addEventListener("stalled", this.boundOnVideoStalled); + video.addEventListener("playing", this.boundOnVideoPlaying); + video.addEventListener("canplay", this.boundOnVideoCanPlay); this.controlTimer = setInterval(() => { this.controlTick(); @@ -253,6 +270,10 @@ export class PCMAudioPlayer { if (this.boundOnVolumeChange) this.videoElement.removeEventListener("volumechange", this.boundOnVolumeChange); if (this.boundOnTimeUpdate) this.videoElement.removeEventListener("timeupdate", this.boundOnTimeUpdate); if (this.boundOnRateChange) this.videoElement.removeEventListener("ratechange", this.boundOnRateChange); + if (this.boundOnVideoWaiting) this.videoElement.removeEventListener("waiting", this.boundOnVideoWaiting); + if (this.boundOnVideoStalled) this.videoElement.removeEventListener("stalled", this.boundOnVideoStalled); + if (this.boundOnVideoPlaying) this.videoElement.removeEventListener("playing", this.boundOnVideoPlaying); + if (this.boundOnVideoCanPlay) this.videoElement.removeEventListener("canplay", this.boundOnVideoCanPlay); } this.boundOnVideoSeeking = null; this.boundOnVideoSeeked = null; @@ -261,6 +282,10 @@ export class PCMAudioPlayer { this.boundOnVolumeChange = null; this.boundOnTimeUpdate = null; this.boundOnRateChange = null; + this.boundOnVideoWaiting = null; + this.boundOnVideoStalled = null; + this.boundOnVideoPlaying = null; + this.boundOnVideoCanPlay = null; this.videoElement = null; } @@ -281,7 +306,7 @@ export class PCMAudioPlayer { this.insertToBuffer(chunk); this.cleanupBuffer(); - if (!this.isSeeking && !this.videoElement?.paused) { + if (this.canScheduleAudio()) { this.pendingChunks.push(chunk); if (this.pendingChunks.length > MAX_PENDING_CHUNKS) { this.pendingChunks.shift(); @@ -348,7 +373,7 @@ export class PCMAudioPlayer { */ private pump(): void { const ctx = this.context; - if (!ctx || !this.gainNode || this.isSeeking || this.pendingChunks.length === 0 || this.videoElement?.paused) { + if (!ctx || !this.gainNode || this.pendingChunks.length === 0 || !this.canScheduleAudio()) { return; } @@ -433,6 +458,59 @@ export class PCMAudioPlayer { this.videoElement?.pause(); } + // ==================== Media readiness ==================== + + private hasPlayableVideoData(): boolean { + const video = this.videoElement; + return ( + !!video && + !video.paused && + !video.seeking && + !this.isSeeking && + video.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA + ); + } + + private canScheduleAudio(): boolean { + return !this.isBuffering && this.hasPlayableVideoData(); + } + + private resetDriftState(): void { + this.driftEma = 0; + this.hasDriftEma = false; + } + + private enterBuffering(reason: "waiting" | "stalled"): void { + const video = this.videoElement; + if (!video || video.paused || video.seeking || this.isSeeking) { + return; + } + + if (!this.isBuffering) { + Log.v(TAG, `Video ${reason}; pausing PCM audio scheduling`); + } + this.isBuffering = true; + this.cancelChain(true); + this.pendingChunks = []; + this.inputCursor = null; + this.resetDriftState(); + } + + private maybeExitBuffering(): void { + const video = this.videoElement; + if (!video || !this.hasPlayableVideoData()) { + return; + } + + if (!this.isBuffering) { + return; + } + + this.isBuffering = false; + Log.v(TAG, "Video playback resumed; resyncing PCM audio"); + this.resyncFromBuffer(video.currentTime); + } + private anchor(time: number): void { this.stretcher?.reset(); // Feedforward the current playback rate immediately: waiting for the next @@ -575,7 +653,7 @@ export class PCMAudioPlayer { private controlTick(): void { const ctx = this.context; const video = this.videoElement; - if (!ctx || !video || ctx.state !== "running" || video.paused || this.isSeeking || !this.stretcher) { + if (!ctx || !video || ctx.state !== "running" || !this.canScheduleAudio() || !this.stretcher) { return; } @@ -702,8 +780,7 @@ export class PCMAudioPlayer { this.cancelChain(true); this.pendingChunks = []; this.inputCursor = null; - this.driftEma = 0; - this.hasDriftEma = false; + this.resetDriftState(); const startIndex = this.findChunkIndexByTime(targetTime); if (startIndex < 0) { @@ -737,6 +814,7 @@ export class PCMAudioPlayer { } private onVideoSeeking(): void { + this.isBuffering = false; this.largeSeekCancelled = false; const video = this.videoElement; if (video) { @@ -786,8 +864,11 @@ export class PCMAudioPlayer { } catch (_e) { Log.w(TAG, "Failed to resume AudioContext on play()"); } - } else if (this.videoElement) { - this.resyncFromBuffer(this.videoElement.currentTime); + } else { + const video = this.videoElement; + if (video && this.canScheduleAudio()) { + this.resyncFromBuffer(video.currentTime); + } } if (this.audioElement) { @@ -800,6 +881,7 @@ export class PCMAudioPlayer { } pause(): void { + this.isBuffering = false; this.cancelChain(); this.pendingChunks = []; this.inputCursor = null; @@ -819,13 +901,13 @@ export class PCMAudioPlayer { this.pendingChunks = []; this.audioBuffer = []; + this.isBuffering = false; this.isSeeking = false; this.inputCursor = null; this.stretcher?.reset(); this.stretcherFailed = false; this.softSyncUntil = 0; - this.driftEma = 0; - this.hasDriftEma = false; + this.resetDriftState(); } setVolume(volume: number): void { From 56884757815a9f5b421a0e54c5c6fcc7e020a2bf Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 04:24:11 +0800 Subject: [PATCH 02/14] fix(player): bridge remux timestamp holes --- web-ui/src/mpegts/remux/mp4-remuxer.ts | 88 ++++++++++++++++++++++---- web-ui/src/mpegts/worker/pipeline.ts | 2 +- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 82abd165..79d1c3e6 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -1,3 +1,4 @@ +import { defaultConfig, type PlayerConfig } from "../config"; import { isFirefox } from "../utils/browser"; import { IllegalStateException } from "../utils/exception"; import Log from "../utils/logger"; @@ -83,6 +84,13 @@ interface MP4Sample { flags: MP4SampleFlags; } +type RemuxerConfig = Pick; + +interface TrackTimingState { + lastOriginalEndDts: number | undefined; + lastOutputEndDts: number | undefined; +} + type InitSegmentCallback = (type: string, segment: InitSegment) => void; type MediaSegmentCallback = (type: string, segment: MediaSegment) => void; @@ -102,6 +110,8 @@ function writeUint32(data: Uint8Array, offset: number, value: number): void { class MP4Remuxer { TAG: string; + private _config: RemuxerConfig; + private _dtsBase: number; private _dtsBaseInited: boolean; private _dtsBaseOffset: number; @@ -111,6 +121,8 @@ class MP4Remuxer { private _videoNextDts: number | undefined; private _audioStashedLastSample: AudioSample | null; private _videoStashedLastSample: VideoSample | null; + private _audioTiming: TrackTimingState; + private _videoTiming: TrackTimingState; private _audioMeta: TrackMetadata | null; private _videoMeta: TrackMetadata | null; @@ -123,9 +135,13 @@ class MP4Remuxer { private _silentAudioMode: boolean; private _silentAudioLastDts: number | undefined; - constructor() { + constructor(config: RemuxerConfig = defaultConfig) { this.TAG = "MP4Remuxer"; + this._config = { + maxBufferHoleMs: config.maxBufferHoleMs ?? defaultConfig.maxBufferHoleMs, + }; + this._dtsBase = -1; this._dtsBaseInited = false; this._dtsBaseOffset = 0; @@ -135,6 +151,8 @@ class MP4Remuxer { this._videoNextDts = undefined; this._audioStashedLastSample = null; this._videoStashedLastSample = null; + this._audioTiming = this._createTrackTimingState(); + this._videoTiming = this._createTrackTimingState(); this._audioMeta = null; this._videoMeta = null; @@ -154,6 +172,8 @@ class MP4Remuxer { this._dtsBaseInited = false; this._silentAudioMode = false; this._silentAudioLastDts = undefined; + this._audioTiming = this._createTrackTimingState(); + this._videoTiming = this._createTrackTimingState(); this._audioMeta = null; this._videoMeta = null; this._onInitSegment = null; @@ -196,13 +216,49 @@ class MP4Remuxer { this._silentAudioLastDts = undefined; } + private _createTrackTimingState(): TrackTimingState { + return { + lastOriginalEndDts: undefined, + lastOutputEndDts: undefined, + }; + } + /** * Map upstream timestamps onto a continuous output timeline. * When `_nextDts` is set (continuous playback), always splice to it. - * After a discontinuity, keep upstream timing relative to the current remux base. + * After a discontinuity, preserve the last timeline correction and bridge + * small forward holes so MSE does not expose tiny buffered range gaps. */ - private _computeDtsCorrection(firstSampleOriginalDts: number, nextDts: number | undefined): number { - return nextDts !== undefined ? firstSampleOriginalDts - nextDts : 0; + private _computeDtsCorrection( + type: "audio" | "video", + firstSampleOriginalDts: number, + nextDts: number | undefined, + timing: TrackTimingState, + ): number { + if (nextDts !== undefined) { + return firstSampleOriginalDts - nextDts; + } + + if (timing.lastOriginalEndDts === undefined || timing.lastOutputEndDts === undefined) { + return 0; + } + + const distance = firstSampleOriginalDts - timing.lastOriginalEndDts; + const maxBufferHoleMs = this._config.maxBufferHoleMs ?? defaultConfig.maxBufferHoleMs; + const bridgedDistance = distance > 0 && distance <= maxBufferHoleMs ? 0 : distance; + if (bridgedDistance !== distance) { + Log.v( + this.TAG, + `${type}: bridging ${Math.round(distance)}ms timestamp hole after discontinuity (max ${maxBufferHoleMs}ms)`, + ); + } + const expectedDts = timing.lastOutputEndDts + bridgedDistance; + return firstSampleOriginalDts - expectedDts; + } + + private _recordTrackTiming(timing: TrackTimingState, sample: MP4Sample): void { + timing.lastOriginalEndDts = sample.originalDts + sample.duration; + timing.lastOutputEndDts = sample.dts + sample.duration; } /** @@ -226,10 +282,6 @@ class MP4Remuxer { if (audioTrack) { this._remuxAudio(audioTrack); } - // In silent audio mode, generate silent frames synced to video - if (this._silentAudioMode && videoTrack?.samples?.length) { - this._generateSilentAudio(videoTrack); - } } /** @@ -237,7 +289,7 @@ class MP4Remuxer { * Used in soft decode mode to keep MSE audio track active (prevents * Safari/Chrome from pausing video when tab goes to background). */ - private _generateSilentAudio(videoTrack: DemuxTrack): void { + private _generateSilentAudio(videoSamples: MP4Sample[]): void { if (!this._audioMeta || !this._onMediaSegment) { return; } @@ -251,11 +303,14 @@ class MP4Remuxer { return; } - const videoSamples = videoTrack.samples as VideoSample[]; - const videoEndDts = videoSamples[videoSamples.length - 1].dts - this._dtsBase; + if (videoSamples.length === 0) { + return; + } + + const videoEndDts = videoSamples[videoSamples.length - 1].dts + videoSamples[videoSamples.length - 1].duration; if (this._silentAudioLastDts === undefined) { - this._silentAudioLastDts = videoSamples[0].dts - this._dtsBase; + this._silentAudioLastDts = videoSamples[0].dts; } const samples: Array<{ unit: Uint8Array; dts: number; pts: number }> = []; @@ -500,7 +555,7 @@ class MP4Remuxer { const firstSampleOriginalDts = (samples[0] as AudioSample).dts - this._dtsBase; - dtsCorrection = this._computeDtsCorrection(firstSampleOriginalDts, this._audioNextDts); + dtsCorrection = this._computeDtsCorrection("audio", firstSampleOriginalDts, this._audioNextDts, this._audioTiming); const mp4Samples: MP4Sample[] = []; @@ -602,6 +657,7 @@ class MP4Remuxer { track.samples = mp4Samples; track.sequenceNumber++; + this._recordTrackTiming(this._audioTiming, mp4Samples[mp4Samples.length - 1]); let moofbox: Uint8Array; @@ -674,7 +730,7 @@ class MP4Remuxer { const firstSampleOriginalDts = (samples[0] as VideoSample).dts - this._dtsBase; - dtsCorrection = this._computeDtsCorrection(firstSampleOriginalDts, this._videoNextDts); + dtsCorrection = this._computeDtsCorrection("video", firstSampleOriginalDts, this._videoNextDts, this._videoTiming); const mp4Samples: MP4Sample[] = []; @@ -751,6 +807,7 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._videoNextDts = latest.dts + latest.duration; + this._recordTrackTiming(this._videoTiming, latest); track.samples = mp4Samples; track.sequenceNumber++; @@ -783,6 +840,9 @@ class MP4Remuxer { type: "video", data: segment.buffer, }); + if (this._silentAudioMode) { + this._generateSilentAudio(mp4Samples); + } } private _mergeBoxes(moof: Uint8Array, mdat: Uint8Array): Uint8Array { diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index c8cde906..4a772308 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -396,7 +396,7 @@ class Pipeline { this._demuxer = demuxer; if (!this._remuxer) { - this._remuxer = new MP4Remuxer(); + this._remuxer = new MP4Remuxer(this._config); if (this._pendingDtsOffsetMs !== 0) { this._remuxer.setDtsBaseOffset(this._pendingDtsOffsetMs); this._pendingDtsOffsetMs = 0; From 6f0f6dbfc64fe3b5e2982ee06d1a2d9279b5782f Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 04:31:14 +0800 Subject: [PATCH 03/14] refactor(player): remove buffer hole threshold --- web-ui/src/mpegts/audio/pcm-audio-player.ts | 37 ++------------------- web-ui/src/mpegts/config.ts | 15 --------- web-ui/src/mpegts/remux/mp4-remuxer.ts | 21 +++--------- web-ui/src/mpegts/worker/pipeline.ts | 2 +- 4 files changed, 8 insertions(+), 67 deletions(-) diff --git a/web-ui/src/mpegts/audio/pcm-audio-player.ts b/web-ui/src/mpegts/audio/pcm-audio-player.ts index 76304c15..477a9650 100644 --- a/web-ui/src/mpegts/audio/pcm-audio-player.ts +++ b/web-ui/src/mpegts/audio/pcm-audio-player.ts @@ -23,7 +23,7 @@ */ import { isIOS } from "../../lib/platform"; -import { maxBufferHoleSec, type PlayerConfig } from "../config"; +import type { PlayerConfig } from "../config"; import Log from "../utils/logger"; import { type Stretcher, WasmStretcher } from "./wasm-stretcher"; @@ -135,10 +135,6 @@ export class PCMAudioPlayer { private isBuffering: boolean = false; private isSeeking: boolean = false; - /** Last `timeupdate` playback position, used to measure seek delta (seeking may already show the target time). */ - private lastKnownVideoTime: number = 0; - /** Set when a large seek already cancelled the scheduling chain in `onVideoSeeking`. */ - private largeSeekCancelled: boolean = false; // Bound event handlers for cleanup private boundOnVideoSeeking: (() => void) | null = null; @@ -216,9 +212,6 @@ export class PCMAudioPlayer { this.setMuted(video.muted); }; this.boundOnTimeUpdate = () => { - if (!video.seeking) { - this.lastKnownVideoTime = video.currentTime; - } this.controlTick(); this.pump(); }; @@ -815,41 +808,17 @@ export class PCMAudioPlayer { private onVideoSeeking(): void { this.isBuffering = false; - this.largeSeekCancelled = false; - const video = this.videoElement; - if (video) { - const delta = Math.abs(video.currentTime - this.lastKnownVideoTime); - if (delta > maxBufferHoleSec(this.config)) { - this.cancelChain(); - this.pendingChunks = []; - this.largeSeekCancelled = true; - } - } + this.cancelChain(); + this.pendingChunks = []; this.isSeeking = true; } private onVideoSeeked(): void { if (!this.videoElement) return; const targetTime = this.videoElement.currentTime; - const prevTime = this.lastKnownVideoTime; - const delta = targetTime - prevTime; - - if (delta > 0 && delta <= maxBufferHoleSec(this.config)) { - Log.v(TAG, `Small forward seek (${(delta * 1000).toFixed(0)}ms), skipping audio resync`); - this.isSeeking = false; - this.lastKnownVideoTime = targetTime; - this.pump(); - return; - } Log.v(TAG, `Video seeked to ${targetTime.toFixed(3)}, resyncing audio`); - if (!this.largeSeekCancelled) { - this.cancelChain(); - this.pendingChunks = []; - } - this.largeSeekCancelled = false; this.isSeeking = false; - this.lastKnownVideoTime = targetTime; this.resyncFromBuffer(targetTime); } diff --git a/web-ui/src/mpegts/config.ts b/web-ui/src/mpegts/config.ts index de600e74..f95ecaea 100644 --- a/web-ui/src/mpegts/config.ts +++ b/web-ui/src/mpegts/config.ts @@ -9,14 +9,6 @@ export interface PlayerConfig { /** PlaybackRate (clamped to [1, 2]) used for latency chasing. Requires `liveSync: true`. @default 1.2 */ liveSyncPlaybackRate: number; - /** - * Maximum media timestamp hole (milliseconds) treated as continuous at remux time; - * gaps at or below this size are bridged onto the output timeline. Also used by the - * PCM audio player to skip resync on small forward seeks. - * @default 300 - */ - maxBufferHoleMs: number; - /** URLs to WASM decoder files, keyed by codec. Omit to disable software decoding for that codec. * e.g. `{ mp2: "/assets/mp2_decoder.wasm" }` */ wasmDecoders: { mp2?: string }; @@ -38,8 +30,6 @@ export const defaultConfig: PlayerConfig = { liveSyncTargetLatency: 1.5, liveSyncPlaybackRate: 1.2, - maxBufferHoleMs: 300, - wasmDecoders: {}, bufferCleanupMaxBackward: 180, @@ -52,8 +42,3 @@ export const defaultConfig: PlayerConfig = { export function createDefaultConfig(): PlayerConfig { return { ...defaultConfig }; } - -/** `maxBufferHoleMs` as seconds for MSE / Web Audio timeline comparisons. */ -export function maxBufferHoleSec(config: Pick): number { - return config.maxBufferHoleMs / 1000; -} diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 79d1c3e6..a9241416 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -1,4 +1,3 @@ -import { defaultConfig, type PlayerConfig } from "../config"; import { isFirefox } from "../utils/browser"; import { IllegalStateException } from "../utils/exception"; import Log from "../utils/logger"; @@ -84,8 +83,6 @@ interface MP4Sample { flags: MP4SampleFlags; } -type RemuxerConfig = Pick; - interface TrackTimingState { lastOriginalEndDts: number | undefined; lastOutputEndDts: number | undefined; @@ -110,8 +107,6 @@ function writeUint32(data: Uint8Array, offset: number, value: number): void { class MP4Remuxer { TAG: string; - private _config: RemuxerConfig; - private _dtsBase: number; private _dtsBaseInited: boolean; private _dtsBaseOffset: number; @@ -135,13 +130,9 @@ class MP4Remuxer { private _silentAudioMode: boolean; private _silentAudioLastDts: number | undefined; - constructor(config: RemuxerConfig = defaultConfig) { + constructor() { this.TAG = "MP4Remuxer"; - this._config = { - maxBufferHoleMs: config.maxBufferHoleMs ?? defaultConfig.maxBufferHoleMs, - }; - this._dtsBase = -1; this._dtsBaseInited = false; this._dtsBaseOffset = 0; @@ -227,7 +218,7 @@ class MP4Remuxer { * Map upstream timestamps onto a continuous output timeline. * When `_nextDts` is set (continuous playback), always splice to it. * After a discontinuity, preserve the last timeline correction and bridge - * small forward holes so MSE does not expose tiny buffered range gaps. + * forward holes so MSE stays on a continuous output timeline. */ private _computeDtsCorrection( type: "audio" | "video", @@ -244,13 +235,9 @@ class MP4Remuxer { } const distance = firstSampleOriginalDts - timing.lastOriginalEndDts; - const maxBufferHoleMs = this._config.maxBufferHoleMs ?? defaultConfig.maxBufferHoleMs; - const bridgedDistance = distance > 0 && distance <= maxBufferHoleMs ? 0 : distance; + const bridgedDistance = distance > 0 ? 0 : distance; if (bridgedDistance !== distance) { - Log.v( - this.TAG, - `${type}: bridging ${Math.round(distance)}ms timestamp hole after discontinuity (max ${maxBufferHoleMs}ms)`, - ); + Log.v(this.TAG, `${type}: bridging ${Math.round(distance)}ms timestamp hole after discontinuity`); } const expectedDts = timing.lastOutputEndDts + bridgedDistance; return firstSampleOriginalDts - expectedDts; diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index 4a772308..c8cde906 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -396,7 +396,7 @@ class Pipeline { this._demuxer = demuxer; if (!this._remuxer) { - this._remuxer = new MP4Remuxer(this._config); + this._remuxer = new MP4Remuxer(); if (this._pendingDtsOffsetMs !== 0) { this._remuxer.setDtsBaseOffset(this._pendingDtsOffsetMs); this._pendingDtsOffsetMs = 0; From d56f6227a5e8ead6585fc8057487949f684c2274 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 05:02:56 +0800 Subject: [PATCH 04/14] fix(player): bridge MP2 presentation timeline gaps --- web-ui/src/mpegts/remux/mp4-generator.ts | 6 +-- web-ui/src/mpegts/remux/mp4-remuxer.ts | 48 ++++++++++++++++++-- web-ui/src/mpegts/worker/pipeline.ts | 57 +++++++++++++++++++++--- 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/web-ui/src/mpegts/remux/mp4-generator.ts b/web-ui/src/mpegts/remux/mp4-generator.ts index 627dd091..72212278 100644 --- a/web-ui/src/mpegts/remux/mp4-generator.ts +++ b/web-ui/src/mpegts/remux/mp4-generator.ts @@ -1176,10 +1176,10 @@ class MP4 { data.set( [ - 0x00, + 0x01, 0x00, 0x0f, - 0x01, // version(0) & flags + 0x01, // version(1) & flags (sampleCount >>> 24) & 0xff, // sample_count (sampleCount >>> 16) & 0xff, (sampleCount >>> 8) & 0xff, @@ -1211,7 +1211,7 @@ class MP4 { (flags.isDependedOn << 6) | (flags.hasRedundancy << 4) | (flags.isNonSync || 0), 0x00, 0x00, // sample_degradation_priority - (cts >>> 24) & 0xff, // sample_composition_time_offset + (cts >>> 24) & 0xff, // sample_composition_time_offset (signed in trun version 1) (cts >>> 16) & 0xff, (cts >>> 8) & 0xff, cts & 0xff, diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index a9241416..f7587f44 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -86,6 +86,7 @@ interface MP4Sample { interface TrackTimingState { lastOriginalEndDts: number | undefined; lastOutputEndDts: number | undefined; + lastOutputEndPts: number | undefined; } type InitSegmentCallback = (type: string, segment: InitSegment) => void; @@ -103,6 +104,8 @@ function writeUint32(data: Uint8Array, offset: number, value: number): void { data[offset + 3] = value & 0xff; } +const PRESENTATION_GAP_TOLERANCE_MS = 1; + // Fragmented mp4 remuxer class MP4Remuxer { TAG: string; @@ -118,6 +121,9 @@ class MP4Remuxer { private _videoStashedLastSample: VideoSample | null; private _audioTiming: TrackTimingState; private _videoTiming: TrackTimingState; + private _videoCtsOffset: number | undefined; + private _videoInitialCtsOffset: number | undefined; + private _videoInitialOutputTime: number | undefined; private _audioMeta: TrackMetadata | null; private _videoMeta: TrackMetadata | null; @@ -144,6 +150,9 @@ class MP4Remuxer { this._videoStashedLastSample = null; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); + this._videoCtsOffset = undefined; + this._videoInitialCtsOffset = undefined; + this._videoInitialOutputTime = undefined; this._audioMeta = null; this._videoMeta = null; @@ -165,6 +174,9 @@ class MP4Remuxer { this._silentAudioLastDts = undefined; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); + this._videoCtsOffset = undefined; + this._videoInitialCtsOffset = undefined; + this._videoInitialOutputTime = undefined; this._audioMeta = null; this._videoMeta = null; this._onInitSegment = null; @@ -211,6 +223,7 @@ class MP4Remuxer { return { lastOriginalEndDts: undefined, lastOutputEndDts: undefined, + lastOutputEndPts: undefined, }; } @@ -243,9 +256,10 @@ class MP4Remuxer { return firstSampleOriginalDts - expectedDts; } - private _recordTrackTiming(timing: TrackTimingState, sample: MP4Sample): void { + private _recordTrackTiming(timing: TrackTimingState, sample: MP4Sample, lastOutputEndPts?: number): void { timing.lastOriginalEndDts = sample.originalDts + sample.duration; timing.lastOutputEndDts = sample.dts + sample.duration; + timing.lastOutputEndPts = lastOutputEndPts ?? sample.pts + sample.duration; } /** @@ -444,6 +458,14 @@ class MP4Remuxer { return this._dtsBase; } + getInitialPresentationOffset(): number { + return this._videoInitialCtsOffset ?? 0; + } + + getInitialOutputTime(): number { + return this._videoInitialOutputTime ?? 0; + } + flushStashedSamples(): void { const videoSample = this._videoStashedLastSample; const audioSample = this._audioStashedLastSample; @@ -720,6 +742,7 @@ class MP4Remuxer { dtsCorrection = this._computeDtsCorrection("video", firstSampleOriginalDts, this._videoNextDts, this._videoTiming); const mp4Samples: MP4Sample[] = []; + let lastOutputEndPts = this._videoTiming.lastOutputEndPts; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples for (let i = 0; i < samples.length; i++) { @@ -727,12 +750,13 @@ class MP4Remuxer { const originalDts = sample.dts - this._dtsBase; const isKeyframe = sample.isKeyframe; const dts = originalDts - (dtsCorrection as number); - const cts = sample.cts; - const pts = dts + cts; if (firstDts === -1) { firstDts = dts; } + if (this._videoInitialOutputTime === undefined) { + this._videoInitialOutputTime = dts; + } let sampleDuration = 0; @@ -773,6 +797,22 @@ class MP4Remuxer { sampleDuration = fallbackDuration; } + if (this._videoCtsOffset === undefined) { + this._videoCtsOffset = sample.cts; + this._videoInitialCtsOffset = sample.cts; + } + + let pts = dts + sample.cts - this._videoCtsOffset; + if (i === 0 && lastOutputEndPts !== undefined && pts > lastOutputEndPts + PRESENTATION_GAP_TOLERANCE_MS) { + const gap = pts - lastOutputEndPts; + this._videoCtsOffset += gap; + pts -= gap; + Log.v(this.TAG, `video: bridging ${Math.round(gap)}ms presentation timestamp hole`); + } + const cts = pts - dts; + const outputEndPts = pts + sampleDuration; + lastOutputEndPts = lastOutputEndPts === undefined ? outputEndPts : Math.max(lastOutputEndPts, outputEndPts); + mp4Samples.push({ dts: dts, pts: pts, @@ -794,7 +834,7 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._videoNextDts = latest.dts + latest.duration; - this._recordTrackTiming(this._videoTiming, latest); + this._recordTrackTiming(this._videoTiming, latest, lastOutputEndPts); track.samples = mp4Samples; track.sequenceNumber++; diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index c8cde906..42b7a512 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -49,6 +49,7 @@ class LoadError extends Error { } const HLS_URL_RE = /\.m3u8?($|\?)/i; +const PCM_TIMELINE_GAP_TOLERANCE_MS = 5; /** Sentinel rejection value for intentionally cancelled segment loads. */ const CANCELLED = Symbol("cancelled"); @@ -108,7 +109,15 @@ class Pipeline { private _audioSamplesSinceAnchor = 0; private _audioSampleRate = 0; /** PCM decoded before the remuxer dts base is known (flushed once available). */ - private _pendingPcm: Array<{ pcm: Float32Array; channels: number; sampleRate: number; ptsMs: number }> = []; + private _pendingPcm: Array<{ + pcm: Float32Array; + channels: number; + sampleRate: number; + ptsMs: number; + durationMs: number; + }> = []; + private _pcmTimelineCorrectionMs: number | null = null; + private _pcmLastOutputEndMs: number | null = null; /** Incremented on audio timing resets to invalidate decode callbacks queued before the reset. */ private _audioGen = 0; @@ -279,7 +288,7 @@ class Pipeline { // frame carried from the previous URL must not be prepended to the // next one, and the PTS anchor must re-establish from the new PES this._workerAudioDecoder?.reset(); - this._resetAudioTiming(); + this._resetAudioTiming(false); } } catch (e) { if (this._runId !== runId || e === CANCELLED) return; @@ -311,12 +320,20 @@ class Pipeline { this._resetAudioTiming(); } - private _resetAudioTiming(): void { + private _resetAudioTiming(resetPcmTimeline = true): void { this._audioGen++; this._audioAnchorPtsMs = null; this._audioSamplesSinceAnchor = 0; this._audioSampleRate = 0; this._pendingPcm = []; + if (resetPcmTimeline) { + this._resetPcmTimeline(); + } + } + + private _resetPcmTimeline(): void { + this._pcmTimelineCorrectionMs = null; + this._pcmLastOutputEndMs = null; } private _loadSegment(meta: SegmentMeta): Promise { @@ -552,7 +569,8 @@ class Pipeline { * PCM decoded before the first remux (dts base unknown) is queued. */ private _emitPcm(pcm: Float32Array, channels: number, sampleRate: number, ptsMs: number): void { - this._pendingPcm.push({ pcm, channels, sampleRate, ptsMs }); + const durationMs = (Math.floor(pcm.length / channels) / sampleRate) * 1000; + this._pendingPcm.push({ pcm, channels, sampleRate, ptsMs, durationMs }); const dtsBase = this._remuxer?.getTimestampBase(); if (dtsBase === undefined) { @@ -564,10 +582,39 @@ class Pipeline { } for (const item of this._pendingPcm) { - this._callbacks.onPCMAudioData(item.pcm, item.channels, item.sampleRate, (item.ptsMs - dtsBase) / 1000); + this._callbacks.onPCMAudioData( + item.pcm, + item.channels, + item.sampleRate, + this._mapPcmTimestamp(item.ptsMs, item.durationMs, dtsBase), + ); } this._pendingPcm = []; } + + private _mapPcmTimestamp(ptsMs: number, durationMs: number, dtsBase: number): number { + const originalTimeMs = ptsMs - dtsBase; + + if (this._pcmTimelineCorrectionMs === null) { + const initialPresentationOffset = this._remuxer?.getInitialPresentationOffset() ?? 0; + const initialOutputTime = this._remuxer?.getInitialOutputTime() ?? 0; + const outputTimeMs = Math.max(initialOutputTime, originalTimeMs - initialPresentationOffset); + this._pcmTimelineCorrectionMs = originalTimeMs - outputTimeMs; + this._pcmLastOutputEndMs = outputTimeMs + durationMs; + return outputTimeMs / 1000; + } + + let outputTimeMs = originalTimeMs - this._pcmTimelineCorrectionMs; + if (this._pcmLastOutputEndMs !== null && outputTimeMs > this._pcmLastOutputEndMs + PCM_TIMELINE_GAP_TOLERANCE_MS) { + const gap = outputTimeMs - this._pcmLastOutputEndMs; + this._pcmTimelineCorrectionMs += gap; + outputTimeMs -= gap; + Log.v(this.TAG, `PCM: bridging ${Math.round(gap)}ms timestamp hole`); + } + + this._pcmLastOutputEndMs = outputTimeMs + durationMs; + return outputTimeMs / 1000; + } } export default Pipeline; From a68e478507c9f92290977c8fedfea9d956950c36 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 05:04:16 +0800 Subject: [PATCH 05/14] fix(player): bridge all positive timestamp gaps --- web-ui/src/mpegts/remux/mp4-remuxer.ts | 4 +--- web-ui/src/mpegts/worker/pipeline.ts | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index f7587f44..0a86dea0 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -104,8 +104,6 @@ function writeUint32(data: Uint8Array, offset: number, value: number): void { data[offset + 3] = value & 0xff; } -const PRESENTATION_GAP_TOLERANCE_MS = 1; - // Fragmented mp4 remuxer class MP4Remuxer { TAG: string; @@ -803,7 +801,7 @@ class MP4Remuxer { } let pts = dts + sample.cts - this._videoCtsOffset; - if (i === 0 && lastOutputEndPts !== undefined && pts > lastOutputEndPts + PRESENTATION_GAP_TOLERANCE_MS) { + if (i === 0 && lastOutputEndPts !== undefined && pts > lastOutputEndPts) { const gap = pts - lastOutputEndPts; this._videoCtsOffset += gap; pts -= gap; diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index 42b7a512..6b5586fa 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -49,8 +49,6 @@ class LoadError extends Error { } const HLS_URL_RE = /\.m3u8?($|\?)/i; -const PCM_TIMELINE_GAP_TOLERANCE_MS = 5; - /** Sentinel rejection value for intentionally cancelled segment loads. */ const CANCELLED = Symbol("cancelled"); @@ -605,7 +603,7 @@ class Pipeline { } let outputTimeMs = originalTimeMs - this._pcmTimelineCorrectionMs; - if (this._pcmLastOutputEndMs !== null && outputTimeMs > this._pcmLastOutputEndMs + PCM_TIMELINE_GAP_TOLERANCE_MS) { + if (this._pcmLastOutputEndMs !== null && outputTimeMs > this._pcmLastOutputEndMs) { const gap = outputTimeMs - this._pcmLastOutputEndMs; this._pcmTimelineCorrectionMs += gap; outputTimeMs -= gap; From 81a8920dec350fc8f1f3b881d57a637424c1927a Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 05:34:34 +0800 Subject: [PATCH 06/14] fix(player): enforce continuous media timeline --- web-ui/src/mpegts/audio/pcm-audio-player.ts | 145 ++++++++++++------- web-ui/src/mpegts/demux/ts-demuxer.ts | 124 +++++++++++++++- web-ui/src/mpegts/player/live-sync.ts | 46 ------ web-ui/src/mpegts/player/mpegts-player.ts | 11 +- web-ui/src/mpegts/player/mse.ts | 5 - web-ui/src/mpegts/remux/mp4-remuxer.ts | 148 +++++++++----------- web-ui/src/mpegts/worker/pipeline.ts | 57 ++------ 7 files changed, 302 insertions(+), 234 deletions(-) diff --git a/web-ui/src/mpegts/audio/pcm-audio-player.ts b/web-ui/src/mpegts/audio/pcm-audio-player.ts index 477a9650..ea493977 100644 --- a/web-ui/src/mpegts/audio/pcm-audio-player.ts +++ b/web-ui/src/mpegts/audio/pcm-audio-player.ts @@ -52,8 +52,6 @@ const CHAIN_RESTART_DELAY = 0.04; const HARD_RESYNC_THRESHOLD = 1.5; /** Input gaps/overlaps within this are absorbed silently (PTS jitter). */ const GAP_SNAP = 0.005; -/** Input gaps up to this long are filled with silence; larger ones re-anchor. */ -const MAX_SILENCE_GAP = 2.0; /** Fade-in length applied when the chain (re)starts, to avoid clicks. */ const FADE_SEC = 0.005; /** Proportional gain for drift correction via stretch ratio. */ @@ -72,6 +70,8 @@ const DRIFT_EMA_ALPHA = 0.4; const CONTROL_INTERVAL_MS = 250; /** Upper bound for pending (not yet scheduled) chunks. */ const MAX_PENDING_CHUNKS = 600; +/** Seconds of decoded PCM to keep in the pending scheduling window after a resync. */ +const PENDING_REFILL_WINDOW_SEC = 2.0; /** Control ticks between verbose drift diagnostics (~10s). */ const DRIFT_LOG_TICKS = 40; @@ -300,10 +300,6 @@ export class PCMAudioPlayer { this.cleanupBuffer(); if (this.canScheduleAudio()) { - this.pendingChunks.push(chunk); - if (this.pendingChunks.length > MAX_PENDING_CHUNKS) { - this.pendingChunks.shift(); - } this.pump(); } } @@ -366,7 +362,7 @@ export class PCMAudioPlayer { */ private pump(): void { const ctx = this.context; - if (!ctx || !this.gainNode || this.pendingChunks.length === 0 || !this.canScheduleAudio()) { + if (!ctx || !this.gainNode || !this.canScheduleAudio()) { return; } @@ -389,7 +385,14 @@ export class PCMAudioPlayer { return; } - while (this.pendingChunks.length > 0) { + while (true) { + if (this.pendingChunks.length === 0) { + this.refillPendingFromBuffer(this.inputCursor ?? this.videoElement?.currentTime ?? 0); + } + if (this.pendingChunks.length === 0) { + break; + } + // Throttle: keep at most SCHEDULE_AHEAD seconds scheduled ahead if (this.nextStartTime - ctx.currentTime >= SCHEDULE_AHEAD) { break; @@ -408,21 +411,10 @@ export class PCMAudioPlayer { const cursor = this.inputCursor as number; const delta = chunk.time - cursor; - if (delta > MAX_SILENCE_GAP) { - // Forward discontinuity: re-anchor at the new position - Log.v(TAG, `Audio stream jump +${delta.toFixed(3)}s, re-anchoring`); - this.cancelChain(true); - this.anchor(chunk.time); - continue; - } - + let chunkEndTime = chunk.endTime; if (delta > GAP_SNAP) { - // Small gap: fill with silence to keep the timeline correct - const gapFrames = Math.round(delta * chunk.sampleRate); - if (gapFrames > 0) { - this.feedStretcher(stretcher, new Float32Array(gapFrames * chunk.channels), chunk.sampleRate); - this.inputCursor = cursor + gapFrames / chunk.sampleRate; - } + Log.w(TAG, `Unexpected PCM pending gap ${delta.toFixed(3)}s; snapping to cursor`); + chunkEndTime = cursor + chunk.duration; } let samples = chunk.samples; @@ -439,7 +431,7 @@ export class PCMAudioPlayer { this.pendingChunks.shift(); this.feedStretcher(stretcher, samples, chunk.sampleRate); - this.inputCursor = chunk.endTime; + this.inputCursor = chunkEndTime; } } @@ -688,6 +680,7 @@ export class PCMAudioPlayer { const correction = Math.min(correctionMax, Math.max(-correctionMax, correctionDrift * correctionGain)); const ratio = Math.min(2, Math.max(0.5, rate * (1 - correction))); this.stretcher.setRatio(ratio); + this.pump(); if (++this.driftLogCounter >= DRIFT_LOG_TICKS) { this.driftLogCounter = 0; @@ -700,6 +693,53 @@ export class PCMAudioPlayer { // ==================== Buffer Management ==================== + private trimChunkStart(chunk: AudioChunk, targetTime: number): AudioChunk | null { + if (targetTime <= chunk.time + GAP_SNAP) { + return chunk; + } + + const cutFrames = Math.round((targetTime - chunk.time) * chunk.sampleRate); + const totalFrames = Math.floor(chunk.samples.length / chunk.channels); + if (cutFrames >= totalFrames) { + return null; + } + + const time = chunk.time + cutFrames / chunk.sampleRate; + return { + samples: chunk.samples.subarray(cutFrames * chunk.channels), + channels: chunk.channels, + sampleRate: chunk.sampleRate, + time, + duration: chunk.endTime - time, + endTime: chunk.endTime, + }; + } + + private refillPendingFromBuffer(startTime: number): void { + if (this.pendingChunks.length >= MAX_PENDING_CHUNKS) return; + + const startIndex = this.findChunkIndexByTime(startTime); + if (startIndex < 0) return; + + const endTime = startTime + PENDING_REFILL_WINDOW_SEC; + for (let i = startIndex; i < this.audioBuffer.length && this.pendingChunks.length < MAX_PENDING_CHUNKS; i++) { + const source = this.audioBuffer[i]; + if (source.endTime <= startTime + GAP_SNAP) { + continue; + } + if (source.time >= endTime) { + break; + } + + const chunk = this.trimChunkStart(source, startTime); + if (!chunk) { + continue; + } + this.pendingChunks.push(chunk); + startTime = chunk.endTime; + } + } + private insertToBuffer(chunk: AudioChunk): void { let low = 0; let high = this.audioBuffer.length; @@ -712,10 +752,41 @@ export class PCMAudioPlayer { } } - if (low < this.audioBuffer.length && Math.abs(this.audioBuffer[low].time - chunk.time) < 0.001) { - this.audioBuffer[low] = chunk; + let normalized = chunk; + if (low > 0) { + const prev = this.audioBuffer[low - 1]; + if (normalized.time > prev.endTime) { + normalized = { ...normalized, time: prev.endTime, endTime: prev.endTime + normalized.duration }; + } else if (normalized.time < prev.endTime) { + const trimmed = this.trimChunkStart(normalized, prev.endTime); + if (!trimmed) { + return; + } + normalized = trimmed; + } + } + + if (low < this.audioBuffer.length && Math.abs(this.audioBuffer[low].time - normalized.time) < 0.001) { + this.audioBuffer[low] = normalized; + } else if (low < this.audioBuffer.length && normalized.endTime > this.audioBuffer[low].time) { + const next = this.audioBuffer[low]; + const keepFrames = Math.max(0, Math.round((next.time - normalized.time) * normalized.sampleRate)); + if (keepFrames === 0) { + return; + } + const totalFrames = Math.floor(normalized.samples.length / normalized.channels); + const frames = Math.min(keepFrames, totalFrames); + const endTime = normalized.time + frames / normalized.sampleRate; + this.audioBuffer.splice(low, 0, { + samples: normalized.samples.subarray(0, frames * normalized.channels), + channels: normalized.channels, + sampleRate: normalized.sampleRate, + time: normalized.time, + duration: endTime - normalized.time, + endTime, + }); } else { - this.audioBuffer.splice(low, 0, chunk); + this.audioBuffer.splice(low, 0, normalized); } } @@ -781,27 +852,7 @@ export class PCMAudioPlayer { return; } - for (let i = startIndex; i < this.audioBuffer.length; i++) { - let chunk = this.audioBuffer[i]; - if (i === startIndex && targetTime > chunk.time + GAP_SNAP) { - // Trim the head so the chain starts exactly at the target position - const cutFrames = Math.round((targetTime - chunk.time) * chunk.sampleRate); - const totalFrames = Math.floor(chunk.samples.length / chunk.channels); - if (cutFrames >= totalFrames) { - continue; - } - const time = chunk.time + cutFrames / chunk.sampleRate; - chunk = { - samples: chunk.samples.subarray(cutFrames * chunk.channels), - channels: chunk.channels, - sampleRate: chunk.sampleRate, - time, - duration: chunk.endTime - time, - endTime: chunk.endTime, - }; - } - this.pendingChunks.push(chunk); - } + this.refillPendingFromBuffer(targetTime); Log.v(TAG, `Resync at ${targetTime.toFixed(3)}s, refilled ${this.pendingChunks.length} chunks`); this.pump(); } diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index 84e0760a..64ad0276 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -103,6 +103,7 @@ type AudioData = export type OnErrorCallback = (type: string, info: string) => void; export type OnTrackMetadataCallback = (type: string, metadata: unknown) => void; export type OnDataAvailableCallback = (audioTrack: unknown, videoTrack: unknown) => void; +export type OnTrackDiscontinuityCallback = (track: "audio" | "video") => void; class TSDemuxer { private readonly TAG: string = "TSDemuxer"; @@ -110,6 +111,7 @@ class TSDemuxer { public onError: OnErrorCallback | null = null; public onTrackMetadata: OnTrackMetadataCallback | null = null; public onDataAvailable: OnDataAvailableCallback | null = null; + public onTrackDiscontinuity: OnTrackDiscontinuityCallback | null = null; /** Software audio decode support (MP2) */ public onRawAudioData: ((frame: { codec: "mp2"; data: Uint8Array; pts: number }) => void) | null = null; @@ -127,6 +129,7 @@ class TSDemuxer { private pes_slice_queues_: PIDToSliceQueues = {}; private section_slice_queues_: PIDToSliceQueues = {}; + private continuity_counters_: Record = {}; private video_metadata_: { vps: H265NaluHVC1 | undefined; @@ -162,6 +165,8 @@ class TSDemuxer { private loas_previous_frame: LOASAACFrame | null = null; private soft_decode_audio_codec_: "mp2" | null = null; + private audio_drop_until_sync_ = false; + private drop_video_until_keyframe_ = false; private video_track_ = { type: "video", @@ -201,6 +206,7 @@ class TSDemuxer { this.onError = null; this.onTrackMetadata = null; this.onDataAvailable = null; + this.onTrackDiscontinuity = null; this.onRawAudioData = null; this.soft_decode_audio_codec_ = null; } @@ -261,6 +267,71 @@ class TSDemuxer { }; } + private isVideoPid(pid: number): boolean { + return pid === this.pmt_?.common_pids.h264 || pid === this.pmt_?.common_pids.h265; + } + + private isAudioPid(pid: number): boolean { + return ( + pid === this.pmt_?.common_pids.adts_aac || + pid === this.pmt_?.common_pids.loas_aac || + pid === this.pmt_?.common_pids.ac3 || + pid === this.pmt_?.common_pids.eac3 || + pid === this.pmt_?.common_pids.mp3 + ); + } + + private resetAudioParserState(): void { + this.audio_last_sample_pts_ = undefined; + this.aac_last_incomplete_data_ = null; + this.loas_previous_frame = null; + this.audio_drop_until_sync_ = true; + } + + private handleTrackDiscontinuity(pid: number, reason: string): void { + delete this.pes_slice_queues_[pid]; + + if (this.isVideoPid(pid)) { + this.drop_video_until_keyframe_ = true; + Log.w(this.TAG, `Video TS discontinuity on pid ${pid}: ${reason}; dropping until keyframe`); + this.onTrackDiscontinuity?.("video"); + return; + } + + if (this.isAudioPid(pid)) { + this.resetAudioParserState(); + Log.w(this.TAG, `Audio TS discontinuity on pid ${pid}: ${reason}; resetting audio parser state`); + this.onTrackDiscontinuity?.("audio"); + } + } + + private shouldProcessPayload(pid: number, continuityCounter: number, discontinuityIndicator?: number): boolean { + if (discontinuityIndicator === 1) { + this.continuity_counters_[pid] = continuityCounter; + this.handleTrackDiscontinuity(pid, "discontinuity indicator"); + return true; + } + + const lastCounter = this.continuity_counters_[pid]; + if (lastCounter === undefined) { + this.continuity_counters_[pid] = continuityCounter; + return true; + } + + if (continuityCounter === lastCounter) { + Log.w(this.TAG, `Duplicate TS packet on pid ${pid} with continuity counter ${continuityCounter}; skipping`); + return false; + } + + const expected = (lastCounter + 1) & 0x0f; + if (continuityCounter !== expected) { + this.handleTrackDiscontinuity(pid, `expected continuity counter ${expected}, got ${continuityCounter}`); + } + + this.continuity_counters_[pid] = continuityCounter; + return true; + } + public parseChunks(chunk: Uint8Array, byte_start: number): number { if (!this.onError || !this.onTrackMetadata || !this.onDataAvailable) { throw new IllegalStateException("onError & onTrackMetadata & onDataAvailable callback must be specified"); @@ -356,6 +427,13 @@ class TSDemuxer { pid === this.pmt_.common_pids.eac3 || pid === this.pmt_.common_pids.mp3 ) { + if (!this.shouldProcessPayload(pid, continuity_conunter, adaptation_field_info.discontinuity_indicator)) { + offset += 188; + if (this.ts_packet_size_ === 204) { + offset += 16; + } + continue; + } this.handlePESSlice(chunk, offset + ts_payload_start_index, ts_payload_length, { pid, stream_type, @@ -630,7 +708,7 @@ class TSDemuxer { this.parseH264Payload(payload, pts, dts, pes_data.file_position, pes_data.random_access_indicator); break; case StreamType.kH265: - this.parseH265Payload(payload, pts, dts, pes_data.file_position); + this.parseH265Payload(payload, pts, dts, pes_data.file_position, pes_data.random_access_indicator); break; default: break; @@ -877,6 +955,14 @@ class TSDemuxer { const pts_ms = Math.floor(pts / this.timescale_); const dts_ms = Math.floor(dts / this.timescale_); + if (this.drop_video_until_keyframe_) { + if (!keyframe) { + return; + } + this.drop_video_until_keyframe_ = false; + Log.v(this.TAG, "Video keyframe found after TS discontinuity; resuming video output"); + } + if (units.length) { const track = this.video_track_; const avc_sample = { @@ -893,7 +979,13 @@ class TSDemuxer { } } - private parseH265Payload(data: Uint8Array, pts: number | undefined, dts: number | undefined, file_position: number) { + private parseH265Payload( + data: Uint8Array, + pts: number | undefined, + dts: number | undefined, + file_position: number, + random_access_indicator: number, + ) { const annexb_parser = new H265AnnexBParser(data); let nalu_payload: H265NaluPayload | null = null; const units: { type: H265NaluType; data: Uint8Array }[] = []; @@ -955,6 +1047,8 @@ class TSDemuxer { nalu_hvc1.type === H265NaluType.kSliceCRA_NUT ) { keyframe = true; + } else if (random_access_indicator === 1) { + keyframe = true; } // Push samples to remuxer only if initialization metadata has been dispatched @@ -972,6 +1066,14 @@ class TSDemuxer { const pts_ms = Math.floor(pts / this.timescale_); const dts_ms = Math.floor(dts / this.timescale_); + if (this.drop_video_until_keyframe_) { + if (!keyframe) { + return; + } + this.drop_video_until_keyframe_ = false; + Log.v(this.TAG, "Video keyframe found after TS discontinuity; resuming video output"); + } + if (units.length) { const track = this.video_track_; const hvc_sample = { @@ -1176,6 +1278,9 @@ class TSDemuxer { let last_sample_pts_ms: number | undefined; aac_frame = adts_parser.readNextAACFrame(); + if (aac_frame != null) { + this.audio_drop_until_sync_ = false; + } while (aac_frame != null) { ref_sample_duration = (1024 / aac_frame.sampling_frequency) * 1000; const audio_sample = { @@ -1271,6 +1376,9 @@ class TSDemuxer { let last_sample_pts_ms: number | undefined; aac_frame = loas_parser.readNextAACFrame(this.loas_previous_frame ?? undefined); + if (aac_frame != null) { + this.audio_drop_until_sync_ = false; + } while (aac_frame != null) { this.loas_previous_frame = aac_frame; ref_sample_duration = (1024 / aac_frame.sampling_frequency) * 1000; @@ -1351,6 +1459,9 @@ class TSDemuxer { let last_sample_pts_ms: number | undefined; ac3_frame = adts_parser.readNextAC3Frame(); + if (ac3_frame != null) { + this.audio_drop_until_sync_ = false; + } while (ac3_frame != null) { ref_sample_duration = (1536 / ac3_frame.sampling_frequency) * 1000; const audio_sample = { @@ -1427,6 +1538,9 @@ class TSDemuxer { let last_sample_pts_ms: number | undefined; eac3_frame = adts_parser.readNextEAC3Frame(); + if (eac3_frame != null) { + this.audio_drop_until_sync_ = false; + } while (eac3_frame != null) { ref_sample_duration = (1536 / eac3_frame.sampling_frequency) * 1000; const audio_sample = { @@ -1523,6 +1637,12 @@ class TSDemuxer { // A payload may start mid-frame (frame straddling a PES boundary); header // fields parsed from such payloads are garbage and must not drive metadata. const sync_at_start = data.length >= 4 && data[0] === 0xff && (data[1] & 0xe0) === 0xe0; + if (this.audio_drop_until_sync_) { + if (!sync_at_start) { + return; + } + this.audio_drop_until_sync_ = false; + } if (this.onRawAudioData && !soft_decode_active && !sync_at_start) { // Can't classify a payload that starts mid-frame; wait for an aligned one return; diff --git a/web-ui/src/mpegts/player/live-sync.ts b/web-ui/src/mpegts/player/live-sync.ts index ccf6b73d..205e1d3b 100644 --- a/web-ui/src/mpegts/player/live-sync.ts +++ b/web-ui/src/mpegts/player/live-sync.ts @@ -3,7 +3,6 @@ import Log from "../utils/logger"; import { type LiveSessionAnchor, lagBehindLiveEdge } from "./wall-clock"; const TAG = "LiveSync"; -const STALL_TAG = "StallJumper"; /** Each live-edge underrun raises the latency floor by this much (seconds). */ const UNDERRUN_BACKOFF_STEP = 1; @@ -104,48 +103,3 @@ export function setupLiveSync( video.playbackRate = 1; }; } - -/** - * Detect and fix stuck playback at startup. - * If the video is stalled or hasn't received canplay and the currentTime is before - * the first buffered range, seek to the start of the buffered range. - */ -export interface StallJumper { - check(): void; - destroy(): void; -} - -export function setupStartupStallJumper(video: HTMLMediaElement): StallJumper { - let canplayReceived = false; - - function onCanPlay(): void { - canplayReceived = true; - video.removeEventListener("canplay", onCanPlay); - } - - function detectAndFix(isStalled?: boolean): void { - const buffered = video.buffered; - if (isStalled || !canplayReceived || video.readyState < 2) { - if (buffered.length > 0 && video.currentTime < buffered.start(0)) { - const target = buffered.start(0); - Log.w(STALL_TAG, `Playback stuck at ${video.currentTime}, seeking to ${target}`); - video.currentTime = target; - } - } - } - - function onStalled(): void { - detectAndFix(true); - } - - video.addEventListener("canplay", onCanPlay); - video.addEventListener("stalled", onStalled); - - return { - check: () => detectAndFix(), - destroy: () => { - video.removeEventListener("canplay", onCanPlay); - video.removeEventListener("stalled", onStalled); - }, - }; -} diff --git a/web-ui/src/mpegts/player/mpegts-player.ts b/web-ui/src/mpegts/player/mpegts-player.ts index fbfdb550..dd768107 100644 --- a/web-ui/src/mpegts/player/mpegts-player.ts +++ b/web-ui/src/mpegts/player/mpegts-player.ts @@ -4,7 +4,7 @@ import type { PlayerImpl, PlayerSegment } from "../types"; import Log from "../utils/logger"; import type { WorkerCommand, WorkerEvent } from "../worker/messages"; import TransmuxWorker from "../worker/transmux-worker.ts?worker&inline"; -import { type StallJumper, setupLiveSync, setupStartupStallJumper } from "./live-sync"; +import { setupLiveSync } from "./live-sync"; import { createMSE, type MSE } from "./mse"; import type { LiveSessionAnchor } from "./wall-clock"; @@ -58,7 +58,6 @@ export function createMpegtsPlayer( let workerInitialized = false; let pendingSegments: PlayerSegment[] | null = null; let destroyLiveSync: (() => void) | null = null; - let stallJumper: StallJumper | null = null; let mseGeneration = 0; let liveSyncEnabled = config.liveSync; /** Live edge assuming continuous playback since session start. */ @@ -267,10 +266,6 @@ export function createMpegtsPlayer( // request, which restarts a live stream mid-flow and corrupts the timeline. // The MSE layer already defers appends while ManagedMediaSource streaming=false. - // Buffered ranges change exactly on SourceBuffer updateend; re-check for startup - // stalls there (iOS does not reliably fire progress/stalled on the media element). - mse.onBufferedChange = () => stallJumper?.check(); - mse.onSourceClose = () => { // The UA killed the media pipeline (e.g. iOS reclaiming resources in // background). Stop fetching — this session cannot be revived. @@ -303,8 +298,6 @@ export function createMpegtsPlayer( if (!destroyLiveSync && liveSyncEnabled) { destroyLiveSync = setupLiveSync(video, config, () => liveSessionAnchor); } - stallJumper?.destroy(); - stallJumper = setupStartupStallJumper(video); destroyVideoDebugLogs?.(); destroyVideoDebugLogs = setupVideoDebugLogs(video); } @@ -369,8 +362,6 @@ export function createMpegtsPlayer( destroyPCMPlayer(); destroyLiveSync?.(); destroyLiveSync = null; - stallJumper?.destroy(); - stallJumper = null; destroyVideoDebugLogs?.(); destroyVideoDebugLogs = null; }, diff --git a/web-ui/src/mpegts/player/mse.ts b/web-ui/src/mpegts/player/mse.ts index 51574b10..12763b60 100644 --- a/web-ui/src/mpegts/player/mse.ts +++ b/web-ui/src/mpegts/player/mse.ts @@ -42,8 +42,6 @@ export interface MSE { onBufferFull: (() => void) | null; /** Fired when buffer space becomes available again after a previous onBufferFull. */ onBufferAvailable: (() => void) | null; - /** Fired after each SourceBuffer update completes (buffered ranges may have changed). */ - onBufferedChange: (() => void) | null; /** ManagedMediaSource: UA wants more media data appended (streaming → true). */ onStartStreaming: (() => void) | null; /** ManagedMediaSource: UA has enough buffered data (streaming → false). */ @@ -291,7 +289,6 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } function onSourceBufferUpdateEnd(): void { - mse.onBufferedChange?.(); tryApplyDuration(); if (hasPendingRemoveRanges()) { doRemoveRanges(); @@ -368,7 +365,6 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { const mse: MSE = { onBufferFull: null, onBufferAvailable: null, - onBufferedChange: null, onStartStreaming: null, onEndStreaming: null, onSourceClose: null, @@ -587,7 +583,6 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { mse.onBufferFull = null; mse.onBufferAvailable = null; - mse.onBufferedChange = null; mse.onStartStreaming = null; mse.onEndStreaming = null; mse.onSourceClose = null; diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 0a86dea0..03f97f77 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -87,6 +87,8 @@ interface TrackTimingState { lastOriginalEndDts: number | undefined; lastOutputEndDts: number | undefined; lastOutputEndPts: number | undefined; + lastOutputDuration: number | undefined; + durationResidual: number; } type InitSegmentCallback = (type: string, segment: InitSegment) => void; @@ -119,6 +121,7 @@ class MP4Remuxer { private _videoStashedLastSample: VideoSample | null; private _audioTiming: TrackTimingState; private _videoTiming: TrackTimingState; + private _pcmTiming: TrackTimingState; private _videoCtsOffset: number | undefined; private _videoInitialCtsOffset: number | undefined; private _videoInitialOutputTime: number | undefined; @@ -148,6 +151,7 @@ class MP4Remuxer { this._videoStashedLastSample = null; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); + this._pcmTiming = this._createTrackTimingState(); this._videoCtsOffset = undefined; this._videoInitialCtsOffset = undefined; this._videoInitialOutputTime = undefined; @@ -172,6 +176,7 @@ class MP4Remuxer { this._silentAudioLastDts = undefined; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); + this._pcmTiming = this._createTrackTimingState(); this._videoCtsOffset = undefined; this._videoInitialCtsOffset = undefined; this._videoInitialOutputTime = undefined; @@ -222,6 +227,8 @@ class MP4Remuxer { lastOriginalEndDts: undefined, lastOutputEndDts: undefined, lastOutputEndPts: undefined, + lastOutputDuration: undefined, + durationResidual: 0, }; } @@ -260,6 +267,18 @@ class MP4Remuxer { timing.lastOutputEndPts = lastOutputEndPts ?? sample.pts + sample.duration; } + private _nextSampleDuration(timing: TrackTimingState, refSampleDuration: unknown, fallbackDuration: number): number { + const reference = + typeof refSampleDuration === "number" && Number.isFinite(refSampleDuration) && refSampleDuration > 0 + ? refSampleDuration + : (timing.lastOutputDuration ?? fallbackDuration); + const withResidual = reference + timing.durationResidual; + const duration = Math.max(1, Math.round(withResidual)); + timing.durationResidual = withResidual - duration; + timing.lastOutputDuration = duration; + return duration; + } + /** * Position the output timeline: the first remuxed sample will be emitted at `offsetMs` * instead of 0. Must be called before any samples are remuxed. @@ -464,6 +483,32 @@ class MP4Remuxer { return this._videoInitialOutputTime ?? 0; } + mapPcmTimestamp(ptsMs: number, durationMs: number): number | undefined { + if (!this._dtsBaseInited) { + return undefined; + } + + const originalTime = ptsMs - this._dtsBase; + const duration = Math.max(0, durationMs); + let outputTime: number; + + if (this._pcmTiming.lastOriginalEndDts === undefined || this._pcmTiming.lastOutputEndDts === undefined) { + outputTime = Math.max(this.getInitialOutputTime(), originalTime - this.getInitialPresentationOffset()); + } else { + const distance = originalTime - this._pcmTiming.lastOriginalEndDts; + outputTime = this._pcmTiming.lastOutputEndDts + (distance > 0 ? 0 : distance); + if (distance > 0) { + Log.v(this.TAG, `PCM: bridging ${Math.round(distance)}ms timestamp hole`); + } + } + + this._pcmTiming.lastOriginalEndDts = originalTime + duration; + this._pcmTiming.lastOutputEndDts = outputTime + duration; + this._pcmTiming.lastOutputEndPts = outputTime + duration; + this._pcmTiming.lastOutputDuration = duration; + return outputTime / 1000; + } + flushStashedSamples(): void { const videoSample = this._videoStashedLastSample; const audioSample = this._audioStashedLastSample; @@ -508,7 +553,6 @@ class MP4Remuxer { const track = audioTrack; const samples = track.samples; - let dtsCorrection: number | undefined; let firstDts = -1; const refSampleDuration = this._audioMeta.refSampleDuration; @@ -562,55 +606,28 @@ class MP4Remuxer { const firstSampleOriginalDts = (samples[0] as AudioSample).dts - this._dtsBase; - dtsCorrection = this._computeDtsCorrection("audio", firstSampleOriginalDts, this._audioNextDts, this._audioTiming); + const dtsCorrection = this._computeDtsCorrection( + "audio", + firstSampleOriginalDts, + this._audioNextDts, + this._audioTiming, + ); const mp4Samples: MP4Sample[] = []; + let nextOutputDts = firstSampleOriginalDts - dtsCorrection; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples for (let i = 0; i < samples.length; i++) { const sample = samples[i] as AudioSample; const originalDts = sample.dts - this._dtsBase; - let sampleDuration = 0; if (originalDts < -0.001) { continue; //pass the first sample with the invalid dts } - const dts = originalDts - (dtsCorrection as number); - - if (i !== samples.length - 1) { - const nextDts = (samples[i + 1] as AudioSample).dts - this._dtsBase - (dtsCorrection as number); - sampleDuration = nextDts - dts; - } else { - // the last sample - if (lastSample != null) { - // use stashed sample's dts to calculate sample duration - const nextDts = lastSample.dts - this._dtsBase - (dtsCorrection as number); - sampleDuration = nextDts - dts; - } else if (mp4Samples.length >= 1) { - // use second last sample duration - sampleDuration = mp4Samples[mp4Samples.length - 1].duration as number; - } else { - // the only one sample, use reference sample duration - sampleDuration = Math.floor(refSampleDuration as number); - } - } - - if (sampleDuration <= 0) { - const fallbackDuration = - Math.floor(refSampleDuration as number) || - (mp4Samples.length >= 1 ? (mp4Samples[mp4Samples.length - 1].duration as number) : 0) || - 26; - Log.w( - this.TAG, - `Audio: non-monotonic dts detected (dts: ${dts} ms, duration: ${Math.round(sampleDuration)} ms), ` + - `clamping sample duration to ${fallbackDuration} ms`, - ); - dtsCorrection = (dtsCorrection as number) + (sampleDuration - fallbackDuration); - sampleDuration = fallbackDuration; - } - - this._audioNextDts = dts + sampleDuration; + const dts = nextOutputDts; + const sampleDuration = this._nextSampleDuration(this._audioTiming, refSampleDuration, 26); + nextOutputDts += sampleDuration; if (firstDts === -1) { firstDts = dts; @@ -664,7 +681,9 @@ class MP4Remuxer { track.samples = mp4Samples; track.sequenceNumber++; - this._recordTrackTiming(this._audioTiming, mp4Samples[mp4Samples.length - 1]); + const latest = mp4Samples[mp4Samples.length - 1]; + this._audioNextDts = latest.dts + latest.duration; + this._recordTrackTiming(this._audioTiming, latest); let moofbox: Uint8Array; @@ -700,7 +719,6 @@ class MP4Remuxer { const track = videoTrack; const samples = track.samples; - let dtsCorrection: number | undefined; let firstDts = -1; if (!samples || samples.length === 0) { @@ -737,17 +755,23 @@ class MP4Remuxer { const firstSampleOriginalDts = (samples[0] as VideoSample).dts - this._dtsBase; - dtsCorrection = this._computeDtsCorrection("video", firstSampleOriginalDts, this._videoNextDts, this._videoTiming); + const dtsCorrection = this._computeDtsCorrection( + "video", + firstSampleOriginalDts, + this._videoNextDts, + this._videoTiming, + ); const mp4Samples: MP4Sample[] = []; let lastOutputEndPts = this._videoTiming.lastOutputEndPts; + let nextOutputDts = firstSampleOriginalDts - dtsCorrection; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples for (let i = 0; i < samples.length; i++) { const sample = samples[i] as VideoSample; const originalDts = sample.dts - this._dtsBase; const isKeyframe = sample.isKeyframe; - const dts = originalDts - (dtsCorrection as number); + const dts = nextOutputDts; if (firstDts === -1) { firstDts = dts; @@ -756,44 +780,8 @@ class MP4Remuxer { this._videoInitialOutputTime = dts; } - let sampleDuration = 0; - - if (i !== samples.length - 1) { - const nextDts = (samples[i + 1] as VideoSample).dts - this._dtsBase - (dtsCorrection as number); - sampleDuration = nextDts - dts; - } else { - // the last sample - if (lastSample != null) { - // use stashed sample's dts to calculate sample duration - const nextDts = lastSample.dts - this._dtsBase - (dtsCorrection as number); - sampleDuration = nextDts - dts; - } else if (mp4Samples.length >= 1) { - // use second last sample duration - sampleDuration = mp4Samples[mp4Samples.length - 1].duration as number; - } else { - // the only one sample, use reference sample duration - sampleDuration = Math.floor(this._videoMeta?.refSampleDuration ?? 0); - } - } - - if (sampleDuration <= 0) { - // Spliced streams (e.g. telco catchup recordings) can regress dts mid-batch. A - // non-positive duration would be written into the trun box as a huge unsigned - // value and trigger a decode error, so clamp it to keep the timeline monotonic. - const fallbackDuration = - Math.floor(this._videoMeta?.refSampleDuration ?? 0) || - (mp4Samples.length >= 1 ? (mp4Samples[mp4Samples.length - 1].duration as number) : 0) || - 40; - Log.w( - this.TAG, - `Video: non-monotonic dts detected (dts: ${dts} ms, duration: ${Math.round(sampleDuration)} ms), ` + - `clamping sample duration to ${fallbackDuration} ms`, - ); - // Re-anchor the remaining samples of this batch so their dts continue right - // after the clamped sample (mirrors the inter-batch dtsCorrection behavior) - dtsCorrection = (dtsCorrection as number) + (sampleDuration - fallbackDuration); - sampleDuration = fallbackDuration; - } + const sampleDuration = this._nextSampleDuration(this._videoTiming, this._videoMeta?.refSampleDuration, 40); + nextOutputDts += sampleDuration; if (this._videoCtsOffset === undefined) { this._videoCtsOffset = sample.cts; diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index 6b5586fa..66acdfcf 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -114,8 +114,6 @@ class Pipeline { ptsMs: number; durationMs: number; }> = []; - private _pcmTimelineCorrectionMs: number | null = null; - private _pcmLastOutputEndMs: number | null = null; /** Incremented on audio timing resets to invalidate decode callbacks queued before the reset. */ private _audioGen = 0; @@ -286,7 +284,7 @@ class Pipeline { // frame carried from the previous URL must not be prepended to the // next one, and the PTS anchor must re-establish from the new PES this._workerAudioDecoder?.reset(); - this._resetAudioTiming(false); + this._resetAudioTiming(); } } catch (e) { if (this._runId !== runId || e === CANCELLED) return; @@ -318,20 +316,12 @@ class Pipeline { this._resetAudioTiming(); } - private _resetAudioTiming(resetPcmTimeline = true): void { + private _resetAudioTiming(): void { this._audioGen++; this._audioAnchorPtsMs = null; this._audioSamplesSinceAnchor = 0; this._audioSampleRate = 0; this._pendingPcm = []; - if (resetPcmTimeline) { - this._resetPcmTimeline(); - } - } - - private _resetPcmTimeline(): void { - this._pcmTimelineCorrectionMs = null; - this._pcmLastOutputEndMs = null; } private _loadSegment(meta: SegmentMeta): Promise { @@ -420,6 +410,11 @@ class Pipeline { demuxer.onError = this._onDemuxException.bind(this); demuxer.timestampBase = meta.timestampBase * 90000; // seconds → 90kHz ticks + demuxer.onTrackDiscontinuity = (track) => { + if (track !== "audio") return; + this._workerAudioDecoder?.reset(); + this._resetAudioTiming(); + }; // Set up software audio decode callback when MP2 WASM URL is configured if (this._config.wasmDecoders.mp2) { @@ -570,8 +565,7 @@ class Pipeline { const durationMs = (Math.floor(pcm.length / channels) / sampleRate) * 1000; this._pendingPcm.push({ pcm, channels, sampleRate, ptsMs, durationMs }); - const dtsBase = this._remuxer?.getTimestampBase(); - if (dtsBase === undefined) { + if (this._remuxer?.getTimestampBase() === undefined) { // Bound the queue: ~25s of audio at one payload per ~72ms is plenty if (this._pendingPcm.length > 512) { this._pendingPcm.shift(); @@ -580,39 +574,14 @@ class Pipeline { } for (const item of this._pendingPcm) { - this._callbacks.onPCMAudioData( - item.pcm, - item.channels, - item.sampleRate, - this._mapPcmTimestamp(item.ptsMs, item.durationMs, dtsBase), - ); + const time = this._remuxer?.mapPcmTimestamp(item.ptsMs, item.durationMs); + if (time === undefined) { + break; + } + this._callbacks.onPCMAudioData(item.pcm, item.channels, item.sampleRate, time); } this._pendingPcm = []; } - - private _mapPcmTimestamp(ptsMs: number, durationMs: number, dtsBase: number): number { - const originalTimeMs = ptsMs - dtsBase; - - if (this._pcmTimelineCorrectionMs === null) { - const initialPresentationOffset = this._remuxer?.getInitialPresentationOffset() ?? 0; - const initialOutputTime = this._remuxer?.getInitialOutputTime() ?? 0; - const outputTimeMs = Math.max(initialOutputTime, originalTimeMs - initialPresentationOffset); - this._pcmTimelineCorrectionMs = originalTimeMs - outputTimeMs; - this._pcmLastOutputEndMs = outputTimeMs + durationMs; - return outputTimeMs / 1000; - } - - let outputTimeMs = originalTimeMs - this._pcmTimelineCorrectionMs; - if (this._pcmLastOutputEndMs !== null && outputTimeMs > this._pcmLastOutputEndMs) { - const gap = outputTimeMs - this._pcmLastOutputEndMs; - this._pcmTimelineCorrectionMs += gap; - outputTimeMs -= gap; - Log.v(this.TAG, `PCM: bridging ${Math.round(gap)}ms timestamp hole`); - } - - this._pcmLastOutputEndMs = outputTimeMs + durationMs; - return outputTimeMs / 1000; - } } export default Pipeline; From 744c6d1565b227bd4411413f6193d8244a1f52dc Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 05:45:39 +0800 Subject: [PATCH 07/14] fix(player): anchor remux tracks to output start --- web-ui/src/mpegts/remux/mp4-remuxer.ts | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 03f97f77..82d14277 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -86,7 +86,6 @@ interface MP4Sample { interface TrackTimingState { lastOriginalEndDts: number | undefined; lastOutputEndDts: number | undefined; - lastOutputEndPts: number | undefined; lastOutputDuration: number | undefined; durationResidual: number; } @@ -226,7 +225,6 @@ class MP4Remuxer { return { lastOriginalEndDts: undefined, lastOutputEndDts: undefined, - lastOutputEndPts: undefined, lastOutputDuration: undefined, durationResidual: 0, }; @@ -249,7 +247,7 @@ class MP4Remuxer { } if (timing.lastOriginalEndDts === undefined || timing.lastOutputEndDts === undefined) { - return 0; + return firstSampleOriginalDts - this._dtsBaseOffset; } const distance = firstSampleOriginalDts - timing.lastOriginalEndDts; @@ -261,10 +259,9 @@ class MP4Remuxer { return firstSampleOriginalDts - expectedDts; } - private _recordTrackTiming(timing: TrackTimingState, sample: MP4Sample, lastOutputEndPts?: number): void { + private _recordTrackTiming(timing: TrackTimingState, sample: MP4Sample): void { timing.lastOriginalEndDts = sample.originalDts + sample.duration; timing.lastOutputEndDts = sample.dts + sample.duration; - timing.lastOutputEndPts = lastOutputEndPts ?? sample.pts + sample.duration; } private _nextSampleDuration(timing: TrackTimingState, refSampleDuration: unknown, fallbackDuration: number): number { @@ -504,7 +501,6 @@ class MP4Remuxer { this._pcmTiming.lastOriginalEndDts = originalTime + duration; this._pcmTiming.lastOutputEndDts = outputTime + duration; - this._pcmTiming.lastOutputEndPts = outputTime + duration; this._pcmTiming.lastOutputDuration = duration; return outputTime / 1000; } @@ -763,7 +759,6 @@ class MP4Remuxer { ); const mp4Samples: MP4Sample[] = []; - let lastOutputEndPts = this._videoTiming.lastOutputEndPts; let nextOutputDts = firstSampleOriginalDts - dtsCorrection; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples @@ -788,16 +783,8 @@ class MP4Remuxer { this._videoInitialCtsOffset = sample.cts; } - let pts = dts + sample.cts - this._videoCtsOffset; - if (i === 0 && lastOutputEndPts !== undefined && pts > lastOutputEndPts) { - const gap = pts - lastOutputEndPts; - this._videoCtsOffset += gap; - pts -= gap; - Log.v(this.TAG, `video: bridging ${Math.round(gap)}ms presentation timestamp hole`); - } + const pts = dts + sample.cts - this._videoCtsOffset; const cts = pts - dts; - const outputEndPts = pts + sampleDuration; - lastOutputEndPts = lastOutputEndPts === undefined ? outputEndPts : Math.max(lastOutputEndPts, outputEndPts); mp4Samples.push({ dts: dts, @@ -820,7 +807,7 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._videoNextDts = latest.dts + latest.duration; - this._recordTrackTiming(this._videoTiming, latest, lastOutputEndPts); + this._recordTrackTiming(this._videoTiming, latest); track.samples = mp4Samples; track.sequenceNumber++; From ed7872ec44402c86c376453d7d6041179663fe18 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 05:55:54 +0800 Subject: [PATCH 08/14] fix(player): start timeline at video keyframe --- web-ui/src/mpegts/demux/ts-demuxer.ts | 65 ++++++++++++++++++++++---- web-ui/src/mpegts/remux/mp4-remuxer.ts | 35 +++++++++----- web-ui/src/mpegts/worker/pipeline.ts | 4 +- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index 64ad0276..c4ba6922 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -162,11 +162,13 @@ class TSDemuxer { private video_init_segment_dispatched_ = false; private audio_init_segment_dispatched_ = false; private video_metadata_changed_ = false; + private video_output_started_ = false; + private video_discontinuity_pending_ = false; private loas_previous_frame: LOASAACFrame | null = null; private soft_decode_audio_codec_: "mp2" | null = null; private audio_drop_until_sync_ = false; - private drop_video_until_keyframe_ = false; + private drop_video_until_keyframe_ = true; private video_track_ = { type: "video", @@ -288,11 +290,34 @@ class TSDemuxer { this.audio_drop_until_sync_ = true; } + private clearAudioTrack(): void { + this.audio_track_.samples = []; + this.audio_track_.length = 0; + } + + private shouldWaitForVideoKeyframe(): boolean { + return this.has_video_ && !this.video_output_started_; + } + + private resumeVideoOutputFromKeyframe(): void { + const reason = this.video_discontinuity_pending_ ? "after TS discontinuity; resuming" : "at stream start; starting"; + this.drop_video_until_keyframe_ = false; + this.video_output_started_ = true; + this.video_discontinuity_pending_ = false; + this.clearAudioTrack(); + this.resetAudioParserState(); + Log.v(this.TAG, `Video keyframe found ${reason} video output timeline`); + } + private handleTrackDiscontinuity(pid: number, reason: string): void { delete this.pes_slice_queues_[pid]; if (this.isVideoPid(pid)) { this.drop_video_until_keyframe_ = true; + this.video_output_started_ = false; + this.video_discontinuity_pending_ = true; + this.clearAudioTrack(); + this.resetAudioParserState(); Log.w(this.TAG, `Video TS discontinuity on pid ${pid}: ${reason}; dropping until keyframe`); this.onTrackDiscontinuity?.("video"); return; @@ -955,12 +980,11 @@ class TSDemuxer { const pts_ms = Math.floor(pts / this.timescale_); const dts_ms = Math.floor(dts / this.timescale_); - if (this.drop_video_until_keyframe_) { - if (!keyframe) { + if (this.drop_video_until_keyframe_ || !this.video_output_started_) { + if (!keyframe || units.length === 0) { return; } - this.drop_video_until_keyframe_ = false; - Log.v(this.TAG, "Video keyframe found after TS discontinuity; resuming video output"); + this.resumeVideoOutputFromKeyframe(); } if (units.length) { @@ -1066,12 +1090,11 @@ class TSDemuxer { const pts_ms = Math.floor(pts / this.timescale_); const dts_ms = Math.floor(dts / this.timescale_); - if (this.drop_video_until_keyframe_) { - if (!keyframe) { + if (this.drop_video_until_keyframe_ || !this.video_output_started_) { + if (!keyframe || units.length === 0) { return; } - this.drop_video_until_keyframe_ = false; - Log.v(this.TAG, "Video keyframe found after TS discontinuity; resuming video output"); + this.resumeVideoOutputFromKeyframe(); } if (units.length) { @@ -1209,6 +1232,9 @@ class TSDemuxer { } private dispatchVideoMediaSegment() { + if (this.shouldWaitForVideoKeyframe()) { + return; + } if (this.isInitSegmentDispatched()) { if (this.video_track_.length) { this.onDataAvailable?.(null, this.video_track_); @@ -1217,6 +1243,9 @@ class TSDemuxer { } private dispatchAudioMediaSegment() { + if (this.shouldWaitForVideoKeyframe()) { + return; + } if (this.isInitSegmentDispatched()) { if (this.audio_track_.length) { this.onDataAvailable?.(this.audio_track_, null); @@ -1225,6 +1254,9 @@ class TSDemuxer { } private dispatchAudioVideoMediaSegment() { + if (this.shouldWaitForVideoKeyframe()) { + return; + } if (this.isInitSegmentDispatched()) { if (this.audio_track_.length || this.video_track_.length) { this.onDataAvailable?.(this.audio_track_, this.video_track_); @@ -1238,6 +1270,9 @@ class TSDemuxer { // Wait for first IDR frame and video init segment being dispatched return; } + if (this.shouldWaitForVideoKeyframe()) { + return; + } if (this.aac_last_incomplete_data_) { const buf = new Uint8Array(data.byteLength + this.aac_last_incomplete_data_.byteLength); @@ -1336,6 +1371,9 @@ class TSDemuxer { // Wait for first IDR frame and video init segment being dispatched return; } + if (this.shouldWaitForVideoKeyframe()) { + return; + } if (this.aac_last_incomplete_data_) { const buf = new Uint8Array(data.byteLength + this.aac_last_incomplete_data_.byteLength); @@ -1435,6 +1473,9 @@ class TSDemuxer { // Wait for first IDR frame and video init segment being dispatched return; } + if (this.shouldWaitForVideoKeyframe()) { + return; + } let ref_sample_duration: number; let base_pts_ms!: number; @@ -1514,6 +1555,9 @@ class TSDemuxer { // Wait for first IDR frame and video init segment being dispatched return; } + if (this.shouldWaitForVideoKeyframe()) { + return; + } let ref_sample_duration: number; let base_pts_ms!: number; @@ -1593,6 +1637,9 @@ class TSDemuxer { // Wait for first IDR frame and video init segment being dispatched return; } + if (this.shouldWaitForVideoKeyframe()) { + return; + } const _mpegAudioV10SampleRateTable = [44100, 48000, 32000, 0]; const _mpegAudioV20SampleRateTable = [22050, 24000, 16000, 0]; diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 82d14277..c4db2264 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -455,11 +455,13 @@ class MP4Remuxer { this._videoDtsBase = (videoTrack.samples[0] as VideoSample).dts; } - // In silent audio mode, use video DTS as base (no real audio samples) - if (this._silentAudioMode) { + // With video present, the output timeline starts at the first emitted + // keyframe. Audio before that point is discarded/compressed onto this same + // video anchor so MSE never starts at an audio-only offset. + if (this._videoDtsBase !== Infinity) { this._dtsBase = this._videoDtsBase; } else { - this._dtsBase = Math.min(this._audioDtsBase, this._videoDtsBase); + this._dtsBase = this._audioDtsBase; } this._dtsBase -= this._dtsBaseOffset; this._dtsBaseInited = true; @@ -760,13 +762,26 @@ class MP4Remuxer { const mp4Samples: MP4Sample[] = []; let nextOutputDts = firstSampleOriginalDts - dtsCorrection; + const dropNegativePts = this._videoTiming.lastOutputEndDts === undefined && this._videoNextDts === undefined; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples for (let i = 0; i < samples.length; i++) { const sample = samples[i] as VideoSample; const originalDts = sample.dts - this._dtsBase; const isKeyframe = sample.isKeyframe; + + if (this._videoCtsOffset === undefined) { + this._videoCtsOffset = sample.cts; + this._videoInitialCtsOffset = sample.cts; + } + const dts = nextOutputDts; + const pts = dts + sample.cts - this._videoCtsOffset; + const cts = pts - dts; + if (dropNegativePts && pts < 0) { + mdatBytes -= sample.length; + continue; + } if (firstDts === -1) { firstDts = dts; @@ -778,14 +793,6 @@ class MP4Remuxer { const sampleDuration = this._nextSampleDuration(this._videoTiming, this._videoMeta?.refSampleDuration, 40); nextOutputDts += sampleDuration; - if (this._videoCtsOffset === undefined) { - this._videoCtsOffset = sample.cts; - this._videoInitialCtsOffset = sample.cts; - } - - const pts = dts + sample.cts - this._videoCtsOffset; - const cts = pts - dts; - mp4Samples.push({ dts: dts, pts: pts, @@ -805,6 +812,12 @@ class MP4Remuxer { }); } + if (mp4Samples.length === 0) { + track.samples = []; + track.length = 0; + return; + } + const latest = mp4Samples[mp4Samples.length - 1]; this._videoNextDts = latest.dts + latest.duration; this._recordTrackTiming(this._videoTiming, latest); diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index 66acdfcf..d2ff7f88 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -411,7 +411,9 @@ class Pipeline { demuxer.onError = this._onDemuxException.bind(this); demuxer.timestampBase = meta.timestampBase * 90000; // seconds → 90kHz ticks demuxer.onTrackDiscontinuity = (track) => { - if (track !== "audio") return; + if (track === "video") { + this._remuxer?.insertDiscontinuity(); + } this._workerAudioDecoder?.reset(); this._resetAudioTiming(); }; From 15fb09ebb36313693cf9c05b55fb06d90346d38a Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 06:06:51 +0800 Subject: [PATCH 09/14] chore(player): add startup media diagnostics --- web-ui/src/mpegts/demux/ts-demuxer.ts | 8 +++++++ web-ui/src/mpegts/player/mse.ts | 26 +++++++++++++++++----- web-ui/src/mpegts/remux/mp4-remuxer.ts | 30 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index c4ba6922..da1dee5e 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -984,6 +984,10 @@ class TSDemuxer { if (!keyframe || units.length === 0) { return; } + Log.v( + this.TAG, + `[startup-debug] h264 keyframe anchor: pts=${pts_ms}ms, dts=${dts_ms}ms, cts=${pts_ms - dts_ms}ms, units=${units.length}, bytes=${length}, randomAccess=${random_access_indicator}`, + ); this.resumeVideoOutputFromKeyframe(); } @@ -1094,6 +1098,10 @@ class TSDemuxer { if (!keyframe || units.length === 0) { return; } + Log.v( + this.TAG, + `[startup-debug] h265 keyframe anchor: pts=${pts_ms}ms, dts=${dts_ms}ms, cts=${pts_ms - dts_ms}ms, units=${units.length}, bytes=${length}, randomAccess=${random_access_indicator}`, + ); this.resumeVideoOutputFromKeyframe(); } diff --git a/web-ui/src/mpegts/player/mse.ts b/web-ui/src/mpegts/player/mse.ts index 12763b60..f746aedf 100644 --- a/web-ui/src/mpegts/player/mse.ts +++ b/web-ui/src/mpegts/player/mse.ts @@ -52,6 +52,7 @@ export interface MSE { } const TAG = "MSE"; +const STARTUP_DEBUG_LOG_LIMIT = 40; export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { // Use ManagedMediaSource only if w3c MediaSource is not available (e.g. iOS Safari) @@ -71,6 +72,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { let isBufferFull = false; let hasPendingEos = false; let pendingDuration: number | null = null; + let startupDebugLogCount = 0; + const appendSequence: Record = { video: 0, audio: 0 }; // Deferred init segments: queued before sourceopen fires let pendingSourceBufferInit: { track: Track; data: ArrayBuffer; codec: string; container: string }[] = []; @@ -193,6 +196,13 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { if (entry.timestampOffset !== undefined) { sb.timestampOffset = entry.timestampOffset / 1000; } + const sequence = ++appendSequence[track]; + if (startupDebugLogCount++ < STARTUP_DEBUG_LOG_LIMIT) { + Log.v( + TAG, + `[startup-debug] append ${track}#${sequence}: bytes=${entry.data.byteLength}, timestampOffset=${entry.timestampOffset ?? "-"}, pending video:${pendingSegments.video.length} audio:${pendingSegments.audio.length}, before=${bufferedSummary()}`, + ); + } sb.appendBuffer(entry.data); if (isBufferFull) { isBufferFull = false; @@ -288,7 +298,13 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } } - function onSourceBufferUpdateEnd(): void { + function onSourceBufferUpdateEnd(track: Track): void { + if (startupDebugLogCount++ < STARTUP_DEBUG_LOG_LIMIT) { + Log.v( + TAG, + `[startup-debug] updateend ${track}: pending video:${pendingSegments.video.length} audio:${pendingSegments.audio.length}, buffered=${bufferedSummary()}`, + ); + } tryApplyDuration(); if (hasPendingRemoveRanges()) { doRemoveRanges(); @@ -303,8 +319,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } } - function onSourceBufferError(e: Event): void { - Log.e(TAG, `SourceBuffer Error:`, e); + function onSourceBufferError(track: Track, e: Event): void { + Log.e(TAG, `SourceBuffer Error (${track}):`, e); } function createSourceBuffer(track: Track, codec: string, container: string): void { @@ -339,8 +355,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { const sb = mediaSource.addSourceBuffer(mimeType); sourceBuffers[track] = sb; - const errorHandler = (e: Event) => onSourceBufferError(e); - const updateEndHandler = () => onSourceBufferUpdateEnd(); + const errorHandler = (e: Event) => onSourceBufferError(track, e); + const updateEndHandler = () => onSourceBufferUpdateEnd(track); sbErrorHandlers[track] = errorHandler; sbUpdateEndHandlers[track] = updateEndHandler; diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index c4db2264..2e40bcce 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -135,6 +135,8 @@ class MP4Remuxer { private _silentAudioMode: boolean; private _silentAudioLastDts: number | undefined; + private _startupDebugAudioSegments: number; + private _startupDebugVideoSegments: number; constructor() { this.TAG = "MP4Remuxer"; @@ -166,6 +168,8 @@ class MP4Remuxer { this._silentAudioMode = false; this._silentAudioLastDts = undefined; + this._startupDebugAudioSegments = 0; + this._startupDebugVideoSegments = 0; } destroy(): void { @@ -173,6 +177,8 @@ class MP4Remuxer { this._dtsBaseInited = false; this._silentAudioMode = false; this._silentAudioLastDts = undefined; + this._startupDebugAudioSegments = 0; + this._startupDebugVideoSegments = 0; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); @@ -465,6 +471,10 @@ class MP4Remuxer { } this._dtsBase -= this._dtsBaseOffset; this._dtsBaseInited = true; + Log.v( + this.TAG, + `[startup-debug] dts base selected=${this._dtsBase.toFixed(3)}ms, videoBase=${this._videoDtsBase === Infinity ? "none" : this._videoDtsBase.toFixed(3)}, audioBase=${this._audioDtsBase === Infinity ? "none" : this._audioDtsBase.toFixed(3)}, offset=${this._dtsBaseOffset.toFixed(3)}`, + ); } getTimestampBase(): number | undefined { @@ -682,6 +692,13 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._audioNextDts = latest.dts + latest.duration; this._recordTrackTiming(this._audioTiming, latest); + if (this._startupDebugAudioSegments++ < 8) { + const first = mp4Samples[0]; + Log.v( + this.TAG, + `[startup-debug] audio segment#${this._startupDebugAudioSegments}: samples=${mp4Samples.length}, firstDts=${first.dts.toFixed(3)}, firstOrig=${first.originalDts.toFixed(3)}, lastEnd=${this._audioNextDts.toFixed(3)}, correction=${dtsCorrection.toFixed(3)}, mdat=${mdatBytes}`, + ); + } let moofbox: Uint8Array; @@ -763,6 +780,7 @@ class MP4Remuxer { const mp4Samples: MP4Sample[] = []; let nextOutputDts = firstSampleOriginalDts - dtsCorrection; const dropNegativePts = this._videoTiming.lastOutputEndDts === undefined && this._videoNextDts === undefined; + let droppedNegativePts = 0; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples for (let i = 0; i < samples.length; i++) { @@ -780,6 +798,7 @@ class MP4Remuxer { const cts = pts - dts; if (dropNegativePts && pts < 0) { mdatBytes -= sample.length; + droppedNegativePts++; continue; } @@ -813,6 +832,10 @@ class MP4Remuxer { } if (mp4Samples.length === 0) { + Log.v( + this.TAG, + `[startup-debug] video segment skipped: raw=${samples.length}, droppedNegativePts=${droppedNegativePts}, firstOrigDts=${firstSampleOriginalDts.toFixed(3)}, correction=${dtsCorrection.toFixed(3)}`, + ); track.samples = []; track.length = 0; return; @@ -821,6 +844,13 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._videoNextDts = latest.dts + latest.duration; this._recordTrackTiming(this._videoTiming, latest); + if (this._startupDebugVideoSegments++ < 8) { + const first = mp4Samples[0]; + Log.v( + this.TAG, + `[startup-debug] video segment#${this._startupDebugVideoSegments}: raw=${samples.length}, out=${mp4Samples.length}, droppedNegativePts=${droppedNegativePts}, firstDts=${first.dts.toFixed(3)}, firstPts=${first.pts.toFixed(3)}, firstCts=${first.cts.toFixed(3)}, firstKey=${first.isKeyframe ? "yes" : "no"}, firstOrig=${first.originalDts.toFixed(3)}, lastEnd=${this._videoNextDts.toFixed(3)}, correction=${dtsCorrection.toFixed(3)}, ctsOffset=${this._videoCtsOffset?.toFixed(3) ?? "-"}, mdat=${mdatBytes}`, + ); + } track.samples = mp4Samples; track.sequenceNumber++; From 2317b6e917974ab31cf5f6290dac171a277e45b7 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 06:10:23 +0800 Subject: [PATCH 10/14] fix(player): drop leading video before timeline start --- web-ui/src/mpegts/demux/ts-demuxer.ts | 8 ------ web-ui/src/mpegts/player/mse.ts | 26 ++++-------------- web-ui/src/mpegts/remux/mp4-remuxer.ts | 38 +++----------------------- 3 files changed, 9 insertions(+), 63 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index da1dee5e..c4ba6922 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -984,10 +984,6 @@ class TSDemuxer { if (!keyframe || units.length === 0) { return; } - Log.v( - this.TAG, - `[startup-debug] h264 keyframe anchor: pts=${pts_ms}ms, dts=${dts_ms}ms, cts=${pts_ms - dts_ms}ms, units=${units.length}, bytes=${length}, randomAccess=${random_access_indicator}`, - ); this.resumeVideoOutputFromKeyframe(); } @@ -1098,10 +1094,6 @@ class TSDemuxer { if (!keyframe || units.length === 0) { return; } - Log.v( - this.TAG, - `[startup-debug] h265 keyframe anchor: pts=${pts_ms}ms, dts=${dts_ms}ms, cts=${pts_ms - dts_ms}ms, units=${units.length}, bytes=${length}, randomAccess=${random_access_indicator}`, - ); this.resumeVideoOutputFromKeyframe(); } diff --git a/web-ui/src/mpegts/player/mse.ts b/web-ui/src/mpegts/player/mse.ts index f746aedf..12763b60 100644 --- a/web-ui/src/mpegts/player/mse.ts +++ b/web-ui/src/mpegts/player/mse.ts @@ -52,7 +52,6 @@ export interface MSE { } const TAG = "MSE"; -const STARTUP_DEBUG_LOG_LIMIT = 40; export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { // Use ManagedMediaSource only if w3c MediaSource is not available (e.g. iOS Safari) @@ -72,8 +71,6 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { let isBufferFull = false; let hasPendingEos = false; let pendingDuration: number | null = null; - let startupDebugLogCount = 0; - const appendSequence: Record = { video: 0, audio: 0 }; // Deferred init segments: queued before sourceopen fires let pendingSourceBufferInit: { track: Track; data: ArrayBuffer; codec: string; container: string }[] = []; @@ -196,13 +193,6 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { if (entry.timestampOffset !== undefined) { sb.timestampOffset = entry.timestampOffset / 1000; } - const sequence = ++appendSequence[track]; - if (startupDebugLogCount++ < STARTUP_DEBUG_LOG_LIMIT) { - Log.v( - TAG, - `[startup-debug] append ${track}#${sequence}: bytes=${entry.data.byteLength}, timestampOffset=${entry.timestampOffset ?? "-"}, pending video:${pendingSegments.video.length} audio:${pendingSegments.audio.length}, before=${bufferedSummary()}`, - ); - } sb.appendBuffer(entry.data); if (isBufferFull) { isBufferFull = false; @@ -298,13 +288,7 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } } - function onSourceBufferUpdateEnd(track: Track): void { - if (startupDebugLogCount++ < STARTUP_DEBUG_LOG_LIMIT) { - Log.v( - TAG, - `[startup-debug] updateend ${track}: pending video:${pendingSegments.video.length} audio:${pendingSegments.audio.length}, buffered=${bufferedSummary()}`, - ); - } + function onSourceBufferUpdateEnd(): void { tryApplyDuration(); if (hasPendingRemoveRanges()) { doRemoveRanges(); @@ -319,8 +303,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } } - function onSourceBufferError(track: Track, e: Event): void { - Log.e(TAG, `SourceBuffer Error (${track}):`, e); + function onSourceBufferError(e: Event): void { + Log.e(TAG, `SourceBuffer Error:`, e); } function createSourceBuffer(track: Track, codec: string, container: string): void { @@ -355,8 +339,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { const sb = mediaSource.addSourceBuffer(mimeType); sourceBuffers[track] = sb; - const errorHandler = (e: Event) => onSourceBufferError(track, e); - const updateEndHandler = () => onSourceBufferUpdateEnd(track); + const errorHandler = (e: Event) => onSourceBufferError(e); + const updateEndHandler = () => onSourceBufferUpdateEnd(); sbErrorHandlers[track] = errorHandler; sbUpdateEndHandlers[track] = updateEndHandler; diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 2e40bcce..c3c83bf8 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -135,8 +135,6 @@ class MP4Remuxer { private _silentAudioMode: boolean; private _silentAudioLastDts: number | undefined; - private _startupDebugAudioSegments: number; - private _startupDebugVideoSegments: number; constructor() { this.TAG = "MP4Remuxer"; @@ -168,8 +166,6 @@ class MP4Remuxer { this._silentAudioMode = false; this._silentAudioLastDts = undefined; - this._startupDebugAudioSegments = 0; - this._startupDebugVideoSegments = 0; } destroy(): void { @@ -177,8 +173,6 @@ class MP4Remuxer { this._dtsBaseInited = false; this._silentAudioMode = false; this._silentAudioLastDts = undefined; - this._startupDebugAudioSegments = 0; - this._startupDebugVideoSegments = 0; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); @@ -471,10 +465,6 @@ class MP4Remuxer { } this._dtsBase -= this._dtsBaseOffset; this._dtsBaseInited = true; - Log.v( - this.TAG, - `[startup-debug] dts base selected=${this._dtsBase.toFixed(3)}ms, videoBase=${this._videoDtsBase === Infinity ? "none" : this._videoDtsBase.toFixed(3)}, audioBase=${this._audioDtsBase === Infinity ? "none" : this._audioDtsBase.toFixed(3)}, offset=${this._dtsBaseOffset.toFixed(3)}`, - ); } getTimestampBase(): number | undefined { @@ -692,13 +682,6 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._audioNextDts = latest.dts + latest.duration; this._recordTrackTiming(this._audioTiming, latest); - if (this._startupDebugAudioSegments++ < 8) { - const first = mp4Samples[0]; - Log.v( - this.TAG, - `[startup-debug] audio segment#${this._startupDebugAudioSegments}: samples=${mp4Samples.length}, firstDts=${first.dts.toFixed(3)}, firstOrig=${first.originalDts.toFixed(3)}, lastEnd=${this._audioNextDts.toFixed(3)}, correction=${dtsCorrection.toFixed(3)}, mdat=${mdatBytes}`, - ); - } let moofbox: Uint8Array; @@ -779,8 +762,7 @@ class MP4Remuxer { const mp4Samples: MP4Sample[] = []; let nextOutputDts = firstSampleOriginalDts - dtsCorrection; - const dropNegativePts = this._videoTiming.lastOutputEndDts === undefined && this._videoNextDts === undefined; - let droppedNegativePts = 0; + const presentationFloor = this._videoInitialOutputTime ?? this._dtsBaseOffset; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples for (let i = 0; i < samples.length; i++) { @@ -794,11 +776,9 @@ class MP4Remuxer { } const dts = nextOutputDts; - const pts = dts + sample.cts - this._videoCtsOffset; - const cts = pts - dts; - if (dropNegativePts && pts < 0) { + const pts = originalDts - dtsCorrection + sample.cts - this._videoCtsOffset; + if (pts < presentationFloor - 0.001) { mdatBytes -= sample.length; - droppedNegativePts++; continue; } @@ -811,6 +791,7 @@ class MP4Remuxer { const sampleDuration = this._nextSampleDuration(this._videoTiming, this._videoMeta?.refSampleDuration, 40); nextOutputDts += sampleDuration; + const cts = pts - dts; mp4Samples.push({ dts: dts, @@ -832,10 +813,6 @@ class MP4Remuxer { } if (mp4Samples.length === 0) { - Log.v( - this.TAG, - `[startup-debug] video segment skipped: raw=${samples.length}, droppedNegativePts=${droppedNegativePts}, firstOrigDts=${firstSampleOriginalDts.toFixed(3)}, correction=${dtsCorrection.toFixed(3)}`, - ); track.samples = []; track.length = 0; return; @@ -844,13 +821,6 @@ class MP4Remuxer { const latest = mp4Samples[mp4Samples.length - 1]; this._videoNextDts = latest.dts + latest.duration; this._recordTrackTiming(this._videoTiming, latest); - if (this._startupDebugVideoSegments++ < 8) { - const first = mp4Samples[0]; - Log.v( - this.TAG, - `[startup-debug] video segment#${this._startupDebugVideoSegments}: raw=${samples.length}, out=${mp4Samples.length}, droppedNegativePts=${droppedNegativePts}, firstDts=${first.dts.toFixed(3)}, firstPts=${first.pts.toFixed(3)}, firstCts=${first.cts.toFixed(3)}, firstKey=${first.isKeyframe ? "yes" : "no"}, firstOrig=${first.originalDts.toFixed(3)}, lastEnd=${this._videoNextDts.toFixed(3)}, correction=${dtsCorrection.toFixed(3)}, ctsOffset=${this._videoCtsOffset?.toFixed(3) ?? "-"}, mdat=${mdatBytes}`, - ); - } track.samples = mp4Samples; track.sequenceNumber++; From 5abe8be901a8db7658ac71bd728a3c3eaf221be8 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 06:38:59 +0800 Subject: [PATCH 11/14] fix(player): normalize hls segment timeline --- web-ui/src/mpegts/demux/ts-demuxer.ts | 20 ++++++++- web-ui/src/mpegts/worker/pipeline.ts | 61 ++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index c4ba6922..d9f38a29 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -53,6 +53,9 @@ type AdaptationFieldInfo = { random_access_indicator?: number; elementary_stream_priority_indicator?: number; }; +type TSDemuxerOptions = { + waitForInitialVideoKeyframe?: boolean; +}; type AACAudioMetadata = { codec: "aac"; audio_object_type: MPEG4AudioObjectTypes; @@ -189,9 +192,13 @@ class TSDemuxer { this.timestamp_offset_ = value; } - public constructor(probe_data: TSProbeResult) { + public constructor(probe_data: TSProbeResult, options: TSDemuxerOptions = {}) { this.ts_packet_size_ = probe_data.ts_packet_size as number; this.sync_offset_ = probe_data.sync_offset as number; + if (options.waitForInitialVideoKeyframe === false) { + this.drop_video_until_keyframe_ = false; + this.video_output_started_ = true; + } } public destroy() { @@ -213,6 +220,17 @@ class TSDemuxer { this.soft_decode_audio_codec_ = null; } + public resetSegmentBoundary(probe_data?: TSProbeResult): void { + if (probe_data) { + this.ts_packet_size_ = probe_data.ts_packet_size as number; + this.sync_offset_ = probe_data.sync_offset as number; + } + this.first_parse_ = true; + this.pes_slice_queues_ = {}; + this.section_slice_queues_ = {}; + this.continuity_counters_ = {}; + } + public static probe(data: Uint8Array): TSProbeResult { let sync_offset = -1; let ts_packet_size = 188; diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index d2ff7f88..e13660eb 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -3,7 +3,7 @@ import { createDefaultConfig } from "../config"; import { WorkerAudioDecoder } from "../decoder/worker-audio-decoder"; import DemuxErrors from "../demux/demux-errors"; import TSDemuxer from "../demux/ts-demuxer"; -import { containsMoov, parseInitSegment, probeFmp4, splitInitFromSegment } from "../hls/fmp4"; +import { containsMoov, getSegmentStartTime, parseInitSegment, probeFmp4, splitInitFromSegment } from "../hls/fmp4"; import { type HlsInfo, HlsSource } from "../hls/hls-source"; import FetchLoader, { LoaderErrors } from "../io/fetch-loader"; import MP4Remuxer from "../remux/mp4-remuxer"; @@ -97,6 +97,8 @@ class Pipeline { private _fmp4InitSent = false; private _fmp4Chunks: Uint8Array[] = []; private _lastInitUrl: string | null = null; + private _fmp4Timescales = new Map(); + private _fmp4TimestampOffsetWarningLogged = false; private _workerAudioDecoder: WorkerAudioDecoder | null = null; private _workerAudioDecoderInitPromise: Promise | null = null; @@ -207,6 +209,8 @@ class Pipeline { this._fmp4InitSent = false; this._fmp4Chunks = []; this._lastInitUrl = null; + this._fmp4Timescales = new Map(); + this._fmp4TimestampOffsetWarningLogged = false; this._paused = false; this._resumeGate?.(); this._resumeGate = null; @@ -273,7 +277,7 @@ class Pipeline { if (this._runId !== runId) return; if (this._fmp4Mode) { - this._flushFmp4Segment(); + this._flushFmp4Segment(meta); } // Flush stashed samples at every segment boundary so the next segment's first // remux batch is not mixed with the previous segment's tail (which would share @@ -316,6 +320,10 @@ class Pipeline { this._resetAudioTiming(); } + private _shouldAnchorSegment(meta: SegmentMeta): boolean { + return meta.resetRemuxer || !this._hlsSource; + } + private _resetAudioTiming(): void { this._audioGen++; this._audioAnchorPtsMs = null; @@ -394,10 +402,19 @@ class Pipeline { // ---- MPEG-TS path ---- private _setupTSDemuxerRemuxer(probeData: unknown, meta: SegmentMeta): void { + const shouldAnchor = this._shouldAnchorSegment(meta); + if (this._hlsSource && !shouldAnchor && this._demuxer && this._remuxer) { + this._demuxer.resetSegmentBoundary(probeData as ConstructorParameters[0]); + return; + } + + const waitForInitialVideoKeyframe = shouldAnchor || !this._demuxer || !this._remuxer; if (this._demuxer) { this._demuxer.destroy(); } - const demuxer = new TSDemuxer(probeData as ConstructorParameters[0]); + const demuxer = new TSDemuxer(probeData as ConstructorParameters[0], { + waitForInitialVideoKeyframe, + }); this._demuxer = demuxer; if (!this._remuxer) { @@ -467,7 +484,10 @@ class Pipeline { } private _sendFmp4Init(data: Uint8Array): void { - const codec = this._hlsSource?.info.codecs ?? parseInitSegment(data).codecs.join(","); + const initInfo = parseInitSegment(data); + this._fmp4Timescales = initInfo.timescales; + this._fmp4TimestampOffsetWarningLogged = false; + const codec = this._hlsSource?.info.codecs ?? initInfo.codecs.join(","); this._callbacks.onInitSegment("video", { type: "video", container: "video/mp4", @@ -477,13 +497,37 @@ class Pipeline { this._fmp4InitSent = true; } + private _warnFmp4TimestampOffsetUnavailable(reason: string): void { + if (this._fmp4TimestampOffsetWarningLogged) { + return; + } + this._fmp4TimestampOffsetWarningLogged = true; + Log.w(this.TAG, `fMP4 timestampOffset unavailable: ${reason}; appending media with original tfdt`); + } + + private _getFmp4TimestampOffset(meta: SegmentMeta, media: Uint8Array): number | undefined { + if (this._fmp4Timescales.size === 0) { + this._warnFmp4TimestampOffsetUnavailable("init segment timescales missing"); + return undefined; + } + + const segmentStart = getSegmentStartTime(media, this._fmp4Timescales); + if (segmentStart === null) { + this._warnFmp4TimestampOffsetUnavailable("media segment has no tfdt"); + return undefined; + } + + const timestampOffset = (meta.start - segmentStart) * 1000; + return Math.abs(timestampOffset) < 0.001 ? 0 : timestampOffset; + } + private _onFmp4Chunk(data: Uint8Array): number { this._fmp4Chunks.push(data); return data.byteLength; } /** Forward a fully buffered fMP4 segment to MSE (extracting the init part on first use). */ - private _flushFmp4Segment(): void { + private _flushFmp4Segment(meta: SegmentMeta): void { if (this._fmp4Chunks.length === 0) { return; } @@ -508,7 +552,12 @@ class Pipeline { } if (media.byteLength > 0) { - this._callbacks.onMediaSegment("video", { type: "video", data: toArrayBuffer(media) }); + this._pendingDtsOffsetMs = 0; + this._callbacks.onMediaSegment("video", { + type: "video", + data: toArrayBuffer(media), + timestampOffset: this._getFmp4TimestampOffset(meta, media), + }); } } From ebf72866b807765ea4ca3c9aa3dda8cfb3b15565 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 07:09:18 +0800 Subject: [PATCH 12/14] fix(player): flush media before TS discontinuity --- web-ui/src/mpegts/demux/ts-demuxer.ts | 32 +++++++++++++++++++++++++- web-ui/src/mpegts/remux/mp4-remuxer.ts | 11 +++++---- web-ui/src/mpegts/worker/pipeline.ts | 1 + 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index d9f38a29..1fd25033 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -105,7 +105,7 @@ type AudioData = export type OnErrorCallback = (type: string, info: string) => void; export type OnTrackMetadataCallback = (type: string, metadata: unknown) => void; -export type OnDataAvailableCallback = (audioTrack: unknown, videoTrack: unknown) => void; +export type OnDataAvailableCallback = (audioTrack: unknown, videoTrack: unknown, force?: boolean) => void; export type OnTrackDiscontinuityCallback = (track: "audio" | "video") => void; class TSDemuxer { @@ -313,10 +313,37 @@ class TSDemuxer { this.audio_track_.length = 0; } + private clearVideoTrack(): void { + this.video_track_.samples = []; + this.video_track_.length = 0; + } + + private clearAudioPESQueues(): void { + const commonPids = this.pmt_?.common_pids; + if (!commonPids) { + return; + } + + for (const pid of [commonPids.adts_aac, commonPids.loas_aac, commonPids.ac3, commonPids.eac3, commonPids.mp3]) { + if (pid !== undefined) { + delete this.pes_slice_queues_[pid]; + } + } + } + private shouldWaitForVideoKeyframe(): boolean { return this.has_video_ && !this.video_output_started_; } + private flushMediaBeforeTrackDiscontinuity(): void { + if (this.shouldWaitForVideoKeyframe() || !this.isInitSegmentDispatched()) { + return; + } + if (this.audio_track_.length || this.video_track_.length) { + this.onDataAvailable?.(this.audio_track_, this.video_track_, true); + } + } + private resumeVideoOutputFromKeyframe(): void { const reason = this.video_discontinuity_pending_ ? "after TS discontinuity; resuming" : "at stream start; starting"; this.drop_video_until_keyframe_ = false; @@ -331,10 +358,13 @@ class TSDemuxer { delete this.pes_slice_queues_[pid]; if (this.isVideoPid(pid)) { + this.flushMediaBeforeTrackDiscontinuity(); + this.clearVideoTrack(); this.drop_video_until_keyframe_ = true; this.video_output_started_ = false; this.video_discontinuity_pending_ = true; this.clearAudioTrack(); + this.clearAudioPESQueues(); this.resetAudioParserState(); Log.w(this.TAG, `Video TS discontinuity on pid ${pid}: ${reason}; dropping until keyframe`); this.onTrackDiscontinuity?.("video"); diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index c3c83bf8..5c3395d4 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -284,7 +284,7 @@ class MP4Remuxer { this._dtsBaseOffset = offsetMs; } - remux(audioTrack: DemuxTrack | undefined, videoTrack: DemuxTrack | undefined): void { + remux(audioTrack: DemuxTrack | null | undefined, videoTrack: DemuxTrack | null | undefined, force = false): void { if (!this._onMediaSegment) { throw new IllegalStateException("MP4Remuxer: onMediaSegment callback must be specificed!"); } @@ -292,10 +292,10 @@ class MP4Remuxer { this._calculateDtsBase(audioTrack, videoTrack); } if (videoTrack) { - this._remuxVideo(videoTrack); + this._remuxVideo(videoTrack, force); } if (audioTrack) { - this._remuxAudio(audioTrack); + this._remuxAudio(audioTrack, force); } } @@ -443,7 +443,10 @@ class MP4Remuxer { }); } - private _calculateDtsBase(audioTrack: DemuxTrack | undefined, videoTrack: DemuxTrack | undefined): void { + private _calculateDtsBase( + audioTrack: DemuxTrack | null | undefined, + videoTrack: DemuxTrack | null | undefined, + ): void { if (this._dtsBaseInited) { return; } diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index e13660eb..b6856cb4 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -429,6 +429,7 @@ class Pipeline { demuxer.timestampBase = meta.timestampBase * 90000; // seconds → 90kHz ticks demuxer.onTrackDiscontinuity = (track) => { if (track === "video") { + this._remuxer?.flushStashedSamples(); this._remuxer?.insertDiscontinuity(); } this._workerAudioDecoder?.reset(); From 9ec7a319cd2230a260f1bfedefbfa68cb30ae318 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 07:19:03 +0800 Subject: [PATCH 13/14] refactor(player): clean up timeline PR internals --- web-ui/src/mpegts/demux/ts-demuxer.ts | 36 +++++++++++------------ web-ui/src/mpegts/player/mpegts-player.ts | 32 -------------------- 2 files changed, 18 insertions(+), 50 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index 1fd25033..8277a0a3 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -53,6 +53,7 @@ type AdaptationFieldInfo = { random_access_indicator?: number; elementary_stream_priority_indicator?: number; }; +type CommonPidKey = keyof PMT["common_pids"]; type TSDemuxerOptions = { waitForInitialVideoKeyframe?: boolean; }; @@ -103,6 +104,9 @@ type AudioData = data: MP3Data; }; +const VIDEO_PID_KEYS: readonly CommonPidKey[] = ["h264", "h265"]; +const AUDIO_PID_KEYS: readonly CommonPidKey[] = ["adts_aac", "loas_aac", "ac3", "eac3", "mp3"]; + export type OnErrorCallback = (type: string, info: string) => void; export type OnTrackMetadataCallback = (type: string, metadata: unknown) => void; export type OnDataAvailableCallback = (audioTrack: unknown, videoTrack: unknown, force?: boolean) => void; @@ -287,18 +291,21 @@ class TSDemuxer { }; } + private isCommonPid(pid: number, keys: readonly CommonPidKey[]): boolean { + const commonPids = this.pmt_?.common_pids; + return !!commonPids && keys.some((key) => commonPids[key] === pid); + } + private isVideoPid(pid: number): boolean { - return pid === this.pmt_?.common_pids.h264 || pid === this.pmt_?.common_pids.h265; + return this.isCommonPid(pid, VIDEO_PID_KEYS); } private isAudioPid(pid: number): boolean { - return ( - pid === this.pmt_?.common_pids.adts_aac || - pid === this.pmt_?.common_pids.loas_aac || - pid === this.pmt_?.common_pids.ac3 || - pid === this.pmt_?.common_pids.eac3 || - pid === this.pmt_?.common_pids.mp3 - ); + return this.isCommonPid(pid, AUDIO_PID_KEYS); + } + + private isMediaPid(pid: number): boolean { + return this.isVideoPid(pid) || this.isAudioPid(pid); } private resetAudioParserState(): void { @@ -324,7 +331,8 @@ class TSDemuxer { return; } - for (const pid of [commonPids.adts_aac, commonPids.loas_aac, commonPids.ac3, commonPids.eac3, commonPids.mp3]) { + for (const key of AUDIO_PID_KEYS) { + const pid = commonPids[key]; if (pid !== undefined) { delete this.pes_slice_queues_[pid]; } @@ -491,15 +499,7 @@ class TSDemuxer { const stream_type = this.pmt_.pid_stream_type[pid]; // process PES only for known common_pids - if ( - pid === this.pmt_.common_pids.h264 || - pid === this.pmt_.common_pids.h265 || - pid === this.pmt_.common_pids.adts_aac || - pid === this.pmt_.common_pids.loas_aac || - pid === this.pmt_.common_pids.ac3 || - pid === this.pmt_.common_pids.eac3 || - pid === this.pmt_.common_pids.mp3 - ) { + if (this.isMediaPid(pid)) { if (!this.shouldProcessPayload(pid, continuity_conunter, adaptation_field_info.discontinuity_indicator)) { offset += 188; if (this.ts_packet_size_ === 204) { diff --git a/web-ui/src/mpegts/player/mpegts-player.ts b/web-ui/src/mpegts/player/mpegts-player.ts index dd768107..3e0634b8 100644 --- a/web-ui/src/mpegts/player/mpegts-player.ts +++ b/web-ui/src/mpegts/player/mpegts-player.ts @@ -1,38 +1,12 @@ import { markPlaybackUnlocked, PCMAudioPlayer } from "../audio/pcm-audio-player"; import type { PlayerConfig } from "../config"; import type { PlayerImpl, PlayerSegment } from "../types"; -import Log from "../utils/logger"; import type { WorkerCommand, WorkerEvent } from "../worker/messages"; import TransmuxWorker from "../worker/transmux-worker.ts?worker&inline"; import { setupLiveSync } from "./live-sync"; import { createMSE, type MSE } from "./mse"; import type { LiveSessionAnchor } from "./wall-clock"; -const TAG = "Player"; - -/** Attach verbose listeners to media element events for diagnosing playback stalls. */ -function setupVideoDebugLogs(video: HTMLVideoElement): () => void { - const events = ["loadedmetadata", "canplay", "playing", "waiting", "stalled", "pause", "seeking", "seeked", "error"]; - const handler = (e: Event) => { - const buffered: string[] = []; - for (let i = 0; i < video.buffered.length; i++) { - buffered.push(`${video.buffered.start(i).toFixed(2)}-${video.buffered.end(i).toFixed(2)}`); - } - Log.v( - TAG, - `video event '${e.type}': currentTime=${video.currentTime.toFixed(2)}, readyState=${video.readyState}, paused=${video.paused}, buffered=[${buffered.join(",")}]${e.type === "error" ? `, error=${video.error?.code}:${video.error?.message}` : ""}`, - ); - }; - for (const ev of events) { - video.addEventListener(ev, handler); - } - return () => { - for (const ev of events) { - video.removeEventListener(ev, handler); - } - }; -} - /** Check if a given time position is within any buffered range of the video element. */ export function isBuffered(video: HTMLMediaElement, seconds: number): boolean { const buffered = video.buffered; @@ -292,14 +266,10 @@ export function createMpegtsPlayer( }; } - let destroyVideoDebugLogs: (() => void) | null = null; - function initLiveHelpers(): void { if (!destroyLiveSync && liveSyncEnabled) { destroyLiveSync = setupLiveSync(video, config, () => liveSessionAnchor); } - destroyVideoDebugLogs?.(); - destroyVideoDebugLogs = setupVideoDebugLogs(video); } const onVideoPlay = () => markPlaybackUnlocked(); @@ -362,8 +332,6 @@ export function createMpegtsPlayer( destroyPCMPlayer(); destroyLiveSync?.(); destroyLiveSync = null; - destroyVideoDebugLogs?.(); - destroyVideoDebugLogs = null; }, destroy() { From 66e10fe2747e02613e095f54060bb9ed1d4c616f Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Sat, 27 Jun 2026 07:28:09 +0800 Subject: [PATCH 14/14] fix(player): address timeline review comments --- web-ui/src/mpegts/demux/ts-demuxer.ts | 2 +- web-ui/src/mpegts/remux/mp4-generator.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index 8277a0a3..016619ca 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -1056,7 +1056,7 @@ class TSDemuxer { pts: number | undefined, dts: number | undefined, file_position: number, - random_access_indicator: number, + random_access_indicator?: number, ) { const annexb_parser = new H265AnnexBParser(data); let nalu_payload: H265NaluPayload | null = null; diff --git a/web-ui/src/mpegts/remux/mp4-generator.ts b/web-ui/src/mpegts/remux/mp4-generator.ts index 72212278..f5d6ba17 100644 --- a/web-ui/src/mpegts/remux/mp4-generator.ts +++ b/web-ui/src/mpegts/remux/mp4-generator.ts @@ -1176,10 +1176,10 @@ class MP4 { data.set( [ - 0x01, + 0x01, // version 0x00, 0x0f, - 0x01, // version(1) & flags + 0x01, // flags: data-offset + sample duration/size/flags/cts (sampleCount >>> 24) & 0xff, // sample_count (sampleCount >>> 16) & 0xff, (sampleCount >>> 8) & 0xff,