From 7a9725aaf0dda385fa3a0a4d4831f55f53114975 Mon Sep 17 00:00:00 2001 From: "H. C. Kruse" Date: Thu, 11 Sep 2025 12:47:26 +0200 Subject: [PATCH] refactor: Pi HW fixes --- pkg/flv/muxer.go | 83 +++++++++++++++++++++++++++++++++++++------ pkg/h264/payloader.go | 18 ++++++++++ pkg/h264/rtp.go | 28 +++++++++++++-- pkg/h264/sps.go | 14 ++++++++ pkg/webrtc/api.go | 2 +- 5 files changed, 131 insertions(+), 14 deletions(-) diff --git a/pkg/flv/muxer.go b/pkg/flv/muxer.go index 98794265c..32b970d09 100644 --- a/pkg/flv/muxer.go +++ b/pkg/flv/muxer.go @@ -30,12 +30,28 @@ func (m *Muxer) GetInit() []byte { obj := map[string]any{} + var metaWidth, metaHeight uint16 + var metaFPS float64 + for _, codec := range m.codecs { switch codec.Name { case core.CodecH264: b[4] |= FlagsVideo obj["videocodecid"] = CodecAVC + // Try to extract width/height and optional FPS from SPS + if sps, _ := h264.GetParameterSet(codec.FmtpLine); len(sps) > 0 { + if s := h264.DecodeSPS(sps); s != nil { + if metaWidth == 0 || metaHeight == 0 { + metaWidth = s.Width() + metaHeight = s.Height() + } + if f := s.FPS(); f > 0 { + metaFPS = f + } + } + } + case core.CodecAAC: b[4] |= FlagsAudio obj["audiocodecid"] = CodecAAC @@ -45,6 +61,15 @@ func (m *Muxer) GetInit() []byte { } } + // Fill optional width/height/framerate if known + if metaWidth > 0 && metaHeight > 0 { + obj["width"] = metaWidth + obj["height"] = metaHeight + } + if metaFPS > 0 { + obj["framerate"] = metaFPS + } + data := amf.EncodeItems("@setDataFrame", "onMetaData", obj) b = append(b, EncodeTag(TagData, 0, data)...) @@ -52,18 +77,12 @@ func (m *Muxer) GetInit() []byte { switch codec.Name { case core.CodecH264: sps, pps := h264.GetParameterSet(codec.FmtpLine) - if len(sps) == 0 { - sps = []byte{0x67, 0x42, 0x00, 0x0a, 0xf8, 0x41, 0xa2} - } else { + if len(sps) > 0 && len(pps) > 0 { h264.FixPixFmt(sps) + config := h264.EncodeConfig(sps, pps) + video := append(encodeAVData(codec, 0), config...) + b = append(b, EncodeTag(TagVideo, 0, video)...) } - if len(pps) == 0 { - pps = []byte{0x68, 0xce, 0x38, 0x80} - } - - config := h264.EncodeConfig(sps, pps) - video := append(encodeAVData(codec, 0), config...) - b = append(b, EncodeTag(TagVideo, 0, video)...) case core.CodecAAC: s := core.Between(codec.FmtpLine, "config=", ";") @@ -85,8 +104,42 @@ func (m *Muxer) GetPayloader(codec *core.Codec) func(packet *rtp.Packet) []byte switch codec.Name { case core.CodecH264: buf := encodeAVData(codec, 1) + // Some RTSP servers (FFmpeg) don't provide sprop-parameter-sets in SDP. + // That makes initial FLV sequence header fallback to a generic SPS/PPS, + // which can confuse some RTMP ingests. Emit a real AVC sequence header + // once we see SPS/PPS inside the first Access Unit. + var sentRealHeader bool return func(packet *rtp.Packet) []byte { + var header []byte + if !sentRealHeader { + // Try to extract SPS/PPS from the current AVCC payload + var sps, pps []byte + for _, nalu := range h264.SplitNALU(packet.Payload) { + switch h264.NALUType(nalu) { + case h264.NALUTypeSPS: + sps = nalu[4:] + case h264.NALUTypePPS: + pps = nalu[4:] + } + } + if len(sps) > 0 && len(pps) > 0 { + conf := h264.EncodeConfig(sps, pps) + hdr := append(encodeAVData(codec, 0), conf...) + // Propagate discovered SPS/PPS into codec fmtp so late joiners (e.g., WebRTC) + // have sprop-parameter-sets available. + if c := h264.ConfigToCodec(conf); c != nil { + codec.FmtpLine = c.FmtpLine + } + if ts0 == 0 { + ts0 = packet.Timestamp + } + timeMS := (packet.Timestamp - ts0) / k + header = EncodeTag(TagVideo, timeMS, hdr) + sentRealHeader = true + } + } + if h264.IsKeyframe(packet.Payload) { buf[0] = 1<<4 | 7 } else { @@ -100,7 +153,15 @@ func (m *Muxer) GetPayloader(codec *core.Codec) func(packet *rtp.Packet) []byte } timeMS := (packet.Timestamp - ts0) / k - return EncodeTag(TagVideo, timeMS, buf) + frame := EncodeTag(TagVideo, timeMS, buf) + if len(header) > 0 { + // Emit real config immediately before the first frame containing SPS/PPS + out := make([]byte, 0, len(header)+len(frame)) + out = append(out, header...) + out = append(out, frame...) + return out + } + return frame } case core.CodecAAC: diff --git a/pkg/h264/payloader.go b/pkg/h264/payloader.go index efc89986d..9212fdc4e 100644 --- a/pkg/h264/payloader.go +++ b/pkg/h264/payloader.go @@ -6,6 +6,9 @@ import "encoding/binary" type Payloader struct { IsAVC bool stapANalu []byte + // Persist latest SPS/PPS across calls to prepend on IDR frames + lastSPS []byte + lastPPS []byte } const ( @@ -100,6 +103,12 @@ func (p *Payloader) Payload(mtu uint16, payload []byte) [][]byte { case audNALUType, fillerNALUType: return case spsNALUType, ppsNALUType: + // Store latest SPS/PPS for future IDR frames + if naluType == spsNALUType { + p.lastSPS = append(p.lastSPS[:0], nalu...) + } else { + p.lastPPS = append(p.lastPPS[:0], nalu...) + } if p.stapANalu == nil { p.stapANalu = []byte{outputStapAHeader} } @@ -108,6 +117,15 @@ func (p *Payloader) Payload(mtu uint16, payload []byte) [][]byte { return } + // If this is an IDR without in-band SPS/PPS, prepend last known SPS/PPS + if naluType == NALUTypeIFrame && p.stapANalu == nil && len(p.lastSPS) > 0 && len(p.lastPPS) > 0 { + p.stapANalu = []byte{outputStapAHeader} + p.stapANalu = append(p.stapANalu, byte(len(p.lastSPS)>>8), byte(len(p.lastSPS))) + p.stapANalu = append(p.stapANalu, p.lastSPS...) + p.stapANalu = append(p.stapANalu, byte(len(p.lastPPS)>>8), byte(len(p.lastPPS))) + p.stapANalu = append(p.stapANalu, p.lastPPS...) + } + if p.stapANalu != nil { // Pack current NALU with SPS and PPS as STAP-A // Supports multiple PPS in a row diff --git a/pkg/h264/rtp.go b/pkg/h264/rtp.go index d093254fd..bda5fa696 100644 --- a/pkg/h264/rtp.go +++ b/pkg/h264/rtp.go @@ -18,6 +18,8 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { sps, pps := GetParameterSet(codec.FmtpLine) ps := JoinNALU(sps, pps) + // Track latest SPS/PPS observed in-band and use them if SDP had none + var spsLast, ppsLast []byte buf := make([]byte, 0, 512*1024) // 512K @@ -35,6 +37,26 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { buf = buf[: 0 : 512*1024] } + // Capture SPS/PPS from current payload (AVCC) to update parameter sets dynamically + // This helps when remote SDP doesn't provide sprop-parameter-sets + for off := 0; off+4 <= len(payload); { + size := 4 + int(binary.BigEndian.Uint32(payload[off:])) + if off+size > len(payload) || size <= 4 { + break + } + nalu := payload[off : off+size] + switch NALUType(nalu) { + case NALUTypeSPS: + spsLast = append(spsLast[:0], nalu[4:]...) + case NALUTypePPS: + ppsLast = append(ppsLast[:0], nalu[4:]...) + } + off += size + } + if len(spsLast) > 0 && len(ppsLast) > 0 { + ps = JoinNALU(spsLast, ppsLast) + } + // Fix TP-Link Tapo TC70: sends SPS and PPS with packet.Marker = true // Reolink Duo 2: sends SPS with Marker and PPS without if packet.Marker && len(payload) < PSMaxSize { @@ -55,8 +77,10 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { // Amcrest IP4M-1051: 9, 6, 1 switch NALUType(payload) { case NALUTypeIFrame: - // fix IFrame without SPS,PPS - buf = append(buf, ps...) + // fix IFrame without SPS,PPS (use latest known) + if len(ps) > 0 { + buf = append(buf, ps...) + } case NALUTypeSEI, NALUTypeAUD: // fix ffmpeg with transcoding first frame i := int(4 + binary.BigEndian.Uint32(payload)) diff --git a/pkg/h264/sps.go b/pkg/h264/sps.go index 1ac739456..8e58a997f 100644 --- a/pkg/h264/sps.go +++ b/pkg/h264/sps.go @@ -364,3 +364,17 @@ func FixPixFmt(sps []byte) { } } } + +// FPS returns frames per second if timing info is present in SPS, otherwise 0. +// Formula: fps = time_scale / (2 * num_units_in_tick) for progressive streams. +// If timing info is absent, returns 0. +func (s *SPS) FPS() float64 { + if s == nil { + return 0 + } + if s.timing_info_present_flag == 0 || s.num_units_in_tick == 0 { + return 0 + } + // Most streams are progressive; the H.264 spec uses the factor 2. + return float64(s.time_scale) / (2.0 * float64(s.num_units_in_tick)) +} diff --git a/pkg/webrtc/api.go b/pkg/webrtc/api.go index fe49ef1e9..1ac53b8ad 100644 --- a/pkg/webrtc/api.go +++ b/pkg/webrtc/api.go @@ -200,7 +200,7 @@ func RegisterDefaultCodecs(m *webrtc.MediaEngine) error { RTPCodecCapability: webrtc.RTPCodecCapability{ MimeType: webrtc.MimeTypeH264, ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032", + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640029", RTCPFeedback: videoRTCPFeedback, }, PayloadType: 98, // Chrome v110 - PayloadType: 112