Possible fix for Raspberry Pi 4 hardware-encoded H.264 streams

pkg/flv/muxer.go

@@ -30,12 +30,28 @@ func (m *Muxer) GetInit() []byte {
 
 	obj := map[string]any{}
 
+	var metaWidth, metaHeight uint16
+	var metaFPS float64
+
 	for _, codec := range m.codecs {
 		switch codec.Name {
 		case core.CodecH264:
 			b[4] |= FlagsVideo
 			obj["videocodecid"] = CodecAVC
 
+			// Try to extract width/height and optional FPS from SPS
+			if sps, _ := h264.GetParameterSet(codec.FmtpLine); len(sps) > 0 {
+				if s := h264.DecodeSPS(sps); s != nil {
+					if metaWidth == 0 || metaHeight == 0 {
+						metaWidth = s.Width()
+						metaHeight = s.Height()
+					}
+					if f := s.FPS(); f > 0 {
+						metaFPS = f
+					}
+				}
+			}
+
 		case core.CodecAAC:
 			b[4] |= FlagsAudio
 			obj["audiocodecid"] = CodecAAC
@@ -45,25 +61,28 @@ func (m *Muxer) GetInit() []byte {
 		}
 	}
 
+	// Fill optional width/height/framerate if known
+	if metaWidth > 0 && metaHeight > 0 {
+		obj["width"] = metaWidth
+		obj["height"] = metaHeight
+	}
+	if metaFPS > 0 {
+		obj["framerate"] = metaFPS
+	}
+
 	data := amf.EncodeItems("@setDataFrame", "onMetaData", obj)
 	b = append(b, EncodeTag(TagData, 0, data)...)
 
 	for _, codec := range m.codecs {
 		switch codec.Name {
 		case core.CodecH264:
 			sps, pps := h264.GetParameterSet(codec.FmtpLine)
-			if len(sps) == 0 {
-				sps = []byte{0x67, 0x42, 0x00, 0x0a, 0xf8, 0x41, 0xa2}
-			} else {
+			if len(sps) > 0 && len(pps) > 0 {
 				h264.FixPixFmt(sps)
+				config := h264.EncodeConfig(sps, pps)
+				video := append(encodeAVData(codec, 0), config...)
+				b = append(b, EncodeTag(TagVideo, 0, video)...)
 			}
-			if len(pps) == 0 {
-				pps = []byte{0x68, 0xce, 0x38, 0x80}
-			}
-
-			config := h264.EncodeConfig(sps, pps)
-			video := append(encodeAVData(codec, 0), config...)
-			b = append(b, EncodeTag(TagVideo, 0, video)...)
 
 		case core.CodecAAC:
 			s := core.Between(codec.FmtpLine, "config=", ";")
@@ -85,8 +104,42 @@ func (m *Muxer) GetPayloader(codec *core.Codec) func(packet *rtp.Packet) []byte
 	switch codec.Name {
 	case core.CodecH264:
 		buf := encodeAVData(codec, 1)
+		// Some RTSP servers (FFmpeg) don't provide sprop-parameter-sets in SDP.
+		// That makes initial FLV sequence header fallback to a generic SPS/PPS,
+		// which can confuse some RTMP ingests. Emit a real AVC sequence header
+		// once we see SPS/PPS inside the first Access Unit.
+		var sentRealHeader bool
 
 		return func(packet *rtp.Packet) []byte {
+			var header []byte
+			if !sentRealHeader {
+				// Try to extract SPS/PPS from the current AVCC payload
+				var sps, pps []byte
+				for _, nalu := range h264.SplitNALU(packet.Payload) {
+					switch h264.NALUType(nalu) {
+					case h264.NALUTypeSPS:
+						sps = nalu[4:]
+					case h264.NALUTypePPS:
+						pps = nalu[4:]
+					}
+				}
+				if len(sps) > 0 && len(pps) > 0 {
+					conf := h264.EncodeConfig(sps, pps)
+					hdr := append(encodeAVData(codec, 0), conf...)
+					// Propagate discovered SPS/PPS into codec fmtp so late joiners (e.g., WebRTC)
+					// have sprop-parameter-sets available.
+					if c := h264.ConfigToCodec(conf); c != nil {
+						codec.FmtpLine = c.FmtpLine
+					}
+					if ts0 == 0 {
+						ts0 = packet.Timestamp
+					}
+					timeMS := (packet.Timestamp - ts0) / k
+					header = EncodeTag(TagVideo, timeMS, hdr)
+					sentRealHeader = true
+				}
+			}
+
 			if h264.IsKeyframe(packet.Payload) {
 				buf[0] = 1<<4 | 7
 			} else {
@@ -100,7 +153,15 @@ func (m *Muxer) GetPayloader(codec *core.Codec) func(packet *rtp.Packet) []byte
 			}
 
 			timeMS := (packet.Timestamp - ts0) / k
-			return EncodeTag(TagVideo, timeMS, buf)
+			frame := EncodeTag(TagVideo, timeMS, buf)
+			if len(header) > 0 {
+				// Emit real config immediately before the first frame containing SPS/PPS
+				out := make([]byte, 0, len(header)+len(frame))
+				out = append(out, header...)
+				out = append(out, frame...)
+				return out
+			}
+			return frame
 		}
 
 	case core.CodecAAC:

pkg/h264/payloader.go

@@ -6,6 +6,9 @@ import "encoding/binary"
 type Payloader struct {
 	IsAVC     bool
 	stapANalu []byte
+	// Persist latest SPS/PPS across calls to prepend on IDR frames
+	lastSPS []byte
+	lastPPS []byte
 }
 
 const (
@@ -100,6 +103,12 @@ func (p *Payloader) Payload(mtu uint16, payload []byte) [][]byte {
 		case audNALUType, fillerNALUType:
 			return
 		case spsNALUType, ppsNALUType:
+			// Store latest SPS/PPS for future IDR frames
+			if naluType == spsNALUType {
+				p.lastSPS = append(p.lastSPS[:0], nalu...)
+			} else {
+				p.lastPPS = append(p.lastPPS[:0], nalu...)
+			}
 			if p.stapANalu == nil {
 				p.stapANalu = []byte{outputStapAHeader}
 			}
@@ -108,6 +117,15 @@ func (p *Payloader) Payload(mtu uint16, payload []byte) [][]byte {
 			return
 		}
 
+		// If this is an IDR without in-band SPS/PPS, prepend last known SPS/PPS
+		if naluType == NALUTypeIFrame && p.stapANalu == nil && len(p.lastSPS) > 0 && len(p.lastPPS) > 0 {
+			p.stapANalu = []byte{outputStapAHeader}
+			p.stapANalu = append(p.stapANalu, byte(len(p.lastSPS)>>8), byte(len(p.lastSPS)))
+			p.stapANalu = append(p.stapANalu, p.lastSPS...)
+			p.stapANalu = append(p.stapANalu, byte(len(p.lastPPS)>>8), byte(len(p.lastPPS)))
+			p.stapANalu = append(p.stapANalu, p.lastPPS...)
+		}
+
 		if p.stapANalu != nil {
 			// Pack current NALU with SPS and PPS as STAP-A
 			// Supports multiple PPS in a row

pkg/h264/rtp.go

@@ -18,6 +18,8 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc {
 
 	sps, pps := GetParameterSet(codec.FmtpLine)
 	ps := JoinNALU(sps, pps)
+	// Track latest SPS/PPS observed in-band and use them if SDP had none
+	var spsLast, ppsLast []byte
 
 	buf := make([]byte, 0, 512*1024) // 512K
 
@@ -35,6 +37,26 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc {
 			buf = buf[: 0 : 512*1024]
 		}
 
+		// Capture SPS/PPS from current payload (AVCC) to update parameter sets dynamically
+		// This helps when remote SDP doesn't provide sprop-parameter-sets
+		for off := 0; off+4 <= len(payload); {
+			size := 4 + int(binary.BigEndian.Uint32(payload[off:]))
+			if off+size > len(payload) || size <= 4 {
+				break
+			}
+			nalu := payload[off : off+size]
+			switch NALUType(nalu) {
+			case NALUTypeSPS:
+				spsLast = append(spsLast[:0], nalu[4:]...)
+			case NALUTypePPS:
+				ppsLast = append(ppsLast[:0], nalu[4:]...)
+			}
+			off += size
+		}
+		if len(spsLast) > 0 && len(ppsLast) > 0 {
+			ps = JoinNALU(spsLast, ppsLast)
+		}
+
 		// Fix TP-Link Tapo TC70: sends SPS and PPS with packet.Marker = true
 		// Reolink Duo 2: sends SPS with Marker and PPS without
 		if packet.Marker && len(payload) < PSMaxSize {
@@ -55,8 +77,10 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc {
 				// Amcrest IP4M-1051: 9, 6, 1
 				switch NALUType(payload) {
 				case NALUTypeIFrame:
-					// fix IFrame without SPS,PPS
-					buf = append(buf, ps...)
+					// fix IFrame without SPS,PPS (use latest known)
+					if len(ps) > 0 {
+						buf = append(buf, ps...)
+					}
 				case NALUTypeSEI, NALUTypeAUD:
 					// fix ffmpeg with transcoding first frame
 					i := int(4 + binary.BigEndian.Uint32(payload))

pkg/h264/sps.go

@@ -364,3 +364,17 @@ func FixPixFmt(sps []byte) {
 		}
 	}
 }
+
+// FPS returns frames per second if timing info is present in SPS, otherwise 0.
+// Formula: fps = time_scale / (2 * num_units_in_tick) for progressive streams.
+// If timing info is absent, returns 0.
+func (s *SPS) FPS() float64 {
+	if s == nil {
+		return 0
+	}
+	if s.timing_info_present_flag == 0 || s.num_units_in_tick == 0 {
+		return 0
+	}
+	// Most streams are progressive; the H.264 spec uses the factor 2.
+	return float64(s.time_scale) / (2.0 * float64(s.num_units_in_tick))
+}

pkg/webrtc/api.go

@@ -200,7 +200,7 @@ func RegisterDefaultCodecs(m *webrtc.MediaEngine) error {
 			RTPCodecCapability: webrtc.RTPCodecCapability{
 				MimeType:     webrtc.MimeTypeH264,
 				ClockRate:    90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032",
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640029",
 				RTCPFeedback: videoRTCPFeedback,
 			},
 			PayloadType: 98, // Chrome v110 - PayloadType: 112