Writing packets to data stream broken in av>=17

[oakaigh]

PyAV version: 17.0.0

The issue appears to be caused by zero-copying introduced in v17; It does not appear in v16.

Steps to reproduce:

• Python:

  import json
  import av
  import numpy
  
  container = av.open("/tmp/sample.mp4", format="mp4", mode="w")
  stream = container.add_data_stream("bin_data")
  vstream = container.add_stream("h264")
  
  for i in range(10):
      packet = av.Packet(json.dumps("This may not work", ensure_ascii=False).encode("utf-8"))
      if i % 2 == 0:
          # works fine when literal; possible memory mgmt issues?
          packet = av.Packet(b"This works fine")    
      vframe = av.VideoFrame.from_ndarray(numpy.random.randint(0, 255, size=(120, 120, 3), dtype=numpy.uint8))
      packet.pts = i
      packet.dts = i
      packet.duration = 1
      packet.stream = stream
      container.mux(packet)
      container.mux(vstream.encode(vframe))
  
  container.close()

• Shell:

  ffprobe -show_packets -select_streams d -show_data /tmp/sample.mp4 

  Output

  ffprobe version 5.1.8-0+deb12u1 Copyright (c) 2007-2025 the FFmpeg developers
    built with gcc 12 (Debian 12.2.0-14+deb12u1)
    configuration: --prefix=/usr --extra-version=0+deb12u1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libdav1d --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libglslang --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librabbitmq --enable-librist --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libsvtav1 --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --disable-sndio --enable-libjxl --enable-pocketsphinx --enable-librsvg --enable-libmfx --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libx264 --enable-libplacebo --enable-librav1e --enable-shared
    libavutil      57. 28.100 / 57. 28.100
    libavcodec     59. 37.100 / 59. 37.100
    libavformat    59. 27.100 / 59. 27.100
    libavdevice    59.  7.100 / 59.  7.100
    libavfilter     8. 44.100 /  8. 44.100
    libswscale      6.  7.100 /  6.  7.100
    libswresample   4.  7.100 /  4.  7.100
    libpostproc    56.  6.100 / 56.  6.100
  Input #0, mov,mp4,m4a,3gp,3g2,mj2, from '/tmp/sample.mp4':
    Metadata:
      major_brand     : isom
      minor_version   : 512
      compatible_brands: isomiso2avc1mp41
      encoder         : Lavf62.3.100
    Duration: 00:00:00.00, start: 0.000000, bitrate: 7656 kb/s
    Stream #0:0[0x1](und): Data: bin_data (gpmd / 0x646D7067), 12240 kb/s
      Metadata:
        handler_name    : GoPro MET
  �[0;33mUnsupported codec with id 98314 for input stream 0
  �[0m[PACKET]
  codec_type=data
  stream_index=0
  pts=0
  pts_time=0.000000
  dts=0
  dts_time=0.000000
  duration=1
  duration_time=0.000011
  size=15
  pos=48
  flags=K_
  data=
  00000000: 5468 6973 2077 6f72 6b73 2066 696e 65    This works fine
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=1
  pts_time=0.000011
  dts=1
  dts_time=0.000011
  duration=1
  duration_time=0.000011
  size=19
  pos=63
  flags=K_
  data=
  00000000: e0de d63f bb7f 0000 0200 0000 0000 0000  ...?............
  00000010: 2096 ef                                   ..
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=2
  pts_time=0.000022
  dts=2
  dts_time=0.000022
  duration=1
  duration_time=0.000011
  size=15
  pos=82
  flags=K_
  data=
  00000000: 5468 6973 2077 6f72 6b73 2066 696e 65    This works fine
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=3
  pts_time=0.000033
  dts=3
  dts_time=0.000033
  duration=1
  duration_time=0.000011
  size=19
  pos=97
  flags=K_
  data=
  00000000: 2039 d93f bb7f 0000 0200 0000 0000 0000   9.?............
  00000010: 2096 ef                                   ..
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=4
  pts_time=0.000044
  dts=4
  dts_time=0.000044
  duration=1
  duration_time=0.000011
  size=15
  pos=116
  flags=K_
  data=
  00000000: 5468 6973 2077 6f72 6b73 2066 696e 65    This works fine
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=5
  pts_time=0.000056
  dts=5
  dts_time=0.000056
  duration=1
  duration_time=0.000011
  size=19
  pos=131
  flags=K_
  data=
  00000000: 103a d93f bb7f 0000 0200 0000 0000 0000  .:.?............
  00000010: 2096 ef                                   ..
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=6
  pts_time=0.000067
  dts=6
  dts_time=0.000067
  duration=1
  duration_time=0.000011
  size=15
  pos=150
  flags=K_
  data=
  00000000: 5468 6973 2077 6f72 6b73 2066 696e 65    This works fine
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=7
  pts_time=0.000078
  dts=7
  dts_time=0.000078
  duration=1
  duration_time=0.000011
  size=19
  pos=165
  flags=K_
  data=
  00000000: b030 d93f bb7f 0000 0200 0000 0000 0000  .0.?............
  00000010: 2096 ef                                   ..
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=8
  pts_time=0.000089
  dts=8
  dts_time=0.000089
  duration=1
  duration_time=0.000011
  size=15
  pos=184
  flags=K_
  data=
  00000000: 5468 6973 2077 6f72 6b73 2066 696e 65    This works fine
  
  [/PACKET]
  [PACKET]
  codec_type=data
  stream_index=0
  pts=9
  pts_time=0.000100
  dts=9
  dts_time=0.000100
  duration=1
  duration_time=0.000011
  size=19
  pos=199
  flags=K_
  data=
  00000000: 2254 6869 7320 6d61 7920 6e6f 7420 776f  "This may not wo
  00000010: 726b 22                                  rk"
  
  [/PACKET]
  

[oakaigh]

Maybe it's not a good idea to make av.Packet zero-copy by default as it has unintended consequences (and breaks backward compatibility). Adding a copy= parameter that defaults to True would be the most appropriate.

[WyattBlue]

I don't this bad data ( .0.?......) even when running your script.
edit: I'm also seeing this on v17. I was unintentionally using v16 before.

Write a red test then submit it as a PR because I want to know if this behavior is platform dependent.

[oakaigh]

@WyattBlue (See comments for more info)

import json
import io
import fractions

import av
import numpy


def generate_container_with_data_packets(file: ... = None):
    if file is None:
        file = io.BytesIO()
    packet_datas_expected = dict[fractions.Fraction, bytes]()

    container = av.open(file, format="mp4", mode="w")
    stream = container.add_data_stream("bin_data")
    vstream = container.add_stream("h264")

    for i in range(10):
        packet_data = json.dumps("This may not work", ensure_ascii=False).encode("utf-8")
        if i % 2 == 0:
            packet_data = b"This works fine"
        packet = av.Packet(packet_data)
        vframe = av.VideoFrame.from_ndarray(numpy.random.randint(0, 255, size=(120, 120, 3), dtype=numpy.uint8))
        packet.pts = i
        packet.dts = i
        packet.duration = 1
        packet.stream = stream
        container.mux(packet)
        container.mux(vstream.encode(vframe))
        # NOTE test passes if this is used
        # packet_datas_expected[packet.pts * packet.time_base] = bytes(packet_data)

    container.close()

    container = av.open(file, format="mp4", mode="r")
    return container, packet_datas_expected


def test_data_packet_bytes():
    container, packet_datas_expected = generate_container_with_data_packets()

    packet_datas = dict[fractions.Fraction, bytes]()
    for packet in container.demux(container.streams.data[0]):
        if packet.pts is None:
            continue
        assert bytes(packet) in (
            b"This works fine", 
            json.dumps("This may not work", ensure_ascii=False).encode("utf-8"),
        ), bytes(packet)
        # NOTE test passes if this is used
        # packet_datas[packet.pts * packet.time_base] = bytes(packet)
    
    assert packet_datas == packet_datas_expected


test_data_packet_bytes()

Output:

---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
Cell In[18], [line 56](vscode-notebook-cell:?execution_count=18&line=56)
     50         # NOTE test passes if this is used
     51         # packet_datas[packet.pts * packet.time_base] = bytes(packet)
     53     assert packet_datas == packet_datas_expected
---> [56](vscode-notebook-cell:?execution_count=18&line=56) test_data_packet_bytes()

Cell In[18], [line 46](vscode-notebook-cell:?execution_count=18&line=46)
     44     if packet.pts is None:
     45         continue
---> [46](vscode-notebook-cell:?execution_count=18&line=46)     assert bytes(packet) in (
     47         b"This works fine", 
     48         json.dumps("This may not work", ensure_ascii=False).encode("utf-8"),
     49     ), bytes(packet)
     50     # NOTE test passes if this is used
     51     # packet_datas[packet.pts * packet.time_base] = bytes(packet)
     53 assert packet_datas == packet_datas_expected

AssertionError: b'\xc0\xa0\x08\x8fD\x7f\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x0c\x96/'

[bdavid-evertz]

I believe this occurs when the source is GC'ed before FFmpeg gets to use it. Holding a reference in the Cython wrapper object is not sufficient as the wrapper will likely be collected shortly after muxing.

Here's the fix I made that seems to work (based on 085b4d2):

diff --git a/av/packet.pxd b/av/packet.pxd
index 9bd077d..1546a13 100644
--- a/av/packet.pxd
+++ b/av/packet.pxd
@@ -19,7 +19,5 @@ cdef class Packet(Buffer):
     cdef lib.AVPacket* ptr
     cdef Stream _stream
     cdef _rebase_time(self, lib.AVRational)
-    # Hold onto the original reference.
-    cdef ByteSource source
     cdef size_t _buffer_size(self)
     cdef void* _buffer_ptr(self)
diff --git a/av/packet.py b/av/packet.py
index 5a97817..3d07a2f 100644
--- a/av/packet.py
+++ b/av/packet.py
@@ -4,8 +4,10 @@ import cython
 from cython.cimports import libav as lib
 from cython.cimports.av.buffer import Buffer, ByteSource, bytesource
 from cython.cimports.av.error import err_check
-from cython.cimports.av.opaque import noop_free, opaque_container
+from cython.cimports.av.opaque import opaque_container
 from cython.cimports.av.utils import avrational_to_fraction, to_avrational
+from cython.cimports.cpython.ref import Py_DECREF, Py_INCREF
+from cython.cimports.libc.stdint import uint8_t
 from cython.cimports.libc.string import memcpy
 
 # Check https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/packet.h#L41
@@ -193,6 +195,18 @@ def _packet_sidedata_from_packet(
     return sdata
 
 
+@cython.cfunc
+@cython.nogil
+@cython.exceptval(check=False)
+def _python_free(
+    opaque: cython.p_void,
+    data: cython.pointer[uint8_t],
+) -> cython.void:
+    if opaque != cython.NULL:
+        with cython.gil:
+            Py_DECREF(cython.cast(object, opaque))
+
+
 @cython.cclass
 class Packet(Buffer):
     """A packet of encoded data within a :class:`~av.format.Stream`.
@@ -226,15 +240,16 @@ class Packet(Buffer):
             size = source.length
             if size:
                 # Create a buffer that references the source data directly.
-                # The noop_free callback is used because Python manages the memory
-                # via self.source keeping the ByteSource alive.
-                buf = lib.av_buffer_create(source.ptr, size, noop_free, cython.NULL, 0)
+                Py_INCREF(source)
+                buf = lib.av_buffer_create(
+                    source.ptr, size, _python_free, cython.cast(cython.p_void, source), 0
+                )
                 if buf == cython.NULL:
+                    Py_DECREF(source)
                     raise MemoryError("Could not allocate AVBufferRef")
                 self.ptr.buf = buf
                 self.ptr.data = source.ptr
                 self.ptr.size = size
-                self.source = source
 
     def __repr__(self):
         stream = self._stream.index if self._stream else 0