Merge pull request #2453 from nift4:fmp4metadata

PiperOrigin-RevId: 777751247

Author: copybara-github

RELEASENOTES.md

@@ -9,6 +9,8 @@
 *   Transformer:
 *   Track Selection:
 *   Extractors:
+    *   Parse metadata from fragmented MP4 files
+        ([#2084](https://github.com/androidx/media/issues/2084)).
     *   JPEG: Support motion photos that don't have an Exif segment at the start
         ([#2552](https://github.com/androidx/media/issues/2552)).
 *   DataSource:

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java

@@ -32,6 +32,7 @@
 import androidx.media3.common.DrmInitData;
 import androidx.media3.common.DrmInitData.SchemeData;
 import androidx.media3.common.Format;
+import androidx.media3.common.Metadata;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.ParserException;
 import androidx.media3.common.util.Log;
@@ -483,6 +484,8 @@ public void init(ExtractorOutput output) {
     enterReadingAtomHeaderState();
     initExtraTracks();
     if (sideloadedTrack != null) {
+      Format.Builder formatBuilder = sideloadedTrack.format.buildUpon();
+      formatBuilder.setContainerMimeType(getContainerMimeType(sideloadedTrack.format));
       TrackBundle bundle =
           new TrackBundle(
               extractorOutput.track(0, sideloadedTrack.type),
@@ -499,7 +502,7 @@ public void init(ExtractorOutput output) {
                   /* duration= */ 0,
                   /* size= */ 0,
                   /* flags= */ 0),
-              getContainerMimeType(sideloadedTrack.format));
+              formatBuilder.build());
       trackBundles.put(0, bundle);
       extractorOutput.endTracks();
     }
@@ -642,6 +645,9 @@ private boolean readAtomHeader(ExtractorInput input) throws IOException {
 
     if (shouldParseContainerAtom(atomType)) {
       long endPosition = input.getPosition() + atomSize - Mp4Box.HEADER_SIZE;
+      if (atomSize != atomHeaderBytesRead && atomType == Mp4Box.TYPE_meta) {
+        maybeSkipRemainingMetaAtomHeaderBytes(input);
+      }
       containerAtoms.push(new ContainerBox(atomType, endPosition));
       if (atomSize == atomHeaderBytesRead) {
         processAtomEnded(endPosition);
@@ -674,6 +680,14 @@ private boolean readAtomHeader(ExtractorInput input) throws IOException {
     return true;
   }
 
+  private void maybeSkipRemainingMetaAtomHeaderBytes(ExtractorInput input) throws IOException {
+    scratch.reset(Mp4Box.HEADER_SIZE);
+    input.peekFully(scratch.getData(), 0, Mp4Box.HEADER_SIZE);
+    BoxParser.maybeSkipRemainingMetaBoxHeaderBytes(scratch);
+    input.skipFully(scratch.getPosition());
+    input.resetPeekPosition();
+  }
+
   private void readAtomPayload(ExtractorInput input) throws IOException {
     int atomPayloadSize = (int) (atomSize - atomHeaderBytesRead);
     @Nullable ParsableByteArray atomData = this.atomData;
@@ -743,11 +757,27 @@ private void onMoovContainerAtomRead(ContainerBox moov) throws ParserException {
       }
     }
 
+    @Nullable Metadata mdtaMetadata = null;
+    @Nullable Mp4Box.ContainerBox meta = moov.getContainerBoxOfType(Mp4Box.TYPE_meta);
+    if (meta != null) {
+      mdtaMetadata = BoxParser.parseMdtaFromMeta(meta);
+    }
+    GaplessInfoHolder gaplessInfoHolder = new GaplessInfoHolder();
+    @Nullable Metadata udtaMetadata = null;
+    @Nullable Mp4Box.LeafBox udta = moov.getLeafBoxOfType(Mp4Box.TYPE_udta);
+    if (udta != null) {
+      udtaMetadata = BoxParser.parseUdta(udta);
+      gaplessInfoHolder.setFromMetadata(udtaMetadata);
+    }
+    Metadata mvhdMetadata =
+        new Metadata(
+            BoxParser.parseMvhd(checkNotNull(moov.getLeafBoxOfType(Mp4Box.TYPE_mvhd)).data));
+
     // Construction of tracks and sample tables.
     List<TrackSampleTable> sampleTables =
         parseTraks(
             moov,
-            new GaplessInfoHolder(),
+            gaplessInfoHolder,
             duration,
             drmInitData,
             /* ignoreEditLists= */ (flags & FLAG_WORKAROUND_IGNORE_EDIT_LISTS) != 0,
@@ -763,12 +793,22 @@ private void onMoovContainerAtomRead(ContainerBox moov) throws ParserException {
         Track track = sampleTable.track;
         TrackOutput output = extractorOutput.track(i, track.type);
         output.durationUs(track.durationUs);
+        Format.Builder formatBuilder = track.format.buildUpon();
+        formatBuilder.setContainerMimeType(containerMimeType);
+        MetadataUtil.setFormatGaplessInfo(track.type, gaplessInfoHolder, formatBuilder);
+        MetadataUtil.setFormatMetadata(
+            track.type,
+            mdtaMetadata,
+            formatBuilder,
+            track.format.metadata,
+            udtaMetadata,
+            mvhdMetadata);
         TrackBundle trackBundle =
             new TrackBundle(
                 output,
                 sampleTable,
                 getDefaultSampleValues(defaultSampleValuesArray, track.id),
-                containerMimeType);
+                formatBuilder.build());
         trackBundles.put(track.id, trackBundle);
         durationUs = max(durationUs, track.durationUs);
       }
@@ -1900,7 +1940,10 @@ private static boolean shouldParseLeafAtom(int atom) {
         || atom == Mp4Box.TYPE_sgpd
         || atom == Mp4Box.TYPE_elst
         || atom == Mp4Box.TYPE_mehd
-        || atom == Mp4Box.TYPE_emsg;
+        || atom == Mp4Box.TYPE_emsg
+        || atom == Mp4Box.TYPE_udta
+        || atom == Mp4Box.TYPE_keys
+        || atom == Mp4Box.TYPE_ilst;
   }
 
   /** Returns whether the extractor should decode a container atom with type {@code atom}. */
@@ -1913,7 +1956,8 @@ private static boolean shouldParseContainerAtom(int atom) {
         || atom == Mp4Box.TYPE_moof
         || atom == Mp4Box.TYPE_traf
         || atom == Mp4Box.TYPE_mvex
-        || atom == Mp4Box.TYPE_edts;
+        || atom == Mp4Box.TYPE_edts
+        || atom == Mp4Box.TYPE_meta;
   }
 
   /** Holds data corresponding to a metadata sample. */
@@ -1946,7 +1990,7 @@ private static final class TrackBundle {
     public int currentTrackRunIndex;
     public int firstSampleToOutputIndex;
 
-    private final String containerMimeType;
+    private final Format baseFormat;
     private final ParsableByteArray encryptionSignalByte;
     private final ParsableByteArray defaultInitializationVector;
 
@@ -1956,11 +2000,11 @@ public TrackBundle(
         TrackOutput output,
         TrackSampleTable moovSampleTable,
         DefaultSampleValues defaultSampleValues,
-        String containerMimeType) {
+        Format baseFormat) {
       this.output = output;
       this.moovSampleTable = moovSampleTable;
       this.defaultSampleValues = defaultSampleValues;
-      this.containerMimeType = containerMimeType;
+      this.baseFormat = baseFormat;
       fragment = new TrackFragment();
       scratch = new ParsableByteArray();
       encryptionSignalByte = new ParsableByteArray(1);
@@ -1971,9 +2015,7 @@ public TrackBundle(
     public void reset(TrackSampleTable moovSampleTable, DefaultSampleValues defaultSampleValues) {
       this.moovSampleTable = moovSampleTable;
       this.defaultSampleValues = defaultSampleValues;
-      Format format =
-          moovSampleTable.track.format.buildUpon().setContainerMimeType(containerMimeType).build();
-      output.format(format);
+      output.format(baseFormat);
       resetFragmentInfo();
     }
 
@@ -1984,14 +2026,7 @@ public void updateDrmInitData(DrmInitData drmInitData) {
               castNonNull(fragment.header).sampleDescriptionIndex);
       @Nullable String schemeType = encryptionBox != null ? encryptionBox.schemeType : null;
       DrmInitData updatedDrmInitData = drmInitData.copyWithSchemeType(schemeType);
-      Format format =
-          moovSampleTable
-              .track
-              .format
-              .buildUpon()
-              .setContainerMimeType(containerMimeType)
-              .setDrmInitData(updatedDrmInitData)
-              .build();
+      Format format = baseFormat.buildUpon().setDrmInitData(updatedDrmInitData).build();
       output.format(format);
     }
 

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java

@@ -1085,8 +1085,8 @@ private void processEndOfStreamReadingAtomHeader() {
   }
 
   private void maybeSkipRemainingMetaAtomHeaderBytes(ExtractorInput input) throws IOException {
-    scratch.reset(8);
-    input.peekFully(scratch.getData(), 0, 8);
+    scratch.reset(Mp4Box.HEADER_SIZE);
+    input.peekFully(scratch.getData(), 0, Mp4Box.HEADER_SIZE);
     BoxParser.maybeSkipRemainingMetaBoxHeaderBytes(scratch);
     input.skipFully(scratch.getPosition());
     input.resetPeekPosition();

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions.mp4.0.dump

@@ -21,6 +21,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 27, hash 9F13E633
       data = length 8, hash 94643657

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions.mp4.reading_within_gop_sample_dependencies.0.dump

@@ -21,6 +21,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 27, hash 9F13E633
       data = length 8, hash 94643657

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions.mp4.reading_within_gop_sample_dependencies.unknown_length.dump

@@ -21,6 +21,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 27, hash 9F13E633
       data = length 8, hash 94643657

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions.mp4.unknown_length.dump

@@ -21,6 +21,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 27, hash 9F13E633
       data = length 8, hash 94643657

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions_h265.mp4.0.dump

@@ -24,6 +24,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 2426, hash 25737613
   sample 0:

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions_h265.mp4.reading_within_gop_sample_dependencies.0.dump

@@ -24,6 +24,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 2426, hash 25737613
   sample 0:

libraries/test_data/src/test/assets/extractordumps/mp4/fragmented_captions_h265.mp4.reading_within_gop_sample_dependencies.unknown_length.dump

@@ -24,6 +24,7 @@ track 0:
       lumaBitdepth = 8
       chromaBitdepth = 8
     language = und
+    metadata = entries=[TSSE: description=null: values=[Lavf60.16.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 2426, hash 25737613
   sample 0: