fix(server): Some MTS videos fail to generate thumbnail (#14134)

* Stop skipping of all frames in MTS video * Only skip flag for mts videos * Fix lint checks * Adds test * Add comment for why flag is removed
immich-app · Nov 14, 2024 · 9203a61 · 9203a61
1 parent 11403ab
commit 9203a61
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 10 deletions.
diff --git a/server/src/interfaces/media.interface.ts b/server/src/interfaces/media.interface.ts
@@ -114,7 +114,12 @@ export interface ImageBuffer {
 }
 
 export interface VideoCodecSWConfig {
-  getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeCommand;
+  getCommand(
+    target: TranscodeTarget,
+    videoStream: VideoStreamInfo,
+    audioStream: AudioStreamInfo,
+    format?: VideoFormat,
+  ): TranscodeCommand;
 }
 
 export interface VideoCodecHWConfig extends VideoCodecSWConfig {

diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
@@ -487,6 +487,22 @@ describe(MediaService.name, () => {
         }),
       );
     });
+    it('should not skip intra frames for MTS file', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamMTS);
+      assetMock.getById.mockResolvedValue(assetStub.video);
+      await sut.handleGenerateThumbnails({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
+        expect.objectContaining({
+          inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'],
+          outputOptions: expect.any(Array),
+          progress: expect.any(Object),
+          twoPass: false,
+        }),
+      );
+    });
 
     it('should use scaling divisible by 2 even when using quick sync', async () => {
       mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);

diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts
@@ -239,7 +239,7 @@ export class MediaService extends BaseService {
     const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
     this.storageCore.ensureFolders(previewPath);
 
-    const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
+    const { format, audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
     const mainVideoStream = this.getMainStream(videoStreams);
     if (!mainVideoStream) {
       throw new Error(`No video streams found for asset ${asset.id}`);
@@ -248,9 +248,14 @@ export class MediaService extends BaseService {
 
     const previewConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.preview.size.toString() });
     const thumbnailConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() });
-
-    const previewOptions = previewConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream);
-    const thumbnailOptions = thumbnailConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream);
+    const previewOptions = previewConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream, format);
+    const thumbnailOptions = thumbnailConfig.getCommand(
+      TranscodeTarget.VIDEO,
+      mainVideoStream,
+      mainAudioStream,
+      format,
+    );
+    this.logger.error(format.formatName);
     await this.mediaRepository.transcode(asset.originalPath, previewPath, previewOptions);
     await this.mediaRepository.transcode(asset.originalPath, thumbnailPath, thumbnailOptions);
 

diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
@@ -6,6 +6,7 @@ import {
   TranscodeCommand,
   VideoCodecHWConfig,
   VideoCodecSWConfig,
+  VideoFormat,
   VideoStreamInfo,
 } from 'src/interfaces/media.interface';
 
@@ -77,9 +78,14 @@ export class BaseConfig implements VideoCodecSWConfig {
     return handler;
   }
 
-  getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+  getCommand(
+    target: TranscodeTarget,
+    videoStream: VideoStreamInfo,
+    audioStream?: AudioStreamInfo,
+    format?: VideoFormat,
+  ) {
     const options = {
-      inputOptions: this.getBaseInputOptions(videoStream),
+      inputOptions: this.getBaseInputOptions(videoStream, format),
       outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'],
       twoPass: this.eligibleForTwoPass(),
       progress: { frameCount: videoStream.frameCount, percentInterval: 5 },
@@ -101,7 +107,7 @@ export class BaseConfig implements VideoCodecSWConfig {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getBaseInputOptions(videoStream: VideoStreamInfo): string[] {
+  getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] {
     return this.getInputThreadOptions();
   }
 
@@ -377,8 +383,11 @@ export class ThumbnailConfig extends BaseConfig {
     return new ThumbnailConfig(config);
   }
 
-  getBaseInputOptions(): string[] {
-    return ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'];
+  getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] {
+    // skip_frame nointra skips all frames for some MPEG-TS files. Look at ffmpeg tickets 7950 and 7895 for more details.
+    return format?.formatName === 'mpegts'
+      ? ['-sws_flags accurate_rnd+full_chroma_int']
+      : ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'];
   }
 
   getBaseOutputOptions() {

diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts
@@ -95,6 +95,13 @@ export const probeStub = {
     ...probeStubDefault,
     videoStreams: [{ ...probeStubDefaultVideoStream[0], bitrate: 40_000_000 }],
   }),
+  videoStreamMTS: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    format: {
+      ...probeStubDefaultFormat,
+      formatName: 'mpegts',
+    },
+  }),
   videoStreamHDR: Object.freeze<VideoInfo>({
     ...probeStubDefault,
     videoStreams: [