handle sidecars in external libraries

immich-app · Dec 19, 2024 · 7af5081 · 7af5081
1 parent e0fc873
commit 7af5081
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 14 deletions.
diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts
@@ -1,5 +1,6 @@
 import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk';
-import { cpSync, existsSync } from 'node:fs';
+import { cpSync, existsSync, unlink, unlinkSync } from 'node:fs';
+import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
 import { Socket } from 'socket.io-client';
 import { userDto, uuidDto } from 'src/fixtures';
 import { errorDto } from 'src/responses';
@@ -601,6 +602,88 @@ describe('/libraries', () => {
 
       expect(assets).toEqual(assetsBefore);
     });
+
+    describe('xmp metadata', async () => {
+      it('should import metadata from file.xmp', async () => {
+        const library = await utils.createLibrary(admin.accessToken, {
+          ownerId: admin.userId,
+          importPaths: [`${testAssetDirInternal}/temp/xmp`],
+        });
+
+        cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.xmp`);
+        cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`);
+
+        await scan(admin.accessToken, library.id);
+
+        await utils.waitForQueueFinish(admin.accessToken, 'library');
+        await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+        const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+        expect(newAssets.items).toEqual([
+          expect.objectContaining({
+            originalFileName: 'glarus.nef',
+            fileCreatedAt: '2000-09-27T12:35:33.000Z', // This time comes from the xmp file, not from the image
+          }),
+        ]);
+
+        unlinkSync(`${testAssetDir}/temp/xmp/glarus.xmp`);
+        unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef`);
+      });
+
+      it('should import metadata from file.ext.xmp', async () => {
+        const library = await utils.createLibrary(admin.accessToken, {
+          ownerId: admin.userId,
+          importPaths: [`${testAssetDirInternal}/temp/xmp`],
+        });
+
+        cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.nef.xmp`);
+        cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`);
+
+        await scan(admin.accessToken, library.id);
+        await utils.waitForQueueFinish(admin.accessToken, 'library');
+        await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+        const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+        expect(newAssets.items).toEqual([
+          expect.objectContaining({
+            originalFileName: 'glarus.nef',
+            fileCreatedAt: '2000-09-27T12:35:33.000Z', // This time comes from the xmp file, not from the image
+          }),
+        ]);
+
+        unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef.xmp`);
+        unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef`);
+      });
+
+      it('should import metadata in file.ext.xmp before file.xmp if both exist', async () => {
+        const library = await utils.createLibrary(admin.accessToken, {
+          ownerId: admin.userId,
+          importPaths: [`${testAssetDirInternal}/temp/xmp`],
+        });
+
+        cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.nef.xmp`);
+        cpSync(`${testAssetDir}/metadata/xmp/dates/2010.xmp`, `${testAssetDir}/temp/xmp/glarus.xmp`);
+        cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`);
+
+        await scan(admin.accessToken, library.id);
+        await utils.waitForQueueFinish(admin.accessToken, 'library');
+        await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+        const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+        expect(newAssets.items).toEqual([
+          expect.objectContaining({
+            originalFileName: 'glarus.nef',
+            fileCreatedAt: '2000-09-27T12:35:33.000Z', // This time comes from the xmp file, not from the image
+          }),
+        ]);
+
+        unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef.xmp`);
+        unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef`);
+      });
+    });
   });
 
   describe('POST /libraries/:id/validate', () => {

diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts
@@ -135,7 +135,7 @@ export interface IDelayedJob extends IBaseJob {
 
 export interface IEntityJob extends IBaseJob {
   id: string;
-  source?: 'upload' | 'sidecar-write' | 'copy';
+  source?: 'upload' | 'library-import' | 'sidecar-write' | 'copy';
   notify?: boolean;
 }
 

diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts
@@ -250,7 +250,7 @@ export class JobService extends BaseService {
       }
 
       case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: {
-        if (item.data.source === 'upload' || item.data.source === 'copy') {
+        if (item.data.source === 'upload' || item.data.source === 'copy' || item.data.source === 'library-import') {
           await this.jobRepository.queue({ name: JobName.GENERATE_THUMBNAILS, data: item.data });
         }
         break;
@@ -266,7 +266,7 @@ export class JobService extends BaseService {
       }
 
       case JobName.GENERATE_THUMBNAILS: {
-        if (!item.data.notify && item.data.source !== 'upload') {
+        if (!item.data.notify && item.data.source !== 'upload' && item.data.source === 'library-import') {
           break;
         }
 

diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts
@@ -396,12 +396,6 @@ export class LibraryService extends BaseService {
 
     const pathHash = this.cryptoRepository.hashSha1(`path:${assetPath}`);
 
-    // TODO: doesn't xmp replace the file extension? Will need investigation
-    let sidecarPath: string | null = null;
-    if (await this.storageRepository.checkFileExists(`${assetPath}.xmp`, R_OK)) {
-      sidecarPath = `${assetPath}.xmp`;
-    }
-
     const assetType = mimeTypes.isVideo(assetPath) ? AssetType.VIDEO : AssetType.IMAGE;
 
     const mtime = stat.mtime;
@@ -418,8 +412,6 @@ export class LibraryService extends BaseService {
       localDateTime: mtime,
       type: assetType,
       originalFileName: parse(assetPath).base,
-
-      sidecarPath,
       isExternal: true,
     });
 
@@ -431,7 +423,10 @@ export class LibraryService extends BaseService {
   async queuePostSyncJobs(asset: AssetEntity) {
     this.logger.debug(`Queueing metadata extraction for: ${asset.originalPath}`);
 
-    await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id, source: 'upload' } });
+    await this.jobRepository.queue({
+      name: JobName.METADATA_EXTRACTION,
+      data: { id: asset.id, source: 'library-import' },
+    });
   }
 
   async queueScan(id: string) {

diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
@@ -148,13 +148,23 @@ export class MetadataService extends BaseService {
   }
 
   @OnJob({ name: JobName.METADATA_EXTRACTION, queue: QueueName.METADATA_EXTRACTION })
-  async handleMetadataExtraction({ id }: JobOf<JobName.METADATA_EXTRACTION>): Promise<JobStatus> {
+  async handleMetadataExtraction({ id, source }: JobOf<JobName.METADATA_EXTRACTION>): Promise<JobStatus> {
+    this.logger.verbose(`Extracting metadata for asset ${id}`);
+
     const { metadata, reverseGeocoding } = await this.getConfig({ withCache: true });
+
+    if (source === 'library-import') {
+      await this.processSidecar(id, false);
+    }
+
     const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } });
+
     if (!asset) {
       return JobStatus.FAILED;
     }
 
+    this.logger.verbose(`Sidecar path: ${asset.sidecarPath}`);
+
     const stats = await this.storageRepository.stat(asset.originalPath);
 
     const exifTags = await this.getExifTags(asset);
@@ -722,6 +732,7 @@ export class MetadataService extends BaseService {
 
     if (sidecarPath) {
       await this.assetRepository.update({ id: asset.id, sidecarPath });
+      this.logger.verbose(`Sidecar discovered at ${sidecarPath} for asset ${asset.id} at `);
       return JobStatus.SUCCESS;
     }