Feature: file event manager

.gitignore

@@ -16,4 +16,5 @@ frontend/out-tsc
 frontend/bazel-out
 /frontend/src/assets/pdfjs/
 circular-dependencies.png
-backend/src/applications/files/assets/ocr-languages/*.traineddata
+backend/src/applications/files/assets/ocr-languages/*.traineddata
+.husky/_/

backend/src/applications/comments/comments.controller.spec.ts

@@ -13,6 +13,7 @@ import { UsersQueries } from '../users/services/users-queries.service'
 import { CommentsController } from './comments.controller'
 import { CommentsManager } from './services/comments-manager.service'
 import { CommentsQueries } from './services/comments-queries.service'
+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'
 
 describe(CommentsController.name, () => {
   let commentsController: CommentsController
@@ -41,6 +42,7 @@ describe(CommentsController.name, () => {
         { provide: Cache, useValue: {} },
         ContextManager,
         { provide: CommentsManager, useValue: commentsManagerMock },
+        { provide: FilesQuotaManager, useValue: {} },
         CommentsQueries,
         SpacesManager,
         SpacesQueries,

backend/src/applications/files/adapters/files-indexer-mysql.service.spec.ts → backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts

@@ -1,11 +1,11 @@
 import { Test, TestingModule } from '@nestjs/testing'
 import { Cache } from '../../../infrastructure/cache/services/cache.service'
 import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
-import { FilesIndexerMySQL } from './files-indexer-mysql.service'
+import { FilesContentStoreMySQL } from './files-content-store-mysql.service'
 
-describe(FilesIndexerMySQL.name, () => {
+describe(FilesContentStoreMySQL.name, () => {
   let module: TestingModule
-  let filesIndexerMySQL: FilesIndexerMySQL
+  let filesIndexerMySQL: FilesContentStoreMySQL
   let db: { execute: jest.Mock }
   let cache: { genSlugKey: jest.Mock; get: jest.Mock; set: jest.Mock }
 
@@ -18,11 +18,11 @@ describe(FilesIndexerMySQL.name, () => {
     }
 
     module = await Test.createTestingModule({
-      providers: [FilesIndexerMySQL, { provide: DB_TOKEN_PROVIDER, useValue: db }, { provide: Cache, useValue: cache }]
+      providers: [FilesContentStoreMySQL, { provide: DB_TOKEN_PROVIDER, useValue: db }, { provide: Cache, useValue: cache }]
     }).compile()
 
     module.useLogger(['fatal'])
-    filesIndexerMySQL = module.get<FilesIndexerMySQL>(FilesIndexerMySQL)
+    filesIndexerMySQL = module.get<FilesContentStoreMySQL>(FilesContentStoreMySQL)
   })
 
   afterAll(async () => {

backend/src/applications/files/adapters/files-indexer-mysql.service.ts → backend/src/applications/files/adapters/files-content-store-mysql.service.ts

@@ -4,18 +4,18 @@ import { MySqlQueryResult } from 'drizzle-orm/mysql2'
 import { CacheDecorator } from '../../../infrastructure/cache/cache.decorator'
 import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
 import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
-import { FilesIndexer } from '../models/files-indexer'
+import { FilesContentStore } from '../models/files-content-store'
 import { FileContent } from '../schemas/file-content.interface'
 import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'
 import { analyzeTerms, genTermsPattern, MaxSortedList } from '../utils/files-search'
 
 @Injectable()
-export class FilesIndexerMySQL implements FilesIndexer {
-  private readonly logger = new Logger(FilesIndexerMySQL.name)
+export class FilesContentStoreMySQL implements FilesContentStore {
+  private readonly logger = new Logger(FilesContentStoreMySQL.name)
 
   constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
 
-  @CacheDecorator(900) // 15 mn
+  @CacheDecorator(300)
   async indexesList(): Promise<string[]> {
     return ((await this.db.execute(sql`SHOW TABLES LIKE '${sql.raw(FILES_CONTENT_TABLE_PREFIX)}%'`))[0] as any).flatMap((r: Record<string, string>) =>
       Object.values(r)
@@ -93,7 +93,7 @@ export class FilesIndexerMySQL implements FilesIndexer {
 
   async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {
     const terms: string[] = analyzeTerms(search)
-    this.logger.debug({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
+    this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
     if (!terms.length) {
       return []
     }

backend/src/applications/files/constants/cache.ts

@@ -5,4 +5,7 @@ export const CACHE_TASK_TTL = 86400 as const // one day
 export const CACHE_LOCK_PREFIX = 'flock' as const
 export const CACHE_LOCK_DEFAULT_TTL = 28800 as const // 8 hours in seconds
 export const CACHE_LOCK_FILE_TTL = 3600 as const
-// cache only office = `office|${fileId}` => docKey
+// cache quota key = `(quota-user|quota-space)-${id}` => number
+export const CACHE_QUOTA_PREFIX = 'quota' as const
+export const CACHE_QUOTA_EVENT_UPDATE_PREFIX = 'event-update-quota' as const
+export const CACHE_QUOTA_TTL = 86400 // 1 day

backend/src/applications/files/constants/compress.ts

@@ -1,2 +1,2 @@
-export const tarExtension = 'tar'
-export const tarGzExtension = 'tgz'
+export const TAR_EXTENSION = 'tar'
+export const TAR_GZ_EXTENSION = 'tgz'

backend/src/applications/files/constants/indexing.ts

@@ -1,5 +1,4 @@
-export const userIndexPrefix = 'user_'
-export const spaceIndexPrefix = 'space_'
-export const shareIndexPrefix = 'share_'
-export const minCharsToSearch = 3
-export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
+export const MIN_CHARS_TO_SEARCH = 3
+export const INDEXABLE_EXTENSIONS = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
+export const CACHE_INDEXING_UPDATE_PREFIX = 'event-update-indexing' as const
+export const CACHE_INDEXING_TTL = 86400 // 1 day

backend/src/applications/files/constants/operations.ts

@@ -20,4 +20,16 @@ export enum FILE_MODE {
   EDIT = 'edit'
 }
 
+export enum FILE_REPOSITORY {
+  USER = 'user',
+  SPACE = 'space',
+  SHARE = 'share'
+}
+
 export const FORCE_AS_FILE_OWNER = 'forceAsFileOwner' as const
+
+export const SEND_FILE_ERROR_MSG = {
+  400: 'The location is a directory',
+  404: 'Location not found',
+  405: 'The location is not readable'
+} as const

backend/src/applications/files/dto/file-operations.dto.ts

@@ -2,7 +2,7 @@ import { Transform } from 'class-transformer'
 import { ArrayMinSize, IsArray, IsBoolean, IsDefined, IsIn, IsInt, IsNotEmpty, IsOptional, IsString, IsUrl } from 'class-validator'
 import { RejectIfMatch } from '../../../common/decorators'
 import { regExpInvalidFileName } from '../../../common/shared'
-import { tarExtension, tarGzExtension } from '../constants/compress'
+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
 
 export class CopyMoveFileDto {
   @IsNotEmpty()
@@ -58,8 +58,8 @@ export class CompressFileDto {
 
   @IsNotEmpty()
   @IsString()
-  @IsIn([tarExtension, tarGzExtension])
-  extension: typeof tarExtension | typeof tarGzExtension
+  @IsIn([TAR_EXTENSION, TAR_GZ_EXTENSION])
+  extension: typeof TAR_EXTENSION | typeof TAR_GZ_EXTENSION
 }
 
 export class SearchFilesDto {

backend/src/applications/files/events/file-events.ts

@@ -0,0 +1,6 @@
+import EventEmitter from 'node:events'
+import type { FileEventEmit, FileTaskEventEmit } from '../interfaces/file-event.interface'
+
+export const FileTaskEvent: EventEmitter<FileTaskEventEmit> = new EventEmitter<FileTaskEventEmit>()
+
+export const FileEvent: EventEmitter<FileEventEmit> = new EventEmitter<FileEventEmit>()

backend/src/applications/files/events/files-events.utils.ts

@@ -0,0 +1,24 @@
+import { FILE_REPOSITORY } from '../constants/operations'
+import type { SpaceEnv } from '../../spaces/models/space-env.model'
+
+export function SpaceToFileRepository(userId: number, space: SpaceEnv): { id: number; type: FILE_REPOSITORY } | null {
+  if (space.inPersonalSpace) {
+    // Personal user space
+    return { id: userId, type: FILE_REPOSITORY.USER }
+  } else if (space.root?.externalPath) {
+    // External paths used as shares or as space roots share the same quota as their origin
+    if (space.inSharesRepository) {
+      return { id: space.root?.externalParentShareId || space.id, type: FILE_REPOSITORY.SHARE }
+    }
+    return { id: space.id, type: FILE_REPOSITORY.SPACE }
+  } else if (space.root?.file?.path && space.root.owner?.login) {
+    // Space root is linked to a user file
+    return { id: space.root.owner.id, type: FILE_REPOSITORY.USER }
+  } else if (space.root?.file?.space?.id) {
+    return { id: space.root.file.space.id, type: FILE_REPOSITORY.SPACE }
+  } else if (space.id) {
+    return { id: space.id, type: FILE_REPOSITORY.SPACE }
+  } else {
+    return null
+  }
+}

backend/src/applications/files/files.module.ts

@@ -1,12 +1,12 @@
 import { Module } from '@nestjs/common'
 import { configuration } from '../../configuration/config.environment'
-import { FilesIndexerMySQL } from './adapters/files-indexer-mysql.service'
+import { FilesContentStoreMySQL } from './adapters/files-content-store-mysql.service'
 import { FilesTasksController } from './files-tasks.controller'
 import { FilesController } from './files.controller'
-import { FilesIndexer } from './models/files-indexer'
+import { FilesContentStore } from './models/files-content-store'
 import { CollaboraOnlineModule } from './modules/collabora-online/collabora-online.module'
 import { OnlyOfficeModule } from './modules/only-office/only-office.module'
-import { FilesContentManager } from './services/files-content-manager.service'
+import { FilesContentIndexer } from './services/files-content-indexer.service'
 import { FilesLockManager } from './services/files-lock-manager.service'
 import { FilesManager } from './services/files-manager.service'
 import { FilesMethods } from './services/files-methods.service'
@@ -16,6 +16,8 @@ import { FilesRecents } from './services/files-recents.service'
 import { FilesScheduler } from './services/files-scheduler.service'
 import { FilesSearchManager } from './services/files-search-manager.service'
 import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesEventManager } from './services/files-event-manager.service'
+import { FilesQuotaManager } from './services/files-quota-manager.service'
 
 @Module({
   imports: [
@@ -32,10 +34,12 @@ import { FilesTasksManager } from './services/files-tasks-manager.service'
     FilesScheduler,
     FilesRecents,
     FilesParser,
-    FilesContentManager,
-    { provide: FilesIndexer, useClass: FilesIndexerMySQL },
-    FilesSearchManager
+    FilesContentIndexer,
+    { provide: FilesContentStore, useClass: FilesContentStoreMySQL },
+    FilesSearchManager,
+    FilesEventManager,
+    FilesQuotaManager
   ],
-  exports: [FilesManager, FilesQueries, FilesLockManager, FilesMethods, FilesRecents]
+  exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]
 })
 export class FilesModule {}

backend/src/applications/files/interfaces/file-event.interface.ts

@@ -0,0 +1,19 @@
+import type { SpaceEnv } from '../../spaces/models/space-env.model'
+import type { FILE_OPERATION } from '../constants/operations'
+import type { ACTION } from '../../../common/constants'
+import type { UserModel } from '../../users/models/user.model'
+
+export interface FileTaskEventEmit {
+  startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string]
+}
+
+export interface FileEventType {
+  user: UserModel
+  space: SpaceEnv
+  action: ACTION
+  rPath: string
+}
+
+export interface FileEventEmit {
+  event: [event: FileEventType]
+}

backend/src/applications/files/interfaces/file-parse-index.ts

@@ -1,5 +1,3 @@
-export type FileParseType = 'user' | 'space' | 'share'
-
 export interface FileParseContext {
   realPath: string
   pathPrefix: string

backend/src/applications/files/models/files-indexer.ts → backend/src/applications/files/models/files-content-store.ts

@@ -1,6 +1,6 @@
 import { FileContent } from '../schemas/file-content.interface'
 
-export abstract class FilesIndexer {
+export abstract class FilesContentStore {
   abstract indexesList(): Promise<string[]>
 
   abstract getIndexName(indexSuffix: string): string

backend/src/applications/files/modules/collabora-online/collabora-online-manager.service.ts

@@ -42,6 +42,8 @@ import {
 import { CollaboraOnlineReqDto, CollaboraSaveDocumentDto } from './collabora-online.dtos'
 import { CollaboraOnlineCheckFileInfo, FastifyCollaboraOnlineSpaceRequest, JwtCollaboraOnlinePayload } from './collabora-online.interface'
 import { API_COLLABORA_ONLINE_FILES } from './collabora-online.routes'
+import { FileEvent } from '../../events/file-events'
+import { ACTION } from '../../../../common/constants'
 
 @Injectable()
 export class CollaboraOnlineManager {
@@ -132,8 +134,9 @@ export class CollaboraOnlineManager {
     }
     // copy contents to avoid inode changes (dbFileHash in some cases)
     try {
-      // todo: versioning
       await copyFileContent(tmpFilePath, req.space.realPath)
+      // emit file event
+      FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath })
       await removeFiles(tmpFilePath)
       const fStats = await fs.stat(req.space.realPath)
       return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDto

backend/src/applications/files/modules/only-office/only-office-manager.service.ts

@@ -49,6 +49,8 @@ import {
 import { OnlyOfficeReqDto } from './only-office.dtos'
 import { OnlyOfficeCallBack, OnlyOfficeConfig, OnlyOfficeConvertForm } from './only-office.interface'
 import { API_ONLY_OFFICE_CALLBACK, API_ONLY_OFFICE_DOCUMENT } from './only-office.routes'
+import { FileEvent } from '../../events/file-events'
+import { ACTION } from '../../../../common/constants'
 
 @Injectable()
 export class OnlyOfficeManager {
@@ -123,14 +125,14 @@ export class OnlyOfficeManager {
             this.logger.debug({ tag: this.callBack.name, msg: `document was edited but closed with no changes : ${space.url}` })
           } else {
             this.logger.debug({ tag: this.callBack.name, msg: `document was edited and closed but not saved (let's do it) : ${space.url}` })
-            await this.saveDocument(space, callBackData.url)
+            await this.saveDocument(user, space, callBackData.url)
           }
           await this.removeFileLock(user.id, space)
           await this.removeDocumentKey(space)
           break
         case 3:
           this.logger.error({ tag: this.callBack.name, msg: `document cannot be saved, an error has occurred (try to save it) : ${space.url}` })
-          await this.saveDocument(space, callBackData.url)
+          await this.saveDocument(user, space, callBackData.url)
           break
         case 4:
           // No active users on the document
@@ -140,11 +142,11 @@ export class OnlyOfficeManager {
           break
         case 6:
           this.logger.debug({ tag: this.callBack.name, msg: `document is edited but save was requested : ${space.url}` })
-          await this.saveDocument(space, callBackData.url)
+          await this.saveDocument(user, space, callBackData.url)
           break
         case 7:
           this.logger.error({ tag: this.callBack.name, msg: `document cannot be force saved, an error has occurred (try to save it) : ${space.url}` })
-          await this.saveDocument(space, callBackData.url)
+          await this.saveDocument(user, space, callBackData.url)
           break
         default:
           this.logger.error({ tag: this.callBack.name, msg: 'unhandled case' })
@@ -317,7 +319,7 @@ export class OnlyOfficeManager {
     return docKey
   }
 
-  private async saveDocument(space: SpaceEnv, url: string): Promise<void> {
+  private async saveDocument(user: UserModel, space: SpaceEnv, url: string): Promise<void> {
     /* url format:
       https://onlyoffice-server.com/cache/files/data/-33120641_7158/output.pptx/output.pptx
       ?md5=duFHKC-5d47s-RRcYn3hAw&expires=1739400549&shardkey=-33120641&filename=output.pptx
@@ -368,8 +370,9 @@ export class OnlyOfficeManager {
     }
     // copy contents to avoid inode changes (`file.id` in some cases)
     try {
-      // todo: versioning
       await copyFileContent(tmpFilePath, space.realPath)
+      // emit file event
+      FileEvent.emit('event', { user: user, space: space, action: ACTION.UPDATE, rPath: space.realPath })
       await removeFiles(tmpFilePath)
     } catch (e) {
       throw new Error(`unable to save document : ${e.message}`)

backend/src/applications/files/modules/only-office/only-office.constants.ts

@@ -2,8 +2,8 @@ export const ONLY_OFFICE_INTERNAL_URI = '/onlyoffice' // used by nginx as a prox
 export const ONLY_OFFICE_CONTEXT = 'OnlyOfficeEnvironment' as const
 export const ONLY_OFFICE_TOKEN_QUERY_PARAM_NAME = 'token' as const
 export const ONLY_OFFICE_APP_LOCK = 'OnlyOffice' as const
+// cache only office = `office|${fileId}` => docKey
 export const ONLY_OFFICE_CACHE_KEY = 'foffice' as const
-
 export const ONLY_OFFICE_EXTENSIONS = new Map<string, 'word' | 'cell' | 'slide' | 'pdf' | 'diagram'>([
   // ─────────────
   // WORD