add(delivery file formatter)

cg/services/fastq_concatenation_service/utils.py

@@ -4,7 +4,7 @@
 import uuid
 
 from cg.services.fastq_concatenation_service.exceptions import ConcatenationError
-from cg.constants.constants import ReadDirection
+from cg.constants.constants import ReadDirection, FileFormat
 from cg.constants import FileExtensions
 
 
@@ -73,7 +73,11 @@ def sort_files_by_name(files: list[Path]) -> list[Path]:
 
 
 def file_can_be_removed(file: Path, forward_file: Path, reverse_file: Path) -> bool:
-    return file.suffix == FileExtensions.GZIP and file != forward_file and file != reverse_file
+    return (
+        f"{FileFormat.FASTQ}{FileExtensions.GZIP}" in file.name
+        and file != forward_file
+        and file != reverse_file
+    )
 
 
 def remove_raw_fastqs(fastq_directory: Path, forward_file: Path, reverse_file: Path) -> None:

cg/services/file_delivery/abstract_classes.py

@@ -3,22 +3,16 @@
 from cg.services.file_delivery.fetch_file_service.fetch_delivery_files_service import (
     FetchDeliveryFilesService,
 )
+
+from cg.services.file_delivery.file_formatter_service.delivery_file_formatting_service import (
+    DeliveryFileFormattingService,
+)
+
 from cg.services.file_delivery.move_files_service.move_delivery_files_service import (
     MoveDeliveryFilesService,
 )
 
 
-class FormatDeliveryFilesService(ABC):
-    """
-    Abstract class that encapsulates the logic required for formatting files to deliver.
-    """
-
-    @abstractmethod
-    def format_files(self, case_id: str) -> None:
-        """Format the files to deliver."""
-        pass
-
-
 class DeliverFilesService(ABC):
     """
     Abstract class that encapsulates the logic required for delivering files to the customer.
@@ -35,9 +29,11 @@ def __init__(
         self,
         delivery_file_manager_service: FetchDeliveryFilesService,
         move_file_service: MoveDeliveryFilesService,
+        file_formatter_service: DeliveryFileFormattingService,
     ):
         self.file_manager = delivery_file_manager_service
         self.file_mover = move_file_service
+        self.file_formatter = file_formatter_service
 
     @abstractmethod
     def deliver_files_for_case(self, case_id: str) -> None:

cg/services/file_delivery/fetch_file_service/fetch_analysis_files_service.py

@@ -51,12 +51,17 @@ def _get_sample_files_from_case_bundle(
         """Return a list of files from a case bundle with a sample id as tag."""
         sample_tags: list[set[str]] = self.tags_fetcher.fetch_tags(workflow).sample_tags
         sample_tags_with_sample_id: list[set[str]] = [tag | {sample_id} for tag in sample_tags]
-
         sample_files: list[File] = self.hk_api.get_files_from_latest_version_containing_tags(
             bundle_name=case_id, tags=sample_tags_with_sample_id
         )
+        sample_name: str = self.status_db.get_sample_by_internal_id(sample_id).name
         return [
-            SampleFile(case_id=case_id, sample_id=sample_id, file_path=sample_file.full_path)
+            SampleFile(
+                case_id=case_id,
+                sample_id=sample_id,
+                sample_name=sample_name,
+                file_path=sample_file.full_path,
+            )
             for sample_file in sample_files
         ]
 
@@ -82,6 +87,10 @@ def get_analysis_case_delivery_files(self, case: Case) -> list[CaseFile]:
             bundle_name=case.internal_id, tags=case_tags, excluded_tags=sample_id_tags
         )
         return [
-            CaseFile(case_id=case.internal_id, file_path=case_file.full_path)
+            CaseFile(
+                case_id=case.internal_id,
+                case_name=case.name,
+                file_path=case_file.full_path,
+            )
             for case_file in case_files
         ]

cg/services/file_delivery/fetch_file_service/fetch_fastq_analysis_files_service.py

@@ -33,13 +33,14 @@ def get_files_to_deliver(self, case_id: str) -> DeliveryFiles:
             self.hk_api,
             tags_fetcher=FetchSampleAndCaseDeliveryFileTagsService(),
         )
-        fastq_files = fetch_fastq_service.get_files_to_deliver(case_id)
-        analysis_files = fetch_analysis_service.get_files_to_deliver(case_id)
+
+        fastq_files: DeliveryFiles = fetch_fastq_service.get_files_to_deliver(case_id)
+        analysis_files: DeliveryFiles = fetch_analysis_service.get_files_to_deliver(case_id)
         delivery_data = DeliveryMetaData(
             customer_internal_id=case.customer.internal_id, ticket_id=case.latest_ticket
         )
         return DeliveryFiles(
             delivery_data=delivery_data,
-            case_files=analysis_case_files,
-            sample_files=analysis_sample_files,
+            case_files=analysis_files.case_files,
+            sample_files=analysis_files.sample_files + fastq_files.sample_files,
         )

cg/services/file_delivery/fetch_file_service/fetch_fastq_files_service.py

@@ -59,7 +59,13 @@ def _get_fastq_files_for_sample(self, case_id: str, sample_id: str) -> list[Samp
         fastq_files: list[File] = self.hk_api.get_files_from_latest_version_containing_tags(
             bundle_name=sample_id, tags=fastq_tags
         )
+        sample_name: str = self.status_db.get_sample_by_internal_id(sample_id).name
         return [
-            SampleFile(case_id=case_id, sample_id=sample_id, file_path=fastq_file.full_path)
+            SampleFile(
+                case_id=case_id,
+                sample_id=sample_id,
+                sample_name=sample_name,
+                file_path=fastq_file.full_path,
+            )
             for fastq_file in fastq_files
         ]

cg/services/file_delivery/fetch_file_service/models.py

@@ -10,12 +10,14 @@ class DeliveryMetaData(BaseModel):
 
 class CaseFile(BaseModel):
     case_id: str
+    case_name: str
     file_path: Path
 
 
 class SampleFile(BaseModel):
     case_id: str
     sample_id: str
+    sample_name: str
     file_path: Path
 
 

cg/services/file_delivery/file_formatter_service/concatenation_delivery_file_formatter_service.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+
+from cg.constants.delivery import INBOX_NAME
+from cg.services.fastq_concatenation_service.fastq_concatenation_service import (
+    FastqConcatenationService,
+)
+from cg.services.file_delivery.fetch_file_service.models import DeliveryFiles
+from cg.services.file_delivery.file_formatter_service.models import FormattedFiles, FormattedFile
+from cg.services.file_delivery.file_formatter_service.utils.case_file_formatter import (
+    CaseFileFormatter,
+)
+from cg.services.file_delivery.file_formatter_service.delivery_file_formatting_service import (
+    DeliveryFileFormattingService,
+)
+from cg.services.file_delivery.file_formatter_service.utils.sample_file_concatenation_formatter import (
+    SampleFileConcatenationFormatter,
+)
+from cg.services.file_delivery.file_formatter_service.utils.sample_file_formatter import (
+    SampleFileFormatter,
+)
+from cg.services.file_delivery.file_formatter_service.utils.utils import (
+    get_ticket_dir_path,
+    create_ticket_dir,
+)
+
+
+class ConcatenationDeliveryFileFormatter(DeliveryFileFormattingService):
+    """
+    Reformat the files to be delivered in the microsalt format.
+    Expected structure:
+    <customer>/inbox/<ticket_id>/<case_name>/<case_files>
+    <customer>/inbox/<ticket_id>/<sample_name>/<sample_files>
+    Extra rule:
+        fastq files are concatenated into a single file per read direction.
+    """
+
+    def __init__(
+        self,
+        case_file_formatter: CaseFileFormatter,
+        sample_file_formatter: SampleFileConcatenationFormatter,
+    ):
+        self.case_file_formatter = case_file_formatter
+        self.sample_file_formatter = sample_file_formatter
+
+    def format_files(self, delivery_files: DeliveryFiles) -> FormattedFiles:
+        """Format the files to be delivered in the concatenated format."""
+        ticket_dir_path: Path = get_ticket_dir_path(delivery_files.sample_files[0].file_path)
+        create_ticket_dir(ticket_dir_path)
+        formatted_files: list[FormattedFile] = []
+        formatter_sample_files = self.sample_file_formatter.format_files(
+            moved_files=delivery_files.sample_files,
+            ticket_dir_path=ticket_dir_path,
+        )
+        formatted_files.extend(formatter_sample_files)
+        if delivery_files.case_files:
+            formatter_case_file = self.case_file_formatter.format_files(
+                moved_files=delivery_files.case_files,
+                ticket_dir_path=ticket_dir_path,
+            )
+            formatted_files.extend(formatter_case_file)
+        return FormattedFiles(files=formatted_files)

cg/services/file_delivery/file_formatter_service/delivery_file_formatting_service.py

@@ -0,0 +1,16 @@
+from abc import abstractmethod, ABC
+
+from cg.services.file_delivery.fetch_file_service.models import DeliveryFiles
+from cg.services.file_delivery.file_formatter_service.models import FormattedFiles
+from cg.store.store import Store
+
+
+class DeliveryFileFormattingService(ABC):
+    """
+    Abstract class that encapsulates the logic required for formatting files to deliver.
+    """
+
+    @abstractmethod
+    def format_files(self, delivery_files: DeliveryFiles) -> FormattedFiles:
+        """Format the files to deliver."""
+        pass

cg/services/file_delivery/file_formatter_service/generic_delivery_file_formatter_service.py

@@ -0,0 +1,51 @@
+import os
+from pathlib import Path
+from cg.constants.delivery import INBOX_NAME
+from cg.services.file_delivery.fetch_file_service.models import DeliveryFiles
+from cg.services.file_delivery.file_formatter_service.utils.case_file_formatter import (
+    CaseFileFormatter,
+)
+from cg.services.file_delivery.file_formatter_service.delivery_file_formatting_service import (
+    DeliveryFileFormattingService,
+)
+from cg.services.file_delivery.file_formatter_service.models import FormattedFiles, FormattedFile
+from cg.services.file_delivery.file_formatter_service.utils.sample_file_formatter import (
+    SampleFileFormatter,
+)
+from cg.services.file_delivery.file_formatter_service.utils.utils import (
+    get_ticket_dir_path,
+    create_ticket_dir,
+)
+
+
+class GenericDeliveryFileFormatter(DeliveryFileFormattingService):
+    """
+    Format the files to be delivered in the generic format.
+    Expected structure:
+    <customer>/inbox/<ticket_id>/<case_name>/<case_files>
+    <customer>/inbox/<ticket_id>/<sample_name>/<sample_files>
+    """
+
+    def __init__(
+        self, case_file_formatter: CaseFileFormatter, sample_file_formatter: SampleFileFormatter
+    ):
+        self.case_file_formatter = case_file_formatter
+        self.sample_file_formatter = sample_file_formatter
+
+    def format_files(self, delivery_files: DeliveryFiles) -> FormattedFiles:
+        """Format the files to be delivered in the generic format."""
+        ticket_dir_path: Path = get_ticket_dir_path(delivery_files.sample_files[0].file_path)
+        create_ticket_dir(ticket_dir_path)
+        formatted_files: list[FormattedFile] = []
+        formatter_sample_files = self.sample_file_formatter.format_files(
+            moved_files=delivery_files.sample_files,
+            ticket_dir_path=get_ticket_dir_path(delivery_files.sample_files[0].file_path),
+        )
+        formatted_files.extend(formatter_sample_files)
+        if delivery_files.case_files:
+            formatter_case_file = self.case_file_formatter.format_files(
+                moved_files=delivery_files.case_files,
+                ticket_dir_path=get_ticket_dir_path(delivery_files.case_files[0].file_path),
+            )
+            formatted_files.extend(formatter_case_file)
+        return FormattedFiles(files=formatted_files)

cg/services/file_delivery/file_formatter_service/models.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+from pydantic import BaseModel
+
+
+class FormattedFile(BaseModel):
+    original_path: Path
+    formatted_path: Path
+
+
+class FormattedFiles(BaseModel):
+    files: list[FormattedFile]

cg/services/file_delivery/file_formatter_service/utils/case_file_formatter.py

@@ -0,0 +1,39 @@
+import os
+from pathlib import Path
+
+from cg.services.file_delivery.fetch_file_service.models import CaseFile
+from cg.services.file_delivery.file_formatter_service.models import FormattedFiles, FormattedFile
+
+
+class CaseFileFormatter:
+
+    def format_files(
+        self, moved_files: list[CaseFile], ticket_dir_path: Path
+    ) -> list[FormattedFile]:
+        """Format the case files to deliver."""
+        self._create_case_name_folder(
+            ticket_path=ticket_dir_path, case_name=moved_files[0].case_name
+        )
+        return self._rename_case_files(moved_files)
+
+    @staticmethod
+    def _rename_case_files(case_files: list[CaseFile]) -> list[FormattedFile]:
+        formatted_files: list[FormattedFile] = []
+        for case_file in case_files:
+            replaced_case_file_name: str = case_file.file_path.name.replace(
+                case_file.case_id, case_file.case_name
+            )
+            formatted_file_path = Path(
+                case_file.file_path.parent, case_file.case_name, replaced_case_file_name
+            )
+            formatted_files.append(
+                FormattedFile(original_path=case_file.file_path, formatted_path=formatted_file_path)
+            )
+        for formatted_file in formatted_files:
+            os.rename(src=formatted_file.original_path, dst=formatted_file.formatted_path)
+        return formatted_files
+
+    @staticmethod
+    def _create_case_name_folder(ticket_path: Path, case_name: str) -> None:
+        case_dir_path = Path(ticket_path, case_name)
+        case_dir_path.mkdir(exist_ok=True)