From f04e95f904ce9e139e0bc759632c24e8b82d0406 Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Thu, 29 Aug 2024 13:56:03 +0200 Subject: [PATCH] add(delivery service factory) (#3647) # Description add factory method --- .../deliver_files_service/__init__.py | 0 .../deliver_files_service.py} | 23 ++-- .../deliver_files_service_factory.py | 106 ++++++++++++++++++ .../fetch_fastq_analysis_files_service.py | 36 +++--- .../test_delivery_file_service_builder.py | 87 ++++++++++++++ 5 files changed, 226 insertions(+), 26 deletions(-) create mode 100644 cg/services/file_delivery/deliver_files_service/__init__.py rename cg/services/file_delivery/{abstract_classes.py => deliver_files_service/deliver_files_service.py} (65%) create mode 100644 cg/services/file_delivery/deliver_files_service/deliver_files_service_factory.py create mode 100644 tests/services/file_delivery/delivery_file_service/test_delivery_file_service_builder.py diff --git a/cg/services/file_delivery/deliver_files_service/__init__.py b/cg/services/file_delivery/deliver_files_service/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cg/services/file_delivery/abstract_classes.py b/cg/services/file_delivery/deliver_files_service/deliver_files_service.py similarity index 65% rename from cg/services/file_delivery/abstract_classes.py rename to cg/services/file_delivery/deliver_files_service/deliver_files_service.py index 0a4b6ec818..977a49dbb2 100644 --- a/cg/services/file_delivery/abstract_classes.py +++ b/cg/services/file_delivery/deliver_files_service/deliver_files_service.py @@ -1,30 +1,24 @@ -from abc import abstractmethod, ABC - +from pathlib import Path from cg.services.file_delivery.fetch_file_service.fetch_delivery_files_service import ( FetchDeliveryFilesService, ) - +from cg.services.file_delivery.fetch_file_service.models import DeliveryFiles from cg.services.file_delivery.file_formatter_service.delivery_file_formatting_service import ( DeliveryFileFormattingService, ) - from cg.services.file_delivery.move_files_service.move_delivery_files_service import ( MoveDeliveryFilesService, ) -class DeliverFilesService(ABC): +class DeliverFilesService: """ - Abstract class that encapsulates the logic required for delivering files to the customer. - + Deliver files to the customer inbox on hasta. 1. Get the files to deliver from Housekeeper based on workflow and data delivery 2. Create a delivery folder structure in the customer folder on Hasta and move the files there 3. Reformatting of output / renaming of files - 4. Start a Rsync job to upload the files to Caesar - 5. Track the status of the Rsync job in Trailblazer """ - @abstractmethod def __init__( self, delivery_file_manager_service: FetchDeliveryFilesService, @@ -35,7 +29,10 @@ def __init__( self.file_mover = move_file_service self.file_formatter = file_formatter_service - @abstractmethod - def deliver_files_for_case(self, case_id: str) -> None: + def deliver_files_for_case(self, case_id: str, delivery_base_path: Path) -> None: """Deliver the files to the customer folder.""" - pass + delivery_files: DeliveryFiles = self.file_manager.get_files_to_deliver(case_id) + moved_files: DeliveryFiles = self.file_mover.move_files( + delivery_files=delivery_files, delivery_base_path=delivery_base_path + ) + self.file_formatter.format_files(moved_files) diff --git a/cg/services/file_delivery/deliver_files_service/deliver_files_service_factory.py b/cg/services/file_delivery/deliver_files_service/deliver_files_service_factory.py new file mode 100644 index 0000000000..77c971855c --- /dev/null +++ b/cg/services/file_delivery/deliver_files_service/deliver_files_service_factory.py @@ -0,0 +1,106 @@ +"""Module for the factory of the deliver files service.""" + +from cg.apps.housekeeper.hk import HousekeeperAPI +from cg.constants import Workflow, DataDelivery +from cg.services.fastq_concatenation_service.fastq_concatenation_service import ( + FastqConcatenationService, +) +from cg.services.file_delivery.deliver_files_service.deliver_files_service import ( + DeliverFilesService, +) +from cg.services.file_delivery.fetch_delivery_files_tags.fetch_delivery_file_tags_service import ( + FetchDeliveryFileTagsService, +) +from cg.services.file_delivery.fetch_delivery_files_tags.fetch_sample_and_case_delivery_file_tags_service import ( + FetchSampleAndCaseDeliveryFileTagsService, +) +from cg.services.file_delivery.fetch_file_service.fetch_analysis_files_service import ( + FetchAnalysisDeliveryFilesService, +) +from cg.services.file_delivery.fetch_file_service.fetch_delivery_files_service import ( + FetchDeliveryFilesService, +) +from cg.services.file_delivery.fetch_file_service.fetch_fastq_analysis_files_service import ( + FetchFastqAndAnalysisDeliveryFilesService, +) +from cg.services.file_delivery.fetch_file_service.fetch_fastq_files_service import ( + FetchFastqDeliveryFilesService, +) +from cg.services.file_delivery.file_formatter_service.delivery_file_formatter import ( + DeliveryFileFormatter, +) + +from cg.services.file_delivery.file_formatter_service.delivery_file_formatting_service import ( + DeliveryFileFormattingService, +) +from cg.services.file_delivery.file_formatter_service.utils.case_file_formatter import ( + CaseFileFormatter, +) +from cg.services.file_delivery.file_formatter_service.utils.sample_file_concatenation_formatter import ( + SampleFileConcatenationFormatter, +) +from cg.services.file_delivery.file_formatter_service.utils.sample_file_formatter import ( + SampleFileFormatter, +) +from cg.services.file_delivery.move_files_service.move_delivery_files_service import ( + MoveDeliveryFilesService, +) +from cg.store.store import Store + + +class DeliveryServiceFactory: + """Class to build the delivery services based on workflow and delivery type.""" + + def __init__(self, store: Store, hk_api: HousekeeperAPI): + self.store = store + self.hk_api = hk_api + + @staticmethod + def _get_file_tag_fetcher(delivery_type: DataDelivery) -> FetchDeliveryFileTagsService: + """Get the file tag fetcher based on the delivery type.""" + service_map: dict[DataDelivery, FetchDeliveryFileTagsService] = { + DataDelivery.FASTQ: FetchSampleAndCaseDeliveryFileTagsService, + DataDelivery.ANALYSIS_FILES: FetchSampleAndCaseDeliveryFileTagsService, + DataDelivery.FASTQ_ANALYSIS: FetchSampleAndCaseDeliveryFileTagsService, + } + return service_map[delivery_type]() + + def _get_file_fetcher(self, delivery_type: DataDelivery) -> FetchDeliveryFilesService: + """Get the file fetcher based on the delivery type.""" + service_map: dict[DataDelivery, FetchDeliveryFilesService] = { + DataDelivery.FASTQ: FetchFastqDeliveryFilesService, + DataDelivery.ANALYSIS_FILES: FetchAnalysisDeliveryFilesService, + DataDelivery.FASTQ_ANALYSIS: FetchFastqAndAnalysisDeliveryFilesService, + } + file_tag_fetcher: FetchDeliveryFileTagsService = self._get_file_tag_fetcher(delivery_type) + return service_map[delivery_type]( + status_db=self.store, + hk_api=self.hk_api, + tags_fetcher=file_tag_fetcher, + ) + + @staticmethod + def _get_sample_file_formatter( + workflow: Workflow, + ) -> SampleFileFormatter | SampleFileConcatenationFormatter: + """Get the file formatter service based on the workflow.""" + if workflow in [Workflow.MICROSALT, Workflow.MUTANT]: + return SampleFileConcatenationFormatter(FastqConcatenationService()) + return SampleFileFormatter() + + def build_delivery_service( + self, workflow: Workflow, delivery_type: DataDelivery + ) -> DeliverFilesService: + """Build a delivery service based on the workflow and delivery type.""" + file_fetcher: FetchDeliveryFilesService = self._get_file_fetcher(delivery_type) + sample_file_formatter: SampleFileFormatter | SampleFileConcatenationFormatter = ( + self._get_sample_file_formatter(workflow) + ) + file_formatter: DeliveryFileFormattingService = DeliveryFileFormatter( + case_file_formatter=CaseFileFormatter(), sample_file_formatter=sample_file_formatter + ) + return DeliverFilesService( + delivery_file_manager_service=file_fetcher, + move_file_service=MoveDeliveryFilesService(), + file_formatter_service=file_formatter, + ) diff --git a/cg/services/file_delivery/fetch_file_service/fetch_fastq_analysis_files_service.py b/cg/services/file_delivery/fetch_file_service/fetch_fastq_analysis_files_service.py index fb99fc0c17..52f6771359 100644 --- a/cg/services/file_delivery/fetch_file_service/fetch_fastq_analysis_files_service.py +++ b/cg/services/file_delivery/fetch_file_service/fetch_fastq_analysis_files_service.py @@ -1,4 +1,7 @@ from cg.apps.housekeeper.hk import HousekeeperAPI +from cg.services.file_delivery.fetch_delivery_files_tags.fetch_delivery_file_tags_service import ( + FetchDeliveryFileTagsService, +) from cg.services.file_delivery.fetch_delivery_files_tags.fetch_sample_and_case_delivery_file_tags_service import ( FetchSampleAndCaseDeliveryFileTagsService, ) @@ -17,30 +20,37 @@ class FetchFastqAndAnalysisDeliveryFilesService(FetchDeliveryFilesService): - def __init__(self, status_db: Store, hk_api: HousekeeperAPI): + + def __init__( + self, status_db: Store, hk_api: HousekeeperAPI, tags_fetcher: FetchDeliveryFileTagsService + ): self.status_db = status_db self.hk_api = hk_api + self.tags_fetcher = tags_fetcher def get_files_to_deliver(self, case_id: str) -> DeliveryFiles: - case: Case = self.status_db.get_case_by_internal_id(internal_id=case_id) - fetch_fastq_service = FetchFastqDeliveryFilesService( - self.status_db, - self.hk_api, - tags_fetcher=FetchSampleAndCaseDeliveryFileTagsService(), + case = self._get_case(case_id) + fastq_files: DeliveryFiles = self._fetch_files( + service_class=FetchFastqDeliveryFilesService, case_id=case_id ) - fetch_analysis_service = FetchAnalysisDeliveryFilesService( - self.status_db, - self.hk_api, - tags_fetcher=FetchSampleAndCaseDeliveryFileTagsService(), + analysis_files: DeliveryFiles = self._fetch_files( + service_class=FetchAnalysisDeliveryFilesService, case_id=case_id ) - - fastq_files: DeliveryFiles = fetch_fastq_service.get_files_to_deliver(case_id) - analysis_files: DeliveryFiles = fetch_analysis_service.get_files_to_deliver(case_id) delivery_data = DeliveryMetaData( customer_internal_id=case.customer.internal_id, ticket_id=case.latest_ticket ) + return DeliveryFiles( delivery_data=delivery_data, case_files=analysis_files.case_files, sample_files=analysis_files.sample_files + fastq_files.sample_files, ) + + def _get_case(self, case_id: str) -> Case: + """Fetch the case from the database.""" + return self.status_db.get_case_by_internal_id(internal_id=case_id) + + def _fetch_files(self, service_class: type, case_id: str) -> DeliveryFiles: + """Fetch files using the provided service class.""" + service = service_class(self.status_db, self.hk_api, tags_fetcher=self.tags_fetcher) + return service.get_files_to_deliver(case_id) diff --git a/tests/services/file_delivery/delivery_file_service/test_delivery_file_service_builder.py b/tests/services/file_delivery/delivery_file_service/test_delivery_file_service_builder.py new file mode 100644 index 0000000000..1965b0a31a --- /dev/null +++ b/tests/services/file_delivery/delivery_file_service/test_delivery_file_service_builder.py @@ -0,0 +1,87 @@ +import pytest +from unittest.mock import MagicMock, patch + +from cg.constants import Workflow, DataDelivery +from cg.services.file_delivery.deliver_files_service.deliver_files_service import ( + DeliverFilesService, +) +from cg.services.file_delivery.deliver_files_service.deliver_files_service_factory import ( + DeliveryServiceFactory, +) +from cg.services.file_delivery.fetch_delivery_files_tags.fetch_sample_and_case_delivery_file_tags_service import ( + FetchSampleAndCaseDeliveryFileTagsService, +) +from cg.services.file_delivery.fetch_file_service.fetch_analysis_files_service import ( + FetchAnalysisDeliveryFilesService, +) +from cg.services.file_delivery.fetch_file_service.fetch_fastq_analysis_files_service import ( + FetchFastqAndAnalysisDeliveryFilesService, +) +from cg.services.file_delivery.fetch_file_service.fetch_fastq_files_service import ( + FetchFastqDeliveryFilesService, +) +from cg.services.file_delivery.file_formatter_service.utils.sample_file_concatenation_formatter import ( + SampleFileConcatenationFormatter, +) +from cg.services.file_delivery.file_formatter_service.utils.sample_file_formatter import ( + SampleFileFormatter, +) +from cg.services.file_delivery.move_files_service.move_delivery_files_service import ( + MoveDeliveryFilesService, +) + + +@pytest.mark.parametrize( + "workflow,delivery_type,expected_tag_fetcher,expected_file_fetcher,expected_file_mover,expected_sample_file_formatter", + [ + ( + Workflow.MICROSALT, + DataDelivery.FASTQ, + FetchSampleAndCaseDeliveryFileTagsService, + FetchFastqDeliveryFilesService, + MoveDeliveryFilesService, + SampleFileConcatenationFormatter, + ), + ( + Workflow.MUTANT, + DataDelivery.ANALYSIS_FILES, + FetchSampleAndCaseDeliveryFileTagsService, + FetchAnalysisDeliveryFilesService, + MoveDeliveryFilesService, + SampleFileConcatenationFormatter, + ), + ( + Workflow.BALSAMIC, + DataDelivery.FASTQ_ANALYSIS, + FetchSampleAndCaseDeliveryFileTagsService, + FetchFastqAndAnalysisDeliveryFilesService, + MoveDeliveryFilesService, + SampleFileFormatter, + ), + ], +) +def test_build_delivery_service( + workflow, + delivery_type, + expected_tag_fetcher, + expected_file_fetcher, + expected_file_mover, + expected_sample_file_formatter, +): + # GIVEN a delivery service builder with mocked store and hk_api + store_mock = MagicMock() + hk_api_mock = MagicMock() + builder = DeliveryServiceFactory(store=store_mock, hk_api=hk_api_mock) + + # WHEN building a delivery service + delivery_service: DeliverFilesService = builder.build_delivery_service( + workflow=workflow, delivery_type=delivery_type + ) + + # THEN the correct file formatter and file fetcher services are used + assert isinstance(delivery_service.file_manager.tags_fetcher, expected_tag_fetcher) + assert isinstance(delivery_service.file_manager, expected_file_fetcher) + assert isinstance(delivery_service.file_mover, expected_file_mover) + assert isinstance( + delivery_service.file_formatter.sample_file_formatter, expected_sample_file_formatter + )