feature(streamline mutant delivery and upload) (#3916) (major)

# Description

Add mutant upload api
refactor delivery
refactor concatenation

cg/apps/lims/api.py

@@ -555,3 +555,11 @@ def _get_negative_controls_from_list(samples: list[Sample]) -> list[Sample]:
             ):
                 negative_controls.append(sample)
         return negative_controls
+
+    def get_sample_region_and_lab_code(self, sample_id: str) -> str:
+        """Return the region code and lab code for a sample formatted as a prefix string."""
+        region_code: str = self.get_sample_attribute(lims_id=sample_id, key="region_code").split(
+            " "
+        )[0]
+        lab_code: str = self.get_sample_attribute(lims_id=sample_id, key="lab_code").split(" ")[0]
+        return f"{region_code}_{lab_code}_"

cg/cli/deliver/base.py

@@ -15,7 +15,7 @@
 from cg.services.deliver_files.deliver_files_service.deliver_files_service import (
     DeliverFilesService,
 )
-from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import (
+from cg.services.deliver_files.factory import (
     DeliveryServiceFactory,
 )
 from cg.services.deliver_files.rsync.service import DeliveryRsyncService
@@ -88,8 +88,7 @@ def deliver_case(
         LOG.error(f"Could not find case with id {case_id}")
         return
     delivery_service: DeliverFilesService = service_builder.build_delivery_service(
-        case=case,
-        delivery_type=delivery_type,
+        case=case, delivery_type=delivery_type
     )
     delivery_service.deliver_files_for_case(
         case=case, delivery_base_path=Path(inbox), dry_run=dry_run
@@ -124,8 +123,7 @@ def deliver_ticket(
         LOG.error(f"Could not find case connected to ticket {ticket}")
         return
     delivery_service: DeliverFilesService = service_builder.build_delivery_service(
-        case=cases[0],
-        delivery_type=delivery_type,
+        case=cases[0], delivery_type=delivery_type
     )
     delivery_service.deliver_files_for_ticket(
         ticket_id=ticket, delivery_base_path=Path(inbox), dry_run=dry_run
@@ -172,8 +170,7 @@ def deliver_sample_raw_data(
         LOG.error(f"Could not find case with id {case_id}")
         return
     delivery_service: DeliverFilesService = service_builder.build_delivery_service(
-        case=case,
-        delivery_type=delivery_type,
+        case=case, delivery_type=delivery_type
     )
     delivery_service.deliver_files_for_sample(
         case=case, sample_id=sample_id, delivery_base_path=Path(inbox), dry_run=dry_run

cg/cli/deliver/utils.py

@@ -5,7 +5,7 @@
 from cg.services.deliver_files.deliver_files_service.deliver_files_service import (
     DeliverFilesService,
 )
-from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import (
+from cg.services.deliver_files.factory import (
     DeliveryServiceFactory,
 )
 from cg.store.models import Analysis, Case
@@ -26,8 +26,7 @@ def deliver_raw_data_for_analyses(
         try:
             case: Case = analysis.case
             delivery_service: DeliverFilesService = service_builder.build_delivery_service(
-                case=case,
-                delivery_type=case.data_delivery,
+                case=case, delivery_type=case.data_delivery
             )
 
             delivery_service.deliver_files_for_case(

cg/cli/upload/base.py

@@ -39,6 +39,7 @@
 from cg.meta.upload.microsalt.microsalt_upload_api import MicrosaltUploadAPI
 from cg.meta.upload.mip.mip_dna import MipDNAUploadAPI
 from cg.meta.upload.mip.mip_rna import MipRNAUploadAPI
+from cg.meta.upload.mutant.mutant import MutantUploadAPI
 from cg.meta.upload.nf_analysis import NfAnalysisUploadAPI
 from cg.meta.upload.tomte.tomte import TomteUploadAPI
 from cg.meta.upload.raredisease.raredisease import RarediseaseUploadAPI
@@ -94,6 +95,8 @@ def upload(context: click.Context, case_id: str | None, restart: bool):
             Workflow.TAXPROFILER,
         }:
             upload_api = NfAnalysisUploadAPI(config_object, case.data_analysis)
+        elif case.data_analysis == Workflow.MUTANT:
+            upload_api = MutantUploadAPI(config_object)
 
         context.obj.meta_apis["upload_api"] = upload_api
         upload_api.upload(ctx=context, case=case, restart=restart)

cg/cli/upload/fohm.py

@@ -41,7 +41,11 @@ def aggregate_delivery(
     context: CGConfig, cases: list, dry_run: bool = False, datestr: str | None = None
 ):
     """Re-aggregates delivery files for FOHM and saves them to default working directory."""
-    fohm_api = FOHMUploadAPI(config=context, dry_run=dry_run, datestr=datestr)
+    fohm_api = FOHMUploadAPI(
+        config=context,
+        dry_run=dry_run,
+        datestr=datestr,
+    )
     try:
         fohm_api.aggregate_delivery(cases)
     except (ValidationError, TypeError) as error:
@@ -57,7 +61,11 @@ def create_komplettering(
     context: CGConfig, cases: list, dry_run: bool = False, datestr: str | None = None
 ):
     """Re-aggregates komplettering files for FOHM and saves them to default working directory."""
-    fohm_api = FOHMUploadAPI(config=context, dry_run=dry_run, datestr=datestr)
+    fohm_api = FOHMUploadAPI(
+        config=context,
+        dry_run=dry_run,
+        datestr=datestr,
+    )
     try:
         fohm_api.create_and_write_complementary_report(cases)
     except ValidationError as error:
@@ -73,7 +81,11 @@ def preprocess_all(
     context: CGConfig, cases: list, dry_run: bool = False, datestr: str | None = None
 ):
     """Create all FOHM upload files, upload to GISAID, sync SFTP and mail reports for all provided cases."""
-    fohm_api = FOHMUploadAPI(config=context, dry_run=dry_run, datestr=datestr)
+    fohm_api = FOHMUploadAPI(
+        config=context,
+        dry_run=dry_run,
+        datestr=datestr,
+    )
     gisaid_api = GisaidAPI(config=context)
     cases = list(cases)
     upload_cases = []
@@ -105,7 +117,11 @@ def preprocess_all(
 @click.pass_obj
 def upload_rawdata(context: CGConfig, dry_run: bool = False, datestr: str | None = None):
     """Deliver files in daily upload directory via sftp."""
-    fohm_api = FOHMUploadAPI(config=context, dry_run=dry_run, datestr=datestr)
+    fohm_api = FOHMUploadAPI(
+        config=context,
+        dry_run=dry_run,
+        datestr=datestr,
+    )
     fohm_api.sync_files_sftp()
 
 
@@ -115,5 +131,9 @@ def upload_rawdata(context: CGConfig, dry_run: bool = False, datestr: str | None
 @click.pass_obj
 def send_reports(context: CGConfig, dry_run: bool = False, datestr: str | None = None):
     """Send all komplettering reports found in the current daily directory to target recipients."""
-    fohm_api = FOHMUploadAPI(config=context, dry_run=dry_run, datestr=datestr)
+    fohm_api = FOHMUploadAPI(
+        config=context,
+        dry_run=dry_run,
+        datestr=datestr,
+    )
     fohm_api.send_mail_reports()

cg/constants/delivery.py

@@ -144,7 +144,6 @@
 ]
 
 MUTANT_ANALYSIS_SAMPLE_TAGS: list[set[str]] = [
-    {"fastq"},
     {"vcf", "vcf-report", "fohm-delivery"},
 ]
 

cg/constants/orderforms.py

@@ -39,7 +39,7 @@ def get_current_orderform_version(order_form: str) -> str:
             Orderform.MIP_DNA: "33",
             Orderform.RML: "19",
             Orderform.MICROSALT: "11",
-            Orderform.SARS_COV_2: "9",
+            Orderform.SARS_COV_2: "10",
             Orderform.MICROBIAL_FASTQ: "1",
             Orderform.PACBIO_LONG_READ: "1",
         }

cg/meta/upload/fohm/fohm.py

@@ -3,22 +3,22 @@
 import logging
 import os
 import re
-import shutil
 from pathlib import Path
-
 import paramiko
-from housekeeper.store.models import Version
-
 from cg.apps.housekeeper.hk import HousekeeperAPI
 from cg.apps.lims import LimsAPI
 from cg.constants import FileExtensions
-from cg.constants.constants import SARS_COV_REGEX
+from cg.constants.constants import SARS_COV_REGEX, DataDelivery
 from cg.constants.housekeeper_tags import FohmTag
 from cg.exc import CgError
 from cg.io.csv import read_csv, write_csv_from_dict
 from cg.models.cg_config import CGConfig
 from cg.models.email import EmailInfo
 from cg.models.fohm.reports import FohmComplementaryReport, FohmPangolinReport
+from cg.services.deliver_files.constants import DeliveryDestination, DeliveryStructure
+from cg.services.deliver_files.factory import (
+    DeliveryServiceFactory,
+)
 from cg.store.models import Case, Sample
 from cg.store.store import Store
 from cg.utils.dict import remove_duplicate_dicts
@@ -28,7 +28,12 @@
 
 
 class FOHMUploadAPI:
-    def __init__(self, config: CGConfig, dry_run: bool = False, datestr: str | None = None):
+    def __init__(
+        self,
+        config: CGConfig,
+        dry_run: bool = False,
+        datestr: str | None = None,
+    ):
         self.config: CGConfig = config
         self.housekeeper_api: HousekeeperAPI = config.housekeeper_api
         self.lims_api: LimsAPI = config.lims_api
@@ -44,6 +49,7 @@ def __init__(self, config: CGConfig, dry_run: bool = False, datestr: str | None
         self._reports_dataframe = None
         self._pangolin_dataframe = None
         self._aggregation_dataframe = None
+        self._delivery_factory: DeliveryServiceFactory = config.delivery_service_factory
 
     @property
     def current_datestr(self) -> str:
@@ -196,16 +202,16 @@ def link_sample_raw_data_files(
             sample: Sample = self.status_db.get_sample_by_internal_id(
                 internal_id=report.internal_id
             )
-            bundle_name: str = sample.links[0].case.internal_id
-            version: Version = self.housekeeper_api.last_version(bundle=bundle_name)
-            files = self.housekeeper_api.files(version=version.id, tags={report.internal_id}).all()
-            for file in files:
-                if self._dry_run:
-                    LOG.info(
-                        f"Would have copied {file.full_path} to {Path(self.daily_rawdata_path)}"
-                    )
-                    continue
-                shutil.copy(file.full_path, Path(self.daily_rawdata_path))
+            case: Case = sample.links[0].case
+            delivery_service = self._delivery_factory.build_delivery_service(
+                case=case,
+                delivery_type=DataDelivery.FASTQ_ANALYSIS,
+                delivery_destination=DeliveryDestination.FOHM,
+                delivery_structure=DeliveryStructure.FLAT,
+            )
+            delivery_service.deliver_files_for_sample_no_rsync(
+                case=case, sample_id=sample.internal_id, delivery_base_path=self.daily_rawdata_path
+            )
 
     def create_pangolin_report(self, reports: list[FohmPangolinReport]) -> None:
         LOG.info("Creating aggregate Pangolin report")
@@ -362,9 +368,13 @@ def parse_and_write_pangolin_report(self) -> list[FohmPangolinReport]:
         self.create_pangolin_report(sars_cov_pangolin_reports)
         return sars_cov_pangolin_reports
 
-    def aggregate_delivery(self, cases: list[str]) -> None:
-        """Aggregate and hardlink reports."""
-        self.set_cases_to_aggregate(cases)
+    def aggregate_delivery(self, case_ids: list[str]) -> None:
+        """
+        Aggregate and hardlink reports.
+        args:
+            case_ids: The internal ids for cases to aggregate.
+        """
+        self.set_cases_to_aggregate(case_ids)
         self.create_daily_delivery_folders()
         sars_cov_complementary_reports: list[FohmComplementaryReport] = (
             self.parse_and_write_complementary_report()

cg/meta/upload/mutant/mutant.py

@@ -0,0 +1,27 @@
+from click import Context
+
+from cg.meta.upload.fohm.fohm import FOHMUploadAPI
+from cg.meta.upload.gisaid import GisaidAPI
+from cg.meta.upload.upload_api import UploadAPI
+from cg.meta.workflow.mutant import MutantAnalysisAPI
+from cg.models.cg_config import CGConfig
+from cg.store.models import Analysis, Case
+
+
+class MutantUploadAPI(UploadAPI):
+
+    def __init__(self, config: CGConfig):
+        self.analysis_api: MutantAnalysisAPI = MutantAnalysisAPI(config)
+        self.fohm_api = FOHMUploadAPI(config)
+        self.gsaid_api = GisaidAPI(config)
+
+        super().__init__(config=config, analysis_api=self.analysis_api)
+
+    def upload(self, ctx: Context, case: Case, restart: bool) -> None:
+        latest_analysis: Analysis = case.analyses[0]
+        self.update_upload_started_at(latest_analysis)
+        self.upload_files_to_customer_inbox(case)
+        self.gsaid_api.upload(case.internal_id)
+        self.fohm_api.aggregate_delivery(case_ids=[case.internal_id])
+        self.fohm_api.sync_files_sftp()
+        self.update_uploaded_at(latest_analysis)

cg/meta/upload/upload_api.py

@@ -15,7 +15,7 @@
 from cg.services.deliver_files.deliver_files_service.deliver_files_service import (
     DeliverFilesService,
 )
-from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import (
+from cg.services.deliver_files.factory import (
     DeliveryServiceFactory,
 )
 from cg.store.models import Analysis, Case
@@ -97,8 +97,7 @@ def upload_files_to_customer_inbox(self, case: Case) -> None:
         """Uploads the analysis files to the customer inbox."""
         factory_service: DeliveryServiceFactory = self.config.delivery_service_factory
         delivery_service: DeliverFilesService = factory_service.build_delivery_service(
-            case=case,
-            delivery_type=case.data_delivery,
+            case=case, delivery_type=case.data_delivery
         )
         delivery_service.deliver_files_for_case(
             case=case, delivery_base_path=Path(self.config.delivery_path)

cg/models/cg_config.py

@@ -27,7 +27,7 @@
 from cg.meta.delivery.delivery import DeliveryAPI
 from cg.services.analysis_service.analysis_service import AnalysisService
 from cg.services.decompression_service.decompressor import Decompressor
-from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import (
+from cg.services.deliver_files.factory import (
     DeliveryServiceFactory,
 )
 from cg.services.deliver_files.rsync.models import RsyncDeliveryConfig
@@ -748,6 +748,7 @@ def delivery_service_factory(self) -> DeliveryServiceFactory:
             LOG.debug("Instantiating delivery service factory")
             factory = DeliveryServiceFactory(
                 store=self.status_db,
+                lims_api=self.lims_api,
                 hk_api=self.housekeeper_api,
                 tb_service=self.trailblazer_api,
                 rsync_service=self.delivery_rsync_service,

cg/services/deliver_files/constants.py

@@ -0,0 +1,23 @@
+from enum import Enum
+
+
+class DeliveryDestination(Enum):
+    """Enum for the DeliveryDestination
+    BASE: Deliver to the base folder provided in the call
+    CUSTOMER: Deliver to the customer folder on hasta
+    FOHM: Deliver to the FOHM folder on hasta
+    """
+
+    BASE = "base"
+    CUSTOMER = "customer"
+    FOHM = "fohm"
+
+
+class DeliveryStructure(Enum):
+    """Enum for the DeliveryStructure
+    FLAT: Deliver the files in a flat structure, i.e. all files in the same folder
+    NESTED: Deliver the files in a nested structure, i.e. files in folders for each sample/case
+    """
+
+    FLAT: str = "flat"
+    NESTED: str = "nested"