Merge branch 'dev-new-delivery-service' of https://github.com/Clinica…

…l-Genomics/cg into dev-new-delivery-service

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 62.1.6
+current_version = 62.2.4
 commit = True
 tag = True
 tag_name = v{new_version}

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "62.1.6"
+__version__ = "62.2.4"

cg/apps/invoice/render.py

@@ -1,8 +1,11 @@
 import datetime as dt
+from pathlib import Path
 
 from openpyxl import Workbook, load_workbook
 from openpyxl.styles import Border, Font, PatternFill, Side
-from pkg_resources import resource_filename
+
+from cg.constants import FileExtensions
+from cg.utils.files import get_project_root_dir
 
 
 def render_xlsx(data: dict) -> Workbook:
@@ -34,11 +37,17 @@ def render_xlsx(data: dict) -> Workbook:
             }]
         }
     """
-    pkg_dir = __name__.rpartition(".")[0]
+    project_root_dir = get_project_root_dir()
     sample_type = "pool" if data["pooled_samples"] else "sample"
     costcenter = data["cost_center"]
-    template_path = resource_filename(pkg_dir, f"templates/{costcenter}_{sample_type}_invoice.xlsx")
-    workbook = load_workbook(template_path)
+    template_path = Path(
+        project_root_dir,
+        "apps",
+        "invoice",
+        "templates",
+        f"{costcenter}_{sample_type}_invoice{FileExtensions.XLSX}",
+    )
+    workbook = load_workbook(template_path.as_posix())
     if data["pooled_samples"]:
         worksheet = workbook["Bilaga Prover"]
         worksheet["C1"] = costcenter.upper()

cg/apps/lims/api.py

@@ -9,8 +9,15 @@
 from genologics.lims import Lims
 from requests.exceptions import HTTPError
 
-from cg.constants import Priority
-from cg.constants.lims import MASTER_STEPS_UDFS, PROP2UDF, DocumentationMethod, LimsArtifactTypes
+from cg.constants.constants import ControlOptions, CustomerId
+from cg.constants.lims import (
+    MASTER_STEPS_UDFS,
+    PROP2UDF,
+    DocumentationMethod,
+    LimsArtifactTypes,
+    LimsProcess,
+)
+from cg.constants.priority import Priority
 from cg.exc import LimsDataError
 
 from .order import OrderHandler
@@ -478,3 +485,73 @@ def get_latest_rna_input_amount(self, sample_id: str) -> float | None:
         )
         input_amount: float | None = self._get_last_used_input_amount(input_amounts=input_amounts)
         return input_amount
+
+    def get_latest_artifact_for_sample(
+        self,
+        process_type: LimsProcess,
+        sample_internal_id: str,
+        artifact_type: LimsArtifactTypes | None = LimsArtifactTypes.ANALYTE,
+    ) -> Artifact:
+        """Return latest artifact for a given sample, process and artifact type."""
+
+        artifacts: list[Artifact] = self.get_artifacts(
+            process_type=process_type,
+            type=artifact_type,
+            samplelimsid=sample_internal_id,
+        )
+
+        if not artifacts:
+            raise LimsDataError(
+                f"No artifacts were found for process {process_type}, type {artifact_type} and sample {sample_internal_id}."
+            )
+
+        latest_artifact: Artifact = self._get_latest_artifact_from_list(artifact_list=artifacts)
+        return latest_artifact
+
+    def _get_latest_artifact_from_list(self, artifact_list: list[Artifact]) -> Artifact:
+        """Returning the latest artifact in a list of artifacts."""
+        artifacts = []
+        for artifact in artifact_list:
+            date = artifact.parent_process.date_run or datetime.today().strftime("%Y-%m-%d")
+            artifacts.append((date, artifact.id, artifact))
+
+        artifacts.sort()
+        date, id, latest_artifact = artifacts[-1]
+        return latest_artifact
+
+    def get_internal_negative_control_id_from_sample_in_pool(
+        self, sample_internal_id: str, pooling_step: LimsProcess
+    ) -> str:
+        """Retrieve from LIMS the sample ID for the internal negative control sample present in the same pool as the given sample."""
+        artifact: Artifact = self.get_latest_artifact_for_sample(
+            process_type=pooling_step,
+            sample_internal_id=sample_internal_id,
+        )
+        negative_controls: list[Sample] = self._get_negative_controls_from_list(
+            samples=artifact.samples
+        )
+
+        if not negative_controls:
+            raise LimsDataError(
+                f"No internal negative controls found in the pool of sample {sample_internal_id}."
+            )
+
+        if len(negative_controls) > 1:
+            sample_ids = [sample.id for sample in negative_controls]
+            raise LimsDataError(
+                f"Multiple internal negative control samples found: {' '.join(sample_ids)}"
+            )
+
+        return negative_controls[0].id
+
+    @staticmethod
+    def _get_negative_controls_from_list(samples: list[Sample]) -> list[Sample]:
+        """Filter and return a list of internal negative controls from a given sample list."""
+        negative_controls = []
+        for sample in samples:
+            if (
+                sample.udf.get("Control") == ControlOptions.NEGATIVE
+                and sample.udf.get("customer") == CustomerId.CG_INTERNAL_CUSTOMER
+            ):
+                negative_controls.append(sample)
+        return negative_controls

cg/apps/tb/api.py

@@ -118,6 +118,7 @@ def add_pending_analysis(
         workflow: Workflow = None,
         ticket: str = None,
         workflow_manager: str = WorkflowManager.Slurm,
+        tower_workflow_id: str | None = None,
     ) -> TrailblazerAnalysis:
         request_body = {
             "case_id": case_id,
@@ -130,6 +131,7 @@ def add_pending_analysis(
             "workflow": workflow.upper(),
             "ticket": ticket,
             "workflow_manager": workflow_manager,
+            "tower_workflow_id": tower_workflow_id,
         }
         LOG.debug(f"Submitting job to Trailblazer: {request_body}")
         if response := self.query_trailblazer(

cg/cli/get.py

@@ -198,7 +198,11 @@ def get_sequencing_run(context: click.Context, samples: bool, flow_cell_id: str)
         sequencing_run.device.internal_id,
         sequencing_run.sequencer_type,
         sequencing_run.sequencer_name,
-        sequencing_run.sequencing_started_at.date(),
+        (
+            sequencing_run.sequencing_started_at.date()
+            if sequencing_run.sequencing_started_at
+            else "Not available"
+        ),
         sequencing_run.sequencing_completed_at.date(),
         sequencing_run.archived_at.date() if sequencing_run.archived_at else "No",
         sequencing_run.data_availability,

cg/cli/post_process/post_process.py

@@ -35,5 +35,4 @@ def post_process_sequencing_run(context: CGConfig, run_name: str, dry_run: bool)
     post_processing_service.post_process(run_name=run_name, dry_run=dry_run)
 
 
-post_process_group: click.Group
 post_process_group.add_command(post_process_sequencing_run)

cg/cli/workflow/mutant/base.py

@@ -9,14 +9,14 @@
     link,
     resolve_compression,
     store,
-    store_available,
 )
 from cg.constants import EXIT_FAIL, EXIT_SUCCESS
 from cg.constants.cli_options import DRY_RUN
 from cg.exc import AnalysisNotReadyError, CgError
 from cg.meta.workflow.analysis import AnalysisAPI
 from cg.meta.workflow.mutant import MutantAnalysisAPI
 from cg.models.cg_config import CGConfig
+from cg.store.models import Case
 
 LOG = logging.getLogger(__name__)
 
@@ -32,7 +32,6 @@ def mutant(context: click.Context) -> None:
 mutant.add_command(resolve_compression)
 mutant.add_command(link)
 mutant.add_command(store)
-mutant.add_command(store_available)
 
 
 @mutant.command("config-case")
@@ -75,7 +74,6 @@ def start(context: click.Context, dry_run: bool, case_id: str, config_artic: str
     context.invoke(link, case_id=case_id, dry_run=dry_run)
     context.invoke(config_case, case_id=case_id, dry_run=dry_run)
     context.invoke(run, case_id=case_id, dry_run=dry_run, config_artic=config_artic)
-    context.invoke(store, case_id=case_id, dry_run=dry_run)
 
 
 @mutant.command("start-available")
@@ -100,3 +98,49 @@ def start_available(context: click.Context, dry_run: bool = False):
             exit_code = EXIT_FAIL
     if exit_code:
         raise click.Abort
+
+
+@mutant.command("store-available")
+@DRY_RUN
+@click.pass_context
+def store_available(context: click.Context, dry_run: bool) -> None:
+    """Run QC checks and store bundles for all finished analyses in Housekeeper."""
+
+    analysis_api: MutantAnalysisAPI = context.obj.meta_apis["analysis_api"]
+
+    exit_code: int = EXIT_SUCCESS
+
+    cases_ready_for_qc: list[Case] = analysis_api.get_cases_to_perform_qc_on()
+    LOG.info(f"Found {len(cases_ready_for_qc)} cases to perform QC on!")
+    for case in cases_ready_for_qc:
+        LOG.info(f"Performing QC on case {case.internal_id}.")
+        try:
+            analysis_api.run_qc_on_case(case=case, dry_run=dry_run)
+        except Exception:
+            exit_code = EXIT_FAIL
+
+    cases_to_store: list[Case] = analysis_api.get_cases_to_store()
+    LOG.info(f"Found {len(cases_to_store)} cases to store!")
+    for case in cases_to_store:
+        LOG.info(f"Storing deliverables for {case.internal_id}")
+        try:
+            context.invoke(store, case_id=case.internal_id, dry_run=dry_run)
+        except Exception as exception_object:
+            LOG.error(f"Error storingc {case.internal_id}: {exception_object}")
+            exit_code = EXIT_FAIL
+
+    if exit_code:
+        raise click.Abort
+
+
+@mutant.command("run-qc")
+@DRY_RUN
+@ARGUMENT_CASE_ID
+@click.pass_context
+def run_qc(context: click.Context, case_id: str, dry_run: bool) -> None:
+    """
+    Run QC on case and generate QC_report file.
+    """
+    analysis_api: MutantAnalysisAPI = context.obj.meta_apis["analysis_api"]
+
+    analysis_api.run_qc(case_id=case_id, dry_run=dry_run)

cg/constants/constants.py

@@ -223,6 +223,7 @@ class FileExtensions(StrEnum):
     TSV: str = ".tsv"
     TXT: str = ".txt"
     VCF: str = ".vcf"
+    XLSX: str = ".xlsx"
     XML: str = ".xml"
     YAML: str = ".yaml"
 
@@ -254,6 +255,13 @@ class MicrosaltAppTags(StrEnum):
     PREP_CATEGORY: str = "mic"
 
 
+class MutantQC:
+    EXTERNAL_NEGATIVE_CONTROL_READS_THRESHOLD: int = 100000
+    INTERNAL_NEGATIVE_CONTROL_READS_THRESHOLD: int = 2000
+    FRACTION_OF_SAMPLES_WITH_FAILED_QC_TRESHOLD: float = 0.2
+    QUALITY_REPORT_FILE_NAME: str = f"QC_report{FileExtensions.JSON}"
+
+
 DRY_RUN_MESSAGE = "Dry run: process call will not be executed!"
 
 

cg/constants/lims.py

@@ -157,3 +157,7 @@ class DocumentationMethod(StrEnum):
 class LimsArtifactTypes(StrEnum):
     ANALYTE: str = "Analyte"
     RESULT_FILE: str = "ResultFile"
+
+
+class LimsProcess(StrEnum):
+    COVID_POOLING_STEP: str = "Pooling and Clean-up (Cov) v1"

cg/constants/report.py

@@ -1,15 +1,17 @@
 """Delivery report constants."""
 
-from importlib.resources import files
 from pathlib import Path
 
 from cg.constants import DataDelivery
 from cg.constants.constants import CancerAnalysisType, FileExtensions, Workflow
 from cg.constants.subject import Sex
+from cg.utils.files import get_project_root_dir
+
+project_root_dir: Path = get_project_root_dir()
 
 DELIVERY_REPORT_FILE_NAME: str = f"delivery-report{FileExtensions.HTML}"
 SWEDAC_LOGO_PATH = Path(
-    files("cg"), "meta", "report", "templates", "static", "images", "SWEDAC_logo.png"
+    project_root_dir, "meta", "report", "templates", "static", "images", "SWEDAC_logo.png"
 )
 
 BALSAMIC_REPORT_ACCREDITED_PANELS: list[str] = ["gmsmyeloid"]

cg/meta/orders/api.py

@@ -13,7 +13,9 @@
 from cg.apps.osticket import OsTicket
 from cg.meta.orders.ticket_handler import TicketHandler
 from cg.models.orders.order import OrderIn, OrderType
-from cg.services.orders.submitters.order_submitter_registry import OrderSubmitterRegistry
+from cg.services.orders.submitters.order_submitter_registry import (
+    OrderSubmitterRegistry,
+)
 from cg.store.store import Store
 
 LOG = logging.getLogger(__name__)
@@ -41,6 +43,7 @@ def submit(self, project: OrderType, order_in: OrderIn, user_name: str, user_mai
         Main entry point for the class towards interfaces that implements it.
         """
         submit_handler = self.submitter_registry.get_order_submitter(project)
+        submit_handler.order_validation_service.validate_order(order_in)
         # detect manual ticket assignment
         ticket_number: str | None = TicketHandler.parse_ticket_number(order_in.name)
         if not ticket_number:

cg/meta/rsync/sbatch.py

@@ -9,8 +9,8 @@
 """
 
 COVID_RSYNC = """
-rsync -rvtL {source_path} {destination_path}
-rsync -rvtL --chmod=777 {covid_report_path} {covid_destination_path}
+rsync -rvL {source_path} {destination_path}
+rsync -rvL --chmod=777 {covid_report_path} {covid_destination_path}
 """
 
 ERROR_RSYNC_FUNCTION = """

cg/meta/workflow/analysis.py

@@ -275,7 +275,11 @@ def get_deliverables_file_path(self, case_id: str) -> Path:
     def get_analysis_finish_path(self, case_id: str) -> Path:
         raise NotImplementedError
 
-    def add_pending_trailblazer_analysis(self, case_id: str) -> None:
+    def add_pending_trailblazer_analysis(
+        self,
+        case_id: str,
+        tower_workflow_id: str | None = None,
+    ) -> None:
         self.check_analysis_ongoing(case_id)
         application_type: str = self.get_application_type(
             self.status_db.get_case_by_internal_id(case_id).links[0].sample
@@ -299,6 +303,7 @@ def add_pending_trailblazer_analysis(self, case_id: str) -> None:
             ticket=ticket,
             workflow=workflow,
             workflow_manager=workflow_manager,
+            tower_workflow_id=tower_workflow_id,
         )
 
     def _get_order_id_from_case_id(self, case_id) -> int:

cg/meta/workflow/mutant/__init__.py

@@ -0,0 +1 @@
+from cg.meta.workflow.mutant.mutant import MutantAnalysisAPI