Merge remote-tracking branch 'origin/archiving-clean-retrieved-spring…

…s' into archiving-clean-retrieved-springs

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 54.10.6
+current_version = 55.0.0
 commit = True
 tag = True
 tag_name = v{new_version}

alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py

@@ -0,0 +1,24 @@
+"""remove_uploaded_to_vogue_at
+
+Revision ID: 584840c706a0
+Revises: 27ec5c4c0380
+Create Date: 2023-12-27 11:50:22.278213
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "584840c706a0"
+down_revision = "27ec5c4c0380"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column("analysis", "uploaded_to_vogue_at")
+
+
+def downgrade():
+    op.add_column("analysis", sa.Column("uploaded_to_vogue_at", sa.DateTime(), nullable=True))

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "54.10.6"
+__version__ = "55.0.0"

cg/meta/workflow/analysis.py

@@ -118,8 +118,8 @@ def get_case_config_path(self, case_id) -> Path:
         """Path to case config file"""
         raise NotImplementedError
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
-        """Path to Trailblazer job id file"""
+    def get_job_ids_path(self, case_id: str) -> Path:
+        """Path to file containing slurm/tower job ids for the case."""
         raise NotImplementedError
 
     def get_sample_name_from_lims_id(self, lims_id: str) -> str:
@@ -206,17 +206,17 @@ def get_analysis_finish_path(self, case_id: str) -> Path:
         raise NotImplementedError
 
     def add_pending_trailblazer_analysis(self, case_id: str) -> None:
-        self.check_analysis_ongoing(case_id=case_id)
-        self.trailblazer_api.mark_analyses_deleted(case_id=case_id)
+        self.check_analysis_ongoing(case_id)
+        self.trailblazer_api.mark_analyses_deleted(case_id)
         self.trailblazer_api.add_pending_analysis(
             case_id=case_id,
             email=environ_email(),
             analysis_type=self.get_application_type(
-                self.status_db.get_case_by_internal_id(internal_id=case_id).links[0].sample
+                self.status_db.get_case_by_internal_id(case_id).links[0].sample
             ),
-            out_dir=self.get_trailblazer_config_path(case_id=case_id).parent.as_posix(),
-            config_path=self.get_trailblazer_config_path(case_id=case_id).as_posix(),
-            slurm_quality_of_service=self.get_slurm_qos_for_case(case_id=case_id),
+            out_dir=self.get_job_ids_path(case_id).parent.as_posix(),
+            config_path=self.get_job_ids_path(case_id).as_posix(),
+            slurm_quality_of_service=self.get_slurm_qos_for_case(case_id),
             data_analysis=str(self.pipeline),
             ticket=self.status_db.get_latest_ticket_from_case(case_id),
             workflow_manager=self.get_workflow_manager(),

cg/meta/workflow/balsamic.py

@@ -117,7 +117,7 @@ def get_case_config_path(self, case_id: str) -> Path:
         """
         return Path(self.root_dir, case_id, case_id + ".json")
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
+    def get_job_ids_path(self, case_id: str) -> Path:
         return Path(self.root_dir, case_id, "analysis", "slurm_jobids.yaml")
 
     def get_bundle_deliverables_type(self, case_id: str) -> str:

cg/meta/workflow/fluffy.py

@@ -130,7 +130,7 @@ def get_deliverables_file_path(self, case_id: str) -> Path:
         """
         return Path(self.get_output_path(case_id), "deliverables.yaml")
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
+    def get_job_ids_path(self, case_id: str) -> Path:
         """
         Location in working directory where SLURM job id file is to be stored.
         This file contains SLURM ID of jobs associated with current analysis ,

cg/meta/workflow/microsalt/microsalt.py

@@ -1,4 +1,3 @@
-import glob
 import logging
 import os
 import re
@@ -88,14 +87,19 @@ def get_case_fastq_path(self, case_id: str) -> Path:
     def get_config_path(self, filename: str) -> Path:
         return Path(self.queries_path, filename).with_suffix(".json")
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
-        """Get trailblazer config path."""
-        case_obj: Case = self.status_db.get_case_by_internal_id(internal_id=case_id)
-        sample_obj: Sample = case_obj.links[0].sample
-        project_id: str = self.get_project(sample_obj.internal_id)
-        return Path(
-            self.root_dir, "results", "reports", "trailblazer", f"{project_id}_slurm_ids.yaml"
-        )
+    def get_job_ids_path(self, case_id: str) -> Path:
+        case_path: Path = self.get_case_path(case_id)
+        job_ids_file_name: str = self.get_job_ids_file_name(case_id)
+        return Path(case_path, job_ids_file_name)
+
+    def get_job_ids_file_name(self, case_id: str) -> str:
+        project_id: str = self.get_lims_project_id(case_id)
+        return f"{project_id}_slurm_ids.yaml"
+
+    def get_lims_project_id(self, case_id: str):
+        case: Case = self.status_db.get_case_by_internal_id(case_id)
+        sample: Sample = case.links[0].sample
+        return self.get_project(sample.internal_id)
 
     def get_deliverables_file_path(self, case_id: str) -> Path:
         """Returns a path where the microSALT deliverables file for the order_id should be
@@ -221,7 +225,7 @@ def get_case_id_from_ticket(self, unique_id: str) -> tuple[str, None]:
         Since sample_id is not specified, nothing is returned as sample_id"""
         case: Case = self.status_db.get_case_by_name(name=unique_id)
         if not case:
-            LOG.error("No case found for ticket number:  %s", unique_id)
+            LOG.error(f"No case found for ticket number:  {unique_id}")
             raise click.Abort
         case_id = case.internal_id
         return case_id, None
@@ -230,19 +234,19 @@ def get_case_id_from_sample(self, unique_id: str) -> tuple[str, str]:
         """If sample is specified, finds the corresponding case_id to which this sample belongs.
         The case_id is to be used for identifying the appropriate path to link fastq files and store the analysis output
         """
-        sample: Sample = self.status_db.get_sample_by_internal_id(internal_id=unique_id)
+        sample: Sample = self.status_db.get_sample_by_internal_id(unique_id)
         if not sample:
-            LOG.error("No sample found with id: %s", unique_id)
+            LOG.error(f"No sample found with id: {unique_id}")
             raise click.Abort
         case_id = sample.links[0].case.internal_id
         sample_id = sample.internal_id
         return case_id, sample_id
 
     def get_case_id_from_case(self, unique_id: str) -> tuple[str, None]:
         """If case_id is specified, validates the presence of case_id in database and returns it"""
-        case_obj: Case = self.status_db.get_case_by_internal_id(internal_id=unique_id)
+        case_obj: Case = self.status_db.get_case_by_internal_id(unique_id)
         if not case_obj:
-            LOG.error("No case found with the id:  %s", unique_id)
+            LOG.error(f"No case found with the id:  {unique_id}")
             raise click.Abort
         case_id = case_obj.internal_id
         return case_id, None

cg/meta/workflow/mip.py

@@ -312,7 +312,7 @@ def run_analysis(self, case_id: str, command_args: dict, dry_run: bool) -> None:
     def get_case_path(self, case_id: str) -> Path:
         return Path(self.root, case_id)
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
+    def get_job_ids_path(self, case_id: str) -> Path:
         return Path(self.get_case_path(case_id=case_id), "analysis", "slurm_job_ids.yaml")
 
     def config_sample(self, link_obj: CaseSample, panel_bed: str) -> dict:

cg/meta/workflow/mutant.py

@@ -55,7 +55,7 @@ def get_case_output_path(self, case_id: str) -> Path:
     def get_case_fastq_dir(self, case_id: str) -> Path:
         return Path(self.get_case_path(case_id=case_id), "fastq")
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
+    def get_job_ids_path(self, case_id: str) -> Path:
         return Path(self.get_case_output_path(case_id=case_id), "trailblazer_config.yaml")
 
     def _is_nanopore(self, application: Application) -> bool:

cg/meta/workflow/nf_analysis.py

@@ -89,7 +89,7 @@ def get_nextflow_config_path(nextflow_config: str | None = None) -> Path | None:
         if nextflow_config:
             return Path(nextflow_config).absolute()
 
-    def get_trailblazer_config_path(self, case_id: str) -> Path:
+    def get_job_ids_path(self, case_id: str) -> Path:
         """Return the path to a Trailblazer config file containing Tower IDs."""
         return Path(self.root_dir, case_id, "tower_ids").with_suffix(FileExtensions.YAML)
 
@@ -197,7 +197,7 @@ def write_deliverables_file(
 
     def write_trailblazer_config(self, case_id: str, tower_id: str) -> None:
         """Write Tower IDs to a file used as the Trailblazer config."""
-        config_path: Path = self.get_trailblazer_config_path(case_id=case_id)
+        config_path: Path = self.get_job_ids_path(case_id=case_id)
         LOG.info(f"Writing Tower ID to {config_path.as_posix()}")
         WriteFile.write_file_from_content(
             content={case_id: [tower_id]},
@@ -246,7 +246,7 @@ def _run_analysis_with_tower(
         if command_args.resume:
             from_tower_id: int = command_args.id or NfTowerHandler.get_last_tower_id(
                 case_id=case_id,
-                trailblazer_config=self.get_trailblazer_config_path(case_id=case_id),
+                trailblazer_config=self.get_job_ids_path(case_id=case_id),
             )
             LOG.info(f"Pipeline will be resumed from run with Tower id: {from_tower_id}.")
             parameters: list[str] = NfTowerHandler.get_tower_relaunch_parameters(

cg/store/models.py

@@ -218,7 +218,6 @@ class Analysis(Model):
 
     created_at = Column(types.DateTime, default=dt.datetime.now, nullable=False)
     case_id = Column(ForeignKey("case.id", ondelete="CASCADE"), nullable=False)
-    uploaded_to_vogue_at = Column(types.DateTime, nullable=True)
 
     case = orm.relationship("Case", back_populates="analyses")
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "cg"
-version = "54.10.6"
+version = "55.0.0"
 description = "Clinical Genomics command center"
 authors = ["Clinical Genomics <[email protected]>"]
 readme = "README.md"

tests/store/api/conftest.py

@@ -465,7 +465,6 @@ def store_with_analyses_for_cases(
             started_at=timestamp_yesterday,
             uploaded_at=timestamp_yesterday,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_yesterday,
             completed_at=timestamp_yesterday,
         )
         helpers.add_analysis(
@@ -474,7 +473,6 @@ def store_with_analyses_for_cases(
             started_at=timestamp_now,
             uploaded_at=timestamp_now,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=None,
             completed_at=timestamp_now,
         )
         sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now)
@@ -505,7 +503,6 @@ def store_with_analyses_for_cases_not_uploaded_fluffy(
             started_at=timestamp_yesterday,
             uploaded_at=timestamp_yesterday,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_yesterday,
             pipeline=Pipeline.FLUFFY,
         )
         helpers.add_analysis(
@@ -514,7 +511,6 @@ def store_with_analyses_for_cases_not_uploaded_fluffy(
             started_at=timestamp_now,
             uploaded_at=None,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_now,
             pipeline=Pipeline.FLUFFY,
         )
         sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now)
@@ -545,7 +541,6 @@ def store_with_analyses_for_cases_not_uploaded_microsalt(
             started_at=timestamp_yesterday,
             uploaded_at=timestamp_yesterday,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_yesterday,
             pipeline=Pipeline.MICROSALT,
         )
         helpers.add_analysis(
@@ -554,7 +549,6 @@ def store_with_analyses_for_cases_not_uploaded_microsalt(
             started_at=timestamp_now,
             uploaded_at=None,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_now,
             pipeline=Pipeline.MICROSALT,
         )
         sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now)
@@ -584,7 +578,6 @@ def store_with_analyses_for_cases_to_deliver(
             started_at=timestamp_yesterday,
             uploaded_at=None,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_yesterday,
             completed_at=timestamp_yesterday,
             pipeline=Pipeline.FLUFFY,
         )
@@ -594,7 +587,6 @@ def store_with_analyses_for_cases_to_deliver(
             started_at=timestamp_now,
             uploaded_at=None,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=None,
             completed_at=timestamp_now,
             pipeline=Pipeline.MIP_DNA,
         )

tests/store/conftest.py

@@ -401,7 +401,6 @@ def store_with_older_and_newer_analyses(
     """Return a store with  older and newer analyses."""
     analysis = base_store._get_query(table=Analysis).first()
     analysis.uploaded_at = timestamp_now
-    analysis.uploaded_to_vogue_at = timestamp_now
     analysis.cleaned_at = timestamp_now
     analysis.started_at = timestamp_now
     analysis.completed_at = timestamp_now
@@ -416,7 +415,6 @@ def store_with_older_and_newer_analyses(
             started_at=time,
             completed_at=time,
             uploaded_at=time,
-            uploaded_to_vogue_at=time,
             cleaned_at=time,
         )
 
@@ -442,15 +440,13 @@ def store_with_analyses_for_cases(
             started_at=timestamp_yesterday,
             uploaded_at=timestamp_yesterday,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_yesterday,
         )
         helpers.add_analysis(
             analysis_store,
             case=case,
             started_at=timestamp_now,
             uploaded_at=timestamp_now,
             delivery_reported_at=None,
-            uploaded_to_vogue_at=timestamp_now,
         )
         sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now)
         link: CaseSample = analysis_store.relate_sample(

tests/store_helpers.py

@@ -322,7 +322,6 @@ def add_analysis(
         data_delivery: DataDelivery = DataDelivery.FASTQ_QC,
         uploading: bool = False,
         config_path: str = None,
-        uploaded_to_vogue_at: datetime = None,
     ) -> Analysis:
         """Utility function to add an analysis for tests."""
 
@@ -346,8 +345,6 @@ def add_analysis(
             analysis.config_path = config_path
         if pipeline:
             analysis.pipeline = str(pipeline)
-        if uploaded_to_vogue_at:
-            analysis.uploaded_to_vogue_at = uploaded_to_vogue_at
 
         analysis.limitations = "A limitation"
         analysis.case = case