Renamed workflow fastq to raw-data (#3708)(minor)

## Description
Closes Clinical-Genomics/add-new-tech#93 and redo of #3676 due to conflicts
The long-read sequencing devices have bam as raw data output format instead of fastq, used by Illumina. We want to standardise the ordering of sequencing data without analysis, so we decided to rename the workflow fastq to raw-data.

Author: diitaz93

alembic/versions/2024_09_11_18dbadd8c436_rename_fastq_to_raw_data.py

@@ -0,0 +1,169 @@
+"""rename_fastq_to_raw_data
+Revision ID: 18dbadd8c436
+Revises: bb4c6dbad991
+Create Date: 2024-09-11 13:15:11.876822
+"""
+
+from enum import StrEnum
+
+from sqlalchemy import orm
+from sqlalchemy.dialects import mysql
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "18dbadd8c436"
+down_revision = "7770dcad8bde"
+branch_labels = None
+depends_on = None
+
+
+base_options = (
+    "balsamic",
+    "balsamic-pon",
+    "balsamic-qc",
+    "balsamic-umi",
+    "demultiplex",
+    "fluffy",
+    "jasen",
+    "microsalt",
+    "mip-dna",
+    "mip-rna",
+    "mutant",
+    "raredisease",
+    "rnafusion",
+    "rsync",
+    "spring",
+    "taxprofiler",
+    "tomte",
+)
+old_options = sorted(base_options + ("fastq",))
+new_options = sorted(base_options + ("raw-data",))
+
+old_enum = mysql.ENUM(*old_options)
+new_enum = mysql.ENUM(*new_options)
+
+
+class Workflow(StrEnum):
+    BALSAMIC: str = "balsamic"
+    BALSAMIC_PON: str = "balsamic-pon"
+    BALSAMIC_QC: str = "balsamic-qc"
+    BALSAMIC_UMI: str = "balsamic-umi"
+    DEMULTIPLEX: str = "demultiplex"
+    FLUFFY: str = "fluffy"
+    JASEN: str = "jasen"
+    MICROSALT: str = "microsalt"
+    MIP_DNA: str = "mip-dna"
+    MIP_RNA: str = "mip-rna"
+    MUTANT: str = "mutant"
+    RAREDISEASE: str = "raredisease"
+    RAW_DATA: str = "raw-data"
+    RNAFUSION: str = "rnafusion"
+    RSYNC: str = "rsync"
+    SPRING: str = "spring"
+    TAXPROFILER: str = "taxprofiler"
+    TOMTE: str = "tomte"
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class Case(Base):
+    __tablename__ = "case"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    data_analysis: Mapped[str | None]
+
+
+class Analysis(Base):
+    __tablename__ = "analysis"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    workflow: Mapped[str | None]
+
+
+class Order(Base):
+    __tablename__ = "order"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    workflow: Mapped[str | None]
+
+
+class ApplicationLimitations(Base):
+    __tablename__ = "application_limitations"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    workflow: Mapped[str | None]
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = orm.Session(bind=bind)
+    try:
+        # Case
+        op.alter_column("case", "data_analysis", type_=mysql.VARCHAR(64), existing_nullable=True)
+        session.query(Case).filter(Case.data_analysis == "fastq").update(
+            {"data_analysis": Workflow.RAW_DATA}, synchronize_session="evaluate"
+        )
+        op.alter_column("case", "data_analysis", type_=new_enum, existing_nullable=True)
+        # Analysis
+        op.alter_column("analysis", "workflow", type_=mysql.VARCHAR(64), existing_nullable=True)
+        session.query(Analysis).filter(Analysis.workflow == "fastq").update(
+            {"workflow": Workflow.RAW_DATA}, synchronize_session="evaluate"
+        )
+        op.alter_column("analysis", "workflow", type_=new_enum, existing_nullable=True)
+        # Order
+        op.alter_column("order", "workflow", type_=mysql.VARCHAR(64), existing_nullable=True)
+        session.query(Order).filter(Order.workflow == "fastq").update(
+            {"workflow": Workflow.RAW_DATA}, synchronize_session="evaluate"
+        )
+        op.alter_column("order", "workflow", type_=new_enum, existing_nullable=True)
+        # Application Limitation
+        op.alter_column(
+            "application_limitations", "workflow", type_=mysql.VARCHAR(64), existing_nullable=True
+        )
+        session.query(ApplicationLimitations).filter(
+            ApplicationLimitations.workflow == "fastq"
+        ).update({"workflow": Workflow.RAW_DATA}, synchronize_session="evaluate")
+        op.alter_column(
+            "application_limitations", "workflow", type_=new_enum, existing_nullable=True
+        )
+        session.commit()
+    finally:
+        session.close()
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = orm.Session(bind=bind)
+    try:
+        # Case
+        op.alter_column("case", "data_analysis", type_=mysql.VARCHAR(64), existing_nullable=True)
+        session.query(Case).filter(Case.data_analysis == Workflow.RAW_DATA).update(
+            {"data_analysis": "fastq"}, synchronize_session="evaluate"
+        )
+        op.alter_column("case", "data_analysis", type_=old_enum, existing_nullable=True)
+        # Analysis
+        op.alter_column("analysis", "workflow", type_=mysql.VARCHAR(64), existing_nullable=True)
+        session.query(Analysis).filter(Analysis.workflow == Workflow.RAW_DATA).update(
+            {"workflow": "fastq"}, synchronize_session="evaluate"
+        )
+        op.alter_column("analysis", "workflow", type_=old_enum, existing_nullable=True)
+        # Order
+        op.alter_column("order", "workflow", type_=mysql.VARCHAR(64), existing_nullable=True)
+        session.query(Order).filter(Order.workflow == Workflow.RAW_DATA).update(
+            {"workflow": "fastq"}, synchronize_session="evaluate"
+        )
+        op.alter_column("order", "workflow", type_=old_enum, existing_nullable=True)
+        # Application Limitation
+        op.alter_column(
+            "application_limitations", "workflow", type_=mysql.VARCHAR(64), existing_nullable=True
+        )
+        session.query(ApplicationLimitations).filter(
+            ApplicationLimitations.workflow == Workflow.RAW_DATA
+        ).update({"workflow": "fastq"}, synchronize_session="evaluate")
+        op.alter_column(
+            "application_limitations", "workflow", type_=old_enum, existing_nullable=True
+        )
+        session.commit()
+    finally:
+        session.close()

cg/cli/deliver/base.py

@@ -11,17 +11,17 @@
 from cg.constants import Workflow
 from cg.constants.cli_options import DRY_RUN
 from cg.constants.delivery import FileDeliveryOption
-from cg.services.deliver_files.delivery_rsync_service.delivery_rsync_service import (
-    DeliveryRsyncService,
-)
 from cg.models.cg_config import CGConfig
 from cg.services.deliver_files.deliver_files_service.deliver_files_service import (
     DeliverFilesService,
 )
 from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import (
     DeliveryServiceFactory,
 )
-from cg.store.models import Case, Analysis
+from cg.services.deliver_files.delivery_rsync_service.delivery_rsync_service import (
+    DeliveryRsyncService,
+)
+from cg.store.models import Analysis, Case
 
 LOG = logging.getLogger(__name__)
 
@@ -148,7 +148,7 @@ def deliver_auto_raw_data(context: CGConfig, dry_run: bool):
        clinical-delivery."""
     service_builder: DeliveryServiceFactory = context.delivery_service_factory
     analyses: list[Analysis] = context.analysis_service.get_analyses_to_upload_for_workflow(
-        workflow=Workflow.FASTQ
+        workflow=Workflow.RAW_DATA
     )
     deliver_raw_data_for_analyses(
         analyses=analyses,

cg/cli/deliver/utils.py

@@ -9,10 +9,9 @@
 from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import (
     DeliveryServiceFactory,
 )
-from cg.store.models import Case, Analysis
+from cg.store.models import Analysis, Case
 from cg.store.store import Store
 
-
 LOG = logging.getLogger(__name__)
 
 
@@ -29,7 +28,7 @@ def deliver_raw_data_for_analyses(
             case: Case = analysis.case
             delivery_service: DeliverFilesService = service_builder.build_delivery_service(
                 delivery_type=case.data_delivery,
-                workflow=Workflow.FASTQ,
+                workflow=Workflow.RAW_DATA,
             )
 
             delivery_service.deliver_files_for_case(

cg/cli/workflow/fastq/base.py

@@ -44,6 +44,6 @@ def store_fastq_analysis(context: click.Context, case_id: str, dry_run: bool = F
 def store_available_fastq_analysis(context: click.Context, dry_run: bool = False):
     """Creates an analysis object in status-db for all fastq cases to be delivered."""
     status_db: Store = context.obj.status_db
-    for case in status_db.cases_to_analyse(workflow=Workflow.FASTQ):
+    for case in status_db.cases_to_analyse(workflow=Workflow.RAW_DATA):
         if SequencingQCService.case_pass_sequencing_qc(case):
             context.invoke(store_fastq_analysis, case_id=case.internal_id, dry_run=dry_run)

cg/cli/workflow/fastq/fastq_service.py

@@ -25,7 +25,7 @@ def _get_case(self, case_id: str) -> Case:
 
     def _add_analysis_to_store(self, case: Case) -> None:
         new_analysis: Analysis = self.store.add_analysis(
-            workflow=Workflow.FASTQ,
+            workflow=Workflow.RAW_DATA,
             completed_at=dt.datetime.now(),
             primary=True,
             started_at=dt.datetime.now(),
@@ -42,7 +42,7 @@ def _add_analysis_to_trailblazer(self, case: Case) -> None:
             order_id=case.latest_order.id,
             out_dir="",
             slurm_quality_of_service=case.slurm_priority,
-            workflow=Workflow.FASTQ,
+            workflow=Workflow.RAW_DATA,
             ticket=case.latest_ticket,
         )
         self.trailblazer_api.set_analysis_status(

cg/constants/constants.py

@@ -135,14 +135,14 @@ class Workflow(StrEnum):
     BALSAMIC_QC: str = "balsamic-qc"
     BALSAMIC_UMI: str = "balsamic-umi"
     DEMULTIPLEX: str = "demultiplex"
-    FASTQ: str = "fastq"
     FLUFFY: str = "fluffy"
     JASEN: str = "jasen"
     MICROSALT: str = "microsalt"
     MIP_DNA: str = "mip-dna"
     MIP_RNA: str = "mip-rna"
     MUTANT: str = "mutant"
     RAREDISEASE: str = "raredisease"
+    RAW_DATA: str = "raw-data"
     RNAFUSION: str = "rnafusion"
     RSYNC: str = "rsync"
     SPRING: str = "spring"

cg/constants/delivery.py

@@ -3,22 +3,18 @@
 from enum import StrEnum
 
 from cg.constants.constants import Workflow
-from cg.constants.housekeeper_tags import (
-    AlignmentFileTag,
-    AnalysisTag,
-    HermesFileTag,
-)
+from cg.constants.housekeeper_tags import AlignmentFileTag, AnalysisTag, HermesFileTag
 
 ONLY_ONE_CASE_PER_TICKET: list[Workflow] = [
-    Workflow.FASTQ,
     Workflow.MICROSALT,
     Workflow.MUTANT,
+    Workflow.RAW_DATA,
 ]
 
 SKIP_MISSING: list[Workflow] = [
-    Workflow.FASTQ,
     Workflow.MICROSALT,
     Workflow.MUTANT,
+    Workflow.RAW_DATA,
 ]
 
 BALSAMIC_ANALYSIS_CASE_TAGS: list[set[str]] = [
@@ -178,10 +174,6 @@
         "case_tags": MICROSALT_ANALYSIS_CASE_TAGS,
         "sample_tags": MICROSALT_ANALYSIS_SAMPLE_TAGS,
     },
-    Workflow.FASTQ: {
-        "case_tags": RAW_DATA_ANALYSIS_CASE_TAGS,
-        "sample_tags": RAW_DATA_ANALYSIS_SAMPLE_TAGS,
-    },
     Workflow.MUTANT: {
         "case_tags": MUTANT_ANALYSIS_CASE_TAGS,
         "sample_tags": MUTANT_ANALYSIS_SAMPLE_TAGS,
@@ -190,6 +182,10 @@
         "case_tags": CLINICAL_DELIVERY_TAGS,
         "sample_tags": CLINICAL_DELIVERY_TAGS,
     },
+    Workflow.RAW_DATA: {
+        "case_tags": RAW_DATA_ANALYSIS_CASE_TAGS,
+        "sample_tags": RAW_DATA_ANALYSIS_SAMPLE_TAGS,
+    },
     Workflow.RNAFUSION: {
         "case_tags": CLINICAL_DELIVERY_TAGS,
         "sample_tags": CLINICAL_DELIVERY_TAGS,

cg/constants/housekeeper_tags.py

@@ -126,7 +126,6 @@ class BalsamicProtectedTags:
     Workflow.BALSAMIC_QC: BalsamicProtectedTags.QC,
     Workflow.BALSAMIC_PON: [],
     Workflow.BALSAMIC_UMI: BalsamicProtectedTags.QC + BalsamicProtectedTags.VARIANT_CALLERS,
-    Workflow.FASTQ: [],
     Workflow.FLUFFY: ["NIPT_csv", "MultiQC"],
     Workflow.MICROSALT: [
         ["microsalt-log"],
@@ -205,6 +204,7 @@ class BalsamicProtectedTags:
     Workflow.RAREDISEASE: [
         [HermesFileTag.LONG_TERM_STORAGE],
     ],
+    Workflow.RAW_DATA: [],
     Workflow.RNAFUSION: [
         [HermesFileTag.LONG_TERM_STORAGE],
         [AnalysisTag.FUSION, AnalysisTag.ARRIBA],  # legacy

cg/meta/delivery/delivery.py

@@ -167,7 +167,7 @@ def get_fastq_delivery_files_by_sample(
     ) -> list[DeliveryFile]:
         """Return a list of FASTQ files to be delivered for a specific sample."""
         delivery_files: list[DeliveryFile] = []
-        fastq_tags: list[set[str]] = self.get_analysis_sample_tags_for_workflow(Workflow.FASTQ)
+        fastq_tags: list[set[str]] = self.get_analysis_sample_tags_for_workflow(Workflow.RAW_DATA)
         if not self.is_sample_deliverable(sample=sample, force=force):
             LOG.warning(f"Sample {sample.internal_id} is not deliverable")
             return delivery_files

cg/meta/transfer/external_data.py

@@ -8,13 +8,13 @@
 from cg.constants import HK_FASTQ_TAGS, FileExtensions
 from cg.constants.constants import CaseActions
 from cg.meta.meta import MetaAPI
+from cg.meta.transfer.utils import are_all_fastq_valid
+from cg.models.cg_config import CGConfig
+from cg.models.slurm.sbatch import Sbatch
 from cg.services.deliver_files.delivery_rsync_service.sbatch import (
     ERROR_RSYNC_FUNCTION,
     RSYNC_CONTENTS_COMMAND,
 )
-from cg.meta.transfer.utils import are_all_fastq_valid
-from cg.models.cg_config import CGConfig
-from cg.models.slurm.sbatch import Sbatch
 from cg.store.models import Case, Customer, Sample
 from cg.utils.files import get_files_matching_pattern
 
@@ -144,7 +144,9 @@ def _get_fastq_paths_to_add(self, sample_id: str) -> list[Path]:
         LOG.debug(f"Checking fastq files in {sample_folder}")
         file_paths: list[Path] = [
             sample_folder.joinpath(path)
-            for path in get_files_matching_pattern(directory=sample_folder, pattern="*.fastq.gz")
+            for path in get_files_matching_pattern(
+                directory=sample_folder, pattern=f"*{FileExtensions.FASTQ_GZ}"
+            )
         ]
         LOG.debug(f"Found {len(file_paths)} fastq files for sample {sample_id}")
         hk_version: Version = self.housekeeper_api.get_or_create_version(bundle_name=sample_id)