Merge branch 'master' into tomte_deliv_v3

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 65.0.1
+current_version = 65.0.11
 commit = True
 tag = True
 tag_name = v{new_version}

Dockerfile

@@ -23,14 +23,16 @@ ENV TRAILBLAZER_SERVICE_ACCOUNT_AUTH_FILE="auth_file"
 
 
 WORKDIR /home/src/app
-COPY pyproject.toml poetry.lock gunicorn.conf.py ./
+COPY pyproject.toml poetry.lock gunicorn.conf.py README.md ./
 
 RUN pip install --no-cache-dir poetry \
     && poetry config virtualenvs.create false \
-    && poetry install --no-interaction --no-ansi
+    && poetry install --no-interaction --no-ansi --no-root
 
 COPY cg ./cg
 
+RUN poetry install --no-interaction --no-ansi
+
 CMD gunicorn \
     --config gunicorn.conf.py \
     cg.server.auto:app

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "65.0.1"
+__version__ = "65.0.11"

cg/apps/housekeeper/hk.py

@@ -7,11 +7,7 @@
 
 from housekeeper.include import checksum as hk_checksum
 from housekeeper.include import include_version
-from housekeeper.store.database import (
-    create_all_tables,
-    drop_all_tables,
-    initialize_database,
-)
+from housekeeper.store.database import create_all_tables, drop_all_tables, initialize_database
 from housekeeper.store.models import Archive, Bundle, File, Tag, Version
 from housekeeper.store.store import Store
 from sqlalchemy.orm import Query
@@ -27,17 +23,13 @@
 
 
 class HousekeeperAPI:
-    """API to decouple cg code from Housekeeper"""
+    """API to decouple cg code from Housekeeper."""
 
     def __init__(self, config: dict) -> None:
         initialize_database(config["housekeeper"]["database"])
         self._store = Store(config["housekeeper"]["root"])
         self.root_dir: str = config["housekeeper"]["root"]
 
-    def __getattr__(self, name):
-        LOG.warning(f"Called undefined {name} on {self.__class__.__name__}, please wrap")
-        return getattr(self._store, name)
-
     def new_bundle(self, name: str, created_at: datetime = None) -> Bundle:
         """Create a new file bundle."""
         return self._store.new_bundle(name, created_at)

cg/apps/tb/api.py

@@ -4,8 +4,8 @@
 import logging
 from typing import Any
 
-from google.auth import jwt
-from google.auth.crypt import RSASigner
+from google.auth.transport.requests import Request
+from google.oauth2 import service_account
 
 from cg.apps.tb.dto.create_job_request import CreateJobRequest
 from cg.apps.tb.dto.summary_response import AnalysisSummary, SummariesResponse
@@ -50,10 +50,12 @@ def __init__(self, config: dict):
 
     @property
     def auth_header(self) -> dict:
-        signer = RSASigner.from_service_account_file(self.service_account_auth_file)
-        payload = {"email": self.service_account}
-        jwt_token = jwt.encode(signer=signer, payload=payload).decode("ascii")
-        return {"Authorization": f"Bearer {jwt_token}"}
+        credentials = service_account.IDTokenCredentials.from_service_account_file(
+            filename=self.service_account_auth_file,
+            target_audience="trailblazer",
+        )
+        credentials.refresh(Request())
+        return {"Authorization": f"Bearer {credentials.token}"}
 
     def query_trailblazer(
         self, command: str, request_body: dict, method: str = APIMethods.POST

cg/cli/workflow/base.py

@@ -14,6 +14,7 @@
 from cg.cli.workflow.mip_dna.base import mip_dna
 from cg.cli.workflow.mip_rna.base import mip_rna
 from cg.cli.workflow.mutant.base import mutant
+from cg.cli.workflow.nallo.base import nallo
 from cg.cli.workflow.raredisease.base import raredisease
 from cg.cli.workflow.rnafusion.base import rnafusion
 from cg.cli.workflow.taxprofiler.base import taxprofiler
@@ -27,16 +28,17 @@ def workflow():
 
 workflow.add_command(balsamic)
 workflow.add_command(balsamic_qc)
-workflow.add_command(balsamic_umi)
 workflow.add_command(balsamic_pon)
+workflow.add_command(balsamic_umi)
+workflow.add_command(fluffy)
+workflow.add_command(jasen)
 workflow.add_command(microsalt)
 workflow.add_command(mip_dna)
 workflow.add_command(mip_rna)
-workflow.add_command(fluffy)
-workflow.add_command(jasen)
 workflow.add_command(mutant)
+workflow.add_command(nallo)
 workflow.add_command(raredisease)
+workflow.add_command(raw_data)
 workflow.add_command(rnafusion)
 workflow.add_command(taxprofiler)
 workflow.add_command(tomte)
-workflow.add_command(raw_data)

cg/cli/workflow/nallo/__init__.py

@@ -0,0 +1 @@
+"""Init for module"""

cg/cli/workflow/nallo/base.py

@@ -0,0 +1,20 @@
+"""CLI support to create config and/or start NALLO."""
+
+import logging
+
+import rich_click as click
+
+from cg.cli.utils import CLICK_CONTEXT_SETTINGS
+from cg.constants.constants import MetaApis
+from cg.meta.workflow.analysis import AnalysisAPI
+from cg.meta.workflow.nallo import NalloAnalysisAPI
+
+LOG = logging.getLogger(__name__)
+
+
+@click.group(invoke_without_command=True, context_settings=CLICK_CONTEXT_SETTINGS)
+@click.pass_context
+def nallo(context: click.Context) -> None:
+    """GMS/Nallo analysis workflow."""
+    AnalysisAPI.get_help(context)
+    context.obj.meta_apis[MetaApis.ANALYSIS_API] = NalloAnalysisAPI(config=context.obj)

cg/constants/nf_analysis.py

@@ -51,7 +51,7 @@ class NfTowerStatus(StrEnum):
         memory = { 4.GB * task.attempt }
         time   = { 4.h  * task.attempt }
         cpus = 2
-        ext.args = ' --data-format json '
+        ext.args = ' --data-format json --cl-config "max_table_rows: 10000" '
     }
 }
 """

cg/meta/upload/scout/uploadscoutapi.py

@@ -398,7 +398,10 @@ def upload_omics_sample_id_to_scout(
                 )
 
     def upload_rna_fraser_outrider_to_scout(
-        self, dry_run: bool, case_id: str, rna_dna_collections: list[RNADNACollection], cust_id: str
+        self,
+        dry_run: bool,
+        case_id: str,
+        rna_dna_collections: list[RNADNACollection],
     ) -> None:
         """Upload omics fraser and outrider file for a case to Scout."""
         status_db: Store = self.status_db
@@ -430,7 +433,7 @@ def upload_rna_fraser_outrider_to_scout(
                     outrider_file_path=rna_outrider.full_path,
                     case_id=dna_case_id,
                     customer_case_name=customer_case.name,
-                    cust_id=cust_id,
+                    cust_id=customer_case.customer.internal_id,
                 )
         for upload_statement in self.get_rna_fraser_outrider_upload_summary(rna_dna_collections):
             LOG.info(upload_statement)
@@ -441,7 +444,6 @@ def upload_rna_genome_build_to_scout(
         dry_run: bool,
         rna_case: str,
         rna_dna_collections: list[RNADNACollection],
-        cust_id: str,
     ) -> None:
         """Upload RNA genome built for a RNA/DNA case to Scout."""
         status_db: Store = self.status_db
@@ -463,7 +465,7 @@ def upload_rna_genome_build_to_scout(
                 self.scout_api.upload_rna_genome_build(
                     case_id=dna_case_id,
                     customer_case_name=customer_case.name,
-                    cust_id=cust_id,
+                    cust_id=customer_case.customer.internal_id,
                     rna_genome_build=rna_genome_build,
                 )
 
@@ -614,21 +616,18 @@ def upload_rna_omics_to_scout(self, dry_run: bool, case_id: str) -> None:
         status_db: Store = self.status_db
         rna_case = status_db.get_case_by_internal_id(case_id)
         rna_dna_collections: list[RNADNACollection] = self.create_rna_dna_collections(rna_case)
-        cust_id: str = rna_case.customer.internal_id
         self.upload_omics_sample_id_to_scout(
             dry_run=dry_run, rna_dna_collections=rna_dna_collections
         )
         self.upload_rna_fraser_outrider_to_scout(
             dry_run=dry_run,
             case_id=case_id,
             rna_dna_collections=rna_dna_collections,
-            cust_id=cust_id,
         )
         self.upload_rna_genome_build_to_scout(
             dry_run=dry_run,
             rna_case=rna_case,
             rna_dna_collections=rna_dna_collections,
-            cust_id=cust_id,
         )
         self.load_rna_variant_outlier_to_scout(
             dry_run=dry_run, rna_dna_collections=rna_dna_collections

cg/meta/workflow/nallo.py

@@ -0,0 +1,20 @@
+"""Module for Nallo Analysis API."""
+
+import logging
+from cg.constants import Workflow
+from cg.meta.workflow.nf_analysis import NfAnalysisAPI
+from cg.models.cg_config import CGConfig
+
+LOG = logging.getLogger(__name__)
+
+
+class NalloAnalysisAPI(NfAnalysisAPI):
+    """Handles communication between Nallo processes
+    and the rest of CG infrastructure."""
+
+    def __init__(
+        self,
+        config: CGConfig,
+        workflow: Workflow = Workflow.NALLO,
+    ):
+        super().__init__(config=config, workflow=workflow)

cg/meta/workflow/nf_analysis.py

@@ -63,7 +63,7 @@ def __init__(self, config: CGConfig, workflow: Workflow):
         self.conda_binary: str | None = None
         self.platform: str | None = None
         self.params: str | None = None
-        self.config: str | None = None
+        self.workflow_config_path: str | None = None
         self.resources: str | None = None
         self.tower_binary_path: str | None = None
         self.tower_workflow: str | None = None
@@ -136,7 +136,7 @@ def get_nextflow_config_content(self, case_id: str) -> str:
         """Return nextflow config content."""
         config_files_list: list[str] = [
             self.platform,
-            self.config,
+            self.workflow_config_path,
             self.resources,
         ]
         extra_parameters_str: list[str] = [

cg/meta/workflow/raredisease.py

@@ -56,7 +56,7 @@ def __init__(
         self.conda_binary: str = config.raredisease.conda_binary
         self.platform: str = config.raredisease.platform
         self.params: str = config.raredisease.params
-        self.config: str = config.raredisease.config
+        self.workflow_config_path: str = config.raredisease.config
         self.resources: str = config.raredisease.resources
         self.tower_binary_path: str = config.tower_binary_path
         self.tower_workflow: str = config.raredisease.tower_workflow

cg/meta/workflow/rnafusion.py

@@ -35,7 +35,7 @@ def __init__(
         self.conda_binary: str = config.rnafusion.conda_binary
         self.platform: str = config.rnafusion.platform
         self.params: str = config.rnafusion.params
-        self.config: str = config.rnafusion.config
+        self.workflow_config_path: str = config.rnafusion.config
         self.resources: str = config.rnafusion.resources
         self.tower_binary_path: str = config.tower_binary_path
         self.tower_workflow: str = config.rnafusion.tower_workflow

cg/meta/workflow/tomte.py

@@ -31,7 +31,7 @@ def __init__(
         self.conda_binary: str = config.tomte.conda_binary
         self.platform: str = config.tomte.platform
         self.params: str = config.tomte.params
-        self.config: str = config.tomte.config
+        self.workflow_config_path: str = config.tomte.config
         self.resources: str = config.tomte.resources
         self.tower_binary_path: str = config.tower_binary_path
         self.tower_workflow: str = config.tomte.tower_workflow

cg/services/orders/store_order_services/constants.py

@@ -0,0 +1,2 @@
+# Hardcoded constant for special order to assign MAF cases to
+MAF_ORDER_ID: int = 12377

cg/services/orders/store_order_services/store_fastq_order_service.py

@@ -8,6 +8,7 @@
 from cg.models.orders.order import OrderIn
 from cg.models.orders.sample_base import StatusEnum
 from cg.services.orders.order_lims_service.order_lims_service import OrderLimsService
+from cg.services.orders.store_order_services.constants import MAF_ORDER_ID
 from cg.services.orders.submitters.order_submitter import StoreOrderService
 from cg.store.models import ApplicationVersion, Case, CaseSample, Customer, Order, Sample
 from cg.store.store import Store
@@ -62,12 +63,14 @@ def order_to_status(order: OrderIn) -> dict:
         }
         return status_data
 
-    def create_maf_case(self, sample_obj: Sample, order: Order) -> None:
+    def create_maf_case(self, sample_obj: Sample, order: Order, case: Case) -> None:
         """Add a MAF case to the Status database."""
+        maf_order = self.status_db.get_order_by_id(MAF_ORDER_ID)
         case: Case = self.status_db.add_case(
+            comment=f"MAF case for {case.internal_id} original order id {order.id}",
             data_analysis=Workflow(Workflow.MIP_DNA),
             data_delivery=DataDelivery(DataDelivery.NO_DELIVERY),
-            name="_".join([sample_obj.name, "MAF"]),
+            name="_".join([sample_obj.internal_id, "MAF"]),
             panels=[GenePanelMasterList.OMIM_AUTO],
             priority=Priority.research,
             ticket=sample_obj.original_ticket,
@@ -78,7 +81,7 @@ def create_maf_case(self, sample_obj: Sample, order: Order) -> None:
         relationship: CaseSample = self.status_db.relate_sample(
             case=case, sample=sample_obj, status=StatusEnum.unknown
         )
-        order.cases.append(case)
+        maf_order.cases.append(case)
         self.status_db.session.add_all([case, relationship])
 
     def store_items_in_status(
@@ -137,7 +140,7 @@ def store_items_in_status(
                     not new_sample.is_tumour
                     and new_sample.prep_category == SeqLibraryPrepCategory.WHOLE_GENOME_SEQUENCING
                 ):
-                    self.create_maf_case(sample_obj=new_sample, order=status_db_order)
+                    self.create_maf_case(sample_obj=new_sample, order=status_db_order, case=case)
                 case.customer = customer
                 new_relationship = self.status_db.relate_sample(
                     case=case, sample=new_sample, status=StatusEnum.unknown

cg/store/crud/create.py

@@ -246,11 +246,13 @@ def add_case(
         priority: Priority | None = Priority.standard,
         synopsis: str | None = None,
         customer_id: int | None = None,
+        comment: str | None = None,
     ) -> Case:
         """Build a new Case record."""
 
         internal_id: str = self.generate_readable_case_id()
         return Case(
+            comment=comment,
             cohorts=cohorts,
             data_analysis=str(data_analysis),
             data_delivery=str(data_delivery),

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "cg"
-version = "65.0.1"
+version = "65.0.11"
 description = "Clinical Genomics command center"
 authors = ["Clinical Genomics <[email protected]>"]
 readme = "README.md"

tests/cli/workflow/nf_analysis/test_cli_workflow_base.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.parametrize(
     "workflow",
-    NEXTFLOW_WORKFLOWS + [Workflow.JASEN],
+    NEXTFLOW_WORKFLOWS + [Workflow.JASEN] + [Workflow.NALLO],
 )
 def test_workflow_no_args(cli_runner: CliRunner, workflow: Workflow, request):
     """Test to see that workflow is added and prints help when no subcommand is specified."""