Merge branch 'add_tomte_scout_upload_to_dna_case' of https://github.c…

…om/Clinical-Genomics/cg into add_tomte_scout_upload_to_dna_case

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 62.2.1
+current_version = 62.2.4
 commit = True
 tag = True
 tag_name = v{new_version}

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "62.2.1"
+__version__ = "62.2.4"

cg/apps/invoice/render.py

@@ -1,8 +1,11 @@
 import datetime as dt
+from pathlib import Path
 
 from openpyxl import Workbook, load_workbook
 from openpyxl.styles import Border, Font, PatternFill, Side
-from pkg_resources import resource_filename
+
+from cg.constants import FileExtensions
+from cg.utils.files import get_project_root_dir
 
 
 def render_xlsx(data: dict) -> Workbook:
@@ -34,11 +37,17 @@ def render_xlsx(data: dict) -> Workbook:
             }]
         }
     """
-    pkg_dir = __name__.rpartition(".")[0]
+    project_root_dir = get_project_root_dir()
     sample_type = "pool" if data["pooled_samples"] else "sample"
     costcenter = data["cost_center"]
-    template_path = resource_filename(pkg_dir, f"templates/{costcenter}_{sample_type}_invoice.xlsx")
-    workbook = load_workbook(template_path)
+    template_path = Path(
+        project_root_dir,
+        "apps",
+        "invoice",
+        "templates",
+        f"{costcenter}_{sample_type}_invoice{FileExtensions.XLSX}",
+    )
+    workbook = load_workbook(template_path.as_posix())
     if data["pooled_samples"]:
         worksheet = workbook["Bilaga Prover"]
         worksheet["C1"] = costcenter.upper()

cg/cli/post_process/post_process.py

@@ -35,5 +35,4 @@ def post_process_sequencing_run(context: CGConfig, run_name: str, dry_run: bool)
     post_processing_service.post_process(run_name=run_name, dry_run=dry_run)
 
 
-post_process_group: click.Group
 post_process_group.add_command(post_process_sequencing_run)

cg/constants/constants.py

@@ -223,6 +223,7 @@ class FileExtensions(StrEnum):
     TSV: str = ".tsv"
     TXT: str = ".txt"
     VCF: str = ".vcf"
+    XLSX: str = ".xlsx"
     XML: str = ".xml"
     YAML: str = ".yaml"
 

cg/constants/report.py

@@ -1,15 +1,17 @@
 """Delivery report constants."""
 
-from importlib.resources import files
 from pathlib import Path
 
 from cg.constants import DataDelivery
 from cg.constants.constants import CancerAnalysisType, FileExtensions, Workflow
 from cg.constants.subject import Sex
+from cg.utils.files import get_project_root_dir
+
+project_root_dir: Path = get_project_root_dir()
 
 DELIVERY_REPORT_FILE_NAME: str = f"delivery-report{FileExtensions.HTML}"
 SWEDAC_LOGO_PATH = Path(
-    files("cg"), "meta", "report", "templates", "static", "images", "SWEDAC_logo.png"
+    project_root_dir, "meta", "report", "templates", "static", "images", "SWEDAC_logo.png"
 )
 
 BALSAMIC_REPORT_ACCREDITED_PANELS: list[str] = ["gmsmyeloid"]

cg/resources/__init__.py

@@ -1,8 +1,9 @@
 from pathlib import Path
 
-import pkg_resources
-
 from cg.constants import FileExtensions
+from cg.utils.files import get_project_root_dir
+
+project_root_dir: Path = get_project_root_dir()
 
 RAREDISEASE_BUNDLE_FILENAMES: str = (
     Path("resources", "raredisease_bundle_filenames").with_suffix(FileExtensions.YAML).as_posix()
@@ -20,18 +21,10 @@
     Path("resources", "tomte_bundle_filenames").with_suffix(FileExtensions.YAML).as_posix()
 )
 
-RAREDISEASE_BUNDLE_FILENAMES_PATH = Path(
-    pkg_resources.resource_filename("cg", RAREDISEASE_BUNDLE_FILENAMES)
-)
+RAREDISEASE_BUNDLE_FILENAMES_PATH = Path(project_root_dir, RAREDISEASE_BUNDLE_FILENAMES)
 
-RNAFUSION_BUNDLE_FILENAMES_PATH: Path = Path(
-    pkg_resources.resource_filename("cg", RNAFUSION_BUNDLE_FILENAMES)
-)
+RNAFUSION_BUNDLE_FILENAMES_PATH = Path(project_root_dir, RNAFUSION_BUNDLE_FILENAMES)
 
-TAXPROFILER_BUNDLE_FILENAMES_PATH: Path = Path(
-    pkg_resources.resource_filename("cg", TAXPROFILER_BUNDLE_FILENAMES)
-)
+TAXPROFILER_BUNDLE_FILENAMES_PATH = Path(project_root_dir, TAXPROFILER_BUNDLE_FILENAMES)
 
-TOMTE_BUNDLE_FILENAMES_PATH: Path = Path(
-    pkg_resources.resource_filename("cg", TOMTE_BUNDLE_FILENAMES)
-)
+TOMTE_BUNDLE_FILENAMES_PATH = Path(project_root_dir, TOMTE_BUNDLE_FILENAMES)

cg/server/admin.py

@@ -551,7 +551,13 @@ class SampleView(BaseView):
         "last_sequenced_at",
         "sex",
     ]
-    column_filters = ["customer.internal_id", "priority", "sex", "application_version.application"]
+    column_filters = [
+        "customer.internal_id",
+        "priority",
+        "sex",
+        "application_version.application",
+        "capture_kit",
+    ]
     column_formatters = {
         "is_external": is_external_application,
         "internal_id": view_case_sample_link,

cg/server/invoices/views.py

@@ -197,7 +197,7 @@ def invoice_template(invoice_id):
     workbook = render_xlsx(invoice_dict)
 
     temp_dir = tempfile.gettempdir()
-    filename = "Invoice_{}_{}.xlsx".format(invoice_obj.id, cost_center)
+    filename = f"Invoice_{invoice_obj.id}_{cost_center}.xlsx"
     excel_path = os.path.join(temp_dir, filename)
     workbook.save(excel_path)
 

cg/services/run_devices/exc.py

@@ -2,28 +2,42 @@
 
 
 class PostProcessingRunFileManagerError(CgError):
+    """Error raised if something goes wrong managing the sequencing run files."""
+
     pass
 
 
 class PostProcessingRunDataGeneratorError(CgError):
+    """Error raised if something goes wrong parsing the run directory data."""
+
     pass
 
 
 class PostProcessingParsingError(CgError):
+    """Error raised if something goes wrong parsing the sequencing run metrics."""
+
     pass
 
 
 class PostProcessingDataTransferError(CgError):
+    """Error raised if something goes wrong creating the DTOs for post-processing."""
+
     pass
 
 
 class PostProcessingStoreDataError(CgError):
+    """Error raised if something goes wrong storing the post-processing data in StatusDB."""
+
     pass
 
 
 class PostProcessingStoreFileError(CgError):
+    """Error raised if something goes wrong storing the post-processing files in Housekeeper."""
+
     pass
 
 
 class PostProcessingError(CgError):
+    """Error raised if something goes wrong during post-processing."""
+
     pass

cg/services/run_devices/pacbio/run_data_generator/pacbio_run_data_generator.py

@@ -1,16 +1,10 @@
 from pathlib import Path
 
 from cg.services.run_devices.abstract_classes import RunDataGenerator
-from cg.services.run_devices.error_handler import (
-    handle_post_processing_errors,
-)
+from cg.services.run_devices.error_handler import handle_post_processing_errors
 from cg.services.run_devices.exc import PostProcessingRunDataGeneratorError
 from cg.services.run_devices.pacbio.run_data_generator.run_data import PacBioRunData
-from cg.services.run_devices.validators import (
-    validate_has_expected_parts,
-    validate_name_pre_fix,
-)
-from cg.utils.string import get_element_from_split
+from cg.services.run_devices.validators import validate_has_expected_parts, validate_name_pre_fix
 
 
 class PacBioRunDataGenerator(RunDataGenerator):
@@ -40,16 +34,16 @@ def get_run_data(self, run_name: str, sequencing_dir: str) -> PacBioRunData:
 
     @staticmethod
     def _get_sequencing_run_name(run_name: str) -> str:
-        return get_element_from_split(value=run_name, element_position=0, split="/")
+        return run_name.split("/")[0]
 
     @staticmethod
     def _get_plate_well(run_name: str) -> str:
-        return get_element_from_split(value=run_name, element_position=-1, split="/")
+        return run_name.split("/")[1]
 
     def _get_plate(self, run_name: str) -> str:
         plate_well: str = self._get_plate_well(run_name)
-        return get_element_from_split(value=plate_well, element_position=0, split="_")
+        return plate_well.split("_")[0]
 
     def _get_well(self, run_name: str) -> str:
         plate_well: str = self._get_plate_well(run_name)
-        return get_element_from_split(value=plate_well, element_position=-1, split="_")
+        return plate_well.split("_")[-1]

cg/services/run_devices/pacbio/run_file_manager/run_file_manager.py

@@ -19,19 +19,15 @@ def get_files_to_parse(self, run_data: PacBioRunData) -> list[Path]:
         """Get the file paths required by the PacBioMetricsParser."""
         run_path: Path = run_data.full_path
         validate_files_or_directories_exist([run_path])
-        files_to_parse: list[Path] = self._get_report_files(run_path)
-        files_to_parse.append(self._get_ccs_report_file(run_path))
-        return files_to_parse
+        return self._get_report_files(run_path)
 
     @handle_post_processing_errors(
         to_except=(FileNotFoundError,), to_raise=PostProcessingRunFileManagerError
     )
     def get_files_to_store(self, run_data: PacBioRunData) -> list[Path]:
         """Get the files to store for the PostProcessingHKService."""
         run_path: Path = run_data.full_path
-        files_to_store: list[Path] = self.get_files_to_parse(run_data)
-        files_to_store.append(self._get_hifi_read_file(run_path))
-        return files_to_store
+        return self.get_files_to_parse(run_data) + self._get_hifi_read_files(run_path)
 
     @staticmethod
     def _get_ccs_report_file(run_path: Path) -> Path:
@@ -44,8 +40,7 @@ def _get_ccs_report_file(run_path: Path) -> Path:
             raise FileNotFoundError(f"No CCS report file found in {statistics_dir}")
         return files[0]
 
-    @staticmethod
-    def _get_report_files(run_path: Path) -> list[Path]:
+    def _get_report_files(self, run_path: Path) -> list[Path]:
         """Return the paths to the unzipped report files."""
         unzipped_dir: Path = Path(
             run_path, PacBioDirsAndFiles.STATISTICS_DIR, PacBioDirsAndFiles.UNZIPPED_REPORTS_DIR
@@ -55,16 +50,17 @@ def _get_report_files(run_path: Path) -> list[Path]:
             Path(unzipped_dir, PacBioDirsAndFiles.LOADING_REPORT),
             Path(unzipped_dir, PacBioDirsAndFiles.RAW_DATA_REPORT),
             Path(unzipped_dir, PacBioDirsAndFiles.SMRTLINK_DATASETS_REPORT),
+            self._get_ccs_report_file(run_path),
         ]
         validate_files_or_directories_exist(report_files)
         return report_files
 
     @staticmethod
-    def _get_hifi_read_file(run_path: Path) -> Path:
+    def _get_hifi_read_files(run_path: Path) -> list[Path]:
         """Return the path to the HiFi read file."""
         hifi_dir = Path(run_path, PacBioDirsAndFiles.HIFI_READS)
-        bam_file: Path = get_files_matching_pattern(
+        bam_files: list[Path] = get_files_matching_pattern(
             directory=hifi_dir, pattern=f"*{FileExtensions.BAM}"
-        )[0]
-        validate_files_or_directories_exist([bam_file])
-        return bam_file
+        )
+        validate_files_or_directories_exist(bam_files)
+        return bam_files

cg/utils/files.py

@@ -1,25 +1,29 @@
-"""Some helper functions for working with files"""
+"""Some helper functions for working with files."""
 
 import logging
 import os
 import shutil
+from importlib.resources import files
 from pathlib import Path
 
 LOG = logging.getLogger(__name__)
 
 
+def get_project_root_dir() -> Path:
+    return Path(files("cg"))
+
+
 def get_file_in_directory(directory: Path, file_name: str) -> Path:
     """Get a file in a directory and subdirectories.
     Raises:
         FileNotFoundError: If the directory does not exist.
     """
     if not directory.is_dir() or not directory.exists():
         raise FileNotFoundError(f"Directory {directory} does not exist")
-    for directory_path, _, files in os.walk(directory):
-        for file in files:
+    for directory_path, _, dir_files in os.walk(directory):
+        for file in dir_files:
             if file_name == file:
-                path_to_file = Path(directory_path, file)
-                return path_to_file
+                return Path(directory_path, file)
     raise FileNotFoundError(f"File {file_name} not found in {directory}")
 
 
@@ -43,10 +47,10 @@ def get_files_in_directory_with_pattern(directory: Path, pattern: str) -> list[P
     files_with_pattern: list[Path] = []
     if not directory.is_dir() or not directory.exists():
         raise FileNotFoundError(f"Directory {directory} does not exist")
-    for directory_path, _, files in os.walk(directory):
-        for file in files:
-            if pattern in file:
-                files_with_pattern.append(Path(directory_path, file))
+    for directory_path, _, dir_files in os.walk(directory):
+        files_with_pattern.extend(
+            Path(directory_path, file) for file in dir_files if pattern in file
+        )
     if not files_with_pattern:
         raise FileNotFoundError(f"No files with pattern {pattern} found in {directory}")
     return files_with_pattern
@@ -117,9 +121,9 @@ def link_or_overwrite_file(src: Path, dst: Path) -> None:
 def get_all_files_in_directory_tree(directory: Path) -> list[Path]:
     """Get the relative paths of all files in a directory and its subdirectories."""
     files_in_directory: list[Path] = []
-    for subdir, _, files in os.walk(directory):
+    for subdir, _, dir_files in os.walk(directory):
         subdir = Path(subdir).relative_to(directory)
-        files_in_directory.extend([Path(subdir, file) for file in files])
+        files_in_directory.extend([Path(subdir, file) for file in dir_files])
     return files_in_directory
 
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "cg"
-version = "62.2.1"
+version = "62.2.4"
 description = "Clinical Genomics command center"
 authors = ["Clinical Genomics <[email protected]>"]
 readme = "README.md"

tests/services/run_devices/pacbio/post_processing/test_post_processing.py

@@ -37,7 +37,7 @@ def test_pac_bio_post_processing_run_name_error(pac_bio_context):
 def test_pac_bio_post_processing_store_data_error(
     pac_bio_context: CGConfig, pac_bio_sequencing_run_name: str
 ):
-    # GIVEN a PacBioPostProcessingService and a wrong run name
+    # GIVEN a PacBioPostProcessingService that raises an error when storing data in StatusDB
 
     post_processing_service: PacBioPostProcessingService = (
         pac_bio_context.post_processing_services.pacbio
@@ -55,7 +55,7 @@ def test_pac_bio_post_processing_store_data_error(
 def test_pac_bio_post_processing_store_files_error(
     pac_bio_context: CGConfig, pac_bio_sequencing_run_name: str
 ):
-    # GIVEN a PacBioPostProcessingService
+    # GIVEN a PacBioPostProcessingService that raises an error when storing files in Housekeeper
     post_processing_service: PacBioPostProcessingService = (
         pac_bio_context.post_processing_services.pacbio
     )