Merge branch 'add-metrics-report' of github.com:Clinical-Genomics/cg …

…into add-metrics-report

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 52.1.5
+current_version = 53.0.0
 commit = True
 tag = True
 tag_name = v{new_version}

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "52.1.5"
+__version__ = "53.0.0"

cg/cli/backup.py

@@ -61,7 +61,7 @@ def backup_flow_cells(context: CGConfig, dry_run: bool):
         flow_cell_encryption_api = FlowCellEncryptionAPI(
             binary_path=context.encryption.binary_path,
             dry_run=dry_run,
-            encryption_dir=Path(context.backup.encryption_directories.current),
+            encryption_dir=Path(context.encryption.encryption_dir),
             flow_cell=flow_cell,
             pigz_binary_path=context.pigz.binary_path,
             slurm_api=SlurmAPI(),
@@ -102,7 +102,7 @@ def encrypt_flow_cells(context: CGConfig, dry_run: bool):
         flow_cell_encryption_api = FlowCellEncryptionAPI(
             binary_path=context.encryption.binary_path,
             dry_run=dry_run,
-            encryption_dir=Path(context.backup.encryption_directories.current),
+            encryption_dir=Path(context.encryption.encryption_dir),
             flow_cell=flow_cell,
             pigz_binary_path=context.pigz.binary_path,
             slurm_api=SlurmAPI(),
@@ -128,7 +128,7 @@ def fetch_flow_cell(context: CGConfig, dry_run: bool, flow_cell_id: Optional[str
     tar_api = TarAPI(binary_path=context.tar.binary_path, dry_run=dry_run)
     context.meta_apis["backup_api"] = BackupAPI(
         encryption_api=encryption_api,
-        encryption_directories=context.backup.encryption_directories,
+        pdc_archiving_directory=context.backup.pdc_archiving_directory,
         status=context.status_db,
         tar_api=tar_api,
         pdc_api=pdc_api,

cg/meta/backup/backup.py

@@ -5,7 +5,6 @@
 from typing import Optional
 
 from housekeeper.store.models import File
-from sqlalchemy.exc import OperationalError
 
 from cg.apps.housekeeper.hk import HousekeeperAPI
 from cg.constants.backup import MAX_PROCESSING_FLOW_CELLS
@@ -18,7 +17,7 @@
 from cg.meta.encryption.encryption import EncryptionAPI, SpringEncryptionAPI
 from cg.meta.tar.tar import TarAPI
 from cg.models import CompressionData
-from cg.models.cg_config import EncryptionDirectories
+from cg.models.cg_config import PDCArchivingDirectory
 from cg.store import Store
 from cg.store.models import Flowcell
 from cg.utils.time import get_elapsed_time, get_start_time
@@ -32,15 +31,15 @@ class BackupAPI:
     def __init__(
         self,
         encryption_api: EncryptionAPI,
-        encryption_directories: EncryptionDirectories,
+        pdc_archiving_directory: PDCArchivingDirectory,
         status: Store,
         tar_api: TarAPI,
         pdc_api: PdcAPI,
         flow_cells_dir: str,
         dry_run: bool = False,
     ):
         self.encryption_api = encryption_api
-        self.encryption_directories: EncryptionDirectories = encryption_directories
+        self.pdc_archiving_directory: PDCArchivingDirectory = pdc_archiving_directory
         self.status: Store = status
         self.tar_api: TarAPI = tar_api
         self.pdc: PdcAPI = pdc_api
@@ -237,7 +236,7 @@ def query_pdc_for_flow_cell(self, flow_cell_id: str) -> list[str]:
         Raise:
             PdcNoFilesMatchingSearchError if no files are found.
         """
-        for _, encryption_directory in self.encryption_directories:
+        for _, encryption_directory in self.pdc_archiving_directory:
             search_pattern = f"{encryption_directory}*{flow_cell_id}*{FileExtensions.GPG}"
             self.pdc.query_pdc(search_pattern)
             if self.pdc.was_file_found(self.pdc.process.stderr):

cg/meta/backup/pdc.py

@@ -43,7 +43,6 @@ def validate_is_dsmc_running(cls) -> bool:
         dsmc_process_count: int = 0
         try:
             for process in psutil.process_iter():
-                LOG.debug(process.name())
                 if "dsmc" == process.name():
                     dsmc_process_count += 1
         except Exception as error:
@@ -115,8 +114,9 @@ def backup_flow_cell(
         for encrypted_file in files_to_archive:
             if not self.dry_run:
                 self.archive_file_to_pdc(file_path=encrypted_file.as_posix())
-                store.update_flow_cell_has_backup(flow_cell=db_flow_cell, has_backup=True)
-                LOG.info(f"Flow cell: {db_flow_cell.name} has been backed up")
+        if not self.dry_run:
+            store.update_flow_cell_has_backup(flow_cell=db_flow_cell, has_backup=True)
+            LOG.info(f"Flow cell: {db_flow_cell.name} has been backed up")
 
     def start_flow_cell_backup(
         self,

cg/models/cg_config.py

@@ -42,14 +42,19 @@ class SlurmConfig(BaseModel):
     qos: SlurmQos = SlurmQos.LOW
 
 
-class EncryptionDirectories(BaseModel):
+class Encryption(BaseModel):
+    encryption_dir: str
+    binary_path: str
+
+
+class PDCArchivingDirectory(BaseModel):
     current: str
     nas: str
     pre_nas: str
 
 
 class BackupConfig(BaseModel):
-    encryption_directories: EncryptionDirectories
+    pdc_archiving_directory: PDCArchivingDirectory
     slurm_flow_cell_encryption: SlurmConfig
 
 
@@ -228,15 +233,15 @@ class DataFlowConfig(BaseModel):
 
 class CGConfig(BaseModel):
     database: str
-    environment: Literal["production", "stage"] = "stage"
-    madeline_exe: str
     delivery_path: str
-    max_flowcells: Optional[int]
-    email_base_settings: EmailBaseSettings
-    flow_cells_dir: str
     demultiplexed_flow_cells_dir: str
     downsample_dir: str
     downsample_script: str
+    email_base_settings: EmailBaseSettings
+    environment: Literal["production", "stage"] = "stage"
+    flow_cells_dir: str
+    madeline_exe: str
+    max_flowcells: Optional[int]
     # Base APIs that always should exist
     status_db_: Store = None
     housekeeper: HousekeeperConfig
@@ -252,7 +257,7 @@ class CGConfig(BaseModel):
     data_flow_config: Optional[DataFlowConfig] = None
     demultiplex: DemultiplexConfig = None
     demultiplex_api_: DemultiplexingAPI = None
-    encryption: Optional[CommonAppConfig] = None
+    encryption: Encryption | None = None
     external: ExternalConfig = None
     genotype: CommonAppConfig = None
     genotype_api_: GenotypeAPI = None

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "cg"
-version = "52.1.5"
+version = "53.0.0"
 description = "Clinical Genomics command center"
 authors = ["Clinical Genomics <[email protected]>"]
 readme = "README.md"

tests/cli/backup/conftest.py

@@ -10,7 +10,7 @@
 def backup_context(cg_context: CGConfig) -> CGConfig:
     cg_context.meta_apis["backup_api"] = BackupAPI(
         encryption_api=EncryptionAPI(binary_path=cg_context.encryption.binary_path),
-        encryption_directories=cg_context.backup.encryption_directories,
+        pdc_archiving_directory=cg_context.backup.pdc_archiving_directory,
         status=cg_context.status_db,
         tar_api=TarAPI(binary_path=cg_context.tar.binary_path),
         pdc_api=cg_context.pdc_api,

tests/cli/backup/test_backup_command.py

@@ -30,9 +30,9 @@ def test_backup_flow_cells(
     )
 
     # GIVEN an encrypted flow cell
-    flow_cells_dir = Path(cg_context.backup.encryption_directories.current, flow_cell_full_name)
-    flow_cells_dir.mkdir(parents=True, exist_ok=True)
-    Path(flow_cells_dir, flow_cell_name).with_suffix(FileExtensions.COMPLETE).touch()
+    flow_cells_encrypt_dir = Path(cg_context.encryption.encryption_dir, flow_cell_full_name)
+    flow_cells_encrypt_dir.mkdir(parents=True, exist_ok=True)
+    Path(flow_cells_encrypt_dir, flow_cell_name).with_suffix(FileExtensions.COMPLETE).touch()
 
     # WHEN backing up flow cells in dry run mode
     result = cli_runner.invoke(backup_flow_cells, ["--dry-run"], obj=cg_context)
@@ -192,7 +192,7 @@ def test_encrypt_flow_cell_when_encryption_already_started(
     cli_runner: CliRunner,
     cg_context: CGConfig,
     caplog,
-    encryption_dir: Path,
+    pdc_archiving_dir: Path,
     flow_cell_name: str,
     flow_cell_full_name: str,
     mocker,
@@ -205,9 +205,9 @@ def test_encrypt_flow_cell_when_encryption_already_started(
     FlowCellDirectoryData.is_flow_cell_ready.return_value = True
 
     # GIVEN a pending flag file
-    flow_cells_dir = Path(cg_context.backup.encryption_directories.current, flow_cell_full_name)
-    flow_cells_dir.mkdir(parents=True, exist_ok=True)
-    Path(flow_cells_dir, flow_cell_name).with_suffix(FileExtensions.PENDING).touch()
+    flow_cells_encrypt_dir = Path(cg_context.encryption.encryption_dir, flow_cell_full_name)
+    flow_cells_encrypt_dir.mkdir(parents=True, exist_ok=True)
+    Path(flow_cells_encrypt_dir, flow_cell_name).with_suffix(FileExtensions.PENDING).touch()
 
     # GIVEN a flow cells directory
 
@@ -225,7 +225,7 @@ def test_encrypt_flow_cell_when_encryption_already_completed(
     cli_runner: CliRunner,
     cg_context: CGConfig,
     caplog,
-    encryption_dir: Path,
+    pdc_archiving_dir: Path,
     flow_cell_name: str,
     flow_cell_full_name: str,
     mocker,
@@ -238,9 +238,9 @@ def test_encrypt_flow_cell_when_encryption_already_completed(
     FlowCellDirectoryData.is_flow_cell_ready.return_value = True
 
     # GIVEN a complete flag file
-    flow_cells_dir = Path(cg_context.backup.encryption_directories.current, flow_cell_full_name)
-    flow_cells_dir.mkdir(parents=True, exist_ok=True)
-    Path(flow_cells_dir, flow_cell_name).with_suffix(FileExtensions.COMPLETE).touch()
+    flow_cells_encrypt_dir = Path(cg_context.encryption.encryption_dir, flow_cell_full_name)
+    flow_cells_encrypt_dir.mkdir(parents=True, exist_ok=True)
+    Path(flow_cells_encrypt_dir, flow_cell_name).with_suffix(FileExtensions.COMPLETE).touch()
 
     # GIVEN a flow cells directory
 

tests/conftest.py

@@ -41,15 +41,18 @@
 from cg.meta.workflow.rnafusion import RnafusionAnalysisAPI
 from cg.meta.workflow.taxprofiler import TaxprofilerAnalysisAPI
 from cg.models import CompressionData
-from cg.models.cg_config import CGConfig, EncryptionDirectories
-from cg.models.demultiplex.run_parameters import RunParametersNovaSeq6000, RunParametersNovaSeqX
+from cg.models.cg_config import CGConfig, PDCArchivingDirectory
+from cg.models.demultiplex.run_parameters import (
+    RunParametersNovaSeq6000,
+    RunParametersNovaSeqX,
+)
 from cg.models.downsample.downsample_data import DownsampleData
 from cg.models.flow_cell.flow_cell import FlowCellDirectoryData
 from cg.models.rnafusion.rnafusion import RnafusionParameters
 from cg.models.taxprofiler.taxprofiler import TaxprofilerParameters
 from cg.store import Store
 from cg.store.database import create_all_tables, drop_all_tables, initialize_database
-from cg.store.models import Bed, BedVersion, Customer, Case, Organism, Sample
+from cg.store.models import Bed, BedVersion, Case, Customer, Organism, Sample
 from cg.utils import Process
 from tests.mocks.crunchy import MockCrunchyAPI
 from tests.mocks.hk_mock import MockHousekeeperAPI
@@ -2158,16 +2161,16 @@ def microsalt_dir(tmpdir_factory) -> Path:
 
 
 @pytest.fixture
-def encryption_dir(tmp_flow_cells_directory: Path) -> Path:
-    """Return a temporary directory for encryption testing."""
-    return Path(tmp_flow_cells_directory, "encrypt")
+def pdc_archiving_dir(tmp_flow_cells_directory: Path) -> Path:
+    """Return a temporary directory for PDC archiving testing."""
+    return Path(tmp_flow_cells_directory, "encrypt", "*")
 
 
 @pytest.fixture
-def encryption_directories(encryption_dir: Path) -> EncryptionDirectories:
-    """Returns different encryption directories."""
-    return EncryptionDirectories(
-        current=f"/{encryption_dir.as_posix()}/", nas="/ENCRYPT/", pre_nas="/OLD_ENCRYPT/"
+def pdc_archiving_directory(pdc_archiving_dir: Path) -> PDCArchivingDirectory:
+    """Returns different PDC archiving directories."""
+    return PDCArchivingDirectory(
+        current=f"/{pdc_archiving_dir.as_posix()}/", nas="/ENCRYPT/", pre_nas="/OLD_ENCRYPT/"
     )
 
 
@@ -2205,7 +2208,7 @@ def context_config(
     flow_cells_dir: Path,
     demultiplexed_runs: Path,
     downsample_dir: Path,
-    encryption_directories: EncryptionDirectories,
+    pdc_archiving_directory: PDCArchivingDirectory,
 ) -> dict:
     """Return a context config."""
     return {
@@ -2224,7 +2227,7 @@ def context_config(
         "madeline_exe": "echo",
         "pon_path": str(cg_dir),
         "backup": {
-            "encryption_directories": encryption_directories.dict(),
+            "pdc_archiving_directory": pdc_archiving_directory.dict(),
             "slurm_flow_cell_encryption": {
                 "account": "development",
                 "hours": 1,
@@ -2277,7 +2280,10 @@ def context_config(
                 "mail_user": email_address,
             },
         },
-        "encryption": {"binary_path": "bin/gpg"},
+        "encryption": {
+            "binary_path": "bin/gpg",
+            "encryption_dir": pdc_archiving_directory.current,
+        },
         "external": {
             "caesar": "server.name.se:/path/%s/on/caesar",
             "hasta": "/path/on/hasta/%s",
@@ -3260,7 +3266,7 @@ def flow_cell_encryption_api(
 ) -> FlowCellEncryptionAPI:
     flow_cell_encryption_api = FlowCellEncryptionAPI(
         binary_path=cg_context.encryption.binary_path,
-        encryption_dir=Path(cg_context.backup.encryption_directories.current),
+        encryption_dir=Path(cg_context.backup.pdc_archiving_directory.current),
         dry_run=True,
         flow_cell=FlowCellDirectoryData(
             flow_cell_path=Path(cg_context.flow_cells_dir, flow_cell_full_name)
@@ -3284,7 +3290,7 @@ def store_with_case_and_sample_with_reads(
     downsample_sample_internal_id_2: str,
 ) -> Store:
     """Return a store with a case and a sample with reads."""
-    case: Family = helpers.add_case(
+    case: Case = helpers.add_case(
         store=store, internal_id=downsample_case_internal_id, name=downsample_case_internal_id
     )
 

tests/meta/backup/conftest.py

@@ -6,7 +6,7 @@
 import pytest
 
 from cg.constants import FileExtensions
-from cg.models.cg_config import EncryptionDirectories
+from cg.models.cg_config import PDCArchivingDirectory
 
 
 @pytest.fixture
@@ -42,12 +42,12 @@ def dsmc_q_archive_output() -> list[str]:
 
 
 @pytest.fixture()
-def archived_flow_cells(encryption_directories: EncryptionDirectories) -> list[str]:
+def archived_flow_cells(pdc_archiving_directory: PDCArchivingDirectory) -> list[str]:
     """Returns a list of archived flow cells."""
     return [
-        f"{encryption_directories.current}/new_flow_cell{FileExtensions.TAR}{FileExtensions.GZIP}{FileExtensions.GPG}",
-        f"{encryption_directories.nas}/old_flow_cell{FileExtensions.TAR}{FileExtensions.GZIP}{FileExtensions.GPG}",
-        f"{encryption_directories.pre_nas}/ancient_flow_cell{FileExtensions.TAR}{FileExtensions.GZIP}{FileExtensions.GPG}",
+        f"{pdc_archiving_directory.current}/new_flow_cell{FileExtensions.TAR}{FileExtensions.GZIP}{FileExtensions.GPG}",
+        f"{pdc_archiving_directory.nas}/old_flow_cell{FileExtensions.TAR}{FileExtensions.GZIP}{FileExtensions.GPG}",
+        f"{pdc_archiving_directory.pre_nas}/ancient_flow_cell{FileExtensions.TAR}{FileExtensions.GZIP}{FileExtensions.GPG}",
     ]