From ad692bcdb8c3d92c63db512a79824b9a84ac77c1 Mon Sep 17 00:00:00 2001 From: Vadym Date: Tue, 9 Jan 2024 09:19:34 +0100 Subject: [PATCH 01/18] Make delivery report mandatory for Scout uploads (#2704) ### Changed * Delivery report mandatory for Scout uploads * Renamed function get_report_accreditation to is_report_accredited --- cg/constants/scout.py | 5 +++ cg/meta/report/balsamic.py | 4 +-- cg/meta/report/mip_dna.py | 4 +-- cg/meta/report/report_api.py | 6 ++-- cg/meta/report/rnafusion.py | 4 +-- .../upload/scout/balsamic_config_builder.py | 9 +++-- .../scout/balsamic_umi_config_builder.py | 16 ++++----- cg/meta/upload/scout/mip_config_builder.py | 15 ++++---- .../upload/scout/rnafusion_config_builder.py | 8 +++-- cg/meta/upload/scout/scout_config_builder.py | 12 +++---- cg/models/scout/scout_load_config.py | 8 +++-- tests/apps/scout/test_scout_load_config.py | 34 +++++++++++-------- tests/cli/upload/conftest.py | 10 ++---- tests/conftest.py | 9 ++++- tests/meta/report/test_balsamic_api.py | 10 +++--- tests/meta/report/test_mip_dna_api.py | 12 +++---- tests/meta/upload/scout/conftest.py | 20 ++++++++--- .../upload/scout/test_meta_upload_scoutapi.py | 12 ++++--- .../upload/scout/test_scout_config_builder.py | 14 -------- 19 files changed, 116 insertions(+), 96 deletions(-) diff --git a/cg/constants/scout.py b/cg/constants/scout.py index ef2bec36b9..39f22145a4 100644 --- a/cg/constants/scout.py +++ b/cg/constants/scout.py @@ -14,6 +14,11 @@ class ScoutExportFileName(StrEnum): PANELS: str = f"gene_panels{FileExtensions.BED}" +class UploadTrack(StrEnum): + RARE_DISEASE: str = "rare" + CANCER: str = "cancer" + + class ScoutCustomCaseReportTags(StrEnum): DELIVERY: str = "delivery_report" CNV: str = "cnv_report" diff --git a/cg/meta/report/balsamic.py b/cg/meta/report/balsamic.py index 25b496c512..e88f531b1f 100644 --- a/cg/meta/report/balsamic.py +++ b/cg/meta/report/balsamic.py @@ -156,10 +156,10 @@ def get_variant_caller_version(var_caller_name: str, var_caller_versions: dict) return versions[0] return None - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: BalsamicAnalysis ) -> bool: - """Checks if the report is accredited or not.""" + """Check if the Balsamic report is accredited.""" if analysis_metadata.config.analysis.sequencing_type == "targeted" and next( ( panel diff --git a/cg/meta/report/mip_dna.py b/cg/meta/report/mip_dna.py index a3763e420c..215daf9b48 100644 --- a/cg/meta/report/mip_dna.py +++ b/cg/meta/report/mip_dna.py @@ -79,10 +79,10 @@ def get_genome_build(self, analysis_metadata: MipAnalysis) -> str: """Return build version of the genome reference of a specific case.""" return analysis_metadata.genome_build - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: MipAnalysis = None ) -> bool: - """Checks if the report is accredited or not by evaluating each of the sample process accreditations.""" + """Check if the MIP-DNA report is accredited by evaluating each of the sample process accreditations.""" for sample in samples: if not sample.application.accredited: return False diff --git a/cg/meta/report/report_api.py b/cg/meta/report/report_api.py index 9ca2057b10..974c282625 100644 --- a/cg/meta/report/report_api.py +++ b/cg/meta/report/report_api.py @@ -174,7 +174,7 @@ def get_report_data(self, case_id: str, analysis_date: datetime) -> ReportModel: version=self.get_report_version(analysis=analysis), date=datetime.today(), case=case_model, - accredited=self.get_report_accreditation( + accredited=self.is_report_accredited( samples=case_model.samples, analysis_metadata=analysis_metadata ), ) @@ -407,10 +407,10 @@ def get_variant_callers(self, _analysis_metadata: AnalysisModel) -> list: """Return list of variant-calling filters used during analysis.""" return [] - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: AnalysisModel ) -> bool: - """Checks if the report is accredited or not.""" + """Check if the report is accredited.""" raise NotImplementedError def get_required_fields(self, case: CaseModel) -> dict: diff --git a/cg/meta/report/rnafusion.py b/cg/meta/report/rnafusion.py index 91a86e700b..4f2eeaaa99 100644 --- a/cg/meta/report/rnafusion.py +++ b/cg/meta/report/rnafusion.py @@ -79,10 +79,10 @@ def get_genome_build(self, analysis_metadata: AnalysisModel) -> str: """Return build version of the genome reference of a specific case.""" return GenomeVersion.hg38.value - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: AnalysisModel ) -> bool: - """Checks if the report is accredited or not. Rnafusion is an accredited workflow.""" + """Check if the report is accredited. Rnafusion is an accredited workflow.""" return True def get_template_name(self) -> str: diff --git a/cg/meta/upload/scout/balsamic_config_builder.py b/cg/meta/upload/scout/balsamic_config_builder.py index b85e25ff5e..9906292db8 100644 --- a/cg/meta/upload/scout/balsamic_config_builder.py +++ b/cg/meta/upload/scout/balsamic_config_builder.py @@ -4,7 +4,8 @@ from cg.apps.lims import LimsAPI from cg.constants.constants import SampleType -from cg.constants.scout import BALSAMIC_CASE_TAGS, BALSAMIC_SAMPLE_TAGS +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import BALSAMIC_CASE_TAGS, BALSAMIC_SAMPLE_TAGS, UploadTrack from cg.constants.subject import PhenotypeStatus from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.meta.upload.scout.scout_config_builder import ScoutConfigBuilder @@ -22,7 +23,10 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li ) self.case_tags: CaseTags = CaseTags(**BALSAMIC_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**BALSAMIC_SAMPLE_TAGS) - self.load_config: BalsamicLoadConfig = BalsamicLoadConfig(track="cancer") + self.load_config: BalsamicLoadConfig = BalsamicLoadConfig( + track=UploadTrack.CANCER.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) def include_case_files(self): LOG.info("Including BALSAMIC specific case level files") @@ -34,7 +38,6 @@ def include_case_files(self): ) self.include_cnv_report() self.include_multiqc_report() - self.include_delivery_report() def include_sample_files(self, config_sample: ScoutCancerIndividual) -> None: LOG.info("Including BALSAMIC specific sample level files.") diff --git a/cg/meta/upload/scout/balsamic_umi_config_builder.py b/cg/meta/upload/scout/balsamic_umi_config_builder.py index a816eede6f..b288417330 100644 --- a/cg/meta/upload/scout/balsamic_umi_config_builder.py +++ b/cg/meta/upload/scout/balsamic_umi_config_builder.py @@ -3,13 +3,11 @@ from housekeeper.store.models import Version from cg.apps.lims import LimsAPI -from cg.constants.scout import BALSAMIC_UMI_CASE_TAGS, BALSAMIC_UMI_SAMPLE_TAGS +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import BALSAMIC_UMI_CASE_TAGS, BALSAMIC_UMI_SAMPLE_TAGS, UploadTrack from cg.meta.upload.scout.balsamic_config_builder import BalsamicConfigBuilder from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags -from cg.models.scout.scout_load_config import ( - BalsamicUmiLoadConfig, - ScoutCancerIndividual, -) +from cg.models.scout.scout_load_config import BalsamicUmiLoadConfig, ScoutCancerIndividual from cg.store.models import Analysis, Sample LOG = logging.getLogger(__name__) @@ -22,12 +20,14 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li ) self.case_tags: CaseTags = CaseTags(**BALSAMIC_UMI_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**BALSAMIC_UMI_SAMPLE_TAGS) - self.load_config: BalsamicUmiLoadConfig = BalsamicUmiLoadConfig(track="cancer") + self.load_config: BalsamicUmiLoadConfig = BalsamicUmiLoadConfig( + track=UploadTrack.CANCER.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) def include_sample_files(self, config_sample: ScoutCancerIndividual) -> None: LOG.info("Including BALSAMIC specific sample level files") def get_balsamic_analysis_type(self, sample: Sample) -> str: - """Returns a formatted balsamic analysis type""" - + """Returns a formatted balsamic analysis type.""" return "panel-umi" diff --git a/cg/meta/upload/scout/mip_config_builder.py b/cg/meta/upload/scout/mip_config_builder.py index 071abaee25..1e0d9bcaaa 100644 --- a/cg/meta/upload/scout/mip_config_builder.py +++ b/cg/meta/upload/scout/mip_config_builder.py @@ -6,17 +6,14 @@ from cg.apps.lims import LimsAPI from cg.apps.madeline.api import MadelineAPI -from cg.constants.scout import MIP_CASE_TAGS, MIP_SAMPLE_TAGS +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import MIP_CASE_TAGS, MIP_SAMPLE_TAGS, UploadTrack from cg.constants.subject import RelationshipStatus from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.meta.upload.scout.scout_config_builder import ScoutConfigBuilder from cg.meta.workflow.mip import MipAnalysisAPI from cg.models.mip.mip_analysis import MipAnalysis -from cg.models.scout.scout_load_config import ( - MipLoadConfig, - ScoutLoadConfig, - ScoutMipIndividual, -) +from cg.models.scout.scout_load_config import MipLoadConfig, ScoutLoadConfig, ScoutMipIndividual from cg.store.models import Analysis, Case, CaseSample LOG = logging.getLogger(__name__) @@ -36,7 +33,10 @@ def __init__( ) self.case_tags: CaseTags = CaseTags(**MIP_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**MIP_SAMPLE_TAGS) - self.load_config: MipLoadConfig = MipLoadConfig(track="rare") + self.load_config: MipLoadConfig = MipLoadConfig( + track=UploadTrack.RARE_DISEASE.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) self.mip_analysis_api: MipAnalysisAPI = mip_analysis_api self.lims_api: LimsAPI = lims_api self.madeline_api: MadelineAPI = madeline_api @@ -116,7 +116,6 @@ def include_case_files(self): self.load_config.vcf_sv = self.get_file_from_hk(self.case_tags.sv_vcf) self.load_config.vcf_sv_research = self.get_file_from_hk(self.case_tags.sv_research_vcf) self.include_multiqc_report() - self.include_delivery_report() def include_sample_files(self, config_sample: ScoutMipIndividual) -> None: """Include sample level files that are optional for mip samples""" diff --git a/cg/meta/upload/scout/rnafusion_config_builder.py b/cg/meta/upload/scout/rnafusion_config_builder.py index b536d5bec1..0bd369ea78 100644 --- a/cg/meta/upload/scout/rnafusion_config_builder.py +++ b/cg/meta/upload/scout/rnafusion_config_builder.py @@ -4,7 +4,8 @@ from cg.apps.lims import LimsAPI from cg.constants.constants import PrepCategory -from cg.constants.scout import RNAFUSION_CASE_TAGS, RNAFUSION_SAMPLE_TAGS, GenomeBuild +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import RNAFUSION_CASE_TAGS, RNAFUSION_SAMPLE_TAGS, GenomeBuild, UploadTrack from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.meta.upload.scout.scout_config_builder import ScoutConfigBuilder from cg.models.scout.scout_load_config import ( @@ -26,7 +27,10 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li ) self.case_tags: CaseTags = CaseTags(**RNAFUSION_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**RNAFUSION_SAMPLE_TAGS) - self.load_config: RnafusionLoadConfig = RnafusionLoadConfig(track="cancer") + self.load_config: RnafusionLoadConfig = RnafusionLoadConfig( + track=UploadTrack.CANCER.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) def build_load_config(self) -> None: """Build a rnafusion-specific load config for uploading a case to scout.""" diff --git a/cg/meta/upload/scout/scout_config_builder.py b/cg/meta/upload/scout/scout_config_builder.py index 1b095889a4..a50d14c205 100644 --- a/cg/meta/upload/scout/scout_config_builder.py +++ b/cg/meta/upload/scout/scout_config_builder.py @@ -6,12 +6,14 @@ from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.lims import LimsAPI +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.models.scout.scout_load_config import ScoutIndividual, ScoutLoadConfig from cg.store.models import Analysis, CaseSample, Sample LOG = logging.getLogger(__name__) + # Maps keys that are used in scout load config on tags that are used in scout @@ -24,7 +26,9 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li self.lims_api: LimsAPI = lims_api self.case_tags: CaseTags self.sample_tags: SampleTags - self.load_config: ScoutLoadConfig = ScoutLoadConfig() + self.load_config: ScoutLoadConfig = ScoutLoadConfig( + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}) + ) def add_common_info_to_load_config(self) -> None: """Add the mandatory common information to a scout load config object""" @@ -137,12 +141,6 @@ def include_multiqc_report(self) -> None: hk_tags=self.case_tags.multiqc_report, latest=True ) - def include_delivery_report(self) -> None: - LOG.info("Include delivery report to case") - self.load_config.delivery_report = self.get_file_from_hk( - hk_tags=self.case_tags.delivery_report, latest=True - ) - def include_sample_alignment_file(self, config_sample: ScoutIndividual) -> None: """Include the alignment file for a sample diff --git a/cg/models/scout/scout_load_config.py b/cg/models/scout/scout_load_config.py index 3e275aad02..de6415cdfa 100644 --- a/cg/models/scout/scout_load_config.py +++ b/cg/models/scout/scout_load_config.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict from typing_extensions import Annotated, Literal +from cg.constants.scout import UploadTrack from cg.models.scout.validators import field_not_none @@ -91,12 +92,13 @@ class ScoutLoadConfig(BaseModel): sv_rank_model_version: str | None = None analysis_date: datetime | None = None samples: list[ScoutIndividual] = [] - - delivery_report: str | None = None + delivery_report: str coverage_qc_report: str | None = None cnv_report: str | None = None multiqc: str | None = None - track: Literal["rare", "cancer"] = "rare" + track: Literal[ + UploadTrack.RARE_DISEASE.value, UploadTrack.CANCER.value + ] = UploadTrack.RARE_DISEASE.value model_config = ConfigDict(validate_assignment=True) diff --git a/tests/apps/scout/test_scout_load_config.py b/tests/apps/scout/test_scout_load_config.py index b97a613233..f732dc0193 100644 --- a/tests/apps/scout/test_scout_load_config.py +++ b/tests/apps/scout/test_scout_load_config.py @@ -1,4 +1,5 @@ -"""Tests for the models in scout load config""" +"""Tests for the models in Scout load config.""" +from pathlib import Path from typing import Any import pytest @@ -11,8 +12,10 @@ @pytest.mark.parametrize("key, value", list(SCOUT_INDIVIDUAL.items())) def test_validate_scout_individual_attributes(scout_individual: dict, key: str, value: Any): - """Test to validate that all attributes of a ScoutMipIndividual are correctly set.""" + """Test that all attributes of a ScoutMipIndividual are correctly set.""" + # GIVEN some sample information + # WHEN instantiating a ScoutMipIndividual ind_obj: ScoutMipIndividual = scout_load_config.ScoutMipIndividual(**scout_individual) @@ -20,26 +23,29 @@ def test_validate_scout_individual_attributes(scout_individual: dict, key: str, assert getattr(ind_obj, key) == value -def test_instantiate_empty_mip_config(): - """Tests whether a MipLoadConfig can be instantiate without arguments.""" - # GIVEN nothing +def test_instantiate_empty_mip_config(delivery_report_html: Path): + """Tests whether a MipLoadConfig can be instantiated only with mandatory arguments.""" + + # GIVEN a delivery report file - # WHEN instantiating a empty mip load config - config: MipLoadConfig = scout_load_config.MipLoadConfig() + # WHEN instantiating an empty MIP load config + config: MipLoadConfig = scout_load_config.MipLoadConfig( + delivery_report=delivery_report_html.as_posix() + ) - # THEN assert it is possible to instantiate without any information + # THEN assert it is possible to instantiate without any not mandatory information assert isinstance(config, scout_load_config.ScoutLoadConfig) -def test_set_mandatory_to_none(): - """The scout load config object should validate fields as they are set. +def test_set_mandatory_to_none(delivery_report_html: Path): + """Test that a value error is raised when a mandatory field is set to None.""" - This test will check that a value error is raised when a mandatory field is set to None. - """ # GIVEN a load config object - config: MipLoadConfig = scout_load_config.MipLoadConfig() + config: MipLoadConfig = scout_load_config.MipLoadConfig( + delivery_report=delivery_report_html.as_posix() + ) # WHEN setting a mandatory field to None with pytest.raises(ValidationError): - # THEN assert a validation error was raised + # THEN a validation error should be raised config.vcf_snv = None diff --git a/tests/cli/upload/conftest.py b/tests/cli/upload/conftest.py index 3bac71741b..d3ad48b1dd 100644 --- a/tests/cli/upload/conftest.py +++ b/tests/cli/upload/conftest.py @@ -28,12 +28,6 @@ from cg.models.scout.scout_load_config import ScoutLoadConfig from cg.store import Store from cg.store.models import Analysis -from tests.cli.workflow.mip.conftest import ( - mip_case_id, - mip_case_ids, - mip_dna_context, - mip_rna_context, -) from tests.meta.upload.scout.conftest import mip_load_config from tests.mocks.hk_mock import MockHousekeeperAPI from tests.mocks.madeline import MockMadelineAPI @@ -248,7 +242,9 @@ def __init__(self, **kwargs): self.housekeeper = None self.madeline_api = MockMadelineAPI() self.analysis = MockAnalysisApi() - self.config = ScoutLoadConfig() + self.config = ScoutLoadConfig( + delivery_report=Path("path", "to", "delivery-report.html").as_posix() + ) self.file_exists = False self.lims = MockLims() self.missing_mandatory_field = False diff --git a/tests/conftest.py b/tests/conftest.py index bc3f593a77..7031d7c179 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,7 @@ from cg.constants import FileExtensions, Pipeline, SequencingFileTag from cg.constants.constants import CaseActions, FileFormat, Strandedness from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.constants.nanopore_files import NanoporeDirsAndFiles from cg.constants.priority import SlurmQos from cg.constants.sequencing import SequencingPlatform @@ -1895,6 +1896,7 @@ def hk_bundle_sample_path(sample_id: str, timestamp: datetime) -> Path: def hk_bundle_data( case_id: str, bed_file: Path, + delivery_report_html: Path, timestamp_yesterday: datetime, sample_id: str, father_sample_id: str, @@ -1910,7 +1912,12 @@ def hk_bundle_data( "path": bed_file.as_posix(), "archive": False, "tags": ["bed", sample_id, father_sample_id, mother_sample_id, "coverage"], - } + }, + { + "path": delivery_report_html.as_posix(), + "archive": False, + "tags": [HK_DELIVERY_REPORT_TAG], + }, ], } diff --git a/tests/meta/report/test_balsamic_api.py b/tests/meta/report/test_balsamic_api.py index 2fc3954873..191b4e95ff 100644 --- a/tests/meta/report/test_balsamic_api.py +++ b/tests/meta/report/test_balsamic_api.py @@ -95,8 +95,8 @@ def test_get_variant_caller_version(report_api_balsamic, case_id): assert version == expected_version -def test_get_report_accreditation(report_api_balsamic, case_id): - """Tests report accreditation for a specific BALSAMIC analysis.""" +def test_is_report_accredited(report_api_balsamic, case_id): + """Test report accreditation for a specific BALSAMIC analysis.""" # GIVEN a mock metadata object and an accredited one balsamic_metadata = report_api_balsamic.analysis_api.get_latest_metadata(case_id) @@ -105,10 +105,8 @@ def test_get_report_accreditation(report_api_balsamic, case_id): balsamic_accredited_metadata.config.panel.capture_kit = "gmsmyeloid" # WHEN performing the accreditation validation - unaccredited_report = report_api_balsamic.get_report_accreditation(None, balsamic_metadata) - accredited_report = report_api_balsamic.get_report_accreditation( - None, balsamic_accredited_metadata - ) + unaccredited_report = report_api_balsamic.is_report_accredited(None, balsamic_metadata) + accredited_report = report_api_balsamic.is_report_accredited(None, balsamic_accredited_metadata) # THEN verify that only the panel "gmsmyeloid" reports are validated assert not unaccredited_report diff --git a/tests/meta/report/test_mip_dna_api.py b/tests/meta/report/test_mip_dna_api.py index c423c81434..e0f3c5e5dc 100644 --- a/tests/meta/report/test_mip_dna_api.py +++ b/tests/meta/report/test_mip_dna_api.py @@ -44,22 +44,22 @@ def test_get_sample_coverage(report_api_mip_dna, sample_store, helpers: StoreHel assert sample_coverage == {"mean_coverage": 37.342, "mean_completeness": 97.1} -def test_get_report_accreditation(report_api_mip_dna, mip_analysis_api, case_mip_dna): - """Verifies the report accreditation extraction workflow.""" +def test_is_report_accredited(report_api_mip_dna, mip_analysis_api, case_mip_dna): + """Test report accreditation extraction workflow.""" # GIVEN a list of accredited samples mip_metadata = mip_analysis_api.get_latest_metadata(case_mip_dna.internal_id) samples = report_api_mip_dna.get_samples_data(case_mip_dna, mip_metadata) # WHEN retrieving the report accreditation - accredited = report_api_mip_dna.get_report_accreditation(samples) + accredited = report_api_mip_dna.is_report_accredited(samples) # THEN check that the report is accredited assert accredited -def test_get_report_accreditation_false(report_api_mip_dna, mip_analysis_api, case_mip_dna): - """Verifies that the report is not accredited if it contains a sample application that is not accredited.""" +def test_is_report_accredited_false(report_api_mip_dna, mip_analysis_api, case_mip_dna): + """Test that the report is not accredited if it contains a sample application that is not accredited.""" # GIVEN a list of samples when one of them is not accredited mip_metadata = mip_analysis_api.get_latest_metadata(case_mip_dna.internal_id) @@ -67,7 +67,7 @@ def test_get_report_accreditation_false(report_api_mip_dna, mip_analysis_api, ca samples[0].application.accredited = False # WHEN retrieving the report accreditation - accredited = report_api_mip_dna.get_report_accreditation(samples) + accredited = report_api_mip_dna.is_report_accredited(samples) # THEN check that the report is not accredited assert not accredited diff --git a/tests/meta/upload/scout/conftest.py b/tests/meta/upload/scout/conftest.py index b0be71f190..b4f2f1e86b 100644 --- a/tests/meta/upload/scout/conftest.py +++ b/tests/meta/upload/scout/conftest.py @@ -1,5 +1,4 @@ """Fixtures for the upload Scout API tests.""" - import logging from datetime import datetime from pathlib import Path @@ -10,6 +9,8 @@ from cg.constants import DataDelivery, Pipeline from cg.constants.constants import FileFormat, PrepCategory +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import UploadTrack from cg.constants.sequencing import SequencingMethod from cg.io.controller import ReadFile from cg.meta.upload.scout.balsamic_config_builder import BalsamicConfigBuilder @@ -241,6 +242,7 @@ def mip_dna_analysis_hk_bundle_data( sv_vcf_file: str, snv_research_vcf_file: str, sv_research_vcf_file: str, + delivery_report_html: Path, ) -> dict: """Return MIP DNA bundle data for Housekeeper.""" return { @@ -415,7 +417,7 @@ def balsamic_analysis_hk_bundle_data( @pytest.fixture(scope="function") def rnafusion_analysis_hk_bundle_data( - case_id: str, timestamp: datetime, rnafusion_analysis_dir: Path + case_id: str, timestamp: datetime, rnafusion_analysis_dir: Path, delivery_report_html: Path ) -> dict: """Get some bundle data for housekeeper.""" return { @@ -433,6 +435,11 @@ def rnafusion_analysis_hk_bundle_data( "archive": False, "tags": ["fusionreport", "research"], }, + { + "path": delivery_report_html.as_posix(), + "archive": False, + "tags": [HK_DELIVERY_REPORT_TAG], + }, ], } @@ -588,14 +595,19 @@ def balsamic_config_builder( @pytest.fixture(name="mip_load_config") def mip_load_config( - mip_dna_analysis_dir: Path, case_id: str, customer_id: str, snv_vcf_file: str + mip_dna_analysis_dir: Path, + case_id: str, + customer_id: str, + snv_vcf_file: str, + delivery_report_html: Path, ) -> MipLoadConfig: """Return a valid MIP load_config.""" return MipLoadConfig( owner=customer_id, family=case_id, vcf_snv=Path(mip_dna_analysis_dir, snv_vcf_file).as_posix(), - track="rare", + track=UploadTrack.RARE_DISEASE.value, + delivery_report=delivery_report_html.as_posix(), ) diff --git a/tests/meta/upload/scout/test_meta_upload_scoutapi.py b/tests/meta/upload/scout/test_meta_upload_scoutapi.py index d0e2f65aa0..7131f03317 100644 --- a/tests/meta/upload/scout/test_meta_upload_scoutapi.py +++ b/tests/meta/upload/scout/test_meta_upload_scoutapi.py @@ -9,7 +9,9 @@ from cg.models.scout.scout_load_config import MipLoadConfig, ScoutLoadConfig -def test_unlinked_family_is_linked(mip_config_builder: MipConfigBuilder): +def test_unlinked_family_is_linked( + mip_config_builder: MipConfigBuilder, delivery_report_html: Path +): """Test that is_family check fails when samples are not linked""" # GIVEN a upload scout api and case data for a case without linked individuals family_data: MipLoadConfig = MipLoadConfig( @@ -17,7 +19,8 @@ def test_unlinked_family_is_linked(mip_config_builder: MipConfigBuilder): "samples": [ {"sample_id": "ADM2", "father": "0", "mother": "0"}, {"sample_id": "ADM3", "father": "0", "mother": "0"}, - ] + ], + "delivery_report": delivery_report_html.as_posix(), } ) # WHEN running the check if case is linked @@ -26,7 +29,7 @@ def test_unlinked_family_is_linked(mip_config_builder: MipConfigBuilder): assert res is False -def test_family_is_linked(mip_config_builder: MipConfigBuilder): +def test_family_is_linked(mip_config_builder: MipConfigBuilder, delivery_report_html: Path): """Test that is_family returns true when samples are linked""" # GIVEN a upload scout api and case data for a linked case family_data: MipLoadConfig = MipLoadConfig( @@ -35,7 +38,8 @@ def test_family_is_linked(mip_config_builder: MipConfigBuilder): {"sample_id": "ADM1", "father": "ADM2", "mother": "ADM3"}, {"sample_id": "ADM2", "father": "0", "mother": "0"}, {"sample_id": "ADM3", "father": "0", "mother": "0"}, - ] + ], + "delivery_report": delivery_report_html.as_posix(), } ) # WHEN running the check if case is linked diff --git a/tests/meta/upload/scout/test_scout_config_builder.py b/tests/meta/upload/scout/test_scout_config_builder.py index 811298718a..eb6eae1b08 100644 --- a/tests/meta/upload/scout/test_scout_config_builder.py +++ b/tests/meta/upload/scout/test_scout_config_builder.py @@ -68,20 +68,6 @@ def test_rnafusion_config_builder( assert isinstance(file_handler.case_tags, CaseTags) -def test_include_delivery_report_mip(mip_config_builder: MipConfigBuilder): - """Test include delivery report.""" - # GIVEN a config builder with data - - # GIVEN a config without a delivery report - assert mip_config_builder.load_config.delivery_report is None - - # WHEN including the delivery report - mip_config_builder.include_delivery_report() - - # THEN assert that the delivery report was added - assert mip_config_builder.load_config.delivery_report is not None - - def test_include_synopsis(mip_config_builder: MipConfigBuilder): """Test include synopsis.""" # GIVEN a config builder with some data From 25f106df60eb0a1fc2675b75a6b5b7218a64eb8d Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Tue, 9 Jan 2024 08:20:05 +0000 Subject: [PATCH 02/18] =?UTF-8?q?Bump=20version:=2054.10.4=20=E2=86=92=205?= =?UTF-8?q?4.10.5=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57ec21029c..f6341d431c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 54.10.4 +current_version = 54.10.5 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 166f881bf7..3caf04e6b6 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "54.10.4" +__version__ = "54.10.5" diff --git a/pyproject.toml b/pyproject.toml index 110356e38a..3011f143fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "54.10.4" +version = "54.10.5" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From a3813263a7f284161d3b41bb1a3d1037435258ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isak=20Ohlsson=20=C3=85ngnell?= <40887124+islean@users.noreply.github.com> Date: Tue, 9 Jan 2024 10:51:20 +0100 Subject: [PATCH 03/18] Export new concentration field (#2785) (patch) ### Fixed - New concentration is uploaded to LIMS --- cg/apps/lims/api.py | 1 + cg/constants/lims.py | 1 + cg/meta/orders/case_submitter.py | 2 +- cg/models/lims/sample.py | 1 + cg/models/orders/samples.py | 1 + tests/fixtures/cgweb_orders/balsamic.json | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cg/apps/lims/api.py b/cg/apps/lims/api.py index 4e02127cc5..caacc6e58d 100644 --- a/cg/apps/lims/api.py +++ b/cg/apps/lims/api.py @@ -90,6 +90,7 @@ def _export_sample(self, lims_sample): else None ), "comment": udfs.get("comment"), + "concentration_ng_ul": udfs.get("Concentration (ng/ul)"), } def get_received_date(self, lims_id: str) -> dt.date: diff --git a/cg/constants/lims.py b/cg/constants/lims.py index 90b1ce5e6f..86671b2fac 100644 --- a/cg/constants/lims.py +++ b/cg/constants/lims.py @@ -12,6 +12,7 @@ "comment": "Comment", "control": "Control", "concentration": "Concentration (nM)", + "concentration_ng_ul": "Concentration (ng/ul)", "concentration_sample": "Sample Conc.", "customer": "customer", "data_analysis": "Data Analysis", diff --git a/cg/meta/orders/case_submitter.py b/cg/meta/orders/case_submitter.py index 3b43dd4963..551edf7201 100644 --- a/cg/meta/orders/case_submitter.py +++ b/cg/meta/orders/case_submitter.py @@ -9,7 +9,7 @@ from cg.meta.orders.submitter import Submitter from cg.models.orders.order import OrderIn from cg.models.orders.samples import Of1508Sample, OrderInSample -from cg.store.models import ApplicationVersion, Customer, Case, CaseSample, Sample +from cg.store.models import ApplicationVersion, Case, CaseSample, Customer, Sample LOG = logging.getLogger(__name__) diff --git a/cg/models/lims/sample.py b/cg/models/lims/sample.py index 474fc746fb..3fbd784837 100644 --- a/cg/models/lims/sample.py +++ b/cg/models/lims/sample.py @@ -13,6 +13,7 @@ class Udf(BaseModel): comment: str | None concentration: str | None concentration_sample: str | None + concentration_ng_ul: str | None customer: str control: str | None data_analysis: str | None diff --git a/cg/models/orders/samples.py b/cg/models/orders/samples.py index e835be055e..bae0d7aa70 100644 --- a/cg/models/orders/samples.py +++ b/cg/models/orders/samples.py @@ -43,6 +43,7 @@ class OrderInSample(BaseModel): priority: PriorityEnum = PriorityEnum.standard require_qc_ok: bool = False volume: str + concentration_ng_ul: str | None @classmethod def is_sample_for(cls, project: OrderType): diff --git a/tests/fixtures/cgweb_orders/balsamic.json b/tests/fixtures/cgweb_orders/balsamic.json index b11371b00c..17179dde03 100644 --- a/tests/fixtures/cgweb_orders/balsamic.json +++ b/tests/fixtures/cgweb_orders/balsamic.json @@ -12,6 +12,7 @@ ], "comment": "other Elution buffer", "container": "96 well plate", + "concentration_ng_ul": "18", "container_name": "p1", "data_analysis": "balsamic", "data_delivery": "fastq-analysis-scout", From 8322d2ec2449c161b54ba4625b90f3d156f3d5de Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Tue, 9 Jan 2024 09:51:52 +0000 Subject: [PATCH 04/18] =?UTF-8?q?Bump=20version:=2054.10.5=20=E2=86=92=205?= =?UTF-8?q?4.10.6=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f6341d431c..24b808e769 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 54.10.5 +current_version = 54.10.6 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 3caf04e6b6..dd66ce9965 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "54.10.5" +__version__ = "54.10.6" diff --git a/pyproject.toml b/pyproject.toml index 3011f143fd..d173a0ee0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "54.10.5" +version = "54.10.6" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 2b87168e57b9cbe1e98e2da7691513b6839afdca Mon Sep 17 00:00:00 2001 From: Sebastian Allard Date: Tue, 9 Jan 2024 11:43:52 +0100 Subject: [PATCH 05/18] Fix missing microsalt jobs (#2813)(patch) --- cg/meta/workflow/analysis.py | 16 ++++++------- cg/meta/workflow/balsamic.py | 2 +- cg/meta/workflow/fluffy.py | 2 +- cg/meta/workflow/microsalt/microsalt.py | 32 ++++++++++++++----------- cg/meta/workflow/mip.py | 2 +- cg/meta/workflow/mutant.py | 2 +- cg/meta/workflow/nf_analysis.py | 6 ++--- 7 files changed, 33 insertions(+), 29 deletions(-) diff --git a/cg/meta/workflow/analysis.py b/cg/meta/workflow/analysis.py index cd82344249..c22df56374 100644 --- a/cg/meta/workflow/analysis.py +++ b/cg/meta/workflow/analysis.py @@ -118,8 +118,8 @@ def get_case_config_path(self, case_id) -> Path: """Path to case config file""" raise NotImplementedError - def get_trailblazer_config_path(self, case_id: str) -> Path: - """Path to Trailblazer job id file""" + def get_job_ids_path(self, case_id: str) -> Path: + """Path to file containing slurm/tower job ids for the case.""" raise NotImplementedError def get_sample_name_from_lims_id(self, lims_id: str) -> str: @@ -206,17 +206,17 @@ def get_analysis_finish_path(self, case_id: str) -> Path: raise NotImplementedError def add_pending_trailblazer_analysis(self, case_id: str) -> None: - self.check_analysis_ongoing(case_id=case_id) - self.trailblazer_api.mark_analyses_deleted(case_id=case_id) + self.check_analysis_ongoing(case_id) + self.trailblazer_api.mark_analyses_deleted(case_id) self.trailblazer_api.add_pending_analysis( case_id=case_id, email=environ_email(), analysis_type=self.get_application_type( - self.status_db.get_case_by_internal_id(internal_id=case_id).links[0].sample + self.status_db.get_case_by_internal_id(case_id).links[0].sample ), - out_dir=self.get_trailblazer_config_path(case_id=case_id).parent.as_posix(), - config_path=self.get_trailblazer_config_path(case_id=case_id).as_posix(), - slurm_quality_of_service=self.get_slurm_qos_for_case(case_id=case_id), + out_dir=self.get_job_ids_path(case_id).parent.as_posix(), + config_path=self.get_job_ids_path(case_id).as_posix(), + slurm_quality_of_service=self.get_slurm_qos_for_case(case_id), data_analysis=str(self.pipeline), ticket=self.status_db.get_latest_ticket_from_case(case_id), workflow_manager=self.get_workflow_manager(), diff --git a/cg/meta/workflow/balsamic.py b/cg/meta/workflow/balsamic.py index 1bcb90e805..bf70942aed 100644 --- a/cg/meta/workflow/balsamic.py +++ b/cg/meta/workflow/balsamic.py @@ -117,7 +117,7 @@ def get_case_config_path(self, case_id: str) -> Path: """ return Path(self.root_dir, case_id, case_id + ".json") - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: return Path(self.root_dir, case_id, "analysis", "slurm_jobids.yaml") def get_bundle_deliverables_type(self, case_id: str) -> str: diff --git a/cg/meta/workflow/fluffy.py b/cg/meta/workflow/fluffy.py index f0c3ff310f..f805c32140 100644 --- a/cg/meta/workflow/fluffy.py +++ b/cg/meta/workflow/fluffy.py @@ -130,7 +130,7 @@ def get_deliverables_file_path(self, case_id: str) -> Path: """ return Path(self.get_output_path(case_id), "deliverables.yaml") - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: """ Location in working directory where SLURM job id file is to be stored. This file contains SLURM ID of jobs associated with current analysis , diff --git a/cg/meta/workflow/microsalt/microsalt.py b/cg/meta/workflow/microsalt/microsalt.py index bbcc1ed6d2..e670bed4e1 100644 --- a/cg/meta/workflow/microsalt/microsalt.py +++ b/cg/meta/workflow/microsalt/microsalt.py @@ -1,4 +1,3 @@ -import glob import logging import os import re @@ -88,14 +87,19 @@ def get_case_fastq_path(self, case_id: str) -> Path: def get_config_path(self, filename: str) -> Path: return Path(self.queries_path, filename).with_suffix(".json") - def get_trailblazer_config_path(self, case_id: str) -> Path: - """Get trailblazer config path.""" - case_obj: Case = self.status_db.get_case_by_internal_id(internal_id=case_id) - sample_obj: Sample = case_obj.links[0].sample - project_id: str = self.get_project(sample_obj.internal_id) - return Path( - self.root_dir, "results", "reports", "trailblazer", f"{project_id}_slurm_ids.yaml" - ) + def get_job_ids_path(self, case_id: str) -> Path: + case_path: Path = self.get_case_path(case_id) + job_ids_file_name: str = self.get_job_ids_file_name(case_id) + return Path(case_path, job_ids_file_name) + + def get_job_ids_file_name(self, case_id: str) -> str: + project_id: str = self.get_lims_project_id(case_id) + return f"{project_id}_slurm_ids.yaml" + + def get_lims_project_id(self, case_id: str): + case: Case = self.status_db.get_case_by_internal_id(case_id) + sample: Sample = case.links[0].sample + return self.get_project(sample.internal_id) def get_deliverables_file_path(self, case_id: str) -> Path: """Returns a path where the microSALT deliverables file for the order_id should be @@ -221,7 +225,7 @@ def get_case_id_from_ticket(self, unique_id: str) -> tuple[str, None]: Since sample_id is not specified, nothing is returned as sample_id""" case: Case = self.status_db.get_case_by_name(name=unique_id) if not case: - LOG.error("No case found for ticket number: %s", unique_id) + LOG.error(f"No case found for ticket number: {unique_id}") raise click.Abort case_id = case.internal_id return case_id, None @@ -230,9 +234,9 @@ def get_case_id_from_sample(self, unique_id: str) -> tuple[str, str]: """If sample is specified, finds the corresponding case_id to which this sample belongs. The case_id is to be used for identifying the appropriate path to link fastq files and store the analysis output """ - sample: Sample = self.status_db.get_sample_by_internal_id(internal_id=unique_id) + sample: Sample = self.status_db.get_sample_by_internal_id(unique_id) if not sample: - LOG.error("No sample found with id: %s", unique_id) + LOG.error(f"No sample found with id: {unique_id}") raise click.Abort case_id = sample.links[0].case.internal_id sample_id = sample.internal_id @@ -240,9 +244,9 @@ def get_case_id_from_sample(self, unique_id: str) -> tuple[str, str]: def get_case_id_from_case(self, unique_id: str) -> tuple[str, None]: """If case_id is specified, validates the presence of case_id in database and returns it""" - case_obj: Case = self.status_db.get_case_by_internal_id(internal_id=unique_id) + case_obj: Case = self.status_db.get_case_by_internal_id(unique_id) if not case_obj: - LOG.error("No case found with the id: %s", unique_id) + LOG.error(f"No case found with the id: {unique_id}") raise click.Abort case_id = case_obj.internal_id return case_id, None diff --git a/cg/meta/workflow/mip.py b/cg/meta/workflow/mip.py index e6dc4993dc..50aee67d54 100644 --- a/cg/meta/workflow/mip.py +++ b/cg/meta/workflow/mip.py @@ -312,7 +312,7 @@ def run_analysis(self, case_id: str, command_args: dict, dry_run: bool) -> None: def get_case_path(self, case_id: str) -> Path: return Path(self.root, case_id) - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: return Path(self.get_case_path(case_id=case_id), "analysis", "slurm_job_ids.yaml") def config_sample(self, link_obj: CaseSample, panel_bed: str) -> dict: diff --git a/cg/meta/workflow/mutant.py b/cg/meta/workflow/mutant.py index 7c99c30082..74ac98e14c 100644 --- a/cg/meta/workflow/mutant.py +++ b/cg/meta/workflow/mutant.py @@ -55,7 +55,7 @@ def get_case_output_path(self, case_id: str) -> Path: def get_case_fastq_dir(self, case_id: str) -> Path: return Path(self.get_case_path(case_id=case_id), "fastq") - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: return Path(self.get_case_output_path(case_id=case_id), "trailblazer_config.yaml") def _is_nanopore(self, application: Application) -> bool: diff --git a/cg/meta/workflow/nf_analysis.py b/cg/meta/workflow/nf_analysis.py index e8642d0e4a..644328b920 100644 --- a/cg/meta/workflow/nf_analysis.py +++ b/cg/meta/workflow/nf_analysis.py @@ -89,7 +89,7 @@ def get_nextflow_config_path(nextflow_config: str | None = None) -> Path | None: if nextflow_config: return Path(nextflow_config).absolute() - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: """Return the path to a Trailblazer config file containing Tower IDs.""" return Path(self.root_dir, case_id, "tower_ids").with_suffix(FileExtensions.YAML) @@ -197,7 +197,7 @@ def write_deliverables_file( def write_trailblazer_config(self, case_id: str, tower_id: str) -> None: """Write Tower IDs to a file used as the Trailblazer config.""" - config_path: Path = self.get_trailblazer_config_path(case_id=case_id) + config_path: Path = self.get_job_ids_path(case_id=case_id) LOG.info(f"Writing Tower ID to {config_path.as_posix()}") WriteFile.write_file_from_content( content={case_id: [tower_id]}, @@ -246,7 +246,7 @@ def _run_analysis_with_tower( if command_args.resume: from_tower_id: int = command_args.id or NfTowerHandler.get_last_tower_id( case_id=case_id, - trailblazer_config=self.get_trailblazer_config_path(case_id=case_id), + trailblazer_config=self.get_job_ids_path(case_id=case_id), ) LOG.info(f"Pipeline will be resumed from run with Tower id: {from_tower_id}.") parameters: list[str] = NfTowerHandler.get_tower_relaunch_parameters( From e4af029042373990e948d7a465041c2a82b8ee45 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Tue, 9 Jan 2024 10:44:20 +0000 Subject: [PATCH 06/18] =?UTF-8?q?Bump=20version:=2054.10.6=20=E2=86=92=205?= =?UTF-8?q?4.10.7=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 24b808e769..043f32957c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 54.10.6 +current_version = 54.10.7 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index dd66ce9965..58623a4fb0 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "54.10.6" +__version__ = "54.10.7" diff --git a/pyproject.toml b/pyproject.toml index d173a0ee0e..2a0cb4b589 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "54.10.6" +version = "54.10.7" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 5b7f8d5f36b4825314e818a41a12ed725f5fad55 Mon Sep 17 00:00:00 2001 From: Henrik Stranneheim Date: Tue, 9 Jan 2024 16:14:38 +0100 Subject: [PATCH 07/18] Rm is uploaded to vogue (#2810) (major) ### Changed - Remove uploaded to vogue from the database and models --- ...84840c706a0_remove_uploaded_to_vogue_at.py | 24 +++++++++++++++++++ cg/store/models.py | 1 - tests/store/api/conftest.py | 8 ------- tests/store/conftest.py | 4 ---- tests/store_helpers.py | 3 --- 5 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py diff --git a/alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py b/alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py new file mode 100644 index 0000000000..1e8207c42b --- /dev/null +++ b/alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py @@ -0,0 +1,24 @@ +"""remove_uploaded_to_vogue_at + +Revision ID: 584840c706a0 +Revises: 27ec5c4c0380 +Create Date: 2023-12-27 11:50:22.278213 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "584840c706a0" +down_revision = "27ec5c4c0380" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("analysis", "uploaded_to_vogue_at") + + +def downgrade(): + op.add_column("analysis", sa.Column("uploaded_to_vogue_at", sa.DateTime(), nullable=True)) diff --git a/cg/store/models.py b/cg/store/models.py index a5adf0dbfb..227e3d68c7 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -218,7 +218,6 @@ class Analysis(Model): created_at = Column(types.DateTime, default=dt.datetime.now, nullable=False) case_id = Column(ForeignKey("case.id", ondelete="CASCADE"), nullable=False) - uploaded_to_vogue_at = Column(types.DateTime, nullable=True) case = orm.relationship("Case", back_populates="analyses") diff --git a/tests/store/api/conftest.py b/tests/store/api/conftest.py index 37e8a046c1..8089d1adba 100644 --- a/tests/store/api/conftest.py +++ b/tests/store/api/conftest.py @@ -465,7 +465,6 @@ def store_with_analyses_for_cases( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, completed_at=timestamp_yesterday, ) helpers.add_analysis( @@ -474,7 +473,6 @@ def store_with_analyses_for_cases( started_at=timestamp_now, uploaded_at=timestamp_now, delivery_reported_at=None, - uploaded_to_vogue_at=None, completed_at=timestamp_now, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) @@ -505,7 +503,6 @@ def store_with_analyses_for_cases_not_uploaded_fluffy( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, pipeline=Pipeline.FLUFFY, ) helpers.add_analysis( @@ -514,7 +511,6 @@ def store_with_analyses_for_cases_not_uploaded_fluffy( started_at=timestamp_now, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_now, pipeline=Pipeline.FLUFFY, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) @@ -545,7 +541,6 @@ def store_with_analyses_for_cases_not_uploaded_microsalt( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, pipeline=Pipeline.MICROSALT, ) helpers.add_analysis( @@ -554,7 +549,6 @@ def store_with_analyses_for_cases_not_uploaded_microsalt( started_at=timestamp_now, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_now, pipeline=Pipeline.MICROSALT, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) @@ -584,7 +578,6 @@ def store_with_analyses_for_cases_to_deliver( started_at=timestamp_yesterday, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, completed_at=timestamp_yesterday, pipeline=Pipeline.FLUFFY, ) @@ -594,7 +587,6 @@ def store_with_analyses_for_cases_to_deliver( started_at=timestamp_now, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=None, completed_at=timestamp_now, pipeline=Pipeline.MIP_DNA, ) diff --git a/tests/store/conftest.py b/tests/store/conftest.py index aebff67ca5..f9e4bed1d3 100644 --- a/tests/store/conftest.py +++ b/tests/store/conftest.py @@ -401,7 +401,6 @@ def store_with_older_and_newer_analyses( """Return a store with older and newer analyses.""" analysis = base_store._get_query(table=Analysis).first() analysis.uploaded_at = timestamp_now - analysis.uploaded_to_vogue_at = timestamp_now analysis.cleaned_at = timestamp_now analysis.started_at = timestamp_now analysis.completed_at = timestamp_now @@ -416,7 +415,6 @@ def store_with_older_and_newer_analyses( started_at=time, completed_at=time, uploaded_at=time, - uploaded_to_vogue_at=time, cleaned_at=time, ) @@ -442,7 +440,6 @@ def store_with_analyses_for_cases( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, ) helpers.add_analysis( analysis_store, @@ -450,7 +447,6 @@ def store_with_analyses_for_cases( started_at=timestamp_now, uploaded_at=timestamp_now, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_now, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) link: CaseSample = analysis_store.relate_sample( diff --git a/tests/store_helpers.py b/tests/store_helpers.py index c1827ba14c..3d57a7e76c 100644 --- a/tests/store_helpers.py +++ b/tests/store_helpers.py @@ -322,7 +322,6 @@ def add_analysis( data_delivery: DataDelivery = DataDelivery.FASTQ_QC, uploading: bool = False, config_path: str = None, - uploaded_to_vogue_at: datetime = None, ) -> Analysis: """Utility function to add an analysis for tests.""" @@ -346,8 +345,6 @@ def add_analysis( analysis.config_path = config_path if pipeline: analysis.pipeline = str(pipeline) - if uploaded_to_vogue_at: - analysis.uploaded_to_vogue_at = uploaded_to_vogue_at analysis.limitations = "A limitation" analysis.case = case From adae4a29de94076d0884a99d60074732cb88e1a9 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Tue, 9 Jan 2024 15:15:06 +0000 Subject: [PATCH 08/18] =?UTF-8?q?Bump=20version:=2054.10.7=20=E2=86=92=205?= =?UTF-8?q?5.0.0=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 043f32957c..a147ddf8ab 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 54.10.7 +current_version = 55.0.0 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 58623a4fb0..7d38e035e4 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "54.10.7" +__version__ = "55.0.0" diff --git a/pyproject.toml b/pyproject.toml index 2a0cb4b589..94f26ceab1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "54.10.7" +version = "55.0.0" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 0fc71b09c941445d163be0552c0a82733b5e5566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isak=20Ohlsson=20=C3=85ngnell?= <40887124+islean@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:51:57 +0100 Subject: [PATCH 09/18] (Archiving) Do not compress archived files (#2814) (minor) ### Added - Method for checking if the Spring file in a CompressionData object is archived. ### Changed - Compression of Fastq -> Spring is not performed when Spring is archived. --- cg/apps/crunchy/crunchy.py | 17 +++-- cg/meta/compress/compress.py | 31 +++++---- tests/apps/crunchy/test_crunchy.py | 65 ++++++++----------- tests/conftest.py | 10 ++- .../meta/compress/test_compress_meta_fastq.py | 64 +++++++++++++----- 5 files changed, 110 insertions(+), 77 deletions(-) diff --git a/cg/apps/crunchy/crunchy.py b/cg/apps/crunchy/crunchy.py index 406515e0be..d0c2c10e04 100644 --- a/cg/apps/crunchy/crunchy.py +++ b/cg/apps/crunchy/crunchy.py @@ -64,25 +64,24 @@ def is_compression_pending(compression_obj: CompressionData) -> bool: def is_fastq_compression_possible(compression_obj: CompressionData) -> bool: """Check if FASTQ compression is possible. - There are three possible answers to this question: - - - Compression is running -> Compression NOT possible - - SPRING archive exists -> Compression NOT possible - - Data is external -> Compression NOT possible - - Not compressed and not running -> Compression IS possible + - Compression is running -> Compression NOT possible + - SPRING file exists on Hasta -> Compression NOT possible + - Data is external -> Compression NOT possible + - Not compressed and + not running -> Compression IS possible """ if CrunchyAPI.is_compression_pending(compression_obj): return False if compression_obj.spring_exists(): - LOG.info("SPRING file found") + LOG.debug("SPRING file found") return False if "external-data" in str(compression_obj.fastq_first): - LOG.info("File is external data and should not be compressed") + LOG.debug("File is external data and should not be compressed") return False - LOG.info("FASTQ compression is possible") + LOG.debug("FASTQ compression is possible") return True diff --git a/cg/meta/compress/compress.py b/cg/meta/compress/compress.py index c2674b9037..f941479e2c 100644 --- a/cg/meta/compress/compress.py +++ b/cg/meta/compress/compress.py @@ -14,7 +14,7 @@ from cg.constants import SequencingFileTag from cg.meta.backup.backup import SpringBackupAPI from cg.meta.compress import files -from cg.models import CompressionData, FileData +from cg.models import CompressionData from cg.store.models import Sample LOG = logging.getLogger(__name__) @@ -73,16 +73,11 @@ def compress_fastq(self, sample_id: str) -> bool: for run_name in sample_fastq: LOG.info(f"Check if compression possible for run {run_name}") compression: CompressionData = sample_fastq[run_name]["compression_data"] - if FileData.is_empty(compression.fastq_first): - LOG.warning(f"Fastq files are empty for {sample_id}: {compression.fastq_first}") - self.delete_fastq_housekeeper( - hk_fastq_first=sample_fastq[run_name]["hk_first"], - hk_fastq_second=sample_fastq[run_name]["hk_second"], - ) - all_ok = False - continue - - if not self.crunchy_api.is_fastq_compression_possible(compression_obj=compression): + is_compression_possible: bool = self._is_fastq_compression_possible( + compression=compression, + sample_id=sample_id, + ) + if not is_compression_possible: LOG.warning(f"FASTQ to SPRING not possible for {sample_id}, run {run_name}") all_ok = False continue @@ -93,6 +88,20 @@ def compress_fastq(self, sample_id: str) -> bool: self.crunchy_api.fastq_to_spring(compression_obj=compression, sample_id=sample_id) return all_ok + def _is_fastq_compression_possible(self, compression: CompressionData, sample_id: str) -> bool: + if self._is_spring_archived(compression): + LOG.debug(f"Found archived Spring file for {sample_id} - compression not possible") + return False + return self.crunchy_api.is_fastq_compression_possible(compression_obj=compression) + + def _is_spring_archived(self, compression_data: CompressionData) -> bool: + spring_file: File | None = self.hk_api.get_file_insensitive_path( + path=compression_data.spring_path + ) + if (not spring_file) or (not spring_file.archive): + return False + return bool(spring_file.archive.archived_at) + def decompress_spring(self, sample_id: str) -> bool: """Decompress SPRING archive for a sample. diff --git a/tests/apps/crunchy/test_crunchy.py b/tests/apps/crunchy/test_crunchy.py index c644a3d2bc..4a4c6e6d1d 100644 --- a/tests/apps/crunchy/test_crunchy.py +++ b/tests/apps/crunchy/test_crunchy.py @@ -56,7 +56,7 @@ def test_is_fastq_compression_possible( assert not spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_possible(compression_object) + result = crunchy_api.is_fastq_compression_possible(compression_obj=compression_object) # THEN result should be True assert result is True @@ -83,7 +83,7 @@ def test_is_fastq_compression_possible_compression_pending( assert not spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_possible(compression_object) + result = crunchy_api.is_fastq_compression_possible(compression_obj=compression_object) # THEN result should be False since the compression flag exists assert result is False @@ -107,7 +107,7 @@ def test_is_fastq_compression_possible_spring_exists( assert spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_possible(compression_object) + result = crunchy_api.is_fastq_compression_possible(compression_obj=compression_object) # THEN result should be False since the compression flag exists assert result is False @@ -116,7 +116,7 @@ def test_is_fastq_compression_possible_spring_exists( def test_is_compression_done( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api, spring_metadata_file: Path, compression_object: CompressionData, caplog, @@ -124,14 +124,13 @@ def test_is_compression_done( """Test if compression is done when everything is correct""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN no SPRING file exists compression_object.spring_path.touch() assert spring_metadata_file == compression_object.spring_metadata_path assert spring_metadata_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be True assert result is True @@ -140,18 +139,17 @@ def test_is_compression_done( def test_is_compression_done_no_spring( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if compression is done when no SPRING archive""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN no SPRING file exists spring_file = compression_object.spring_path assert not spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be false assert not result @@ -160,20 +158,19 @@ def test_is_compression_done_no_spring( def test_is_compression_done_no_flag_spring( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if SPRING compression is done when no metadata file""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() - # GIVEN a non existing flag file + # GIVEN a non-existing flag file assert not compression_object.spring_metadata_path.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be false assert not result @@ -182,7 +179,7 @@ def test_is_compression_done_no_flag_spring( def test_is_compression_done_spring( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, spring_metadata_file: Path, caplog, @@ -190,7 +187,6 @@ def test_is_compression_done_spring( """Test if compression is done when SPRING files exists""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -199,7 +195,7 @@ def test_is_compression_done_spring( assert compression_object.spring_metadata_path.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be True assert result @@ -208,7 +204,7 @@ def test_is_compression_done_spring( def test_is_compression_done_spring_new_files( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, spring_metadata_file: Path, caplog, @@ -219,7 +215,6 @@ def test_is_compression_done_spring_new_files( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -237,7 +232,7 @@ def test_is_compression_done_spring_new_files( assert "updated" in file_info # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be False since the updated date < 3 weeks assert result is False @@ -246,7 +241,7 @@ def test_is_compression_done_spring_new_files( def test_is_compression_done_spring_old_files( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, spring_metadata_file: Path, caplog, @@ -257,7 +252,6 @@ def test_is_compression_done_spring_old_files( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -277,7 +271,7 @@ def test_is_compression_done_spring_old_files( ) # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be True since the updated date > 3 weeks assert result is True @@ -286,7 +280,7 @@ def test_is_compression_done_spring_old_files( def test_is_spring_decompression_possible_no_fastq( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if decompression is possible when there are no FASTQ files @@ -294,7 +288,6 @@ def test_is_spring_decompression_possible_no_fastq( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -305,7 +298,7 @@ def test_is_spring_decompression_possible_no_fastq( assert not compression_object.fastq_second.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_spring_decompression_possible(compression_object) + result = real_crunchy_api.is_spring_decompression_possible(compression_object) # THEN result should be True since there are no fastq files assert result is True @@ -314,7 +307,7 @@ def test_is_spring_decompression_possible_no_fastq( def test_is_spring_decompression_possible_no_spring( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if decompression is possible when there are no SPRING archive @@ -322,10 +315,9 @@ def test_is_spring_decompression_possible_no_spring( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # WHEN checking if SPRING compression is done - result = crunchy_api.is_spring_decompression_possible(compression_object) + result = real_crunchy_api.is_spring_decompression_possible(compression_object) # THEN result should be False since there is no SPRING archive assert result is False @@ -334,7 +326,7 @@ def test_is_spring_decompression_possible_no_spring( def test_is_spring_decompression_possible_fastq( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if decompression is possible when there are existing FASTQ files @@ -342,7 +334,6 @@ def test_is_spring_decompression_possible_fastq( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() # GIVEN that the FASTQ files exists @@ -350,7 +341,7 @@ def test_is_spring_decompression_possible_fastq( compression_object.fastq_second.touch() # WHEN checking if SPRING decompression is possible - result = crunchy_api.is_spring_decompression_possible(compression_object) + result = real_crunchy_api.is_spring_decompression_possible(compression_object) # THEN result should be False since the FASTQ files already exists assert result is False @@ -359,31 +350,29 @@ def test_is_spring_decompression_possible_fastq( def test_is_not_pending_when_no_flag_file( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData + real_crunchy_api: CrunchyAPI, compression_object: CompressionData ): """Test if SPRING compression is pending when no flag file""" # GIVEN a crunchy-api, and a FASTQ file - crunchy_api = CrunchyAPI(crunchy_config) - # GIVEN a non existing pending flag + # GIVEN a non-existing pending flag assert not compression_object.pending_path.exists() # WHEN checking if SPRING compression is ongoing - result = crunchy_api.is_compression_pending(compression_object) + result = real_crunchy_api.is_compression_pending(compression_object) # THEN result should be False since the pending flag is not there assert result is False -def test_is_pending(crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData): +def test_is_pending(real_crunchy_api: CrunchyAPI, compression_object: CompressionData): """Test if SPRING compression is pending when pending file exists""" # GIVEN a crunchy-api, and FASTQ files - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing pending flag compression_object.pending_path.touch() assert compression_object.pending_path.exists() # WHEN checking if SPRING compression is pending - result = crunchy_api.is_compression_pending(compression_object) + result = real_crunchy_api.is_compression_pending(compression_object) # THEN result should be True since the pending_path exists assert result is True diff --git a/tests/conftest.py b/tests/conftest.py index 7031d7c179..470cb447ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,12 +14,15 @@ from housekeeper.store.models import File, Version from requests import Response +from cg.apps.crunchy import CrunchyAPI from cg.apps.demultiplex.demultiplex_api import DemultiplexingAPI from cg.apps.demultiplex.sample_sheet.sample_models import ( FlowCellSampleBcl2Fastq, FlowCellSampleBCLConvert, ) -from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import SampleSheetCreatorBCLConvert +from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import ( + SampleSheetCreatorBCLConvert, +) from cg.apps.downsample.downsample import DownsampleAPI from cg.apps.gens import GensAPI from cg.apps.gt import GenotypeAPI @@ -381,6 +384,11 @@ def crunchy_config() -> dict[str, dict[str, Any]]: } +@pytest.fixture +def real_crunchy_api(crunchy_config) -> CrunchyAPI: + return CrunchyAPI(crunchy_config) + + @pytest.fixture def hk_config_dict(root_path: Path): """Housekeeper configs.""" diff --git a/tests/meta/compress/test_compress_meta_fastq.py b/tests/meta/compress/test_compress_meta_fastq.py index 1431fb6796..49fb08cf95 100644 --- a/tests/meta/compress/test_compress_meta_fastq.py +++ b/tests/meta/compress/test_compress_meta_fastq.py @@ -1,5 +1,8 @@ """Tests for FASTQ part of meta compress api""" import logging +from unittest import mock + +from cg.meta.compress import CompressAPI def test_compress_case_fastq_one_sample(populated_compress_fastq_api, sample, caplog): @@ -10,14 +13,15 @@ def test_compress_case_fastq_one_sample(populated_compress_fastq_api, sample, ca # GIVEN a populated compress api # WHEN Compressing the bam files for the case - res = compress_api.compress_fastq(sample) + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=False): + result = compress_api.compress_fastq(sample) - # THEN assert compression succeded - assert res is True - # THEN assert that the correct information is communicated - assert "Compressing" in caplog.text - # THEN assert that the correct information is communicated - assert "to SPRING format" in caplog.text + # THEN assert compression succeded + assert result is True + # THEN assert that the correct information is communicated + assert "Compressing" in caplog.text + # THEN assert that the correct information is communicated + assert "to SPRING format" in caplog.text def test_compress_fastq_compression_done( @@ -34,15 +38,16 @@ def test_compress_fastq_compression_done( compression_object.spring_path.touch() # WHEN Compressing the bam files for the case - res = compress_api.compress_fastq(sample) + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=False): + result = compress_api.compress_fastq(sample) - # THEN assert compression succeded - assert res is False - # THEN assert that the correct information is communicated - assert f"FASTQ to SPRING not possible for {sample}" in caplog.text + # THEN assert compression succeded + assert result is False + # THEN assert that the correct information is communicated + assert f"FASTQ to SPRING not possible for {sample}" in caplog.text -def test_compress_case_fastq_compression_pending( +def test_compress_sample_fastq_compression_pending( populated_compress_fastq_api, sample, compression_object, caplog ): """Test to compress all FASTQ files for a sample when compression is pending @@ -55,9 +60,32 @@ def test_compress_case_fastq_compression_pending( compression_object.pending_path.touch() # WHEN compressing the FASTQ files for the case - res = compress_api.compress_fastq(sample) + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=False): + result = compress_api.compress_fastq(sample) + + # THEN assert compression returns False + assert result is False + # THEN assert that the correct information is communicated + assert f"FASTQ to SPRING not possible for {sample}" in caplog.text + + +def test_compress_sample_fastq_archived_spring_file( + populated_compress_fastq_api, sample, compression_object, caplog +): + """Test to compress all FASTQ files for a sample when the Spring file is archived + + The program should not compress any files since the Spring file already exists + """ + caplog.set_level(logging.DEBUG) + compress_api = populated_compress_fastq_api + # GIVEN that the pending flag exists + compression_object.pending_path.touch() + + # WHEN compressing the FASTQ files for the case + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=True): + result = compress_api.compress_fastq(sample) - # THEN assert compression returns False - assert res is False - # THEN assert that the correct information is communicated - assert f"FASTQ to SPRING not possible for {sample}" in caplog.text + # THEN assert compression returns False + assert result is False + # THEN assert that the correct information is communicated + assert f"FASTQ to SPRING not possible for {sample}" in caplog.text From 18f37a18bd8d5ff1e33811955d1f2217ddff9813 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Wed, 10 Jan 2024 12:52:25 +0000 Subject: [PATCH 10/18] =?UTF-8?q?Bump=20version:=2055.0.0=20=E2=86=92=2055?= =?UTF-8?q?.1.0=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a147ddf8ab..5c8fdb1b05 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 55.0.0 +current_version = 55.1.0 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 7d38e035e4..c3144b4344 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "55.0.0" +__version__ = "55.1.0" diff --git a/pyproject.toml b/pyproject.toml index 94f26ceab1..1afed31be1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "55.0.0" +version = "55.1.0" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From dd803dec393d1e6b889c883f82f6b599fe0308cc Mon Sep 17 00:00:00 2001 From: Sebastian Allard Date: Wed, 10 Jan 2024 14:30:17 +0100 Subject: [PATCH 11/18] Remove mark_analyses_deleted (#2817)(patch) --- cg/apps/tb/api.py | 14 -------------- cg/meta/workflow/analysis.py | 1 - tests/cli/workflow/conftest.py | 10 ---------- tests/mocks/tb_mock.py | 3 --- 4 files changed, 28 deletions(-) diff --git a/cg/apps/tb/api.py b/cg/apps/tb/api.py index b4c836b2eb..107051809c 100644 --- a/cg/apps/tb/api.py +++ b/cg/apps/tb/api.py @@ -91,20 +91,6 @@ def is_latest_analysis_completed(self, case_id: str) -> bool: def is_latest_analysis_qc(self, case_id: str) -> bool: return self.get_latest_analysis_status(case_id=case_id) == AnalysisStatus.QC - def mark_analyses_deleted(self, case_id: str) -> list | None: - """Mark all analyses for case deleted without removing analysis files""" - request_body = { - "case_id": case_id, - } - response = self.query_trailblazer( - command="mark-analyses-deleted", request_body=request_body - ) - if response: - if isinstance(response, list): - return [TrailblazerAnalysis.model_validate(analysis) for analysis in response] - if isinstance(response, dict): - return [TrailblazerAnalysis.model_validate(response)] - def add_pending_analysis( self, case_id: str, diff --git a/cg/meta/workflow/analysis.py b/cg/meta/workflow/analysis.py index c22df56374..84b10c0839 100644 --- a/cg/meta/workflow/analysis.py +++ b/cg/meta/workflow/analysis.py @@ -207,7 +207,6 @@ def get_analysis_finish_path(self, case_id: str) -> Path: def add_pending_trailblazer_analysis(self, case_id: str) -> None: self.check_analysis_ongoing(case_id) - self.trailblazer_api.mark_analyses_deleted(case_id) self.trailblazer_api.add_pending_analysis( case_id=case_id, email=environ_email(), diff --git a/tests/cli/workflow/conftest.py b/tests/cli/workflow/conftest.py index 8f676c960f..2d3c07014a 100644 --- a/tests/cli/workflow/conftest.py +++ b/tests/cli/workflow/conftest.py @@ -135,7 +135,6 @@ class MockTB: def __init__(self): self._link_was_called = False - self._mark_analyses_deleted_called = False self._add_pending_was_called = False self._add_pending_analysis_was_called = False self._family = None @@ -187,11 +186,6 @@ def first_was_called(self): return Row() - def mark_analyses_deleted(self, case_id: str): - """Mock this function""" - self._case_id = case_id - self._mark_analyses_deleted_called = True - def add_pending(self, case_id: str, email: str): """Mock this function""" self._case_id = case_id @@ -204,10 +198,6 @@ def add_pending_analysis(self, case_id: str, email: str): self._email = email self._add_pending_analysis_was_called = True - def mark_analyses_deleted_called(self): - """check if mark_analyses_deleted was called""" - return self._mark_analyses_deleted_called - def add_pending_was_called(self): """check if add_pending was called""" return self._add_pending_was_called diff --git a/tests/mocks/tb_mock.py b/tests/mocks/tb_mock.py index 3355265db2..705e20ef44 100644 --- a/tests/mocks/tb_mock.py +++ b/tests/mocks/tb_mock.py @@ -12,9 +12,6 @@ def is_latest_analysis_ongoing(self, *args, **kwargs) -> bool: def add_pending_analysis(self, *args, **kwargs) -> None: return None - def mark_analyses_deleted(self, *args, **kwargs) -> None: - return None - def add_commit(self, *args, **kwargs) -> None: return None From 92394bcf0b52a06ed5f972f4298d821beab90b5c Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Wed, 10 Jan 2024 13:30:45 +0000 Subject: [PATCH 12/18] =?UTF-8?q?Bump=20version:=2055.1.0=20=E2=86=92=2055?= =?UTF-8?q?.1.1=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5c8fdb1b05..cefe97934f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 55.1.0 +current_version = 55.1.1 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index c3144b4344..1fc22e2fdb 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "55.1.0" +__version__ = "55.1.1" diff --git a/pyproject.toml b/pyproject.toml index 1afed31be1..2f04e3c652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "55.1.0" +version = "55.1.1" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From fc14a35ca1e8c32635d6fd2e54c370e21bf8965c Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 11 Jan 2024 09:49:46 +0100 Subject: [PATCH 13/18] feat - use fixture plugins instead of conftest files (#2816)(minor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #2095 and continue #2259. ### TL;DR The main `conftest.py` was too crowded, now the fixtures are separated into different modules available to all tests. ### Details The current test system looks for fixtures defined in the `conftest.py` files, either in the local directory or in any directory above, but it is impossible to use a fixture defined in a directory below it. For example, a test function in `test_module_top.py` can't access a fixture defined in `tests/subfolder/subsubfolder/conftest.py`: ```shell tests ├── conftest.py ├── subfolder │ ├── sub_subfolder │ │ ├── __init__.py │ │ ├── conftest.py │ │ └── test_module_sub.py └── test_module_top.py ``` To solve this, we started writing all our fixtures in the first contest, i.e. `tests/conftest.py`. This made this file extremely crowded and difficult to navigate. This PR changes the conftest structure for a plugin structure, in which the fixtures are grouped by topic in a folder called `fixture_plugins` and imported in the conftest as a plugin. In this way, the structure of the tests is ```shell tests ├── conftest.py ├── fixture_plugins │ ├── fixture_subfolder │ │ ├── fixtures_topic_1.py │ │ └── __init__.py │ ├── fixtures_topic_2.py │ └── __init__.py ├── test_subfolder │ ├── __init__.py │ └── test_module_sub.py └── test_module_top.py ``` and both `test_module_top.py` and `test_module_sub.py` have access to all the fixtures defined in `fixture_plugins`. ### Added - Fixture plugin modules with fixtures in the new folder `tests/fixture_plugins/` with the following structure: ```shell tests ├── conftest.py ├── fixture_plugins │ ├── demultiplex_fixtures │ │ ├── __init__.py │ │ ├── flow_cell_fixtures.py │ │ ├── name_fixtures.py │ │ ├── path_fixtures.py │ │ ├── run_parameters_fixtures.py │ │ └── sample_fixtures.py │ ├── __init__.py │ └── timestamp_fixtures.py ``` ### Changed - Moved demultiplexing and timestamp fixtures from `conftest.py` to separate modules in the new fixture plugin directory. --- tests/conftest.py | 1601 ++++------------- tests/fixture_plugins/__init__.py | 0 .../demultiplex_fixtures/__init__.py | 0 .../flow_cell_fixtures.py | 172 ++ .../demultiplex_fixtures/name_fixtures.py | 116 ++ .../demultiplex_fixtures/path_fixtures.py | 458 +++++ .../run_parameters_fixtures.py | 98 + .../demultiplex_fixtures/sample_fixtures.py | 135 ++ tests/fixture_plugins/timestamp_fixtures.py | 46 + 9 files changed, 1336 insertions(+), 1290 deletions(-) create mode 100644 tests/fixture_plugins/__init__.py create mode 100644 tests/fixture_plugins/demultiplex_fixtures/__init__.py create mode 100644 tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py create mode 100644 tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py create mode 100644 tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py create mode 100644 tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py create mode 100644 tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py create mode 100644 tests/fixture_plugins/timestamp_fixtures.py diff --git a/tests/conftest.py b/tests/conftest.py index 470cb447ad..12b16a06cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import os import shutil from copy import deepcopy -from datetime import MAXYEAR, datetime, timedelta +from datetime import datetime from pathlib import Path from subprocess import CompletedProcess from typing import Any, Generator @@ -16,29 +16,21 @@ from cg.apps.crunchy import CrunchyAPI from cg.apps.demultiplex.demultiplex_api import DemultiplexingAPI -from cg.apps.demultiplex.sample_sheet.sample_models import ( - FlowCellSampleBcl2Fastq, - FlowCellSampleBCLConvert, -) -from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import ( - SampleSheetCreatorBCLConvert, -) from cg.apps.downsample.downsample import DownsampleAPI from cg.apps.gens import GensAPI from cg.apps.gt import GenotypeAPI from cg.apps.hermes.hermes_api import HermesApi from cg.apps.housekeeper.hk import HousekeeperAPI -from cg.apps.lims.api import LimsAPI +from cg.apps.lims import LimsAPI from cg.apps.slurm.slurm_api import SlurmAPI from cg.constants import FileExtensions, Pipeline, SequencingFileTag from cg.constants.constants import CaseActions, FileFormat, Strandedness -from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles +from cg.constants.demultiplexing import DemultiplexingDirsAndFiles from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG -from cg.constants.nanopore_files import NanoporeDirsAndFiles from cg.constants.priority import SlurmQos from cg.constants.sequencing import SequencingPlatform from cg.constants.subject import Sex -from cg.io.controller import ReadFile, WriteFile +from cg.io.controller import WriteFile from cg.io.json import read_json, write_json from cg.io.yaml import write_yaml from cg.meta.encryption.encryption import FlowCellEncryptionAPI @@ -50,11 +42,6 @@ from cg.meta.workflow.taxprofiler import TaxprofilerAnalysisAPI from cg.models import CompressionData from cg.models.cg_config import CGConfig, PDCArchivingDirectory -from cg.models.demultiplex.run_parameters import ( - RunParametersHiSeq, - RunParametersNovaSeq6000, - RunParametersNovaSeqX, -) from cg.models.downsample.downsample_data import DownsampleData from cg.models.flow_cell.flow_cell import FlowCellDirectoryData from cg.models.rnafusion.rnafusion import RnafusionParameters @@ -76,51 +63,14 @@ LOG = logging.getLogger(__name__) - -# Timestamp fixture - - -@pytest.fixture(scope="session") -def old_timestamp() -> datetime: - """Return a time stamp in date time format.""" - return datetime(1900, 1, 1) - - -@pytest.fixture(scope="session") -def timestamp() -> datetime: - """Return a time stamp in date time format.""" - return datetime(2020, 5, 1) - - -@pytest.fixture(scope="session") -def later_timestamp() -> datetime: - """Return a time stamp in date time format.""" - return datetime(2020, 6, 1) - - -@pytest.fixture(scope="session") -def future_date() -> datetime: - """Return a distant date in the future for which no events happen later.""" - return datetime(MAXYEAR, 1, 1, 1, 1, 1) - - -@pytest.fixture(scope="session") -def timestamp_now() -> datetime: - """Return a time stamp of today's date in date time format.""" - return datetime.now() - - -@pytest.fixture(scope="session") -def timestamp_yesterday(timestamp_now: datetime) -> datetime: - """Return a time stamp of yesterday's date in date time format.""" - return timestamp_now - timedelta(days=1) - - -@pytest.fixture(scope="session") -def timestamp_in_2_weeks(timestamp_now: datetime) -> datetime: - """Return a time stamp 14 days ahead in time.""" - return timestamp_now + timedelta(days=14) - +pytest_plugins = [ + "tests.fixture_plugins.timestamp_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.flow_cell_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.name_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.path_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.run_parameters_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.sample_fixtures", +] # Case fixtures @@ -384,6 +334,59 @@ def crunchy_config() -> dict[str, dict[str, Any]]: } +@pytest.fixture +def demultiplexing_context_for_demux( + demultiplexing_api_for_demux: DemultiplexingAPI, + cg_context: CGConfig, + store_with_demultiplexed_samples: Store, +) -> CGConfig: + """Return cg context with a demultiplex context.""" + cg_context.demultiplex_api_ = demultiplexing_api_for_demux + cg_context.housekeeper_api_ = demultiplexing_api_for_demux.hk_api + cg_context.status_db_ = store_with_demultiplexed_samples + return cg_context + + +@pytest.fixture(name="demultiplex_context") +def demultiplex_context( + demultiplexing_api: DemultiplexingAPI, + real_housekeeper_api: HousekeeperAPI, + cg_context: CGConfig, + store_with_demultiplexed_samples: Store, +) -> CGConfig: + """Return cg context with a demultiplex context.""" + cg_context.demultiplex_api_ = demultiplexing_api + cg_context.housekeeper_api_ = real_housekeeper_api + cg_context.status_db_ = store_with_demultiplexed_samples + return cg_context + + +@pytest.fixture(name="demultiplex_configs_for_demux") +def demultiplex_configs_for_demux( + tmp_flow_cells_demux_all_directory: Path, + tmp_empty_demultiplexed_runs_directory: Path, +) -> dict: + """Return demultiplex configs.""" + return { + "flow_cells_dir": tmp_flow_cells_demux_all_directory.as_posix(), + "demultiplexed_flow_cells_dir": tmp_empty_demultiplexed_runs_directory.as_posix(), + "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, + } + + +@pytest.fixture(name="demultiplex_configs") +def demultiplex_configs( + tmp_flow_cells_directory: Path, + tmp_demultiplexed_runs_directory: Path, +) -> dict: + """Return demultiplex configs.""" + return { + "flow_cells_dir": tmp_flow_cells_directory.as_posix(), + "demultiplexed_flow_cells_dir": tmp_demultiplexed_runs_directory.as_posix(), + "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, + } + + @pytest.fixture def real_crunchy_api(crunchy_config) -> CrunchyAPI: return CrunchyAPI(crunchy_config) @@ -423,9 +426,46 @@ def gens_config() -> dict[str, dict[str, str]]: } +@pytest.fixture(name="sample_sheet_context") +def sample_sheet_context( + cg_context: CGConfig, lims_api: LimsAPI, populated_housekeeper_api: HousekeeperAPI +) -> CGConfig: + """Return cg context with added Lims and Housekeeper API.""" + cg_context.lims_api_ = lims_api + cg_context.housekeeper_api_ = populated_housekeeper_api + return cg_context + + # Api fixtures +@pytest.fixture(name="demultiplexing_api_for_demux") +def demultiplexing_api_for_demux( + demultiplex_configs_for_demux: dict, + sbatch_process: Process, + populated_housekeeper_api: HousekeeperAPI, +) -> DemultiplexingAPI: + """Return demultiplex API.""" + demux_api = DemultiplexingAPI( + config=demultiplex_configs_for_demux, + housekeeper_api=populated_housekeeper_api, + ) + demux_api.slurm_api.process = sbatch_process + return demux_api + + +@pytest.fixture +def demultiplexing_api( + demultiplex_configs: dict, sbatch_process: Process, populated_housekeeper_api: HousekeeperAPI +) -> DemultiplexingAPI: + """Return demultiplex API.""" + demux_api = DemultiplexingAPI( + config=demultiplex_configs, housekeeper_api=populated_housekeeper_api + ) + demux_api.slurm_api.process = sbatch_process + return demux_api + + @pytest.fixture def rsync_api(cg_context: CGConfig) -> RsyncAPI: """RsyncAPI fixture.""" @@ -514,1325 +554,272 @@ def sv_research_vcf_file() -> str: # Common file fixtures @pytest.fixture(scope="session") -def fixtures_dir() -> Path: - """Return the path to the fixtures dir.""" - return Path("tests", "fixtures") - - -@pytest.fixture(scope="session") -def analysis_dir(fixtures_dir: Path) -> Path: - """Return the path to the analysis dir.""" - return Path(fixtures_dir, "analysis") - - -@pytest.fixture(scope="session") -def microsalt_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the analysis dir.""" - return Path(analysis_dir, "microsalt") - - -@pytest.fixture(scope="session") -def apps_dir(fixtures_dir: Path) -> Path: - """Return the path to the apps dir.""" - return Path(fixtures_dir, "apps") - - -@pytest.fixture(scope="session") -def cgweb_orders_dir(fixtures_dir: Path) -> Path: - """Return the path to the cgweb_orders dir.""" - return Path(fixtures_dir, "cgweb_orders") - - -@pytest.fixture(scope="session") -def data_dir(fixtures_dir: Path) -> Path: - """Return the path to the data dir.""" - return Path(fixtures_dir, "data") - - -@pytest.fixture -def fastq_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the fastq files dir.""" - return Path(demultiplex_fixtures, "fastq") - - -@pytest.fixture -def spring_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the fastq files dir.""" - return Path(demultiplex_fixtures, "spring") - - -@pytest.fixture -def project_dir(tmpdir_factory) -> Generator[Path, None, None]: - """Path to a temporary directory where intermediate files can be stored.""" - yield Path(tmpdir_factory.mktemp("data")) - - -@pytest.fixture -def tmp_file(project_dir) -> Path: - """Return a temp file path.""" - return Path(project_dir, "test") - - -@pytest.fixture -def non_existing_file_path(project_dir: Path) -> Path: - """Return the path to a non-existing file.""" - return Path(project_dir, "a_file.txt") - - -@pytest.fixture(scope="session") -def content() -> str: - """Return some content for a file.""" - return ( - "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt" - " ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ull" - "amco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehende" - "rit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaec" - "at cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." - ) - - -@pytest.fixture -def filled_file(non_existing_file_path: Path, content: str) -> Path: - """Return the path to a existing file with some content.""" - with open(non_existing_file_path, "w") as outfile: - outfile.write(content) - return non_existing_file_path - - -@pytest.fixture(scope="session") -def orderforms(fixtures_dir: Path) -> Path: - """Return the path to the directory with order forms.""" - return Path(fixtures_dir, "orderforms") - - -@pytest.fixture -def hk_file(filled_file: Path, case_id: str) -> File: - """Return a housekeeper File object.""" - return File(id=case_id, path=filled_file.as_posix()) - - -@pytest.fixture -def mip_dna_store_files(apps_dir: Path) -> Path: - """Return the path to the directory with mip dna store files.""" - return Path(apps_dir, "mip", "dna", "store") - - -@pytest.fixture -def case_qc_sample_info_path(mip_dna_store_files: Path) -> Path: - """Return path to case_qc_sample_info.yaml.""" - return Path(mip_dna_store_files, "case_qc_sample_info.yaml") - - -@pytest.fixture -def delivery_report_html(mip_dna_store_files: Path) -> Path: - """Return the path to a qc metrics deliverables file with case data.""" - return Path(mip_dna_store_files, "empty_delivery_report.html") - - -@pytest.fixture -def mip_deliverables_file(mip_dna_store_files: Path) -> Path: - """Fixture for general deliverables file in mip.""" - return Path(mip_dna_store_files, "case_id_deliverables.yaml") - - -@pytest.fixture -def case_qc_metrics_deliverables(apps_dir: Path) -> Path: - """Return the path to a qc metrics deliverables file with case data.""" - return Path(apps_dir, "mip", "case_metrics_deliverables.yaml") - - -@pytest.fixture -def mip_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the directory with mip analysis files.""" - return Path(analysis_dir, "mip") - - -@pytest.fixture -def balsamic_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the directory with balsamic analysis files.""" - return Path(analysis_dir, "balsamic") - - -@pytest.fixture -def balsamic_wgs_analysis_dir(balsamic_analysis_dir: Path) -> Path: - """Return the path to the directory with balsamic analysis files.""" - return Path(balsamic_analysis_dir, "tn_wgs") - - -@pytest.fixture -def mip_dna_analysis_dir(mip_analysis_dir: Path) -> Path: - """Return the path to the directory with mip dna analysis files.""" - return Path(mip_analysis_dir, "dna") - - -@pytest.fixture -def rnafusion_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the directory with rnafusion analysis files.""" - return Path(analysis_dir, "rnafusion") - - -@pytest.fixture -def sample_cram(mip_dna_analysis_dir: Path) -> Path: - """Return the path to the cram file for a sample.""" - return Path(mip_dna_analysis_dir, "adm1.cram") - - -@pytest.fixture(name="father_sample_cram") -def father_sample_cram( - mip_dna_analysis_dir: Path, - father_sample_id: str, -) -> Path: - """Return the path to the cram file for the father sample.""" - return Path(mip_dna_analysis_dir, father_sample_id + FileExtensions.CRAM) - - -@pytest.fixture(name="mother_sample_cram") -def mother_sample_cram(mip_dna_analysis_dir: Path, mother_sample_id: str) -> Path: - """Return the path to the cram file for the mother sample.""" - return Path(mip_dna_analysis_dir, mother_sample_id + FileExtensions.CRAM) - - -@pytest.fixture(name="sample_cram_files") -def sample_crams( - sample_cram: Path, father_sample_cram: Path, mother_sample_cram: Path -) -> list[Path]: - """Return a list of cram paths for three samples.""" - return [sample_cram, father_sample_cram, mother_sample_cram] - - -@pytest.fixture(name="vcf_file") -def vcf_file(mip_dna_store_files: Path) -> Path: - """Return the path to a VCF file.""" - return Path(mip_dna_store_files, "yellowhog_clinical_selected.vcf") - - -@pytest.fixture(name="fastq_file") -def fastq_file(fastq_dir: Path) -> Path: - """Return the path to a FASTQ file.""" - return Path(fastq_dir, "dummy_run_R1_001.fastq.gz") - - -@pytest.fixture(name="fastq_file_father") -def fastq_file_father(fastq_dir: Path) -> Path: - """Return the path to a FASTQ file.""" - return Path(fastq_dir, "fastq_run_R1_001.fastq.gz") - - -@pytest.fixture(name="spring_file") -def spring_file(spring_dir: Path) -> Path: - """Return the path to an existing spring file.""" - return Path(spring_dir, "dummy_run_001.spring") - - -@pytest.fixture(name="spring_meta_data_file") -def spring_meta_data_file(spring_dir: Path) -> Path: - """Return the path to an existing spring file.""" - return Path(spring_dir, "dummy_spring_meta_data.json") - - -@pytest.fixture(name="spring_file_father") -def spring_file_father(spring_dir: Path) -> Path: - """Return the path to a second existing spring file.""" - return Path(spring_dir, "dummy_run_002.spring") - - -@pytest.fixture(name="madeline_output") -def madeline_output(apps_dir: Path) -> Path: - """Return str of path for file with Madeline output.""" - return Path(apps_dir, "madeline", "madeline.xml") - - -@pytest.fixture(name="file_does_not_exist") -def file_does_not_exist() -> Path: - """Return a file path that does not exist.""" - return Path("file", "does", "not", "exist") - - -# Compression fixtures - - -@pytest.fixture(name="run_name") -def run_name() -> str: - """Return the name of a fastq run.""" - return "fastq_run" - - -@pytest.fixture(name="original_fastq_data") -def original_fastq_data(fastq_dir: Path, run_name) -> CompressionData: - """Return a compression object with a path to the original fastq files.""" - return CompressionData(Path(fastq_dir, run_name)) - - -@pytest.fixture(name="fastq_stub") -def fastq_stub(project_dir: Path, run_name: str) -> Path: - """Creates a path to the base format of a fastq run.""" - return Path(project_dir, run_name) - - -@pytest.fixture(name="compression_object") -def compression_object(fastq_stub: Path, original_fastq_data: CompressionData) -> CompressionData: - """Creates compression data object with information about files used in fastq compression.""" - working_files: CompressionData = CompressionData(fastq_stub) - working_file_map: dict[str, str] = { - original_fastq_data.fastq_first.as_posix(): working_files.fastq_first.as_posix(), - original_fastq_data.fastq_second.as_posix(): working_files.fastq_second.as_posix(), - } - for original_file, working_file in working_file_map.items(): - shutil.copy(original_file, working_file) - return working_files - - -# Demultiplex fixtures - - -@pytest.fixture -def lims_novaseq_bcl_convert_samples( - lims_novaseq_samples_raw: list[dict], -) -> list[FlowCellSampleBCLConvert]: - """Return a list of parsed flow cell samples demultiplexed with BCL convert.""" - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in lims_novaseq_samples_raw] - - -@pytest.fixture -def lims_novaseq_bcl2fastq_samples( - lims_novaseq_samples_raw: list[dict], -) -> list[FlowCellSampleBcl2Fastq]: - """Return a list of parsed Bcl2fastq flow cell samples""" - return [FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_samples_raw] - - -@pytest.fixture -def lims_novaseq_6000_bcl2fastq_samples( - lims_novaseq_6000_sample_raw: list[dict], -) -> list[FlowCellSampleBcl2Fastq]: - """Return a list of parsed Bcl2fastq flow cell samples""" - return [ - FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_6000_sample_raw - ] - - -@pytest.fixture(name="tmp_flow_cells_directory") -def tmp_flow_cells_directory(tmp_path: Path, flow_cells_dir: Path) -> Path: - """ - Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. - Generates a copy of the original flow cells directory - """ - original_dir = flow_cells_dir - tmp_dir = Path(tmp_path, "flow_cells") - - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="tmp_flow_cells_demux_all_directory") -def tmp_flow_cells_demux_all_directory(tmp_path: Path, flow_cells_demux_all_dir: Path) -> Path: - """ - Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. - Generates a copy of the original flow cells directory. - This fixture is used for testing of the cg demutliplex all cmd. - """ - original_dir = flow_cells_demux_all_dir - tmp_dir = Path(tmp_path, "flow_cells_demux_all") - - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="tmp_flow_cell_directory_bcl2fastq") -def flow_cell_working_directory_bcl2fastq( - bcl2fastq_flow_cell_dir: Path, tmp_flow_cells_directory: Path -) -> Path: - """Return the path to a working directory that will be deleted after test is run. - - This is a path to a flow cell directory with the run parameters present. - """ - return Path(tmp_flow_cells_directory, bcl2fastq_flow_cell_dir.name) - - -@pytest.fixture(name="tmp_flow_cell_directory_bclconvert") -def flow_cell_working_directory_bclconvert( - bcl_convert_flow_cell_dir: Path, tmp_flow_cells_directory: Path -) -> Path: - """Return the path to a working directory that will be deleted after test is run. - This is a path to a flow cell directory with the run parameters present. - """ - return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_dir.name) - - -@pytest.fixture -def tmp_flow_cell_name_no_run_parameters() -> str: - """This is the name of a flow cell directory with the run parameters missing.""" - return "180522_A00689_0200_BHLCKNCCXY" - - -@pytest.fixture -def tmp_flow_cell_name_malformed_sample_sheet() -> str: - """ "Returns the name of a flow cell directory ready for demultiplexing with BCL convert. - Contains a sample sheet with malformed headers. - """ - return "201203_A00689_0200_AHVKJCDRXY" - - -@pytest.fixture -def tmp_flow_cell_name_no_sample_sheet() -> str: - """Return the name of a flow cell directory with the run parameters and sample sheet missing.""" - return "170407_A00689_0209_BHHKVCALXX" - - -@pytest.fixture(name="tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq") -def tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq() -> str: - """Returns the name of a flow cell directory ready for demultiplexing with bcl2fastq.""" - return "211101_D00483_0615_AHLG5GDRXY" - - -@pytest.fixture -def tmp_flow_cells_directory_no_run_parameters( - tmp_flow_cell_name_no_run_parameters: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the run parameters missing.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_run_parameters) - - -@pytest.fixture(name="tmp_flow_cells_directory_no_sample_sheet") -def tmp_flow_cells_directory_no_sample_sheet( - tmp_flow_cell_name_no_sample_sheet: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the sample sheet and run parameters missing.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_sample_sheet) - - -@pytest.fixture -def tmp_flow_cells_directory_malformed_sample_sheet( - tmp_flow_cell_name_malformed_sample_sheet: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with a sample sheet with malformed headers.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_malformed_sample_sheet) - - -@pytest.fixture -def tmp_flow_cells_directory_ready_for_demultiplexing_bcl_convert( - bcl_convert_flow_cell_full_name: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the run parameters missing.""" - return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_full_name) - - -@pytest.fixture -def tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq( - tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the run parameters missing.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq) - - -# Temporary demultiplexed runs fixtures -@pytest.fixture(name="tmp_demultiplexed_runs_directory") -def tmp_demultiplexed_flow_cells_directory(tmp_path: Path, demultiplexed_runs: Path) -> Path: - """Return the path to a temporary demultiplex-runs directory. - Generates a copy of the original demultiplexed-runs - """ - original_dir = demultiplexed_runs - tmp_dir = Path(tmp_path, "demultiplexed-runs") - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="tmp_demultiplexed_runs_bcl2fastq_directory") -def tmp_demultiplexed_runs_bcl2fastq_directory( - tmp_demultiplexed_runs_directory: Path, bcl2fastq_flow_cell_dir: Path -) -> Path: - """Return the path to a temporary demultiplex-runs bcl2fastq flow cell directory.""" - return Path(tmp_demultiplexed_runs_directory, bcl2fastq_flow_cell_dir.name) - - -@pytest.fixture(name="tmp_bcl2fastq_flow_cell") -def tmp_bcl2fastq_flow_cell( - tmp_demultiplexed_runs_bcl2fastq_directory: Path, -) -> FlowCellDirectoryData: - """Create a flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=tmp_demultiplexed_runs_bcl2fastq_directory, - bcl_converter=BclConverter.BCL2FASTQ, - ) - - -@pytest.fixture -def novaseq6000_flow_cell( - tmp_flow_cells_directory_malformed_sample_sheet: Path, -) -> FlowCellDirectoryData: - """Return a NovaSeq6000 flow cell.""" - return FlowCellDirectoryData( - flow_cell_path=tmp_flow_cells_directory_malformed_sample_sheet, - bcl_converter=BclConverter.BCLCONVERT, - ) - - -@pytest.fixture(name="tmp_bcl_convert_flow_cell") -def tmp_bcl_convert_flow_cell( - tmp_flow_cell_directory_bclconvert: Path, -) -> FlowCellDirectoryData: - """Create a flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=tmp_flow_cell_directory_bclconvert, - bcl_converter=BclConverter.DRAGEN, - ) - - -@pytest.fixture(name="tmp_demultiplexed_runs_not_finished_directory") -def tmp_demultiplexed_runs_not_finished_flow_cells_directory( - tmp_path: Path, demux_results_not_finished_dir: Path -) -> Path: - """ - Return a temporary demultiplex-runs-unfinished path with an unfinished flow cell directory. - Generates a copy of the original demultiplexed-runs-unfinished directory. - """ - original_dir = demux_results_not_finished_dir - tmp_dir = Path(tmp_path, "demultiplexed-runs-unfinished") - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory") -def demultiplexed_runs_bcl2fastq_flow_cell_directory( - tmp_demultiplexed_runs_not_finished_directory: Path, - bcl2fastq_flow_cell_full_name: str, -) -> Path: - """Copy the content of a demultiplexed but not finished directory to a temporary location.""" - return Path(tmp_demultiplexed_runs_not_finished_directory, bcl2fastq_flow_cell_full_name) - - -@pytest.fixture(name="tmp_unfinished_bcl2fastq_flow_cell") -def unfinished_bcl2fastq_flow_cell( - demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory: Path, - bcl2fastq_flow_cell_full_name: str, -) -> FlowCellDirectoryData: - """Copy the content of a demultiplexed but not finished directory to a temporary location.""" - return FlowCellDirectoryData( - flow_cell_path=demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory, - bcl_converter=BclConverter.BCL2FASTQ, - ) - - -@pytest.fixture(name="sample_sheet_context") -def sample_sheet_context( - cg_context: CGConfig, lims_api: LimsAPI, populated_housekeeper_api: HousekeeperAPI -) -> CGConfig: - """Return cg context with added Lims and Housekeeper API.""" - cg_context.lims_api_ = lims_api - cg_context.housekeeper_api_ = populated_housekeeper_api - return cg_context - - -@pytest.fixture -def bcl_convert_sample_sheet_creator( - bcl_convert_flow_cell: FlowCellDirectoryData, - lims_novaseq_bcl_convert_samples: list[FlowCellSampleBCLConvert], -) -> SampleSheetCreatorBCLConvert: - """Returns a sample sheet creator for version 2 sample sheets with dragen format.""" - return SampleSheetCreatorBCLConvert( - flow_cell=bcl_convert_flow_cell, - lims_samples=lims_novaseq_bcl_convert_samples, - ) - - -@pytest.fixture(scope="session") -def bcl_convert_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: - """ - Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in - cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/230504_A00689_0804_BHY7FFDRX2. - """ - return ["ACC11927A2", "ACC11927A5"] - - -@pytest.fixture(scope="session") -def bcl2fastq_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: - """ - Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in - cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/170407_A00689_0209_BHHKVCALXX. - """ - return ["SVE2528A1"] - - -@pytest.fixture(scope="session") -def flow_cell_name_demultiplexed_with_bcl2fastq() -> str: - """Return the name of a flow cell that has been demultiplexed with BCL2Fastq.""" - return "HHKVCALXX" - - -@pytest.fixture(scope="session") -def flow_cell_directory_name_demultiplexed_with_bcl2fastq( - flow_cell_name_demultiplexed_with_bcl2fastq: str, -): - """Return the name of a flow cell directory that has been demultiplexed with BCL2Fastq.""" - return f"170407_ST-E00198_0209_B{flow_cell_name_demultiplexed_with_bcl2fastq}" - - -@pytest.fixture(scope="session") -def flow_cell_name_demultiplexed_with_bcl_convert() -> str: - return "HY7FFDRX2" - - -@pytest.fixture(scope="session") -def flow_cell_directory_name_demultiplexed_with_bcl_convert( - flow_cell_name_demultiplexed_with_bcl_convert: str, -): - return f"230504_A00689_0804_B{flow_cell_name_demultiplexed_with_bcl_convert}" - - -# Fixtures for test demultiplex flow cell -@pytest.fixture -def tmp_empty_demultiplexed_runs_directory(tmp_demultiplexed_runs_directory) -> Path: - return Path(tmp_demultiplexed_runs_directory, "empty") - - -@pytest.fixture -def store_with_demultiplexed_samples( - store: Store, - helpers: StoreHelpers, - bcl_convert_demultiplexed_flow_cell_sample_internal_ids: list[str], - bcl2fastq_demultiplexed_flow_cell_sample_internal_ids: list[str], - flow_cell_name_demultiplexed_with_bcl2fastq: str, - flow_cell_name_demultiplexed_with_bcl_convert: str, -) -> Store: - """Return a store with samples that have been demultiplexed with BCL Convert and BCL2Fastq.""" - helpers.add_flow_cell( - store, flow_cell_name_demultiplexed_with_bcl_convert, sequencer_type="novaseq" - ) - helpers.add_flow_cell( - store, flow_cell_name_demultiplexed_with_bcl2fastq, sequencer_type="hiseqx" - ) - for i, sample_internal_id in enumerate(bcl_convert_demultiplexed_flow_cell_sample_internal_ids): - helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl_convert_{i}") - helpers.add_sample_lane_sequencing_metrics( - store, - sample_internal_id=sample_internal_id, - flow_cell_name=flow_cell_name_demultiplexed_with_bcl_convert, - ) - - for i, sample_internal_id in enumerate(bcl2fastq_demultiplexed_flow_cell_sample_internal_ids): - helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl2fastq_{i}") - helpers.add_sample_lane_sequencing_metrics( - store, - sample_internal_id=sample_internal_id, - flow_cell_name=flow_cell_name_demultiplexed_with_bcl2fastq, - ) - return store - - -@pytest.fixture -def demultiplexing_context_for_demux( - demultiplexing_api_for_demux: DemultiplexingAPI, - cg_context: CGConfig, - store_with_demultiplexed_samples: Store, -) -> CGConfig: - """Return cg context with a demultiplex context.""" - cg_context.demultiplex_api_ = demultiplexing_api_for_demux - cg_context.housekeeper_api_ = demultiplexing_api_for_demux.hk_api - cg_context.status_db_ = store_with_demultiplexed_samples - return cg_context - - -@pytest.fixture(name="demultiplex_context") -def demultiplex_context( - demultiplexing_api: DemultiplexingAPI, - real_housekeeper_api: HousekeeperAPI, - cg_context: CGConfig, - store_with_demultiplexed_samples: Store, -) -> CGConfig: - """Return cg context with a demultiplex context.""" - cg_context.demultiplex_api_ = demultiplexing_api - cg_context.housekeeper_api_ = real_housekeeper_api - cg_context.status_db_ = store_with_demultiplexed_samples - return cg_context - - -@pytest.fixture(name="demultiplex_configs_for_demux") -def demultiplex_configs_for_demux( - tmp_flow_cells_demux_all_directory: Path, - tmp_empty_demultiplexed_runs_directory: Path, -) -> dict: - """Return demultiplex configs.""" - return { - "flow_cells_dir": tmp_flow_cells_demux_all_directory.as_posix(), - "demultiplexed_flow_cells_dir": tmp_empty_demultiplexed_runs_directory.as_posix(), - "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, - } - - -@pytest.fixture(name="demultiplex_configs") -def demultiplex_configs( - tmp_flow_cells_directory: Path, - tmp_demultiplexed_runs_directory: Path, -) -> dict: - """Return demultiplex configs.""" - return { - "flow_cells_dir": tmp_flow_cells_directory.as_posix(), - "demultiplexed_flow_cells_dir": tmp_demultiplexed_runs_directory.as_posix(), - "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, - } - - -@pytest.fixture(name="demultiplexing_api_for_demux") -def demultiplexing_api_for_demux( - demultiplex_configs_for_demux: dict, - sbatch_process: Process, - populated_housekeeper_api: HousekeeperAPI, -) -> DemultiplexingAPI: - """Return demultiplex API.""" - demux_api = DemultiplexingAPI( - config=demultiplex_configs_for_demux, - housekeeper_api=populated_housekeeper_api, - ) - demux_api.slurm_api.process = sbatch_process - return demux_api - - -@pytest.fixture -def demultiplexing_api( - demultiplex_configs: dict, sbatch_process: Process, populated_housekeeper_api: HousekeeperAPI -) -> DemultiplexingAPI: - """Return demultiplex API.""" - demux_api = DemultiplexingAPI( - config=demultiplex_configs, housekeeper_api=populated_housekeeper_api - ) - demux_api.slurm_api.process = sbatch_process - return demux_api - - -@pytest.fixture(name="novaseq6000_bcl_convert_sample_sheet_path") -def novaseq6000_sample_sheet_path() -> Path: - """Return the path to a NovaSeq 6000 BCL convert sample sheet.""" - return Path( - "tests", - "fixtures", - "apps", - "sequencing_metrics_parser", - "230622_A00621_0864_AHY7FFDRX2", - "Unaligned", - "Reports", - "SampleSheet.csv", - ) - - -@pytest.fixture(scope="session") -def demultiplex_fixtures(apps_dir: Path) -> Path: - """Return the path to the demultiplex fixture directory.""" - return Path(apps_dir, "demultiplexing") - - -@pytest.fixture(scope="session") -def raw_lims_sample_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the raw samples fixture directory.""" - return Path(demultiplex_fixtures, "raw_lims_samples") - - -@pytest.fixture(scope="session") -def run_parameters_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the run parameters fixture directory.""" - return Path(demultiplex_fixtures, "run_parameters") - - -@pytest.fixture(scope="session") -def demultiplexed_runs(demultiplex_fixtures: Path) -> Path: - """Return the path to the demultiplexed flow cells fixture directory.""" - return Path(demultiplex_fixtures, "demultiplexed-runs") - - -@pytest.fixture(scope="session") -def flow_cells_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the sequenced flow cells fixture directory.""" - return Path(demultiplex_fixtures, DemultiplexingDirsAndFiles.FLOW_CELLS_DIRECTORY_NAME) - - -@pytest.fixture(scope="session") -def nanopore_flow_cells_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the sequenced flow cells fixture directory.""" - return Path(demultiplex_fixtures, NanoporeDirsAndFiles.DATA_DIRECTORY) - - -@pytest.fixture(scope="session") -def flow_cells_demux_all_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the sequenced flow cells fixture directory.""" - return Path(demultiplex_fixtures, "flow_cells_demux_all") - - -@pytest.fixture(scope="session") -def demux_results_not_finished_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to a dir with demultiplexing results where nothing has been cleaned.""" - return Path(demultiplex_fixtures, "demultiplexed-runs-unfinished") - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: - return Path(tmp_flow_cells_directory, "230912_A00187_1009_AHK33MDRX3") - - -@pytest.fixture() -def novaseq_6000_post_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: - return FlowCellDirectoryData(Path(flow_cells_dir, "230912_A00187_1009_AHK33MDRX3")) - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_correct_sample_sheet( - novaseq_6000_post_1_5_kits_flow_cell: Path, -) -> Path: - return Path(novaseq_6000_post_1_5_kits_flow_cell, "CorrectSampleSheet.csv") - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_raw_lims_samples( - novaseq_6000_post_1_5_kits_flow_cell: Path, -) -> Path: - return Path(novaseq_6000_post_1_5_kits_flow_cell, "HK33MDRX3_raw.json") - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_lims_samples( - novaseq_6000_post_1_5_kits_raw_lims_samples: Path, -) -> list[FlowCellSampleBCLConvert]: - return [ - FlowCellSampleBCLConvert.model_validate(sample) - for sample in read_json(novaseq_6000_post_1_5_kits_raw_lims_samples) - ] - - -@pytest.fixture() -def novaseq_6000_pre_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: - return FlowCellDirectoryData(Path(flow_cells_dir, "190927_A00689_0069_BHLYWYDSXX")) - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: - return Path(tmp_flow_cells_directory, "190927_A00689_0069_BHLYWYDSXX") - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_correct_sample_sheet( - novaseq_6000_pre_1_5_kits_flow_cell: Path, -) -> Path: - return Path(novaseq_6000_pre_1_5_kits_flow_cell, "CorrectSampleSheet.csv") - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_raw_lims_samples(novaseq_6000_pre_1_5_kits_flow_cell: Path) -> Path: - return Path(novaseq_6000_pre_1_5_kits_flow_cell, "HLYWYDSXX_raw.json") - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_lims_samples( - novaseq_6000_pre_1_5_kits_raw_lims_samples: Path, -) -> list[FlowCellSampleBCLConvert]: - return [ - FlowCellSampleBCLConvert.model_validate(sample) - for sample in read_json(novaseq_6000_pre_1_5_kits_raw_lims_samples) - ] - - -@pytest.fixture -def novaseq_x_flow_cell_directory(tmp_flow_cells_directory: Path) -> Path: - return Path(tmp_flow_cells_directory, "20231108_LH00188_0028_B22F52TLT3") - - -@pytest.fixture() -def novaseq_x_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: - return FlowCellDirectoryData(Path(flow_cells_dir, "20231108_LH00188_0028_B22F52TLT3")) - - -@pytest.fixture -def novaseq_x_correct_sample_sheet(novaseq_x_flow_cell_directory: Path) -> Path: - return Path(novaseq_x_flow_cell_directory, "CorrectSampleSheet.csv") - - -@pytest.fixture -def novaseq_x_raw_lims_samples(novaseq_x_flow_cell_directory: Path) -> Path: - return Path(novaseq_x_flow_cell_directory, "22F52TLT3_raw.json") - - -@pytest.fixture -def novaseq_x_lims_samples(novaseq_x_raw_lims_samples: Path) -> list[FlowCellSampleBCLConvert]: - return [ - FlowCellSampleBCLConvert.model_validate(sample) - for sample in read_json(novaseq_x_raw_lims_samples) - ] - - -@pytest.fixture(scope="session") -def hiseq_x_single_index_flow_cell_name() -> str: - """Return the full name of a HiSeqX flow cell with only one index.""" - return "170517_ST-E00266_0210_BHJCFFALXX" - - -@pytest.fixture(scope="session") -def hiseq_x_dual_index_flow_cell_name() -> str: - """Return the full name of a HiSeqX flow cell with two indexes.""" - return "180508_ST-E00269_0269_AHL32LCCXY" - - -@pytest.fixture(scope="session") -def hiseq_2500_dual_index_flow_cell_name() -> str: - """Return the full name of a HiSeq2500 flow cell with double indexes.""" - return "181005_D00410_0735_BHM2LNBCX2" - - -@pytest.fixture(scope="session") -def hiseq_2500_custom_index_flow_cell_name() -> str: - """Return the full name of a HiSeq2500 flow cell with double indexes.""" - return "180509_D00450_0598_BHGYFNBCX2" - - -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell_full_name() -> str: - """Return full flow cell name.""" - return "201203_D00483_0200_AHVKJCDRXX" - - -@pytest.fixture(scope="session") -def bcl_convert_flow_cell_full_name() -> str: - """Return the full name of a bcl_convert flow cell.""" - return "211101_A00187_0615_AHLG5GDRZZ" - - -@pytest.fixture(scope="session") -def novaseq_x_flow_cell_full_name() -> str: - """Return the full name of a NovaSeqX flow cell.""" - return "20230508_LH00188_0003_A22522YLT3" - - -@pytest.fixture(scope="session") -def novaseq_x_manifest_file(novaseq_x_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeqX manifest file.""" - return Path(novaseq_x_flow_cell_dir, "Manifest.tsv") - - -@pytest.fixture(scope="session") -def hiseq_x_single_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_x_single_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeqX flow cell.""" - return Path(flow_cells_dir, hiseq_x_single_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def hiseq_x_dual_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_x_dual_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeqX flow cell.""" - return Path(flow_cells_dir, hiseq_x_dual_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def hiseq_2500_dual_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_2500_dual_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeq2500 flow cell.""" - return Path(flow_cells_dir, hiseq_2500_dual_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def hiseq_2500_custom_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_2500_custom_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeq2500 flow cell.""" - return Path(flow_cells_dir, hiseq_2500_custom_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell_dir(flow_cells_dir: Path, bcl2fastq_flow_cell_full_name: str) -> Path: - """Return the path to the bcl2fastq flow cell demultiplex fixture directory.""" - return Path(flow_cells_dir, bcl2fastq_flow_cell_full_name) - - -@pytest.fixture(scope="session") -def bcl_convert_flow_cell_dir(flow_cells_dir: Path, bcl_convert_flow_cell_full_name: str) -> Path: - """Return the path to the bcl_convert flow cell demultiplex fixture directory.""" - return Path(flow_cells_dir, bcl_convert_flow_cell_full_name) - - -@pytest.fixture(scope="session") -def novaseq_x_flow_cell_dir(flow_cells_dir: Path, novaseq_x_flow_cell_full_name: str) -> Path: - """Return the path to the NovaSeqX flow cell demultiplex fixture directory.""" - return Path(flow_cells_dir, novaseq_x_flow_cell_full_name) - - -@pytest.fixture -def hiseq_x_single_index_bcl_convert_lims_samples( - hiseq_x_single_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeqX single index flow cell.""" - path = Path( - hiseq_x_single_index_flow_cell_dir, f"HJCFFALXX_bcl_convert_raw{FileExtensions.JSON}" - ) - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture -def hiseq_x_dual_index_bcl_convert_lims_samples( - hiseq_x_dual_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeqX dual index flow cell.""" - path = Path(hiseq_x_dual_index_flow_cell_dir, f"HL32LCCXY_bcl_convert_raw{FileExtensions.JSON}") - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture -def hiseq_2500_dual_index_bcl_convert_lims_samples( - hiseq_2500_dual_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeq2500 dual index flow cell.""" - path = Path(hiseq_2500_dual_index_flow_cell_dir, "HM2LNBCX2_bcl_convert_raw.json") - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture -def hiseq_2500_custom_index_bcl_convert_lims_samples( - hiseq_2500_custom_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeq2500 custom index flow cell.""" - path = Path(hiseq_2500_custom_index_flow_cell_dir, "HGYFNBCX2_bcl_convert_raw.json") - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture(scope="session") -def novaseq_bcl2fastq_sample_sheet_path(bcl2fastq_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeq6000 Bcl2fastq sample sheet.""" - return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) - - -@pytest.fixture(scope="session") -def novaseq_bcl_convert_sample_sheet_path(bcl_convert_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeq6000 bcl_convert sample sheet.""" - return Path(bcl_convert_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) - - -@pytest.fixture(scope="session") -def run_parameters_wrong_instrument(run_parameters_dir: Path) -> Path: - """Return a NovaSeqX run parameters file path with a wrong instrument value.""" - return Path(run_parameters_dir, "RunParameters_novaseq_X_wrong_instrument.xml") +def fixtures_dir() -> Path: + """Return the path to the fixtures dir.""" + return Path("tests", "fixtures") @pytest.fixture(scope="session") -def hiseq_x_single_index_run_parameters_path( - hiseq_x_single_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeqX run parameters file with single index.""" - return Path( - hiseq_x_single_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def analysis_dir(fixtures_dir: Path) -> Path: + """Return the path to the analysis dir.""" + return Path(fixtures_dir, "analysis") @pytest.fixture(scope="session") -def hiseq_x_dual_index_run_parameters_path( - hiseq_x_dual_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeqX run parameters file with dual index.""" - return Path( - hiseq_x_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def microsalt_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the analysis dir.""" + return Path(analysis_dir, "microsalt") @pytest.fixture(scope="session") -def hiseq_2500_dual_index_run_parameters_path( - hiseq_2500_dual_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeq2500 run parameters file with dual index.""" - return Path( - hiseq_2500_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def apps_dir(fixtures_dir: Path) -> Path: + """Return the path to the apps dir.""" + return Path(fixtures_dir, "apps") @pytest.fixture(scope="session") -def hiseq_2500_custom_index_run_parameters_path( - hiseq_2500_custom_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeq2500 run parameters file with custom index.""" - return Path( - hiseq_2500_custom_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def cgweb_orders_dir(fixtures_dir: Path) -> Path: + """Return the path to the cgweb_orders dir.""" + return Path(fixtures_dir, "cgweb_orders") @pytest.fixture(scope="session") -def novaseq_6000_run_parameters_path(bcl2fastq_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeq6000 run parameters file.""" - return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) +def data_dir(fixtures_dir: Path) -> Path: + """Return the path to the data dir.""" + return Path(fixtures_dir, "data") @pytest.fixture -def novaseq_6000_run_parameters_pre_1_5_kits_path( - novaseq_6000_pre_1_5_kits_flow_cell: Path, -) -> Path: - """Return the path to a NovaSeq6000 pre 1.5 kit run parameters file.""" - return Path( - novaseq_6000_pre_1_5_kits_flow_cell, - DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, - ) +def fastq_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the fastq files dir.""" + return Path(demultiplex_fixtures, "fastq") @pytest.fixture -def novaseq_6000_run_parameters_post_1_5_kits_path( - novaseq_6000_post_1_5_kits_flow_cell: Path, -) -> Path: - """Return the path to a NovaSeq6000 post 1.5 kit run parameters file.""" - return Path( - novaseq_6000_post_1_5_kits_flow_cell, - DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, - ) +def spring_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the fastq files dir.""" + return Path(demultiplex_fixtures, "spring") -@pytest.fixture(scope="session") -def novaseq_x_run_parameters_path(novaseq_x_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeqX run parameters file.""" - return Path(novaseq_x_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) +@pytest.fixture +def project_dir(tmpdir_factory) -> Generator[Path, None, None]: + """Path to a temporary directory where intermediate files can be stored.""" + yield Path(tmpdir_factory.mktemp("data")) -@pytest.fixture(scope="function") -def run_parameters_hiseq_different_index(run_parameters_dir: Path) -> RunParametersHiSeq: - """Return a HiSeq RunParameters object with different index cycles.""" - path = Path(run_parameters_dir, "RunParameters_hiseq_2500_different_index_cycles.xml") - return RunParametersHiSeq(run_parameters_path=path) +@pytest.fixture +def tmp_file(project_dir) -> Path: + """Return a temp file path.""" + return Path(project_dir, "test") -@pytest.fixture(scope="function") -def run_parameters_novaseq_6000_different_index( - run_parameters_dir: Path, -) -> RunParametersNovaSeq6000: - """Return a NovaSeq6000 RunParameters object with different index cycles.""" - path = Path(run_parameters_dir, "RunParameters_novaseq_6000_different_index_cycles.xml") - return RunParametersNovaSeq6000(run_parameters_path=path) +@pytest.fixture +def non_existing_file_path(project_dir: Path) -> Path: + """Return the path to a non-existing file.""" + return Path(project_dir, "a_file.txt") -@pytest.fixture(scope="function") -def run_parameters_novaseq_x_different_index(run_parameters_dir: Path) -> RunParametersNovaSeqX: - """Return a NovaSeqX RunParameters object with different index cycles.""" - path = Path(run_parameters_dir, "RunParameters_novaseq_X_different_index_cycles.xml") - return RunParametersNovaSeqX(run_parameters_path=path) +@pytest.fixture(scope="session") +def content() -> str: + """Return some content for a file.""" + return ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt" + " ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ull" + "amco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehende" + "rit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaec" + "at cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." + ) -@pytest.fixture(scope="module") -def run_parameters_missing_versions_path( - run_parameters_dir: Path, -) -> Path: - """Return a NovaSeq6000 run parameters path without software and reagent kit versions.""" - return Path(run_parameters_dir, "RunParameters_novaseq_no_software_nor_reagent_version.xml") +@pytest.fixture +def filled_file(non_existing_file_path: Path, content: str) -> Path: + """Return the path to a existing file with some content.""" + with open(non_existing_file_path, "w") as outfile: + outfile.write(content) + return non_existing_file_path @pytest.fixture(scope="session") -def hiseq_x_single_index_run_parameters( - hiseq_x_single_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeqX run parameters object with single index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_x_single_index_run_parameters_path) +def orderforms(fixtures_dir: Path) -> Path: + """Return the path to the directory with order forms.""" + return Path(fixtures_dir, "orderforms") -@pytest.fixture(scope="session") -def hiseq_x_dual_index_run_parameters( - hiseq_x_dual_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeqX run parameters object with dual index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_x_dual_index_run_parameters_path) +@pytest.fixture +def hk_file(filled_file: Path, case_id: str) -> File: + """Return a housekeeper File object.""" + return File(id=case_id, path=filled_file.as_posix()) -@pytest.fixture(scope="session") -def hiseq_2500_dual_index_run_parameters( - hiseq_2500_dual_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeq2500 run parameters object with dual index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_2500_dual_index_run_parameters_path) +@pytest.fixture +def mip_dna_store_files(apps_dir: Path) -> Path: + """Return the path to the directory with mip dna store files.""" + return Path(apps_dir, "mip", "dna", "store") -@pytest.fixture(scope="session") -def hiseq_2500_custom_index_run_parameters( - hiseq_2500_custom_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeq2500 run parameters object with custom index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_2500_custom_index_run_parameters_path) +@pytest.fixture +def case_qc_sample_info_path(mip_dna_store_files: Path) -> Path: + """Return path to case_qc_sample_info.yaml.""" + return Path(mip_dna_store_files, "case_qc_sample_info.yaml") -@pytest.fixture(scope="session") -def novaseq_6000_run_parameters( - novaseq_6000_run_parameters_path: Path, -) -> RunParametersNovaSeq6000: - """Return a NovaSeq6000 run parameters object.""" - return RunParametersNovaSeq6000(run_parameters_path=novaseq_6000_run_parameters_path) +@pytest.fixture +def delivery_report_html(mip_dna_store_files: Path) -> Path: + """Return the path to a qc metrics deliverables file with case data.""" + return Path(mip_dna_store_files, "empty_delivery_report.html") @pytest.fixture -def novaseq_6000_run_parameters_pre_1_5_kits( - novaseq_6000_run_parameters_pre_1_5_kits_path: Path, -) -> RunParametersNovaSeq6000: - """Return a NovaSeq6000 run parameters pre 1.5 kit object.""" - return RunParametersNovaSeq6000( - run_parameters_path=novaseq_6000_run_parameters_pre_1_5_kits_path - ) +def mip_deliverables_file(mip_dna_store_files: Path) -> Path: + """Fixture for general deliverables file in mip.""" + return Path(mip_dna_store_files, "case_id_deliverables.yaml") @pytest.fixture -def novaseq_6000_run_parameters_post_1_5_kits(novaseq_6000_run_parameters_post_1_5_kits_path: Path): - """Return a NovaSeq6000 run parameters post 1.5 kit object.""" - return RunParametersNovaSeq6000( - run_parameters_path=novaseq_6000_run_parameters_post_1_5_kits_path - ) +def case_qc_metrics_deliverables(apps_dir: Path) -> Path: + """Return the path to a qc metrics deliverables file with case data.""" + return Path(apps_dir, "mip", "case_metrics_deliverables.yaml") -@pytest.fixture(scope="session") -def novaseq_x_run_parameters( - novaseq_x_run_parameters_path: Path, -) -> RunParametersNovaSeqX: - """Return a NovaSeqX run parameters object.""" - return RunParametersNovaSeqX(run_parameters_path=novaseq_x_run_parameters_path) +@pytest.fixture +def mip_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the directory with mip analysis files.""" + return Path(analysis_dir, "mip") -@pytest.fixture(scope="module") -def hiseq_x_single_index_flow_cell( - hiseq_x_single_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a single-index HiSeqX flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_x_single_index_flow_cell_dir) +@pytest.fixture +def balsamic_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the directory with balsamic analysis files.""" + return Path(analysis_dir, "balsamic") -@pytest.fixture(scope="module") -def hiseq_x_dual_index_flow_cell( - hiseq_x_dual_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a dual-index HiSeqX flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_x_dual_index_flow_cell_dir) +@pytest.fixture +def balsamic_wgs_analysis_dir(balsamic_analysis_dir: Path) -> Path: + """Return the path to the directory with balsamic analysis files.""" + return Path(balsamic_analysis_dir, "tn_wgs") -@pytest.fixture(scope="module") -def hiseq_2500_dual_index_flow_cell( - hiseq_2500_dual_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a dual-index HiSeq2500 flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_2500_dual_index_flow_cell_dir) +@pytest.fixture +def mip_dna_analysis_dir(mip_analysis_dir: Path) -> Path: + """Return the path to the directory with mip dna analysis files.""" + return Path(mip_analysis_dir, "dna") -@pytest.fixture(scope="module") -def hiseq_2500_custom_index_flow_cell( - hiseq_2500_custom_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a custom-index HiSeq2500 flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_2500_custom_index_flow_cell_dir) +@pytest.fixture +def rnafusion_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the directory with rnafusion analysis files.""" + return Path(analysis_dir, "rnafusion") -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell(bcl2fastq_flow_cell_dir: Path) -> FlowCellDirectoryData: - """Create a flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=bcl2fastq_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ - ) +@pytest.fixture +def sample_cram(mip_dna_analysis_dir: Path) -> Path: + """Return the path to the cram file for a sample.""" + return Path(mip_dna_analysis_dir, "adm1.cram") -@pytest.fixture(scope="session") -def novaseq_flow_cell_demultiplexed_with_bcl2fastq( - bcl_convert_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a Novaseq6000 flow cell object demultiplexed using Bcl2fastq.""" - return FlowCellDirectoryData( - flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ - ) +@pytest.fixture(name="father_sample_cram") +def father_sample_cram( + mip_dna_analysis_dir: Path, + father_sample_id: str, +) -> Path: + """Return the path to the cram file for the father sample.""" + return Path(mip_dna_analysis_dir, father_sample_id + FileExtensions.CRAM) -@pytest.fixture(scope="module") -def bcl_convert_flow_cell(bcl_convert_flow_cell_dir: Path) -> FlowCellDirectoryData: - """Create a bcl_convert flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.DRAGEN - ) +@pytest.fixture(name="mother_sample_cram") +def mother_sample_cram(mip_dna_analysis_dir: Path, mother_sample_id: str) -> Path: + """Return the path to the cram file for the mother sample.""" + return Path(mip_dna_analysis_dir, mother_sample_id + FileExtensions.CRAM) -@pytest.fixture(scope="function") -def novaseq_6000_flow_cell(bcl_convert_flow_cell: FlowCellDirectoryData) -> FlowCellDirectoryData: - """Return a NovaSeq6000 flow cell object.""" - return bcl_convert_flow_cell +@pytest.fixture(name="sample_cram_files") +def sample_crams( + sample_cram: Path, father_sample_cram: Path, mother_sample_cram: Path +) -> list[Path]: + """Return a list of cram paths for three samples.""" + return [sample_cram, father_sample_cram, mother_sample_cram] -@pytest.fixture(scope="function") -def novaseq_x_flow_cell(novaseq_x_flow_cell_dir: Path) -> FlowCellDirectoryData: - """Create a NovaSeqX flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=novaseq_x_flow_cell_dir, bcl_converter=BclConverter.DRAGEN - ) +@pytest.fixture(name="vcf_file") +def vcf_file(mip_dna_store_files: Path) -> Path: + """Return the path to a VCF file.""" + return Path(mip_dna_store_files, "yellowhog_clinical_selected.vcf") -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell_id(bcl2fastq_flow_cell: FlowCellDirectoryData) -> str: - """Return flow cell id from bcl2fastq flow cell object.""" - return bcl2fastq_flow_cell.id +@pytest.fixture(name="fastq_file") +def fastq_file(fastq_dir: Path) -> Path: + """Return the path to a FASTQ file.""" + return Path(fastq_dir, "dummy_run_R1_001.fastq.gz") -@pytest.fixture(scope="module") -def bcl_convert_flow_cell_id(bcl_convert_flow_cell: FlowCellDirectoryData) -> str: - """Return flow cell id from bcl_convert flow cell object.""" - return bcl_convert_flow_cell.id +@pytest.fixture(name="fastq_file_father") +def fastq_file_father(fastq_dir: Path) -> Path: + """Return the path to a FASTQ file.""" + return Path(fastq_dir, "fastq_run_R1_001.fastq.gz") -@pytest.fixture(name="demultiplexing_delivery_file") -def demultiplexing_delivery_file(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: - """Return demultiplexing delivery started file.""" - return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.DELIVERY) +@pytest.fixture(name="spring_file") +def spring_file(spring_dir: Path) -> Path: + """Return the path to an existing spring file.""" + return Path(spring_dir, "dummy_run_001.spring") -@pytest.fixture(name="hiseq_x_tile_dir") -def hiseq_x_tile_dir(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: - """Return HiSeqX tile dir.""" - return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.HISEQ_X_TILE_DIR) +@pytest.fixture(name="spring_meta_data_file") +def spring_meta_data_file(spring_dir: Path) -> Path: + """Return the path to an existing spring file.""" + return Path(spring_dir, "dummy_spring_meta_data.json") -@pytest.fixture(name="lims_novaseq_samples_file") -def lims_novaseq_samples_file(raw_lims_sample_dir: Path) -> Path: - """Return the path to a file with sample info in lims format.""" - return Path(raw_lims_sample_dir, "raw_samplesheet_novaseq.json") +@pytest.fixture(name="spring_file_father") +def spring_file_father(spring_dir: Path) -> Path: + """Return the path to a second existing spring file.""" + return Path(spring_dir, "dummy_run_002.spring") -@pytest.fixture -def lims_novaseq_6000_samples_file(bcl2fastq_flow_cell_dir: Path) -> Path: - """Return the path to the file with the raw samples of HVKJCDRXX flow cell in lims format.""" - return Path(bcl2fastq_flow_cell_dir, "HVKJCDRXX_raw.json") +@pytest.fixture(name="madeline_output") +def madeline_output(apps_dir: Path) -> Path: + """Return str of path for file with Madeline output.""" + return Path(apps_dir, "madeline", "madeline.xml") -@pytest.fixture -def lims_novaseq_samples_raw(lims_novaseq_samples_file: Path) -> list[dict]: - """Return a list of raw flow cell samples.""" - return ReadFile.get_content_from_file( - file_format=FileFormat.JSON, file_path=lims_novaseq_samples_file - ) +@pytest.fixture(name="file_does_not_exist") +def file_does_not_exist() -> Path: + """Return a file path that does not exist.""" + return Path("file", "does", "not", "exist") -@pytest.fixture -def lims_novaseq_6000_sample_raw(lims_novaseq_6000_samples_file: Path) -> list[dict]: - """Return the list of raw samples from flow cell HVKJCDRXX.""" - return ReadFile.get_content_from_file( - file_format=FileFormat.JSON, file_path=lims_novaseq_6000_samples_file - ) +# Compression fixtures -@pytest.fixture(name="demultiplexed_flow_cell") -def demultiplexed_flow_cell(demultiplexed_runs: Path, bcl2fastq_flow_cell_full_name: str) -> Path: - """Return the path to a demultiplexed flow cell with bcl2fastq.""" - return Path(demultiplexed_runs, bcl2fastq_flow_cell_full_name) +@pytest.fixture(name="run_name") +def run_name() -> str: + """Return the name of a fastq run.""" + return "fastq_run" -@pytest.fixture(name="bcl_convert_demultiplexed_flow_cell") -def bcl_convert_demultiplexed_flow_cell( - demultiplexed_runs: Path, bcl_convert_flow_cell_full_name: str -) -> Path: - """Return the path to a demultiplexed flow cell with BCLConvert.""" - return Path(demultiplexed_runs, bcl_convert_flow_cell_full_name) +@pytest.fixture(name="original_fastq_data") +def original_fastq_data(fastq_dir: Path, run_name) -> CompressionData: + """Return a compression object with a path to the original fastq files.""" + return CompressionData(Path(fastq_dir, run_name)) -@pytest.fixture(name="novaseqx_demultiplexed_flow_cell") -def novaseqx_demultiplexed_flow_cell(demultiplexed_runs: Path, novaseq_x_flow_cell_full_name: str): - """Return the path to a demultiplexed NovaSeqX flow cell.""" - return Path(demultiplexed_runs, novaseq_x_flow_cell_full_name) +@pytest.fixture(name="fastq_stub") +def fastq_stub(project_dir: Path, run_name: str) -> Path: + """Creates a path to the base format of a fastq run.""" + return Path(project_dir, run_name) -@pytest.fixture() -def novaseqx_flow_cell_with_sample_sheet_no_fastq( - novaseqx_flow_cell_directory: Path, novaseqx_demultiplexed_flow_cell: Path -) -> FlowCellDirectoryData: - """Return a flow cell from a tmp dir with a sample sheet and no sample fastq files.""" - novaseqx_flow_cell_directory.mkdir(parents=True, exist_ok=True) - flow_cell = FlowCellDirectoryData(novaseqx_flow_cell_directory) - sample_sheet_path = Path( - novaseqx_demultiplexed_flow_cell, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME - ) - flow_cell._sample_sheet_path_hk = sample_sheet_path - return flow_cell +@pytest.fixture(name="compression_object") +def compression_object(fastq_stub: Path, original_fastq_data: CompressionData) -> CompressionData: + """Creates compression data object with information about files used in fastq compression.""" + working_files: CompressionData = CompressionData(fastq_stub) + working_file_map: dict[str, str] = { + original_fastq_data.fastq_first.as_posix(): working_files.fastq_first.as_posix(), + original_fastq_data.fastq_second.as_posix(): working_files.fastq_second.as_posix(), + } + for original_file, working_file in working_file_map.items(): + shutil.copy(original_file, working_file) + return working_files # Genotype file fixture @@ -2143,6 +1130,40 @@ def analysis_store_single_case( yield base_store +@pytest.fixture +def store_with_demultiplexed_samples( + store: Store, + helpers: StoreHelpers, + bcl_convert_demultiplexed_flow_cell_sample_internal_ids: list[str], + bcl2fastq_demultiplexed_flow_cell_sample_internal_ids: list[str], + flow_cell_name_demultiplexed_with_bcl2fastq: str, + flow_cell_name_demultiplexed_with_bcl_convert: str, +) -> Store: + """Return a store with samples that have been demultiplexed with BCL Convert and BCL2Fastq.""" + helpers.add_flow_cell( + store, flow_cell_name_demultiplexed_with_bcl_convert, sequencer_type="novaseq" + ) + helpers.add_flow_cell( + store, flow_cell_name_demultiplexed_with_bcl2fastq, sequencer_type="hiseqx" + ) + for i, sample_internal_id in enumerate(bcl_convert_demultiplexed_flow_cell_sample_internal_ids): + helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl_convert_{i}") + helpers.add_sample_lane_sequencing_metrics( + store, + sample_internal_id=sample_internal_id, + flow_cell_name=flow_cell_name_demultiplexed_with_bcl_convert, + ) + + for i, sample_internal_id in enumerate(bcl2fastq_demultiplexed_flow_cell_sample_internal_ids): + helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl2fastq_{i}") + helpers.add_sample_lane_sequencing_metrics( + store, + sample_internal_id=sample_internal_id, + flow_cell_name=flow_cell_name_demultiplexed_with_bcl2fastq, + ) + return store + + @pytest.fixture(name="collaboration_id") def collaboration_id() -> str: """Return a default customer group.""" @@ -3260,7 +2281,6 @@ def rnafusion_context( case_id_not_enough_reads: str, sample_id_not_enough_reads: str, total_sequenced_reads_not_pass: int, - timestamp_yesterday: datetime, ) -> CGConfig: """context to use in cli""" cg_context.housekeeper_api_ = nf_analysis_housekeeper @@ -3484,6 +2504,7 @@ def nf_analysis_housekeeper( helpers: StoreHelpers, mock_fastq_files: list[Path], sample_id: str, + timestamp_now: datetime, ): """Create populated Housekeeper sample bundle mock.""" diff --git a/tests/fixture_plugins/__init__.py b/tests/fixture_plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fixture_plugins/demultiplex_fixtures/__init__.py b/tests/fixture_plugins/demultiplex_fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py new file mode 100644 index 0000000000..09213170bd --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py @@ -0,0 +1,172 @@ +"""Fixtures for flow cell objects.""" +from pathlib import Path + +import pytest + +from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData + +# Functional flow cells + + +@pytest.fixture(scope="module") +def hiseq_x_single_index_flow_cell( + hiseq_x_single_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a single-index HiSeqX flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_x_single_index_flow_cell_dir) + + +@pytest.fixture(scope="module") +def hiseq_x_dual_index_flow_cell( + hiseq_x_dual_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a dual-index HiSeqX flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_x_dual_index_flow_cell_dir) + + +@pytest.fixture(scope="module") +def hiseq_2500_dual_index_flow_cell( + hiseq_2500_dual_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a dual-index HiSeq2500 flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_2500_dual_index_flow_cell_dir) + + +@pytest.fixture(scope="module") +def hiseq_2500_custom_index_flow_cell( + hiseq_2500_custom_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a custom-index HiSeq2500 flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_2500_custom_index_flow_cell_dir) + + +@pytest.fixture() +def novaseq_6000_post_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: + return FlowCellDirectoryData(Path(flow_cells_dir, "230912_A00187_1009_AHK33MDRX3")) + + +@pytest.fixture() +def novaseq_6000_pre_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: + return FlowCellDirectoryData(Path(flow_cells_dir, "190927_A00689_0069_BHLYWYDSXX")) + + +@pytest.fixture() +def novaseq_x_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: + return FlowCellDirectoryData(Path(flow_cells_dir, "20231108_LH00188_0028_B22F52TLT3")) + + +# Broken flow cells + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell(bcl2fastq_flow_cell_dir: Path) -> FlowCellDirectoryData: + """Create a flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=bcl2fastq_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ + ) + + +@pytest.fixture(scope="session") +def novaseq_flow_cell_demultiplexed_with_bcl2fastq( + bcl_convert_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a Novaseq6000 flow cell object demultiplexed using Bcl2fastq.""" + return FlowCellDirectoryData( + flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ + ) + + +@pytest.fixture(scope="module") +def bcl_convert_flow_cell(bcl_convert_flow_cell_dir: Path) -> FlowCellDirectoryData: + """Create a bcl_convert flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.DRAGEN + ) + + +@pytest.fixture(scope="function") +def novaseq_6000_flow_cell(bcl_convert_flow_cell: FlowCellDirectoryData) -> FlowCellDirectoryData: + """Return a NovaSeq6000 flow cell object.""" + return bcl_convert_flow_cell + + +@pytest.fixture(scope="function") +def novaseq_x_flow_cell(novaseq_x_flow_cell_dir: Path) -> FlowCellDirectoryData: + """Create a NovaSeqX flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=novaseq_x_flow_cell_dir, bcl_converter=BclConverter.DRAGEN + ) + + +@pytest.fixture() +def novaseqx_flow_cell_with_sample_sheet_no_fastq( + novaseqx_flow_cell_directory: Path, novaseqx_demultiplexed_flow_cell: Path +) -> FlowCellDirectoryData: + """Return a flow cell from a tmp dir with a sample sheet and no sample fastq files.""" + novaseqx_flow_cell_directory.mkdir(parents=True, exist_ok=True) + flow_cell = FlowCellDirectoryData(novaseqx_flow_cell_directory) + sample_sheet_path = Path( + novaseqx_demultiplexed_flow_cell, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME + ) + flow_cell._sample_sheet_path_hk = sample_sheet_path + return flow_cell + + +@pytest.fixture(name="tmp_bcl2fastq_flow_cell") +def tmp_bcl2fastq_flow_cell( + tmp_demultiplexed_runs_bcl2fastq_directory: Path, +) -> FlowCellDirectoryData: + """Create a flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=tmp_demultiplexed_runs_bcl2fastq_directory, + bcl_converter=BclConverter.BCL2FASTQ, + ) + + +@pytest.fixture +def novaseq6000_flow_cell( + tmp_flow_cells_directory_malformed_sample_sheet: Path, +) -> FlowCellDirectoryData: + """Return a NovaSeq6000 flow cell.""" + return FlowCellDirectoryData( + flow_cell_path=tmp_flow_cells_directory_malformed_sample_sheet, + bcl_converter=BclConverter.BCLCONVERT, + ) + + +@pytest.fixture(name="tmp_bcl_convert_flow_cell") +def tmp_bcl_convert_flow_cell( + tmp_flow_cell_directory_bclconvert: Path, +) -> FlowCellDirectoryData: + """Create a flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=tmp_flow_cell_directory_bclconvert, + bcl_converter=BclConverter.DRAGEN, + ) + + +@pytest.fixture(name="tmp_unfinished_bcl2fastq_flow_cell") +def unfinished_bcl2fastq_flow_cell( + demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory: Path, +) -> FlowCellDirectoryData: + """Copy the content of a demultiplexed but not finished directory to a temporary location.""" + return FlowCellDirectoryData( + flow_cell_path=demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory, + bcl_converter=BclConverter.BCL2FASTQ, + ) + + +# Flow cell attributes + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell_id(bcl2fastq_flow_cell: FlowCellDirectoryData) -> str: + """Return flow cell id from bcl2fastq flow cell object.""" + return bcl2fastq_flow_cell.id + + +@pytest.fixture(scope="module") +def bcl_convert_flow_cell_id(bcl_convert_flow_cell: FlowCellDirectoryData) -> str: + """Return flow cell id from bcl_convert flow cell object.""" + return bcl_convert_flow_cell.id diff --git a/tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py new file mode 100644 index 0000000000..c4bbb1d3b4 --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py @@ -0,0 +1,116 @@ +import pytest + + +@pytest.fixture +def tmp_flow_cell_name_no_run_parameters() -> str: + """This is the name of a flow cell directory with the run parameters missing.""" + return "180522_A00689_0200_BHLCKNCCXY" + + +@pytest.fixture +def tmp_flow_cell_name_malformed_sample_sheet() -> str: + """ "Returns the name of a flow cell directory ready for demultiplexing with BCL convert. + Contains a sample sheet with malformed headers. + """ + return "201203_A00689_0200_AHVKJCDRXY" + + +@pytest.fixture +def tmp_flow_cell_name_no_sample_sheet() -> str: + """Return the name of a flow cell directory with the run parameters and sample sheet missing.""" + return "170407_A00689_0209_BHHKVCALXX" + + +@pytest.fixture(name="tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq") +def tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq() -> str: + """Returns the name of a flow cell directory ready for demultiplexing with bcl2fastq.""" + return "211101_D00483_0615_AHLG5GDRXY" + + +@pytest.fixture(scope="session") +def flow_cell_name_demultiplexed_with_bcl2fastq() -> str: + """Return the name of a flow cell that has been demultiplexed with BCL2Fastq.""" + return "HHKVCALXX" + + +@pytest.fixture(scope="session") +def flow_cell_directory_name_demultiplexed_with_bcl2fastq( + flow_cell_name_demultiplexed_with_bcl2fastq: str, +) -> str: + """Return the name of a flow cell directory that has been demultiplexed with BCL2Fastq.""" + return f"170407_ST-E00198_0209_B{flow_cell_name_demultiplexed_with_bcl2fastq}" + + +@pytest.fixture(scope="session") +def flow_cell_name_demultiplexed_with_bcl_convert() -> str: + return "HY7FFDRX2" + + +@pytest.fixture(scope="session") +def flow_cell_directory_name_demultiplexed_with_bcl_convert( + flow_cell_name_demultiplexed_with_bcl_convert: str, +) -> str: + return f"230504_A00689_0804_B{flow_cell_name_demultiplexed_with_bcl_convert}" + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_flow_cell_name() -> str: + """Return the full name of a HiSeqX flow cell with only one index.""" + return "170517_ST-E00266_0210_BHJCFFALXX" + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_flow_cell_name() -> str: + """Return the full name of a HiSeqX flow cell with two indexes.""" + return "180508_ST-E00269_0269_AHL32LCCXY" + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_flow_cell_name() -> str: + """Return the full name of a HiSeq2500 flow cell with double indexes.""" + return "181005_D00410_0735_BHM2LNBCX2" + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_flow_cell_name() -> str: + """Return the full name of a HiSeq2500 flow cell with double indexes.""" + return "180509_D00450_0598_BHGYFNBCX2" + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell_full_name() -> str: + """Return full flow cell name.""" + return "201203_D00483_0200_AHVKJCDRXX" + + +@pytest.fixture(scope="session") +def bcl_convert_flow_cell_full_name() -> str: + """Return the full name of a bcl_convert flow cell.""" + return "211101_A00187_0615_AHLG5GDRZZ" + + +@pytest.fixture(scope="session") +def novaseq_x_flow_cell_full_name() -> str: + """Return the full name of a NovaSeqX flow cell.""" + return "20230508_LH00188_0003_A22522YLT3" + + +# Lists + + +@pytest.fixture(scope="session") +def bcl_convert_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: + """ + Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in + cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/230504_A00689_0804_BHY7FFDRX2. + """ + return ["ACC11927A2", "ACC11927A5"] + + +@pytest.fixture(scope="session") +def bcl2fastq_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: + """ + Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in + cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/170407_A00689_0209_BHHKVCALXX. + """ + return ["SVE2528A1"] diff --git a/tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py new file mode 100644 index 0000000000..807c34ee41 --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py @@ -0,0 +1,458 @@ +"""Path fixtures for demultiplex tests.""" +import shutil +from pathlib import Path + +import pytest + +from cg.constants.demultiplexing import DemultiplexingDirsAndFiles +from cg.constants.nanopore_files import NanoporeDirsAndFiles +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData + +CORRECT_SAMPLE_SHEET: str = "CorrectSampleSheet.csv" + + +@pytest.fixture(name="tmp_flow_cells_directory") +def tmp_flow_cells_directory(tmp_path: Path, flow_cells_dir: Path) -> Path: + """ + Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. + Generates a copy of the original flow cells directory + """ + original_dir = flow_cells_dir + tmp_dir = Path(tmp_path, "flow_cells") + + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="tmp_flow_cells_demux_all_directory") +def tmp_flow_cells_demux_all_directory(tmp_path: Path, flow_cells_demux_all_dir: Path) -> Path: + """ + Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. + Generates a copy of the original flow cells directory. + This fixture is used for testing of the cg demutliplex all cmd. + """ + original_dir = flow_cells_demux_all_dir + tmp_dir = Path(tmp_path, "flow_cells_demux_all") + + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="tmp_flow_cell_directory_bcl2fastq") +def flow_cell_working_directory_bcl2fastq( + bcl2fastq_flow_cell_dir: Path, tmp_flow_cells_directory: Path +) -> Path: + """Return the path to a working directory that will be deleted after test is run. + + This is a path to a flow cell directory with the run parameters present. + """ + return Path(tmp_flow_cells_directory, bcl2fastq_flow_cell_dir.name) + + +@pytest.fixture(name="tmp_flow_cell_directory_bclconvert") +def flow_cell_working_directory_bclconvert( + bcl_convert_flow_cell_dir: Path, tmp_flow_cells_directory: Path +) -> Path: + """Return the path to a working directory that will be deleted after test is run. + This is a path to a flow cell directory with the run parameters present. + """ + return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_dir.name) + + +@pytest.fixture +def tmp_flow_cells_directory_no_run_parameters( + tmp_flow_cell_name_no_run_parameters: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the run parameters missing.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_run_parameters) + + +@pytest.fixture(name="tmp_flow_cells_directory_no_sample_sheet") +def tmp_flow_cells_directory_no_sample_sheet( + tmp_flow_cell_name_no_sample_sheet: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the sample sheet and run parameters missing.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_sample_sheet) + + +@pytest.fixture +def tmp_flow_cells_directory_malformed_sample_sheet( + tmp_flow_cell_name_malformed_sample_sheet: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with a sample sheet with malformed headers.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_malformed_sample_sheet) + + +@pytest.fixture +def tmp_flow_cells_directory_ready_for_demultiplexing_bcl_convert( + bcl_convert_flow_cell_full_name: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the run parameters missing.""" + return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_full_name) + + +@pytest.fixture +def tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq( + tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the run parameters missing.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq) + + +# Temporary demultiplexed runs fixtures +@pytest.fixture(name="tmp_demultiplexed_runs_directory") +def tmp_demultiplexed_flow_cells_directory(tmp_path: Path, demultiplexed_runs: Path) -> Path: + """Return the path to a temporary demultiplex-runs directory. + Generates a copy of the original demultiplexed-runs + """ + original_dir = demultiplexed_runs + tmp_dir = Path(tmp_path, "demultiplexed-runs") + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="tmp_demultiplexed_runs_bcl2fastq_directory") +def tmp_demultiplexed_runs_bcl2fastq_directory( + tmp_demultiplexed_runs_directory: Path, bcl2fastq_flow_cell_dir: Path +) -> Path: + """Return the path to a temporary demultiplex-runs bcl2fastq flow cell directory.""" + return Path(tmp_demultiplexed_runs_directory, bcl2fastq_flow_cell_dir.name) + + +@pytest.fixture(name="tmp_demultiplexed_runs_not_finished_directory") +def tmp_demultiplexed_runs_not_finished_flow_cells_directory( + tmp_path: Path, demux_results_not_finished_dir: Path +) -> Path: + """ + Return a temporary demultiplex-runs-unfinished path with an unfinished flow cell directory. + Generates a copy of the original demultiplexed-runs-unfinished directory. + """ + original_dir = demux_results_not_finished_dir + tmp_dir = Path(tmp_path, "demultiplexed-runs-unfinished") + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory") +def demultiplexed_runs_bcl2fastq_flow_cell_directory( + tmp_demultiplexed_runs_not_finished_directory: Path, + bcl2fastq_flow_cell_full_name: str, +) -> Path: + """Copy the content of a demultiplexed but not finished directory to a temporary location.""" + return Path(tmp_demultiplexed_runs_not_finished_directory, bcl2fastq_flow_cell_full_name) + + +@pytest.fixture(name="novaseq6000_bcl_convert_sample_sheet_path") +def novaseq6000_sample_sheet_path() -> Path: + """Return the path to a NovaSeq 6000 BCL convert sample sheet.""" + return Path( + "tests", + "fixtures", + "apps", + "sequencing_metrics_parser", + "230622_A00621_0864_AHY7FFDRX2", + "Unaligned", + "Reports", + "SampleSheet.csv", + ) + + +@pytest.fixture(scope="session") +def demultiplex_fixtures(apps_dir: Path) -> Path: + """Return the path to the demultiplex fixture directory.""" + return Path(apps_dir, "demultiplexing") + + +@pytest.fixture(scope="session") +def raw_lims_sample_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the raw samples fixture directory.""" + return Path(demultiplex_fixtures, "raw_lims_samples") + + +@pytest.fixture(scope="session") +def run_parameters_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the run parameters fixture directory.""" + return Path(demultiplex_fixtures, "run_parameters") + + +@pytest.fixture(scope="session") +def demultiplexed_runs(demultiplex_fixtures: Path) -> Path: + """Return the path to the demultiplexed flow cells fixture directory.""" + return Path(demultiplex_fixtures, "demultiplexed-runs") + + +@pytest.fixture(scope="session") +def flow_cells_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the sequenced flow cells fixture directory.""" + return Path(demultiplex_fixtures, DemultiplexingDirsAndFiles.FLOW_CELLS_DIRECTORY_NAME) + + +@pytest.fixture(scope="session") +def nanopore_flow_cells_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the sequenced flow cells fixture directory.""" + return Path(demultiplex_fixtures, NanoporeDirsAndFiles.DATA_DIRECTORY) + + +@pytest.fixture(scope="session") +def flow_cells_demux_all_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the sequenced flow cells fixture directory.""" + return Path(demultiplex_fixtures, "flow_cells_demux_all") + + +@pytest.fixture(scope="session") +def demux_results_not_finished_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to a dir with demultiplexing results where nothing has been cleaned.""" + return Path(demultiplex_fixtures, "demultiplexed-runs-unfinished") + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: + return Path(tmp_flow_cells_directory, "230912_A00187_1009_AHK33MDRX3") + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_correct_sample_sheet( + novaseq_6000_post_1_5_kits_flow_cell: Path, +) -> Path: + return Path(novaseq_6000_post_1_5_kits_flow_cell, CORRECT_SAMPLE_SHEET) + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_raw_lims_samples( + novaseq_6000_post_1_5_kits_flow_cell: Path, +) -> Path: + return Path(novaseq_6000_post_1_5_kits_flow_cell, "HK33MDRX3_raw.json") + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: + return Path(tmp_flow_cells_directory, "190927_A00689_0069_BHLYWYDSXX") + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_correct_sample_sheet( + novaseq_6000_pre_1_5_kits_flow_cell: Path, +) -> Path: + return Path(novaseq_6000_pre_1_5_kits_flow_cell, CORRECT_SAMPLE_SHEET) + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_raw_lims_samples(novaseq_6000_pre_1_5_kits_flow_cell: Path) -> Path: + return Path(novaseq_6000_pre_1_5_kits_flow_cell, "HLYWYDSXX_raw.json") + + +@pytest.fixture +def novaseq_x_flow_cell_directory(tmp_flow_cells_directory: Path) -> Path: + return Path(tmp_flow_cells_directory, "20231108_LH00188_0028_B22F52TLT3") + + +@pytest.fixture +def novaseq_x_correct_sample_sheet(novaseq_x_flow_cell_directory: Path) -> Path: + return Path(novaseq_x_flow_cell_directory, CORRECT_SAMPLE_SHEET) + + +@pytest.fixture +def novaseq_x_raw_lims_samples(novaseq_x_flow_cell_directory: Path) -> Path: + return Path(novaseq_x_flow_cell_directory, "22F52TLT3_raw.json") + + +@pytest.fixture(scope="session") +def novaseq_x_manifest_file(novaseq_x_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeqX manifest file.""" + return Path(novaseq_x_flow_cell_dir, "Manifest.tsv") + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_x_single_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeqX flow cell.""" + return Path(flow_cells_dir, hiseq_x_single_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_x_dual_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeqX flow cell.""" + return Path(flow_cells_dir, hiseq_x_dual_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_2500_dual_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeq2500 flow cell.""" + return Path(flow_cells_dir, hiseq_2500_dual_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_2500_custom_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeq2500 flow cell.""" + return Path(flow_cells_dir, hiseq_2500_custom_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell_dir(flow_cells_dir: Path, bcl2fastq_flow_cell_full_name: str) -> Path: + """Return the path to the bcl2fastq flow cell demultiplex fixture directory.""" + return Path(flow_cells_dir, bcl2fastq_flow_cell_full_name) + + +@pytest.fixture(scope="session") +def bcl_convert_flow_cell_dir(flow_cells_dir: Path, bcl_convert_flow_cell_full_name: str) -> Path: + """Return the path to the bcl_convert flow cell demultiplex fixture directory.""" + return Path(flow_cells_dir, bcl_convert_flow_cell_full_name) + + +@pytest.fixture(scope="session") +def novaseq_x_flow_cell_dir(flow_cells_dir: Path, novaseq_x_flow_cell_full_name: str) -> Path: + """Return the path to the NovaSeqX flow cell demultiplex fixture directory.""" + return Path(flow_cells_dir, novaseq_x_flow_cell_full_name) + + +@pytest.fixture(scope="session") +def novaseq_bcl2fastq_sample_sheet_path(bcl2fastq_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeq6000 Bcl2fastq sample sheet.""" + return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) + + +@pytest.fixture(scope="session") +def novaseq_bcl_convert_sample_sheet_path(bcl_convert_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeq6000 bcl_convert sample sheet.""" + return Path(bcl_convert_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) + + +@pytest.fixture(scope="session") +def run_parameters_wrong_instrument(run_parameters_dir: Path) -> Path: + """Return a NovaSeqX run parameters file path with a wrong instrument value.""" + return Path(run_parameters_dir, "RunParameters_novaseq_X_wrong_instrument.xml") + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_run_parameters_path( + hiseq_x_single_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeqX run parameters file with single index.""" + return Path( + hiseq_x_single_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_run_parameters_path( + hiseq_x_dual_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeqX run parameters file with dual index.""" + return Path( + hiseq_x_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_run_parameters_path( + hiseq_2500_dual_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeq2500 run parameters file with dual index.""" + return Path( + hiseq_2500_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_run_parameters_path( + hiseq_2500_custom_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeq2500 run parameters file with custom index.""" + return Path( + hiseq_2500_custom_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def novaseq_6000_run_parameters_path(bcl2fastq_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeq6000 run parameters file.""" + return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) + + +@pytest.fixture +def novaseq_6000_run_parameters_pre_1_5_kits_path( + novaseq_6000_pre_1_5_kits_flow_cell: Path, +) -> Path: + """Return the path to a NovaSeq6000 pre 1.5 kit run parameters file.""" + return Path( + novaseq_6000_pre_1_5_kits_flow_cell, + DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, + ) + + +@pytest.fixture +def novaseq_6000_run_parameters_post_1_5_kits_path( + novaseq_6000_post_1_5_kits_flow_cell: Path, +) -> Path: + """Return the path to a NovaSeq6000 post 1.5 kit run parameters file.""" + return Path( + novaseq_6000_post_1_5_kits_flow_cell, + DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, + ) + + +@pytest.fixture(scope="session") +def novaseq_x_run_parameters_path(novaseq_x_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeqX run parameters file.""" + return Path(novaseq_x_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) + + +@pytest.fixture(scope="module") +def run_parameters_missing_versions_path( + run_parameters_dir: Path, +) -> Path: + """Return a NovaSeq6000 run parameters path without software and reagent kit versions.""" + return Path(run_parameters_dir, "RunParameters_novaseq_no_software_nor_reagent_version.xml") + + +@pytest.fixture(name="demultiplexing_delivery_file") +def demultiplexing_delivery_file(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: + """Return demultiplexing delivery started file.""" + return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.DELIVERY) + + +@pytest.fixture(name="hiseq_x_tile_dir") +def hiseq_x_tile_dir(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: + """Return HiSeqX tile dir.""" + return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.HISEQ_X_TILE_DIR) + + +@pytest.fixture(name="lims_novaseq_samples_file") +def lims_novaseq_samples_file(raw_lims_sample_dir: Path) -> Path: + """Return the path to a file with sample info in lims format.""" + return Path(raw_lims_sample_dir, "raw_samplesheet_novaseq.json") + + +@pytest.fixture +def lims_novaseq_6000_samples_file(bcl2fastq_flow_cell_dir: Path) -> Path: + """Return the path to the file with the raw samples of HVKJCDRXX flow cell in lims format.""" + return Path(bcl2fastq_flow_cell_dir, "HVKJCDRXX_raw.json") + + +@pytest.fixture(name="demultiplexed_flow_cell") +def demultiplexed_flow_cell(demultiplexed_runs: Path, bcl2fastq_flow_cell_full_name: str) -> Path: + """Return the path to a demultiplexed flow cell with bcl2fastq.""" + return Path(demultiplexed_runs, bcl2fastq_flow_cell_full_name) + + +@pytest.fixture(name="bcl_convert_demultiplexed_flow_cell") +def bcl_convert_demultiplexed_flow_cell( + demultiplexed_runs: Path, bcl_convert_flow_cell_full_name: str +) -> Path: + """Return the path to a demultiplexed flow cell with BCLConvert.""" + return Path(demultiplexed_runs, bcl_convert_flow_cell_full_name) + + +# Fixtures for test demultiplex flow cell +@pytest.fixture +def tmp_empty_demultiplexed_runs_directory(tmp_demultiplexed_runs_directory) -> Path: + return Path(tmp_demultiplexed_runs_directory, "empty") + + +@pytest.fixture(name="novaseqx_demultiplexed_flow_cell") +def novaseqx_demultiplexed_flow_cell(demultiplexed_runs: Path, novaseq_x_flow_cell_full_name: str): + """Return the path to a demultiplexed NovaSeqX flow cell.""" + return Path(demultiplexed_runs, novaseq_x_flow_cell_full_name) diff --git a/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py new file mode 100644 index 0000000000..e8c7459cec --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py @@ -0,0 +1,98 @@ +from pathlib import Path + +import pytest + +from cg.models.demultiplex.run_parameters import ( + RunParametersHiSeq, + RunParametersNovaSeq6000, + RunParametersNovaSeqX, +) + + +@pytest.fixture(scope="function") +def run_parameters_hiseq_different_index(run_parameters_dir: Path) -> RunParametersHiSeq: + """Return a HiSeq RunParameters object with different index cycles.""" + path = Path(run_parameters_dir, "RunParameters_hiseq_2500_different_index_cycles.xml") + return RunParametersHiSeq(run_parameters_path=path) + + +@pytest.fixture(scope="function") +def run_parameters_novaseq_6000_different_index( + run_parameters_dir: Path, +) -> RunParametersNovaSeq6000: + """Return a NovaSeq6000 RunParameters object with different index cycles.""" + path = Path(run_parameters_dir, "RunParameters_novaseq_6000_different_index_cycles.xml") + return RunParametersNovaSeq6000(run_parameters_path=path) + + +@pytest.fixture(scope="function") +def run_parameters_novaseq_x_different_index(run_parameters_dir: Path) -> RunParametersNovaSeqX: + """Return a NovaSeqX RunParameters object with different index cycles.""" + path = Path(run_parameters_dir, "RunParameters_novaseq_X_different_index_cycles.xml") + return RunParametersNovaSeqX(run_parameters_path=path) + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_run_parameters( + hiseq_x_single_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeqX run parameters object with single index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_x_single_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_run_parameters( + hiseq_x_dual_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeqX run parameters object with dual index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_x_dual_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_run_parameters( + hiseq_2500_dual_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeq2500 run parameters object with dual index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_2500_dual_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_run_parameters( + hiseq_2500_custom_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeq2500 run parameters object with custom index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_2500_custom_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def novaseq_6000_run_parameters( + novaseq_6000_run_parameters_path: Path, +) -> RunParametersNovaSeq6000: + """Return a NovaSeq6000 run parameters object.""" + return RunParametersNovaSeq6000(run_parameters_path=novaseq_6000_run_parameters_path) + + +@pytest.fixture +def novaseq_6000_run_parameters_pre_1_5_kits( + novaseq_6000_run_parameters_pre_1_5_kits_path: Path, +) -> RunParametersNovaSeq6000: + """Return a NovaSeq6000 run parameters pre 1.5 kit object.""" + return RunParametersNovaSeq6000( + run_parameters_path=novaseq_6000_run_parameters_pre_1_5_kits_path + ) + + +@pytest.fixture +def novaseq_6000_run_parameters_post_1_5_kits(novaseq_6000_run_parameters_post_1_5_kits_path: Path): + """Return a NovaSeq6000 run parameters post 1.5 kit object.""" + return RunParametersNovaSeq6000( + run_parameters_path=novaseq_6000_run_parameters_post_1_5_kits_path + ) + + +@pytest.fixture(scope="session") +def novaseq_x_run_parameters( + novaseq_x_run_parameters_path: Path, +) -> RunParametersNovaSeqX: + """Return a NovaSeqX run parameters object.""" + return RunParametersNovaSeqX(run_parameters_path=novaseq_x_run_parameters_path) diff --git a/tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py new file mode 100644 index 0000000000..9973ddf58e --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py @@ -0,0 +1,135 @@ +"""Demultiplex sample fixtures.""" +from pathlib import Path + +import pytest + +from cg.apps.demultiplex.sample_sheet.sample_models import ( + FlowCellSampleBcl2Fastq, + FlowCellSampleBCLConvert, +) +from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import SampleSheetCreatorBCLConvert +from cg.constants import FileExtensions +from cg.constants.constants import FileFormat +from cg.io.controller import ReadFile +from cg.io.json import read_json +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData + + +@pytest.fixture +def lims_novaseq_bcl_convert_samples( + lims_novaseq_samples_raw: list[dict], +) -> list[FlowCellSampleBCLConvert]: + """Return a list of parsed flow cell samples demultiplexed with BCL convert.""" + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in lims_novaseq_samples_raw] + + +@pytest.fixture +def lims_novaseq_bcl2fastq_samples( + lims_novaseq_samples_raw: list[dict], +) -> list[FlowCellSampleBcl2Fastq]: + """Return a list of parsed Bcl2fastq flow cell samples""" + return [FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_samples_raw] + + +@pytest.fixture +def lims_novaseq_6000_bcl2fastq_samples( + lims_novaseq_6000_sample_raw: list[dict], +) -> list[FlowCellSampleBcl2Fastq]: + """Return a list of parsed Bcl2fastq flow cell samples""" + return [ + FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_6000_sample_raw + ] + + +@pytest.fixture +def bcl_convert_sample_sheet_creator( + bcl_convert_flow_cell: FlowCellDirectoryData, + lims_novaseq_bcl_convert_samples: list[FlowCellSampleBCLConvert], +) -> SampleSheetCreatorBCLConvert: + """Returns a sample sheet creator for version 2 sample sheets with dragen format.""" + return SampleSheetCreatorBCLConvert( + flow_cell=bcl_convert_flow_cell, + lims_samples=lims_novaseq_bcl_convert_samples, + ) + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_lims_samples( + novaseq_6000_post_1_5_kits_raw_lims_samples: Path, +) -> list[FlowCellSampleBCLConvert]: + return [ + FlowCellSampleBCLConvert.model_validate(sample) + for sample in read_json(novaseq_6000_post_1_5_kits_raw_lims_samples) + ] + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_lims_samples( + novaseq_6000_pre_1_5_kits_raw_lims_samples: Path, +) -> list[FlowCellSampleBCLConvert]: + return [ + FlowCellSampleBCLConvert.model_validate(sample) + for sample in read_json(novaseq_6000_pre_1_5_kits_raw_lims_samples) + ] + + +@pytest.fixture +def novaseq_x_lims_samples(novaseq_x_raw_lims_samples: Path) -> list[FlowCellSampleBCLConvert]: + return [ + FlowCellSampleBCLConvert.model_validate(sample) + for sample in read_json(novaseq_x_raw_lims_samples) + ] + + +@pytest.fixture +def hiseq_x_single_index_bcl_convert_lims_samples( + hiseq_x_single_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeqX single index flow cell.""" + path = Path( + hiseq_x_single_index_flow_cell_dir, f"HJCFFALXX_bcl_convert_raw{FileExtensions.JSON}" + ) + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def hiseq_x_dual_index_bcl_convert_lims_samples( + hiseq_x_dual_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeqX dual index flow cell.""" + path = Path(hiseq_x_dual_index_flow_cell_dir, f"HL32LCCXY_bcl_convert_raw{FileExtensions.JSON}") + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def hiseq_2500_dual_index_bcl_convert_lims_samples( + hiseq_2500_dual_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeq2500 dual index flow cell.""" + path = Path(hiseq_2500_dual_index_flow_cell_dir, "HM2LNBCX2_bcl_convert_raw.json") + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def hiseq_2500_custom_index_bcl_convert_lims_samples( + hiseq_2500_custom_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeq2500 custom index flow cell.""" + path = Path(hiseq_2500_custom_index_flow_cell_dir, "HGYFNBCX2_bcl_convert_raw.json") + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def lims_novaseq_samples_raw(lims_novaseq_samples_file: Path) -> list[dict]: + """Return a list of raw flow cell samples.""" + return ReadFile.get_content_from_file( + file_format=FileFormat.JSON, file_path=lims_novaseq_samples_file + ) + + +@pytest.fixture +def lims_novaseq_6000_sample_raw(lims_novaseq_6000_samples_file: Path) -> list[dict]: + """Return the list of raw samples from flow cell HVKJCDRXX.""" + return ReadFile.get_content_from_file( + file_format=FileFormat.JSON, file_path=lims_novaseq_6000_samples_file + ) diff --git a/tests/fixture_plugins/timestamp_fixtures.py b/tests/fixture_plugins/timestamp_fixtures.py new file mode 100644 index 0000000000..6e77e289bb --- /dev/null +++ b/tests/fixture_plugins/timestamp_fixtures.py @@ -0,0 +1,46 @@ +"""Timestamp fixtures.""" +from datetime import MAXYEAR, datetime, timedelta + +import pytest + + +@pytest.fixture(scope="session") +def old_timestamp() -> datetime: + """Return a time stamp in date time format.""" + return datetime(1900, 1, 1) + + +@pytest.fixture(scope="session") +def timestamp() -> datetime: + """Return a time stamp in date time format.""" + return datetime(2020, 5, 1) + + +@pytest.fixture(scope="session") +def later_timestamp() -> datetime: + """Return a time stamp in date time format.""" + return datetime(2020, 6, 1) + + +@pytest.fixture(scope="session") +def future_date() -> datetime: + """Return a distant date in the future for which no events happen later.""" + return datetime(MAXYEAR, 1, 1, 1, 1, 1) + + +@pytest.fixture(scope="session") +def timestamp_now() -> datetime: + """Return a time stamp of today's date in date time format.""" + return datetime.now() + + +@pytest.fixture(scope="session") +def timestamp_yesterday(timestamp_now: datetime) -> datetime: + """Return a time stamp of yesterday's date in date time format.""" + return timestamp_now - timedelta(days=1) + + +@pytest.fixture(scope="session") +def timestamp_in_2_weeks(timestamp_now: datetime) -> datetime: + """Return a time stamp 14 days ahead in time.""" + return timestamp_now + timedelta(days=14) From 2172cdcc0766638f0a67f04bb5f5ef3f2391f685 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 11 Jan 2024 08:50:13 +0000 Subject: [PATCH 14/18] =?UTF-8?q?Bump=20version:=2055.1.1=20=E2=86=92=2055?= =?UTF-8?q?.2.0=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cefe97934f..98ccac2f77 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 55.1.1 +current_version = 55.2.0 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 1fc22e2fdb..1caaab50d0 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "55.1.1" +__version__ = "55.2.0" diff --git a/pyproject.toml b/pyproject.toml index 2f04e3c652..03c6563115 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "55.1.1" +version = "55.2.0" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 979e9cb31d2afe66f34a0b553a56b8cfcd1afaed Mon Sep 17 00:00:00 2001 From: Sebastian Allard Date: Thu, 11 Jan 2024 12:48:29 +0100 Subject: [PATCH 15/18] Deliver the correct microsalt files (#2800)(patch) --- cg/constants/delivery.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cg/constants/delivery.py b/cg/constants/delivery.py index c0ca4bd7cd..8504e9991e 100644 --- a/cg/constants/delivery.py +++ b/cg/constants/delivery.py @@ -124,14 +124,7 @@ {"salmon-quant"}, ] -MICROSALT_ANALYSIS_CASE_TAGS: list[set[str]] = [ - {"microsalt-qc"}, - {"microsalt-type"}, - {"assembly"}, - {"trimmed-forward-reads"}, - {"trimmed-reverse-reads"}, - {"reference-alignment-deduplicated"}, -] +MICROSALT_ANALYSIS_CASE_TAGS = [{"qc-report"}, {"typing-report"}] MICROSALT_ANALYSIS_SAMPLE_TAGS: list[set[str]] = [] From cfb771cb94a48ce4c9d0bf772bcb76c18543fff1 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 11 Jan 2024 11:48:56 +0000 Subject: [PATCH 16/18] =?UTF-8?q?Bump=20version:=2055.2.0=20=E2=86=92=2055?= =?UTF-8?q?.2.1=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 98ccac2f77..cf0dfe360d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 55.2.0 +current_version = 55.2.1 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 1caaab50d0..1250496ba7 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "55.2.0" +__version__ = "55.2.1" diff --git a/pyproject.toml b/pyproject.toml index 03c6563115..b4a7821ee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "55.2.0" +version = "55.2.1" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 62a1840fed46c584d5abf489f4d98c2627753d86 Mon Sep 17 00:00:00 2001 From: Sebastian Allard Date: Thu, 11 Jan 2024 14:30:56 +0100 Subject: [PATCH 17/18] Add microsalt upload command (#2823)(patch) --- cg/cli/upload/base.py | 17 +++++++------ .../upload/microsalt/microsalt_upload_api.py | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 cg/meta/upload/microsalt/microsalt_upload_api.py diff --git a/cg/cli/upload/base.py b/cg/cli/upload/base.py index e1cb57c8d3..c4aeb16e42 100644 --- a/cg/cli/upload/base.py +++ b/cg/cli/upload/base.py @@ -32,6 +32,7 @@ from cg.constants import Pipeline from cg.exc import AnalysisAlreadyUploadedError from cg.meta.upload.balsamic.balsamic import BalsamicUploadAPI +from cg.meta.upload.microsalt.microsalt_upload_api import MicrosaltUploadAPI from cg.meta.upload.mip.mip_dna import MipDNAUploadAPI from cg.meta.upload.mip.mip_rna import MipRNAUploadAPI from cg.meta.upload.rnafusion.rnafusion import RnafusionUploadAPI @@ -57,7 +58,7 @@ def upload(context: click.Context, case_id: str | None, restart: bool): """Upload results from analyses""" config_object: CGConfig = context.obj - upload_api: UploadAPI = MipDNAUploadAPI(config=config_object) # default upload API + upload_api: UploadAPI = MipDNAUploadAPI(config_object) LOG.info("----------------- UPLOAD -----------------") @@ -65,21 +66,21 @@ def upload(context: click.Context, case_id: str | None, restart: bool): context.obj.meta_apis["upload_api"] = upload_api elif case_id: # Provided case ID without a subcommand: upload everything try: - upload_api.analysis_api.status_db.verify_case_exists(case_internal_id=case_id) - case: Case = upload_api.status_db.get_case_by_internal_id(internal_id=case_id) + upload_api.analysis_api.status_db.verify_case_exists(case_id) + case: Case = upload_api.status_db.get_case_by_internal_id(case_id) upload_api.verify_analysis_upload(case_obj=case, restart=restart) except AnalysisAlreadyUploadedError: # Analysis being uploaded or it has been already uploaded return - # Update the upload API based on the data analysis type (MIP-DNA by default) - # Upload for balsamic, balsamic-umi and balsamic-qc if Pipeline.BALSAMIC in case.data_analysis: - upload_api = BalsamicUploadAPI(config=config_object) + upload_api = BalsamicUploadAPI(config_object) elif case.data_analysis == Pipeline.RNAFUSION: - upload_api = RnafusionUploadAPI(config=config_object) + upload_api = RnafusionUploadAPI(config_object) elif case.data_analysis == Pipeline.MIP_RNA: - upload_api: UploadAPI = MipRNAUploadAPI(config=config_object) + upload_api = MipRNAUploadAPI(config_object) + elif case.data_analysis == Pipeline.MICROSALT: + upload_api = MicrosaltUploadAPI(config_object) context.obj.meta_apis["upload_api"] = upload_api upload_api.upload(ctx=context, case=case, restart=restart) diff --git a/cg/meta/upload/microsalt/microsalt_upload_api.py b/cg/meta/upload/microsalt/microsalt_upload_api.py new file mode 100644 index 0000000000..b4f31a48ec --- /dev/null +++ b/cg/meta/upload/microsalt/microsalt_upload_api.py @@ -0,0 +1,25 @@ +import logging + +import click + +from cg.cli.upload.clinical_delivery import upload_clinical_delivery +from cg.meta.upload.upload_api import UploadAPI +from cg.meta.workflow.microsalt.microsalt import MicrosaltAnalysisAPI +from cg.models.cg_config import CGConfig +from cg.store.models import Analysis, Case + +LOG = logging.getLogger(__name__) + + +class MicrosaltUploadAPI(UploadAPI): + def __init__(self, config: CGConfig): + self.analysis_api = MicrosaltAnalysisAPI(config) + super().__init__(config=config, analysis_api=self.analysis_api) + + def upload(self, ctx: click.Context, case: Case, restart: bool) -> None: + """Uploads MIP-DNA analysis data and files.""" + analysis: Analysis = case.analyses[0] + self.update_upload_started_at(analysis) + + ctx.invoke(upload_clinical_delivery, case_id=case.internal_id) + self.update_uploaded_at(analysis=analysis) From 571fade658377cf531c7994eaaeb5e6e08a00c10 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 11 Jan 2024 13:31:25 +0000 Subject: [PATCH 18/18] =?UTF-8?q?Bump=20version:=2055.2.1=20=E2=86=92=2055?= =?UTF-8?q?.2.2=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cf0dfe360d..7a8f99accf 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 55.2.1 +current_version = 55.2.2 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 1250496ba7..9f890933d9 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "55.2.1" +__version__ = "55.2.2" diff --git a/pyproject.toml b/pyproject.toml index b4a7821ee9..f1e6fe26c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "55.2.1" +version = "55.2.2" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md"